From 82db23cf2a8996963dfb7b19ae25c152855cd614 Mon Sep 17 00:00:00 2001 From: Vacha Shah Date: Thu, 4 Jan 2024 03:02:10 +0000 Subject: [PATCH 01/36] Adding proto message structure for QuerySearchResult Signed-off-by: Vacha Shah --- server/build.gradle | 1 + .../search/QuerySearchResultProto.proto | 79 +++++++++++++++++++ .../search/ShardSearchRequestProto.proto | 75 ++++++++++++++++++ 3 files changed, 155 insertions(+) create mode 100644 server/src/main/proto/server/search/QuerySearchResultProto.proto create mode 100644 server/src/main/proto/server/search/ShardSearchRequestProto.proto diff --git a/server/build.gradle b/server/build.gradle index cb48142a61159..120aa7ffa45ea 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -364,6 +364,7 @@ tasks.named("licenseHeaders").configure { excludes << 'org/opensearch/client/documentation/placeholder.txt' // Ignore for protobuf generated code excludes << 'org/opensearch/extensions/proto/*' + excludes << 'org/opensearch/server/proto/*' } tasks.test { diff --git a/server/src/main/proto/server/search/QuerySearchResultProto.proto b/server/src/main/proto/server/search/QuerySearchResultProto.proto new file mode 100644 index 0000000000000..d4d8501a3c927 --- /dev/null +++ b/server/src/main/proto/server/search/QuerySearchResultProto.proto @@ -0,0 +1,79 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +syntax = "proto3"; +package org.opensearch.server.proto; + +import "server/search/ShardSearchRequestProto.proto"; + +option java_outer_classname = "QuerySearchResultProto"; + +message QuerySearchResult { + ShardSearchContextId contextId = 1; + optional int32 from = 2; + optional int32 size = 3; + optional TopDocsAndMaxScore topDocsAndMaxScore = 4; + optional bool hasScoreDocs = 5; + optional TotalHits totalHits = 6; + optional float maxScore = 7; + optional TopDocs topDocs = 8; + optional bool hasAggs = 9; + optional bool hasSuggest = 10; + optional bool searchTimedOut = 11; + optional bool terminatedEarly = 12; + optional bytes profileShardResults = 13; + optional int64 serviceTimeEWMA = 14; + optional int32 nodeQueueSize = 15; + ShardSearchRequest shardSearchRequest = 16; + SearchShardTarget searchShardTarget = 17; + + + message TopDocsAndMaxScore { + TopDocs topDocs = 1; + float maxScore = 2; + } + + message TopDocs { + TotalHits totalHits = 1; + repeated ScoreDoc scoreDocs = 2; + + message ScoreDoc { + int32 doc = 1; + float score = 2; + int32 shardIndex = 3; + } + } + + message RescoreDocIds { + map docIds = 1; + + message setInteger { + repeated int32 values = 1; + } + } + +} + +message SearchShardTarget { + string nodeId = 1; + ShardId shardId = 2; + string clusterAlias = 3; +} + +message TotalHits { + int64 value = 1; + Relation relation = 2; + + enum Relation { + EQUAL_TO = 0; + GREATER_THAN_OR_EQUAL_TO = 1; + } +} diff --git a/server/src/main/proto/server/search/ShardSearchRequestProto.proto b/server/src/main/proto/server/search/ShardSearchRequestProto.proto new file mode 100644 index 0000000000000..633df85fcf1d8 --- /dev/null +++ b/server/src/main/proto/server/search/ShardSearchRequestProto.proto @@ -0,0 +1,75 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +syntax = "proto3"; +package org.opensearch.server.proto; + +option java_outer_classname = "ShardSearchRequestProto"; + +message ShardSearchRequest { + OriginalIndices originalIndices = 1; + ShardId shardId = 2; + int32 numberOfShards = 3; + SearchType searchType = 4; + bytes source = 5; + bool requestCache = 6; + bytes aliasFilter = 7; + float indexBoost = 8; + bool allowPartialSearchResults = 9; + repeated string indexRoutings = 10; + string preference = 11; + Scroll scroll = 12; + int64 nowInMillis = 13; + optional string clusterAlias = 14; + optional ShardSearchContextId readerId = 15; + optional string timeValue = 16; + int64 inboundNetworkTime = 17; + int64 outboundNetworkTime = 18; + bool canReturnNullResponseIfMatchNoDocs = 19; + + enum SearchType { + QUERY_THEN_FETCH = 0; + DFS_QUERY_THEN_FETCH = 1; + } +} + +message ShardSearchContextId { + string sessionId = 1; + int64 id = 2; +} + +message ShardId { + int32 shardId = 1; + int32 hashCode = 2; + string indexName = 3; + string indexUUID = 4; +} + +message Scroll { + string keepAlive = 1; +} + +message OriginalIndices { + repeated string indices = 1; + IndicesOptions indicesOptions = 2; + + message IndicesOptions { + bool ignoreUnavailable = 1; + bool allowNoIndices = 2; + bool expandWildcardsOpen = 3; + bool expandWildcardsClosed = 4; + bool expandWildcardsHidden = 5; + bool allowAliasesToMultipleIndices = 6; + bool forbidClosedIndices = 7; + bool ignoreAliases = 8; + bool ignoreThrottled = 9; + } +} From fe1fd9061d6118d31a580316dabc46e622adecb9 Mon Sep 17 00:00:00 2001 From: Vacha Shah Date: Thu, 4 Jan 2024 06:36:50 +0000 Subject: [PATCH 02/36] Adding ProtobufWriteable interface and integrating with transport requests and responses Signed-off-by: Vacha Shah --- .../common/io/stream/ProtobufWriteable.java | 69 +++++++++++++++++++ .../core/transport/TransportMessage.java | 9 ++- .../core/transport/TransportResponse.java | 22 ++++++ .../transport/TransportRequest.java | 16 +++++ 4 files changed, 115 insertions(+), 1 deletion(-) create mode 100644 libs/core/src/main/java/org/opensearch/core/common/io/stream/ProtobufWriteable.java diff --git a/libs/core/src/main/java/org/opensearch/core/common/io/stream/ProtobufWriteable.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/ProtobufWriteable.java new file mode 100644 index 0000000000000..da8ba03ccaaa8 --- /dev/null +++ b/libs/core/src/main/java/org/opensearch/core/common/io/stream/ProtobufWriteable.java @@ -0,0 +1,69 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.core.common.io.stream; + +import org.opensearch.common.annotation.ExperimentalApi; + +import java.io.IOException; +import java.io.OutputStream; + +/** + * Implementers can be written to a {@linkplain OutputStream} and read from a byte array. This allows them to be "thrown + * across the wire" using OpenSearch's internal protocol with protobuf bytes. + * + * @opensearch.api + */ +@ExperimentalApi +public interface ProtobufWriteable { + + /** + * Write this into the {@linkplain OutputStream}. + */ + void writeTo(OutputStream out) throws IOException; + + /** + * Reference to a method that can write some object to a {@link OutputStream}. + * + * @opensearch.experimental + */ + @FunctionalInterface + @ExperimentalApi + interface Writer { + + /** + * Write {@code V}-type {@code value} to the {@code out}put stream. + * + * @param out Output to write the {@code value} too + * @param value The value to add + */ + void write(final OutputStream out, V value) throws IOException; + } + + /** + * Reference to a method that can read some object from a byte array. + * + * @opensearch.experimental + */ + @FunctionalInterface + @ExperimentalApi + interface Reader { + + /** + * Read {@code V}-type value from a byte array. + * + * @param in byte array to read the value from + */ + V read(final byte[] in) throws IOException; + } +} diff --git a/libs/core/src/main/java/org/opensearch/core/transport/TransportMessage.java b/libs/core/src/main/java/org/opensearch/core/transport/TransportMessage.java index 941babda40aa3..69be6cbecc96a 100644 --- a/libs/core/src/main/java/org/opensearch/core/transport/TransportMessage.java +++ b/libs/core/src/main/java/org/opensearch/core/transport/TransportMessage.java @@ -32,6 +32,7 @@ package org.opensearch.core.transport; +import org.opensearch.core.common.io.stream.ProtobufWriteable; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.common.transport.TransportAddress; @@ -41,7 +42,7 @@ * * @opensearch.internal */ -public abstract class TransportMessage implements Writeable { +public abstract class TransportMessage implements Writeable, ProtobufWriteable { private TransportAddress remoteAddress; @@ -63,4 +64,10 @@ public TransportMessage() {} * currently a no-op */ public TransportMessage(StreamInput in) {} + + /** + * Constructs a new transport message with the data from the byte array. This is + * currently a no-op + */ + public TransportMessage(byte[] in) {} } diff --git a/libs/core/src/main/java/org/opensearch/core/transport/TransportResponse.java b/libs/core/src/main/java/org/opensearch/core/transport/TransportResponse.java index 4ae01e140a89c..2537997c239b3 100644 --- a/libs/core/src/main/java/org/opensearch/core/transport/TransportResponse.java +++ b/libs/core/src/main/java/org/opensearch/core/transport/TransportResponse.java @@ -37,6 +37,7 @@ import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; +import java.io.OutputStream; /** * Response over the transport interface @@ -60,6 +61,24 @@ public TransportResponse(StreamInput in) throws IOException { super(in); } + /** + * Constructs a new transport response with the data from the byte array. This is + * currently a no-op. However, this exists to allow extenders to call super(in) + * so that reading can mirror writing where we often call super.writeTo(out). + */ + public TransportResponse(byte[] in) throws IOException { + super(in); + } + + /** + * Writes this response to the {@linkplain OutputStream}. This is added here so that classes + * don't have to implement since it is an experimental feature and only being added for + * search apis incrementally. + */ + public void writeTo(OutputStream out) throws IOException { + // no-op + } + /** * Empty transport response * @@ -75,5 +94,8 @@ public String toString() { @Override public void writeTo(StreamOutput out) throws IOException {} + + @Override + public void writeTo(OutputStream out) throws IOException {} } } diff --git a/server/src/main/java/org/opensearch/transport/TransportRequest.java b/server/src/main/java/org/opensearch/transport/TransportRequest.java index c62cf59d3be2f..fb5d6fd858eb4 100644 --- a/server/src/main/java/org/opensearch/transport/TransportRequest.java +++ b/server/src/main/java/org/opensearch/transport/TransportRequest.java @@ -40,6 +40,7 @@ import org.opensearch.tasks.TaskAwareRequest; import java.io.IOException; +import java.io.OutputStream; /** * A transport request @@ -61,6 +62,10 @@ public Empty() {} public Empty(StreamInput in) throws IOException { super(in); } + + public Empty(byte[] in) throws IOException { + super(in); + } } /** @@ -74,6 +79,11 @@ public TransportRequest(StreamInput in) throws IOException { parentTaskId = TaskId.readFromStream(in); } + /** + * This is added here so that classes don't have to implement since it is an experimental feature and only being added for search apis incrementally. + */ + public TransportRequest(byte[] in) throws IOException {} + /** * Set a reference to task that created this request. */ @@ -94,4 +104,10 @@ public TaskId getParentTask() { public void writeTo(StreamOutput out) throws IOException { parentTaskId.writeTo(out); } + + /** + * This is added here so that classes don't have to implement since it is an experimental feature and only being added for search apis incrementally. + */ + @Override + public void writeTo(OutputStream out) throws IOException {} } From 931e1d9d658d0129d87feb296497ef497b4ee62d Mon Sep 17 00:00:00 2001 From: Vacha Shah Date: Fri, 5 Jan 2024 06:07:18 +0000 Subject: [PATCH 03/36] Adding experimental feature flag for protobuf integration Signed-off-by: Vacha Shah --- .../common/settings/FeatureFlagSettings.java | 3 ++- .../org/opensearch/common/util/FeatureFlags.java | 12 ++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java b/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java index 985eb40711e16..832138739d44b 100644 --- a/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java @@ -36,6 +36,7 @@ protected FeatureFlagSettings( FeatureFlags.DATETIME_FORMATTER_CACHING_SETTING, FeatureFlags.WRITEABLE_REMOTE_INDEX_SETTING, FeatureFlags.REMOTE_STORE_MIGRATION_EXPERIMENTAL_SETTING, - FeatureFlags.PLUGGABLE_CACHE_SETTING + FeatureFlags.PLUGGABLE_CACHE_SETTING, + FeatureFlags.PROTOBUF_SETTING ); } diff --git a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java index bdfce72d106d3..cfbd68a349e33 100644 --- a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java +++ b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java @@ -61,6 +61,16 @@ public class FeatureFlags { */ public static final String WRITEABLE_REMOTE_INDEX = "opensearch.experimental.feature.writeable_remote_index.enabled"; + /** + * Gates the optimization to enable bloom filters for doc id lookup. + */ + public static final String DOC_ID_FUZZY_SET = "opensearch.experimental.optimize_doc_id_lookup.fuzzy_set.enabled"; + + /** + * Gates the functionality of integrating protobuf within search API and node-to-node communication. + */ + public static final String PROTOBUF = "opensearch.experimental.feature.protobuf.enabled"; + /** * Gates the functionality of pluggable cache. * Enables OpenSearch to use pluggable caches with respective store names via setting. @@ -155,4 +165,6 @@ public static boolean isEnabled(Setting featureFlag) { return featureFlag.getDefault(Settings.EMPTY); } } + + public static final Setting PROTOBUF_SETTING = Setting.boolSetting(PROTOBUF, false, Property.NodeScope); } From 7bbf23a6a2a93987e3b5f68d9c16b7db426fc41b Mon Sep 17 00:00:00 2001 From: Vacha Shah Date: Tue, 9 Jan 2024 02:52:57 +0000 Subject: [PATCH 04/36] Removing ShardSearchRequest from proto and modifying QuerySearchResult Signed-off-by: Vacha Shah --- .../opensearch/search/SearchPhaseResult.java | 4 + .../search/internal/ShardSearchRequest.java | 2 +- .../search/query/QuerySearchResult.java | 66 ++++++++++++++++ .../search/QuerySearchResultProto.proto | 15 +++- .../search/ShardSearchRequestProto.proto | 75 ------------------- 5 files changed, 83 insertions(+), 79 deletions(-) delete mode 100644 server/src/main/proto/server/search/ShardSearchRequestProto.proto diff --git a/server/src/main/java/org/opensearch/search/SearchPhaseResult.java b/server/src/main/java/org/opensearch/search/SearchPhaseResult.java index a351b3bd2dda6..bac518079da63 100644 --- a/server/src/main/java/org/opensearch/search/SearchPhaseResult.java +++ b/server/src/main/java/org/opensearch/search/SearchPhaseResult.java @@ -71,6 +71,10 @@ protected SearchPhaseResult(StreamInput in) throws IOException { super(in); } + protected SearchPhaseResult(byte[] in) throws IOException { + super(in); + } + /** * Returns the search context ID that is used to reference the search context on the executing node * or null if no context was created. diff --git a/server/src/main/java/org/opensearch/search/internal/ShardSearchRequest.java b/server/src/main/java/org/opensearch/search/internal/ShardSearchRequest.java index de1d5fb8b4098..ec470343f6cb2 100644 --- a/server/src/main/java/org/opensearch/search/internal/ShardSearchRequest.java +++ b/server/src/main/java/org/opensearch/search/internal/ShardSearchRequest.java @@ -99,7 +99,7 @@ public class ShardSearchRequest extends TransportRequest implements IndicesReque private final long nowInMillis; private long inboundNetworkTime; private long outboundNetworkTime; - private final boolean allowPartialSearchResults; + private final Boolean allowPartialSearchResults; private final String[] indexRoutings; private final String preference; private final OriginalIndices originalIndices; diff --git a/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java b/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java index f3ac953ab9d1d..f7345672a4d3c 100644 --- a/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java +++ b/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java @@ -33,6 +33,7 @@ package org.opensearch.search.query; import org.apache.lucene.search.FieldDoc; +import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TotalHits; import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.io.stream.DelayableWriteable; @@ -50,8 +51,11 @@ import org.opensearch.search.profile.NetworkTime; import org.opensearch.search.profile.ProfileShardResult; import org.opensearch.search.suggest.Suggest; +import org.opensearch.server.proto.QuerySearchResultProto; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import static org.opensearch.common.lucene.Lucene.readTopDocs; import static org.opensearch.common.lucene.Lucene.writeTopDocs; @@ -90,6 +94,8 @@ public final class QuerySearchResult extends SearchPhaseResult { private final boolean isNull; + private QuerySearchResultProto.QuerySearchResult querySearchResultProto; + public QuerySearchResult() { this(false); } @@ -103,11 +109,38 @@ public QuerySearchResult(StreamInput in) throws IOException { } } + public QuerySearchResult(byte[] in) throws IOException { + super(in); + this.querySearchResultProto = QuerySearchResultProto.QuerySearchResult.parseFrom(in); + isNull = false; + } + public QuerySearchResult(ShardSearchContextId contextId, SearchShardTarget shardTarget, ShardSearchRequest shardSearchRequest) { this.contextId = contextId; setSearchShardTarget(shardTarget); isNull = false; setShardSearchRequest(shardSearchRequest); + + QuerySearchResultProto.ShardId shardIdProto = QuerySearchResultProto.ShardId.newBuilder() + .setShardId(shardTarget.getShardId().getId()) + .setHashCode(shardTarget.getShardId().hashCode()) + .setIndexName(shardTarget.getShardId().getIndexName()) + .setIndexUUID(shardTarget.getShardId().getIndex().getUUID()) + .build(); + QuerySearchResultProto.SearchShardTarget searchShardTarget = QuerySearchResultProto.SearchShardTarget.newBuilder() + .setNodeId(shardTarget.getNodeId()) + .setShardId(shardIdProto) + .setClusterAlias(shardTarget.getClusterAlias()) + .build(); + this.querySearchResultProto = QuerySearchResultProto.QuerySearchResult.newBuilder() + .setContextId( + QuerySearchResultProto.ShardSearchContextId.newBuilder() + .setSessionId(contextId.getSessionId()) + .setId(contextId.getId()) + .build() + ) + .setSearchShardTarget(searchShardTarget) + .build(); } private QuerySearchResult(boolean isNull) { @@ -201,6 +234,39 @@ private void setTopDocs(TopDocsAndMaxScore topDocsAndMaxScore) { this.totalHits = topDocsAndMaxScore.topDocs.totalHits; this.maxScore = topDocsAndMaxScore.maxScore; this.hasScoreDocs = topDocsAndMaxScore.topDocs.scoreDocs.length > 0; + + List scoreDocs = new ArrayList<>(); + if (this.hasScoreDocs) { + for (ScoreDoc scoreDoc : topDocsAndMaxScore.topDocs.scoreDocs) { + scoreDocs.add( + QuerySearchResultProto.QuerySearchResult.TopDocs.ScoreDoc.newBuilder() + .setDoc(scoreDoc.doc) + .setScore(scoreDoc.score) + .setShardIndex(scoreDoc.shardIndex) + .build() + ); + } + } + QuerySearchResultProto.QuerySearchResult.TopDocs topDocsBuilder = QuerySearchResultProto.QuerySearchResult.TopDocs.newBuilder() + .setTotalHits( + QuerySearchResultProto.TotalHits.newBuilder() + .setValue(topDocsAndMaxScore.topDocs.totalHits.value) + .setRelation(QuerySearchResultProto.TotalHits.Relation.valueOf(topDocsAndMaxScore.topDocs.totalHits.relation.name())) + .build() + ) + .addAllScoreDocs(scoreDocs) + .build(); + QuerySearchResultProto.QuerySearchResult.TopDocsAndMaxScore topDocsAndMaxScoreBuilder = + QuerySearchResultProto.QuerySearchResult.TopDocsAndMaxScore.newBuilder() + .setMaxScore(topDocsAndMaxScore.maxScore) + .setTopDocs(topDocsBuilder) + .build(); + this.querySearchResultProto.toBuilder() + .setTopDocsAndMaxScore(topDocsAndMaxScoreBuilder) + .setMaxScore(this.maxScore) + .setTotalHits(topDocsBuilder.getTotalHits()) + .setHasScoreDocs(this.hasScoreDocs) + .build(); } public DocValueFormat[] sortValueFormats() { diff --git a/server/src/main/proto/server/search/QuerySearchResultProto.proto b/server/src/main/proto/server/search/QuerySearchResultProto.proto index d4d8501a3c927..8caf561bf2bdd 100644 --- a/server/src/main/proto/server/search/QuerySearchResultProto.proto +++ b/server/src/main/proto/server/search/QuerySearchResultProto.proto @@ -12,8 +12,6 @@ syntax = "proto3"; package org.opensearch.server.proto; -import "server/search/ShardSearchRequestProto.proto"; - option java_outer_classname = "QuerySearchResultProto"; message QuerySearchResult { @@ -32,7 +30,6 @@ message QuerySearchResult { optional bytes profileShardResults = 13; optional int64 serviceTimeEWMA = 14; optional int32 nodeQueueSize = 15; - ShardSearchRequest shardSearchRequest = 16; SearchShardTarget searchShardTarget = 17; @@ -77,3 +74,15 @@ message TotalHits { GREATER_THAN_OR_EQUAL_TO = 1; } } + +message ShardSearchContextId { + string sessionId = 1; + int64 id = 2; +} + +message ShardId { + int32 shardId = 1; + int32 hashCode = 2; + string indexName = 3; + string indexUUID = 4; +} diff --git a/server/src/main/proto/server/search/ShardSearchRequestProto.proto b/server/src/main/proto/server/search/ShardSearchRequestProto.proto deleted file mode 100644 index 633df85fcf1d8..0000000000000 --- a/server/src/main/proto/server/search/ShardSearchRequestProto.proto +++ /dev/null @@ -1,75 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -syntax = "proto3"; -package org.opensearch.server.proto; - -option java_outer_classname = "ShardSearchRequestProto"; - -message ShardSearchRequest { - OriginalIndices originalIndices = 1; - ShardId shardId = 2; - int32 numberOfShards = 3; - SearchType searchType = 4; - bytes source = 5; - bool requestCache = 6; - bytes aliasFilter = 7; - float indexBoost = 8; - bool allowPartialSearchResults = 9; - repeated string indexRoutings = 10; - string preference = 11; - Scroll scroll = 12; - int64 nowInMillis = 13; - optional string clusterAlias = 14; - optional ShardSearchContextId readerId = 15; - optional string timeValue = 16; - int64 inboundNetworkTime = 17; - int64 outboundNetworkTime = 18; - bool canReturnNullResponseIfMatchNoDocs = 19; - - enum SearchType { - QUERY_THEN_FETCH = 0; - DFS_QUERY_THEN_FETCH = 1; - } -} - -message ShardSearchContextId { - string sessionId = 1; - int64 id = 2; -} - -message ShardId { - int32 shardId = 1; - int32 hashCode = 2; - string indexName = 3; - string indexUUID = 4; -} - -message Scroll { - string keepAlive = 1; -} - -message OriginalIndices { - repeated string indices = 1; - IndicesOptions indicesOptions = 2; - - message IndicesOptions { - bool ignoreUnavailable = 1; - bool allowNoIndices = 2; - bool expandWildcardsOpen = 3; - bool expandWildcardsClosed = 4; - bool expandWildcardsHidden = 5; - bool allowAliasesToMultipleIndices = 6; - bool forbidClosedIndices = 7; - bool ignoreAliases = 8; - bool ignoreThrottled = 9; - } -} From 044b07f19edc1ea436d9ed0ef23743ca606c7c77 Mon Sep 17 00:00:00 2001 From: Vacha Shah Date: Fri, 12 Jan 2024 02:16:11 +0000 Subject: [PATCH 05/36] Making protobuf feature flag dynamic Signed-off-by: Vacha Shah --- .../opensearch/common/util/FeatureFlags.java | 4 +- .../search/query/QuerySearchResult.java | 105 +++++++++--------- 2 files changed, 57 insertions(+), 52 deletions(-) diff --git a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java index cfbd68a349e33..7b009772b6acc 100644 --- a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java +++ b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java @@ -69,7 +69,7 @@ public class FeatureFlags { /** * Gates the functionality of integrating protobuf within search API and node-to-node communication. */ - public static final String PROTOBUF = "opensearch.experimental.feature.protobuf.enabled"; + public static final String PROTOBUF = "opensearch.experimental.feature.search_with_protobuf.enabled"; /** * Gates the functionality of pluggable cache. @@ -166,5 +166,5 @@ public static boolean isEnabled(Setting featureFlag) { } } - public static final Setting PROTOBUF_SETTING = Setting.boolSetting(PROTOBUF, false, Property.NodeScope); + public static final Setting PROTOBUF_SETTING = Setting.boolSetting(PROTOBUF, false, Property.NodeScope, Property.Dynamic); } diff --git a/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java b/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java index f7345672a4d3c..9e87597610462 100644 --- a/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java +++ b/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java @@ -38,6 +38,7 @@ import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.io.stream.DelayableWriteable; import org.opensearch.common.lucene.search.TopDocsAndMaxScore; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.search.DocValueFormat; @@ -121,26 +122,28 @@ public QuerySearchResult(ShardSearchContextId contextId, SearchShardTarget shard isNull = false; setShardSearchRequest(shardSearchRequest); - QuerySearchResultProto.ShardId shardIdProto = QuerySearchResultProto.ShardId.newBuilder() - .setShardId(shardTarget.getShardId().getId()) - .setHashCode(shardTarget.getShardId().hashCode()) - .setIndexName(shardTarget.getShardId().getIndexName()) - .setIndexUUID(shardTarget.getShardId().getIndex().getUUID()) - .build(); - QuerySearchResultProto.SearchShardTarget searchShardTarget = QuerySearchResultProto.SearchShardTarget.newBuilder() - .setNodeId(shardTarget.getNodeId()) - .setShardId(shardIdProto) - .setClusterAlias(shardTarget.getClusterAlias()) - .build(); - this.querySearchResultProto = QuerySearchResultProto.QuerySearchResult.newBuilder() - .setContextId( - QuerySearchResultProto.ShardSearchContextId.newBuilder() - .setSessionId(contextId.getSessionId()) - .setId(contextId.getId()) - .build() - ) - .setSearchShardTarget(searchShardTarget) - .build(); + if (FeatureFlags.isEnabled(FeatureFlags.PROTOBUF_SETTING)) { + QuerySearchResultProto.ShardId shardIdProto = QuerySearchResultProto.ShardId.newBuilder() + .setShardId(shardTarget.getShardId().getId()) + .setHashCode(shardTarget.getShardId().hashCode()) + .setIndexName(shardTarget.getShardId().getIndexName()) + .setIndexUUID(shardTarget.getShardId().getIndex().getUUID()) + .build(); + QuerySearchResultProto.SearchShardTarget searchShardTarget = QuerySearchResultProto.SearchShardTarget.newBuilder() + .setNodeId(shardTarget.getNodeId()) + .setShardId(shardIdProto) + .setClusterAlias(shardTarget.getClusterAlias()) + .build(); + this.querySearchResultProto = QuerySearchResultProto.QuerySearchResult.newBuilder() + .setContextId( + QuerySearchResultProto.ShardSearchContextId.newBuilder() + .setSessionId(contextId.getSessionId()) + .setId(contextId.getId()) + .build() + ) + .setSearchShardTarget(searchShardTarget) + .build(); + } } private QuerySearchResult(boolean isNull) { @@ -235,38 +238,40 @@ private void setTopDocs(TopDocsAndMaxScore topDocsAndMaxScore) { this.maxScore = topDocsAndMaxScore.maxScore; this.hasScoreDocs = topDocsAndMaxScore.topDocs.scoreDocs.length > 0; - List scoreDocs = new ArrayList<>(); - if (this.hasScoreDocs) { - for (ScoreDoc scoreDoc : topDocsAndMaxScore.topDocs.scoreDocs) { - scoreDocs.add( - QuerySearchResultProto.QuerySearchResult.TopDocs.ScoreDoc.newBuilder() - .setDoc(scoreDoc.doc) - .setScore(scoreDoc.score) - .setShardIndex(scoreDoc.shardIndex) - .build() - ); + if (FeatureFlags.isEnabled(FeatureFlags.PROTOBUF_SETTING)) { + List scoreDocs = new ArrayList<>(); + if (this.hasScoreDocs) { + for (ScoreDoc scoreDoc : topDocsAndMaxScore.topDocs.scoreDocs) { + scoreDocs.add( + QuerySearchResultProto.QuerySearchResult.TopDocs.ScoreDoc.newBuilder() + .setDoc(scoreDoc.doc) + .setScore(scoreDoc.score) + .setShardIndex(scoreDoc.shardIndex) + .build() + ); + } } - } - QuerySearchResultProto.QuerySearchResult.TopDocs topDocsBuilder = QuerySearchResultProto.QuerySearchResult.TopDocs.newBuilder() - .setTotalHits( - QuerySearchResultProto.TotalHits.newBuilder() - .setValue(topDocsAndMaxScore.topDocs.totalHits.value) - .setRelation(QuerySearchResultProto.TotalHits.Relation.valueOf(topDocsAndMaxScore.topDocs.totalHits.relation.name())) - .build() - ) - .addAllScoreDocs(scoreDocs) - .build(); - QuerySearchResultProto.QuerySearchResult.TopDocsAndMaxScore topDocsAndMaxScoreBuilder = - QuerySearchResultProto.QuerySearchResult.TopDocsAndMaxScore.newBuilder() - .setMaxScore(topDocsAndMaxScore.maxScore) - .setTopDocs(topDocsBuilder) + QuerySearchResultProto.QuerySearchResult.TopDocs topDocsBuilder = QuerySearchResultProto.QuerySearchResult.TopDocs.newBuilder() + .setTotalHits( + QuerySearchResultProto.TotalHits.newBuilder() + .setValue(topDocsAndMaxScore.topDocs.totalHits.value) + .setRelation(QuerySearchResultProto.TotalHits.Relation.valueOf(topDocsAndMaxScore.topDocs.totalHits.relation.name())) + .build() + ) + .addAllScoreDocs(scoreDocs) .build(); - this.querySearchResultProto.toBuilder() - .setTopDocsAndMaxScore(topDocsAndMaxScoreBuilder) - .setMaxScore(this.maxScore) - .setTotalHits(topDocsBuilder.getTotalHits()) - .setHasScoreDocs(this.hasScoreDocs) - .build(); + QuerySearchResultProto.QuerySearchResult.TopDocsAndMaxScore topDocsAndMaxScoreBuilder = + QuerySearchResultProto.QuerySearchResult.TopDocsAndMaxScore.newBuilder() + .setMaxScore(topDocsAndMaxScore.maxScore) + .setTopDocs(topDocsBuilder) + .build(); + this.querySearchResultProto.toBuilder() + .setTopDocsAndMaxScore(topDocsAndMaxScoreBuilder) + .setMaxScore(this.maxScore) + .setTotalHits(topDocsBuilder.getTotalHits()) + .setHasScoreDocs(this.hasScoreDocs) + .build(); + } } public DocValueFormat[] sortValueFormats() { From b198451a32990b4db9de40d5bb297f6202d8c86d Mon Sep 17 00:00:00 2001 From: Vacha Shah Date: Sun, 14 Jan 2024 00:52:52 +0000 Subject: [PATCH 06/36] Merging latest changes from main Signed-off-by: Vacha Shah --- .../java/org/opensearch/search/query/QuerySearchResult.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java b/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java index 9e87597610462..a6c79e0f06f60 100644 --- a/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java +++ b/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java @@ -255,7 +255,9 @@ private void setTopDocs(TopDocsAndMaxScore topDocsAndMaxScore) { .setTotalHits( QuerySearchResultProto.TotalHits.newBuilder() .setValue(topDocsAndMaxScore.topDocs.totalHits.value) - .setRelation(QuerySearchResultProto.TotalHits.Relation.valueOf(topDocsAndMaxScore.topDocs.totalHits.relation.name())) + .setRelation( + QuerySearchResultProto.TotalHits.Relation.valueOf(topDocsAndMaxScore.topDocs.totalHits.relation.name()) + ) .build() ) .addAllScoreDocs(scoreDocs) From 4f569c0d36b883b5e78121d6f49276b88b1628c4 Mon Sep 17 00:00:00 2001 From: Vacha Shah Date: Sun, 14 Jan 2024 06:52:19 +0000 Subject: [PATCH 07/36] Adding feature flag to cluster settings Signed-off-by: Vacha Shah --- .../opensearch/common/settings/ClusterSettings.java | 4 +++- .../org/opensearch/search/DefaultSearchContext.java | 2 +- .../opensearch/search/query/QuerySearchResult.java | 12 +++++++++--- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index 9d763c970c3e7..83b2186fa6cee 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -750,6 +750,8 @@ public void apply(Settings value, Settings current, Settings previous) { TelemetrySettings.METRICS_FEATURE_ENABLED_SETTING ), List.of(FeatureFlags.PLUGGABLE_CACHE), - List.of(CacheSettings.getConcreteStoreNameSettingForCacheType(CacheType.INDICES_REQUEST_CACHE)) + List.of(CacheSettings.getConcreteStoreNameSettingForCacheType(CacheType.INDICES_REQUEST_CACHE)), + List.of(FeatureFlags.PROTOBUF), + List.of(FeatureFlags.PROTOBUF_SETTING) ); } diff --git a/server/src/main/java/org/opensearch/search/DefaultSearchContext.java b/server/src/main/java/org/opensearch/search/DefaultSearchContext.java index c76ea71c0a094..36be1fb2936ec 100644 --- a/server/src/main/java/org/opensearch/search/DefaultSearchContext.java +++ b/server/src/main/java/org/opensearch/search/DefaultSearchContext.java @@ -211,7 +211,7 @@ final class DefaultSearchContext extends SearchContext { // SearchContexts use a BigArrays that can circuit break this.bigArrays = bigArrays.withCircuitBreaking(); this.dfsResult = new DfsSearchResult(readerContext.id(), shardTarget, request); - this.queryResult = new QuerySearchResult(readerContext.id(), shardTarget, request); + this.queryResult = new QuerySearchResult(readerContext.id(), shardTarget, request, clusterService.getSettings()); this.fetchResult = new FetchSearchResult(readerContext.id(), shardTarget); this.indexService = readerContext.indexService(); this.indexShard = readerContext.indexShard(); diff --git a/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java b/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java index a6c79e0f06f60..af2ae612772a3 100644 --- a/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java +++ b/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java @@ -38,6 +38,7 @@ import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.io.stream.DelayableWriteable; import org.opensearch.common.lucene.search.TopDocsAndMaxScore; +import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; @@ -116,13 +117,18 @@ public QuerySearchResult(byte[] in) throws IOException { isNull = false; } - public QuerySearchResult(ShardSearchContextId contextId, SearchShardTarget shardTarget, ShardSearchRequest shardSearchRequest) { + public QuerySearchResult( + ShardSearchContextId contextId, + SearchShardTarget shardTarget, + ShardSearchRequest shardSearchRequest, + Settings settings + ) { this.contextId = contextId; setSearchShardTarget(shardTarget); isNull = false; setShardSearchRequest(shardSearchRequest); - if (FeatureFlags.isEnabled(FeatureFlags.PROTOBUF_SETTING)) { + if (FeatureFlags.PROTOBUF_SETTING.get(settings)) { QuerySearchResultProto.ShardId shardIdProto = QuerySearchResultProto.ShardId.newBuilder() .setShardId(shardTarget.getShardId().getId()) .setHashCode(shardTarget.getShardId().hashCode()) @@ -238,7 +244,7 @@ private void setTopDocs(TopDocsAndMaxScore topDocsAndMaxScore) { this.maxScore = topDocsAndMaxScore.maxScore; this.hasScoreDocs = topDocsAndMaxScore.topDocs.scoreDocs.length > 0; - if (FeatureFlags.isEnabled(FeatureFlags.PROTOBUF_SETTING)) { + if (this.querySearchResultProto != null) { List scoreDocs = new ArrayList<>(); if (this.hasScoreDocs) { for (ScoreDoc scoreDoc : topDocsAndMaxScore.topDocs.scoreDocs) { From 891a085a6beb416d27f25b072104998df02e3b3f Mon Sep 17 00:00:00 2001 From: Vacha Shah Date: Sun, 14 Jan 2024 06:53:04 +0000 Subject: [PATCH 08/36] Converting FetchSearchResult to proto message Signed-off-by: Vacha Shah --- .../search/fetch/FetchSearchResult.java | 32 ++++++++ .../search/FetchSearchResultProto.proto | 74 +++++++++++++++++++ 2 files changed, 106 insertions(+) create mode 100644 server/src/main/proto/server/search/FetchSearchResultProto.proto diff --git a/server/src/main/java/org/opensearch/search/fetch/FetchSearchResult.java b/server/src/main/java/org/opensearch/search/fetch/FetchSearchResult.java index 26fa90141c2a9..667b022091a85 100644 --- a/server/src/main/java/org/opensearch/search/fetch/FetchSearchResult.java +++ b/server/src/main/java/org/opensearch/search/fetch/FetchSearchResult.java @@ -41,8 +41,11 @@ import org.opensearch.search.SearchShardTarget; import org.opensearch.search.internal.ShardSearchContextId; import org.opensearch.search.query.QuerySearchResult; +import org.opensearch.server.proto.FetchSearchResultProto; +import org.opensearch.server.proto.QuerySearchResultProto; import java.io.IOException; +import java.io.OutputStream; /** * Result from a fetch @@ -56,6 +59,8 @@ public final class FetchSearchResult extends SearchPhaseResult { // client side counter private transient int counter; + private FetchSearchResultProto.FetchSearchResult fetchSearchResultProto; + public FetchSearchResult() {} public FetchSearchResult(StreamInput in) throws IOException { @@ -64,9 +69,23 @@ public FetchSearchResult(StreamInput in) throws IOException { hits = new SearchHits(in); } + public FetchSearchResult(byte[] in) throws IOException { + super(in); + this.fetchSearchResultProto = FetchSearchResultProto.FetchSearchResult.parseFrom(in); + contextId = new ShardSearchContextId( + this.fetchSearchResultProto.getContextId().getSessionId(), + this.fetchSearchResultProto.getContextId().getId() + ); + } + public FetchSearchResult(ShardSearchContextId id, SearchShardTarget shardTarget) { this.contextId = id; setSearchShardTarget(shardTarget); + this.fetchSearchResultProto = FetchSearchResultProto.FetchSearchResult.newBuilder() + .setContextId( + QuerySearchResultProto.ShardSearchContextId.newBuilder().setSessionId(id.getSessionId()).setId(id.getId()).build() + ) + .build(); } @Override @@ -109,4 +128,17 @@ public void writeTo(StreamOutput out) throws IOException { contextId.writeTo(out); hits.writeTo(out); } + + @Override + public void writeTo(OutputStream out) throws IOException { + out.write(fetchSearchResultProto.toByteArray()); + } + + public FetchSearchResultProto.FetchSearchResult response() { + return this.fetchSearchResultProto; + } + + public FetchSearchResult(FetchSearchResultProto.FetchSearchResult fetchSearchResult) { + this.fetchSearchResultProto = fetchSearchResult; + } } diff --git a/server/src/main/proto/server/search/FetchSearchResultProto.proto b/server/src/main/proto/server/search/FetchSearchResultProto.proto new file mode 100644 index 0000000000000..1074725c9654c --- /dev/null +++ b/server/src/main/proto/server/search/FetchSearchResultProto.proto @@ -0,0 +1,74 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +syntax = "proto3"; +package org.opensearch.server.proto; + +import "server/search/QuerySearchResultProto.proto"; + +option java_outer_classname = "FetchSearchResultProto"; + +message FetchSearchResult { + ShardSearchContextId contextId = 1; + optional SearchHits hits = 2; +} + +message SearchHits { + TotalHits totalHits = 1; + float maxScore = 2; + int32 size = 3; + repeated SearchHit hits = 4; + /* optional repeated SortField sortFields = 5; */ + optional string collapseField = 5; + repeated bytes collapseValues = 6; +} + +message SearchHit { + int32 docId = 1; + float score = 2; + string id = 3; + NestedIdentity nestedIdentity = 4; + int64 version = 5; + int64 seqNo = 6; + int64 primaryTerm = 7; + bytes source = 8; + map documentFields = 9; + map metaFields = 10; + map highlightFields = 11; + SearchSortValues sortValues = 12; + repeated string matchedQueries = 13; + /* Explanation explanation = 14;*/ + SearchShardTarget shard = 14; + string index = 15; + string clusterAlias = 16; + map sourceAsMap = 17; + + message NestedIdentity { + string field = 1; + int32 offset = 2; + NestedIdentity child = 3; + } + + message DocumentField { + string name = 1; + repeated bytes values = 2; + } + + message HighlightField { + string name = 1; + repeated string fragments = 2; + } + + message SearchSortValues { + repeated bytes formattedSortValues = 1; + repeated bytes rawSortValues = 2; + } +} From e00c1a88138876936976d64717a8cdda38dcac28 Mon Sep 17 00:00:00 2001 From: Vacha Shah Date: Tue, 16 Jan 2024 02:54:34 +0000 Subject: [PATCH 09/36] Converting QueryFetchSearchResult to a proto message Signed-off-by: Vacha Shah --- .../search/fetch/QueryFetchSearchResult.java | 16 +++++++++++++ .../search/query/QuerySearchResult.java | 9 ++++++++ .../search/QueryFetchSearchResultProto.proto | 23 +++++++++++++++++++ 3 files changed, 48 insertions(+) create mode 100644 server/src/main/proto/server/search/QueryFetchSearchResultProto.proto diff --git a/server/src/main/java/org/opensearch/search/fetch/QueryFetchSearchResult.java b/server/src/main/java/org/opensearch/search/fetch/QueryFetchSearchResult.java index ce4c59fc77489..b57434a99edb4 100644 --- a/server/src/main/java/org/opensearch/search/fetch/QueryFetchSearchResult.java +++ b/server/src/main/java/org/opensearch/search/fetch/QueryFetchSearchResult.java @@ -38,6 +38,7 @@ import org.opensearch.search.SearchShardTarget; import org.opensearch.search.internal.ShardSearchContextId; import org.opensearch.search.query.QuerySearchResult; +import org.opensearch.server.proto.QueryFetchSearchResultProto; import java.io.IOException; @@ -51,15 +52,30 @@ public final class QueryFetchSearchResult extends SearchPhaseResult { private final QuerySearchResult queryResult; private final FetchSearchResult fetchResult; + private QueryFetchSearchResultProto.QueryFetchSearchResult queryFetchSearchResultProto; + public QueryFetchSearchResult(StreamInput in) throws IOException { super(in); queryResult = new QuerySearchResult(in); fetchResult = new FetchSearchResult(in); } + public QueryFetchSearchResult(byte[] in) throws IOException { + super(in); + this.queryFetchSearchResultProto = QueryFetchSearchResultProto.QueryFetchSearchResult.parseFrom(in); + queryResult = new QuerySearchResult(in); + fetchResult = new FetchSearchResult(in); + } + public QueryFetchSearchResult(QuerySearchResult queryResult, FetchSearchResult fetchResult) { this.queryResult = queryResult; this.fetchResult = fetchResult; + if (queryResult.response() != null && fetchResult.response() != null) { + this.queryFetchSearchResultProto = QueryFetchSearchResultProto.QueryFetchSearchResult.newBuilder() + .setQueryResult(queryResult.response()) + .setFetchResult(fetchResult.response()) + .build(); + } } @Override diff --git a/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java b/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java index af2ae612772a3..b181d6c2462fb 100644 --- a/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java +++ b/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java @@ -496,4 +496,13 @@ public TotalHits getTotalHits() { public float getMaxScore() { return maxScore; } + + public QuerySearchResultProto.QuerySearchResult response() { + return this.querySearchResultProto; + } + + public QuerySearchResult(QuerySearchResultProto.QuerySearchResult querySearchResult) { + this.querySearchResultProto = querySearchResult; + this.isNull = false; + } } diff --git a/server/src/main/proto/server/search/QueryFetchSearchResultProto.proto b/server/src/main/proto/server/search/QueryFetchSearchResultProto.proto new file mode 100644 index 0000000000000..08d7d9941022c --- /dev/null +++ b/server/src/main/proto/server/search/QueryFetchSearchResultProto.proto @@ -0,0 +1,23 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +syntax = "proto3"; +package org.opensearch.server.proto; + +import "server/search/QuerySearchResultProto.proto"; +import "server/search/FetchSearchResultProto.proto"; + +option java_outer_classname = "QueryFetchSearchResultProto"; + +message QueryFetchSearchResult { + QuerySearchResult queryResult = 1; + FetchSearchResult fetchResult = 2; +} From 6274355176a8fe990b82e4ed46f7df5b68e9cac2 Mon Sep 17 00:00:00 2001 From: Vacha Shah Date: Tue, 16 Jan 2024 02:57:04 +0000 Subject: [PATCH 10/36] Adding ActionListenerResponseHandler for protobuf Signed-off-by: Vacha Shah --- .../node/tasks/CancellableTasksIT.java | 6 + .../action/ActionListenerResponseHandler.java | 6 + ...ProtobufActionListenerResponseHandler.java | 78 ++++++++ ...TransportFieldCapabilitiesIndexAction.java | 6 + .../TransportResyncReplicationAction.java | 6 + .../opensearch/action/search/PitService.java | 6 + .../action/search/SearchTransportService.java | 89 ++++++++- .../broadcast/TransportBroadcastAction.java | 6 + .../node/TransportBroadcastByNodeAction.java | 7 + .../support/nodes/TransportNodesAction.java | 6 + .../TransportReplicationAction.java | 6 + ...ransportInstanceSingleOperationAction.java | 6 + .../shard/TransportSingleShardAction.java | 12 ++ .../support/tasks/TransportTasksAction.java | 8 + .../coordination/FollowersChecker.java | 6 + .../cluster/coordination/JoinHelper.java | 12 ++ .../cluster/coordination/LeaderChecker.java | 6 + .../coordination/PreVoteCollector.java | 6 + .../PublicationTransportHandler.java | 13 ++ .../decommission/DecommissionController.java | 6 + .../org/opensearch/discovery/PeerFinder.java | 6 + .../extensions/ExtensionsManager.java | 6 + .../UpdateSettingsResponseHandler.java | 6 + .../ExtensionTransportActionsHandler.java | 12 ++ .../rest/RestSendToExtensionAction.java | 6 + .../RetentionLeaseBackgroundSyncAction.java | 6 + .../index/seqno/RetentionLeaseSyncAction.java | 6 + .../recovery/PeerRecoveryTargetService.java | 6 + .../checkpoint/PublishCheckpointAction.java | 6 + .../indices/store/IndicesStore.java | 6 + .../snapshots/SnapshotShardsService.java | 6 + .../TraceableTransportResponseHandler.java | 6 + .../EmptyTransportResponseHandler.java | 9 + .../transport/PlainTransportFuture.java | 6 + .../transport/RemoteClusterConnection.java | 6 + .../transport/SniffConnectionStrategy.java | 6 + .../transport/TransportActionProxy.java | 6 + .../transport/TransportHandshaker.java | 6 + .../transport/TransportResponseHandler.java | 7 + .../transport/TransportService.java | 12 ++ ...tAddVotingConfigExclusionsActionTests.java | 6 + ...learVotingConfigExclusionsActionTests.java | 6 + .../action/search/DfsQueryPhaseTests.java | 13 +- .../action/search/FetchSearchPhaseTests.java | 28 ++- .../search/SearchPhaseControllerTests.java | 43 +++-- .../SearchQueryThenFetchAsyncActionTests.java | 4 +- ...ReplicationAllPermitsAcquisitionTests.java | 7 + .../coordination/FollowersCheckerTests.java | 25 +++ .../coordination/LeaderCheckerTests.java | 7 + .../coordination/PreVoteCollectorTests.java | 6 + .../opensearch/discovery/PeerFinderTests.java | 6 + .../SegmentReplicationSourceServiceTests.java | 18 ++ .../search/query/QuerySearchResultTests.java | 3 +- .../transport/InboundHandlerTests.java | 30 ++++ .../transport/TransportActionProxyTests.java | 12 ++ ...ortServiceDeserializationFailureTests.java | 14 ++ .../AbstractSimpleTransportTestCase.java | 169 ++++++++++++++++++ .../DisruptableMockTransportTests.java | 18 ++ .../test/disruption/NetworkDisruptionIT.java | 6 + 59 files changed, 817 insertions(+), 37 deletions(-) create mode 100644 server/src/main/java/org/opensearch/action/ProtobufActionListenerResponseHandler.java diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/node/tasks/CancellableTasksIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/node/tasks/CancellableTasksIT.java index bdb36b62ada21..30128ac73ba44 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/node/tasks/CancellableTasksIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/node/tasks/CancellableTasksIT.java @@ -560,6 +560,12 @@ public String executor() { public TestResponse read(StreamInput in) throws IOException { return new TestResponse(in); } + + @Override + public TestResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } ); } diff --git a/server/src/main/java/org/opensearch/action/ActionListenerResponseHandler.java b/server/src/main/java/org/opensearch/action/ActionListenerResponseHandler.java index 86cbb8a6307be..af431619f894c 100644 --- a/server/src/main/java/org/opensearch/action/ActionListenerResponseHandler.java +++ b/server/src/main/java/org/opensearch/action/ActionListenerResponseHandler.java @@ -89,4 +89,10 @@ public Response read(StreamInput in) throws IOException { public String toString() { return super.toString() + "/" + listener; } + + @Override + public Response read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } diff --git a/server/src/main/java/org/opensearch/action/ProtobufActionListenerResponseHandler.java b/server/src/main/java/org/opensearch/action/ProtobufActionListenerResponseHandler.java new file mode 100644 index 0000000000000..10272ca797517 --- /dev/null +++ b/server/src/main/java/org/opensearch/action/ProtobufActionListenerResponseHandler.java @@ -0,0 +1,78 @@ +/* +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +*/ + +package org.opensearch.action; + +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.common.io.stream.ProtobufWriteable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.transport.TransportResponse; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportException; +import org.opensearch.transport.TransportResponseHandler; + +import java.io.IOException; +import java.util.Objects; + +/** + * A simple base class for action response listeners, defaulting to using the SAME executor (as its +* very common on response handlers). +* +* @opensearch.api +*/ +public class ProtobufActionListenerResponseHandler implements TransportResponseHandler { + + private final ActionListener listener; + private final ProtobufWriteable.Reader reader; + private final String executor; + + public ProtobufActionListenerResponseHandler( + ActionListener listener, + ProtobufWriteable.Reader reader, + String executor + ) { + this.listener = Objects.requireNonNull(listener); + this.reader = Objects.requireNonNull(reader); + this.executor = Objects.requireNonNull(executor); + } + + public ProtobufActionListenerResponseHandler(ActionListener listener, ProtobufWriteable.Reader reader) { + this(listener, reader, ThreadPool.Names.SAME); + } + + @Override + public void handleResponse(Response response) { + listener.onResponse(response); + } + + @Override + public void handleException(TransportException e) { + listener.onFailure(e); + } + + @Override + public String executor() { + return executor; + } + + @Override + public String toString() { + return super.toString() + "/" + listener; + } + + @Override + public Response read(StreamInput in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } + + @Override + public Response read(byte[] in) throws IOException { + return reader.read(in); + } +} diff --git a/server/src/main/java/org/opensearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java b/server/src/main/java/org/opensearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java index 10bf4975311d6..491473116d843 100644 --- a/server/src/main/java/org/opensearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java +++ b/server/src/main/java/org/opensearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java @@ -339,6 +339,12 @@ public void handleResponse(final FieldCapabilitiesIndexResponse response) { public void handleException(TransportException exp) { onFailure(shardRouting, exp); } + + @Override + public FieldCapabilitiesIndexResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } ); } diff --git a/server/src/main/java/org/opensearch/action/resync/TransportResyncReplicationAction.java b/server/src/main/java/org/opensearch/action/resync/TransportResyncReplicationAction.java index 9d60706d1f100..e79053e2a19e4 100644 --- a/server/src/main/java/org/opensearch/action/resync/TransportResyncReplicationAction.java +++ b/server/src/main/java/org/opensearch/action/resync/TransportResyncReplicationAction.java @@ -253,6 +253,12 @@ public void handleResponse(ResyncReplicationResponse response) { public void handleException(TransportException exp) { listener.onFailure(exp); } + + @Override + public ResyncReplicationResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } ); } diff --git a/server/src/main/java/org/opensearch/action/search/PitService.java b/server/src/main/java/org/opensearch/action/search/PitService.java index b6480ce63f827..4f67b3414b3a4 100644 --- a/server/src/main/java/org/opensearch/action/search/PitService.java +++ b/server/src/main/java/org/opensearch/action/search/PitService.java @@ -202,6 +202,12 @@ public String executor() { public GetAllPitNodesResponse read(StreamInput in) throws IOException { return new GetAllPitNodesResponse(in); } + + @Override + public GetAllPitNodesResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } ); } diff --git a/server/src/main/java/org/opensearch/action/search/SearchTransportService.java b/server/src/main/java/org/opensearch/action/search/SearchTransportService.java index 64c738f633f2e..ddc2400c74dfc 100644 --- a/server/src/main/java/org/opensearch/action/search/SearchTransportService.java +++ b/server/src/main/java/org/opensearch/action/search/SearchTransportService.java @@ -35,12 +35,15 @@ import org.opensearch.action.ActionListenerResponseHandler; import org.opensearch.action.IndicesRequest; import org.opensearch.action.OriginalIndices; +import org.opensearch.action.ProtobufActionListenerResponseHandler; import org.opensearch.action.support.ChannelActionListener; import org.opensearch.action.support.IndicesOptions; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.Nullable; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.util.concurrent.ConcurrentCollections; import org.opensearch.core.action.ActionListener; +import org.opensearch.core.common.io.stream.ProtobufWriteable; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; @@ -241,16 +244,31 @@ public void sendExecuteQuery( // we optimize this and expect a QueryFetchSearchResult if we only have a single shard in the search request // this used to be the QUERY_AND_FETCH which doesn't exist anymore. final boolean fetchDocuments = request.numberOfShards() == 1; - Writeable.Reader reader = fetchDocuments ? QueryFetchSearchResult::new : QuerySearchResult::new; + // System.setProperty("opensearch.experimental.feature.search_with_protobuf.enabled", "true"); + if (FeatureFlags.isEnabled(FeatureFlags.PROTOBUF_SETTING)) { + // System.out.println("Feature flag enabled"); + ProtobufWriteable.Reader reader = fetchDocuments ? QueryFetchSearchResult::new : QuerySearchResult::new; + + final ActionListener handler = responseWrapper.apply(connection, listener); + transportService.sendChildRequest( + connection, + QUERY_ACTION_NAME, + request, + task, + new ProtobufConnectionCountingHandler<>(handler, reader, clientConnections, connection.getNode().getId()) + ); + } else { + Writeable.Reader reader = fetchDocuments ? QueryFetchSearchResult::new : QuerySearchResult::new; - final ActionListener handler = responseWrapper.apply(connection, listener); - transportService.sendChildRequest( - connection, - QUERY_ACTION_NAME, - request, - task, - new ConnectionCountingHandler<>(handler, reader, clientConnections, connection.getNode().getId()) - ); + final ActionListener handler = responseWrapper.apply(connection, listener); + transportService.sendChildRequest( + connection, + QUERY_ACTION_NAME, + request, + task, + new ConnectionCountingHandler<>(handler, reader, clientConnections, connection.getNode().getId()) + ); + } } public void sendExecuteQuery( @@ -775,4 +793,57 @@ private boolean assertNodePresent() { return true; } } + + /** + * A handler that counts connections for protobuf + * + * @opensearch.internal + */ + final class ProtobufConnectionCountingHandler extends ProtobufActionListenerResponseHandler< + Response> { + private final Map clientConnections; + private final String nodeId; + + ProtobufConnectionCountingHandler( + final ActionListener listener, + final ProtobufWriteable.Reader responseReader, + final Map clientConnections, + final String nodeId + ) { + super(listener, responseReader); + this.clientConnections = clientConnections; + this.nodeId = nodeId; + // Increment the number of connections for this node by one + clientConnections.compute(nodeId, (id, conns) -> conns == null ? 1 : conns + 1); + } + + @Override + public void handleResponse(Response response) { + super.handleResponse(response); + // Decrement the number of connections or remove it entirely if there are no more connections + // We need to remove the entry here so we don't leak when nodes go away forever + assert assertNodePresent(); + clientConnections.computeIfPresent(nodeId, (id, conns) -> conns.longValue() == 1 ? null : conns - 1); + } + + @Override + public void handleException(TransportException e) { + super.handleException(e); + // Decrement the number of connections or remove it entirely if there are no more connections + // We need to remove the entry here so we don't leak when nodes go away forever + assert assertNodePresent(); + clientConnections.computeIfPresent(nodeId, (id, conns) -> conns.longValue() == 1 ? null : conns - 1); + } + + private boolean assertNodePresent() { + clientConnections.compute(nodeId, (id, conns) -> { + assert conns != null : "number of connections for " + id + " is null, but should be an integer"; + assert conns >= 1 : "number of connections for " + id + " should be >= 1 but was " + conns; + return conns; + }); + // Always return true, there is additional asserting here, the boolean is just so this + // can be skipped when assertions are not enabled + return true; + } + } } diff --git a/server/src/main/java/org/opensearch/action/support/broadcast/TransportBroadcastAction.java b/server/src/main/java/org/opensearch/action/support/broadcast/TransportBroadcastAction.java index 8bf8555194976..4cd26bd739f1b 100644 --- a/server/src/main/java/org/opensearch/action/support/broadcast/TransportBroadcastAction.java +++ b/server/src/main/java/org/opensearch/action/support/broadcast/TransportBroadcastAction.java @@ -229,6 +229,12 @@ public void handleResponse(ShardResponse response) { public void handleException(TransportException e) { onOperation(shard, shardIt, shardIndex, e); } + + @Override + public ShardResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } ); } diff --git a/server/src/main/java/org/opensearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java b/server/src/main/java/org/opensearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java index c08cfb7af0e3d..7bab85d3b68cd 100644 --- a/server/src/main/java/org/opensearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java +++ b/server/src/main/java/org/opensearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java @@ -392,6 +392,13 @@ public void handleException(TransportException exp) { public String executor() { return ThreadPool.Names.SAME; } + + @Override + public TransportBroadcastByNodeAction.NodeResponse read(byte[] in) + throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } ); } catch (Exception e) { diff --git a/server/src/main/java/org/opensearch/action/support/nodes/TransportNodesAction.java b/server/src/main/java/org/opensearch/action/support/nodes/TransportNodesAction.java index 9a1a28dd70636..3d88bd623135b 100644 --- a/server/src/main/java/org/opensearch/action/support/nodes/TransportNodesAction.java +++ b/server/src/main/java/org/opensearch/action/support/nodes/TransportNodesAction.java @@ -286,6 +286,12 @@ public void handleException(TransportException exp) { public String executor() { return ThreadPool.Names.SAME; } + + @Override + public NodeResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } ); } catch (Exception e) { diff --git a/server/src/main/java/org/opensearch/action/support/replication/TransportReplicationAction.java b/server/src/main/java/org/opensearch/action/support/replication/TransportReplicationAction.java index 49a96603f6802..98e75944fc2ff 100644 --- a/server/src/main/java/org/opensearch/action/support/replication/TransportReplicationAction.java +++ b/server/src/main/java/org/opensearch/action/support/replication/TransportReplicationAction.java @@ -1175,6 +1175,12 @@ public void handleException(TransportException exp) { finishWithUnexpectedFailure(e); } } + + @Override + public Response read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } }); } diff --git a/server/src/main/java/org/opensearch/action/support/single/instance/TransportInstanceSingleOperationAction.java b/server/src/main/java/org/opensearch/action/support/single/instance/TransportInstanceSingleOperationAction.java index 21d4ba726e86f..8600613dbe54c 100644 --- a/server/src/main/java/org/opensearch/action/support/single/instance/TransportInstanceSingleOperationAction.java +++ b/server/src/main/java/org/opensearch/action/support/single/instance/TransportInstanceSingleOperationAction.java @@ -245,6 +245,12 @@ public void handleException(TransportException exp) { listener.onFailure(exp); } } + + @Override + public Response read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } }); } diff --git a/server/src/main/java/org/opensearch/action/support/single/shard/TransportSingleShardAction.java b/server/src/main/java/org/opensearch/action/support/single/shard/TransportSingleShardAction.java index df91559a2f8cb..3a80dbabb535f 100644 --- a/server/src/main/java/org/opensearch/action/support/single/shard/TransportSingleShardAction.java +++ b/server/src/main/java/org/opensearch/action/support/single/shard/TransportSingleShardAction.java @@ -225,6 +225,12 @@ public void handleResponse(final Response response) { public void handleException(TransportException exp) { listener.onFailure(exp); } + + @Override + public Response read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } ); } else { @@ -302,6 +308,12 @@ public void handleResponse(final Response response) { public void handleException(TransportException exp) { onFailure(finalShardRouting, exp); } + + @Override + public Response read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } ); } diff --git a/server/src/main/java/org/opensearch/action/support/tasks/TransportTasksAction.java b/server/src/main/java/org/opensearch/action/support/tasks/TransportTasksAction.java index f33d7161660a3..fd960df77d492 100644 --- a/server/src/main/java/org/opensearch/action/support/tasks/TransportTasksAction.java +++ b/server/src/main/java/org/opensearch/action/support/tasks/TransportTasksAction.java @@ -316,6 +316,14 @@ public void handleException(TransportException exp) { public String executor() { return ThreadPool.Names.SAME; } + + @Override + public + TransportTasksAction.NodeTasksResponse + read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } ); } diff --git a/server/src/main/java/org/opensearch/cluster/coordination/FollowersChecker.java b/server/src/main/java/org/opensearch/cluster/coordination/FollowersChecker.java index 70bb0515bb022..08c3c58dacc2d 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/FollowersChecker.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/FollowersChecker.java @@ -408,6 +408,12 @@ public void handleException(TransportException exp) { public String executor() { return Names.SAME; } + + @Override + public Empty read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } ); } diff --git a/server/src/main/java/org/opensearch/cluster/coordination/JoinHelper.java b/server/src/main/java/org/opensearch/cluster/coordination/JoinHelper.java index 9bf6bac07da53..f6f2bd029ca2e 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/JoinHelper.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/JoinHelper.java @@ -421,6 +421,12 @@ public void handleException(TransportException exp) { public String executor() { return Names.SAME; } + + @Override + public Empty read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } ); } else { @@ -451,6 +457,12 @@ public void handleException(TransportException exp) { public String executor() { return ThreadPool.Names.SAME; } + + @Override + public Empty read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } }); } diff --git a/server/src/main/java/org/opensearch/cluster/coordination/LeaderChecker.java b/server/src/main/java/org/opensearch/cluster/coordination/LeaderChecker.java index 8d4373b865f62..dee8619f3b213 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/LeaderChecker.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/LeaderChecker.java @@ -349,6 +349,12 @@ public void handleException(TransportException exp) { public String executor() { return Names.SAME; } + + @Override + public Empty read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } ); } diff --git a/server/src/main/java/org/opensearch/cluster/coordination/PreVoteCollector.java b/server/src/main/java/org/opensearch/cluster/coordination/PreVoteCollector.java index cc4d1ac156c53..b6bfe54973718 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/PreVoteCollector.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/PreVoteCollector.java @@ -217,6 +217,12 @@ public String executor() { public String toString() { return "TransportResponseHandler{" + PreVoteCollector.this + ", node=" + n + '}'; } + + @Override + public PreVoteResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } ) ); diff --git a/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java b/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java index 1fdaeead0d28d..359ee0f91482f 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java @@ -47,6 +47,7 @@ import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.transport.TransportResponse; +import org.opensearch.core.transport.TransportResponse.Empty; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.BytesTransportRequest; import org.opensearch.transport.TransportChannel; @@ -380,6 +381,12 @@ public void handleException(TransportException exp) { public String executor() { return ThreadPool.Names.GENERIC; } + + @Override + public Empty read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } ); } @@ -451,6 +458,12 @@ public void handleException(TransportException exp) { public String executor() { return ThreadPool.Names.GENERIC; } + + @Override + public PublishWithJoinResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } }; transportService.sendRequest(destination, PUBLISH_STATE_ACTION_NAME, request, stateRequestOptions, responseHandler); } catch (Exception e) { diff --git a/server/src/main/java/org/opensearch/cluster/decommission/DecommissionController.java b/server/src/main/java/org/opensearch/cluster/decommission/DecommissionController.java index fec313b4b0b73..307155f779742 100644 --- a/server/src/main/java/org/opensearch/cluster/decommission/DecommissionController.java +++ b/server/src/main/java/org/opensearch/cluster/decommission/DecommissionController.java @@ -257,6 +257,12 @@ public String executor() { public NodesStatsResponse read(StreamInput in) throws IOException { return new NodesStatsResponse(in); } + + @Override + public NodesStatsResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } ); } diff --git a/server/src/main/java/org/opensearch/discovery/PeerFinder.java b/server/src/main/java/org/opensearch/discovery/PeerFinder.java index 1d997c8cbabe8..8eb65cfe12b24 100644 --- a/server/src/main/java/org/opensearch/discovery/PeerFinder.java +++ b/server/src/main/java/org/opensearch/discovery/PeerFinder.java @@ -504,6 +504,12 @@ public void handleException(TransportException exp) { public String executor() { return Names.GENERIC; } + + @Override + public PeersResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } }; transportService.sendRequest( discoveryNode, diff --git a/server/src/main/java/org/opensearch/extensions/ExtensionsManager.java b/server/src/main/java/org/opensearch/extensions/ExtensionsManager.java index b531abcb845d7..d2fecaf53d5b4 100644 --- a/server/src/main/java/org/opensearch/extensions/ExtensionsManager.java +++ b/server/src/main/java/org/opensearch/extensions/ExtensionsManager.java @@ -384,6 +384,12 @@ public void handleException(TransportException exp) { public String executor() { return ThreadPool.Names.GENERIC; } + + @Override + public InitializeExtensionResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } }; logger.info("Sending extension request type: " + REQUEST_EXTENSION_ACTION_NAME); diff --git a/server/src/main/java/org/opensearch/extensions/UpdateSettingsResponseHandler.java b/server/src/main/java/org/opensearch/extensions/UpdateSettingsResponseHandler.java index 8c7c3c4cb9bd9..c8e7d8e28e86c 100644 --- a/server/src/main/java/org/opensearch/extensions/UpdateSettingsResponseHandler.java +++ b/server/src/main/java/org/opensearch/extensions/UpdateSettingsResponseHandler.java @@ -48,4 +48,10 @@ public void handleException(TransportException exp) { public String executor() { return ThreadPool.Names.GENERIC; } + + @Override + public AcknowledgedResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } diff --git a/server/src/main/java/org/opensearch/extensions/action/ExtensionTransportActionsHandler.java b/server/src/main/java/org/opensearch/extensions/action/ExtensionTransportActionsHandler.java index ac60df1b73764..700a66c2babce 100644 --- a/server/src/main/java/org/opensearch/extensions/action/ExtensionTransportActionsHandler.java +++ b/server/src/main/java/org/opensearch/extensions/action/ExtensionTransportActionsHandler.java @@ -219,6 +219,12 @@ public void handleException(TransportException exp) { public String executor() { return ThreadPool.Names.GENERIC; } + + @Override + public ExtensionActionResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } }; try { transportService.sendRequest( @@ -283,6 +289,12 @@ public void handleException(TransportException exp) { public String executor() { return ThreadPool.Names.GENERIC; } + + @Override + public RemoteExtensionActionResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } }; try { transportService.sendRequest( diff --git a/server/src/main/java/org/opensearch/extensions/rest/RestSendToExtensionAction.java b/server/src/main/java/org/opensearch/extensions/rest/RestSendToExtensionAction.java index f4503ce55e6bc..ec03c9015e695 100644 --- a/server/src/main/java/org/opensearch/extensions/rest/RestSendToExtensionAction.java +++ b/server/src/main/java/org/opensearch/extensions/rest/RestSendToExtensionAction.java @@ -240,6 +240,12 @@ public void handleException(TransportException exp) { public String executor() { return ThreadPool.Names.GENERIC; } + + @Override + public RestExecuteOnExtensionResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } }; try { diff --git a/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseBackgroundSyncAction.java b/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseBackgroundSyncAction.java index 5fa0a1a6459e7..c00931bf4eb15 100644 --- a/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseBackgroundSyncAction.java +++ b/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseBackgroundSyncAction.java @@ -167,6 +167,12 @@ public void handleException(TransportException e) { } getLogger().warn(new ParameterizedMessage("{} retention lease background sync failed", shardId), e); } + + @Override + public ReplicationResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } ); } diff --git a/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseSyncAction.java b/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseSyncAction.java index ca3c7e1d49700..88d415af0cd68 100644 --- a/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseSyncAction.java +++ b/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseSyncAction.java @@ -178,6 +178,12 @@ public void handleException(TransportException e) { taskManager.unregister(task); listener.onFailure(e); } + + @Override + public ReplicationResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } ); } diff --git a/server/src/main/java/org/opensearch/indices/recovery/PeerRecoveryTargetService.java b/server/src/main/java/org/opensearch/indices/recovery/PeerRecoveryTargetService.java index c24840d0c1333..fa8c8b24d86d4 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/PeerRecoveryTargetService.java +++ b/server/src/main/java/org/opensearch/indices/recovery/PeerRecoveryTargetService.java @@ -762,5 +762,11 @@ public String executor() { public RecoveryResponse read(StreamInput in) throws IOException { return new RecoveryResponse(in); } + + @Override + public RecoveryResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } } diff --git a/server/src/main/java/org/opensearch/indices/replication/checkpoint/PublishCheckpointAction.java b/server/src/main/java/org/opensearch/indices/replication/checkpoint/PublishCheckpointAction.java index 8f39aa194b06c..600063b218eae 100644 --- a/server/src/main/java/org/opensearch/indices/replication/checkpoint/PublishCheckpointAction.java +++ b/server/src/main/java/org/opensearch/indices/replication/checkpoint/PublishCheckpointAction.java @@ -172,6 +172,12 @@ public void handleException(TransportException e) { e ); } + + @Override + public ReplicationResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } ); logger.trace( diff --git a/server/src/main/java/org/opensearch/indices/store/IndicesStore.java b/server/src/main/java/org/opensearch/indices/store/IndicesStore.java index 1efaca09204da..67f44332c40c4 100644 --- a/server/src/main/java/org/opensearch/indices/store/IndicesStore.java +++ b/server/src/main/java/org/opensearch/indices/store/IndicesStore.java @@ -352,6 +352,12 @@ private void allNodesResponded() { ); } + @Override + public ShardActiveResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } + } private class ShardActiveRequestHandler implements TransportRequestHandler { diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotShardsService.java b/server/src/main/java/org/opensearch/snapshots/SnapshotShardsService.java index 89f1ea142336e..7528e3a09142e 100644 --- a/server/src/main/java/org/opensearch/snapshots/SnapshotShardsService.java +++ b/server/src/main/java/org/opensearch/snapshots/SnapshotShardsService.java @@ -623,6 +623,12 @@ public void handleException(TransportException exp) { public String executor() { return ThreadPool.Names.SAME; } + + @Override + public UpdateIndexShardSnapshotStatusResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } ) ); diff --git a/server/src/main/java/org/opensearch/telemetry/tracing/handler/TraceableTransportResponseHandler.java b/server/src/main/java/org/opensearch/telemetry/tracing/handler/TraceableTransportResponseHandler.java index eb9d53d2df51b..f938fe6cd1799 100644 --- a/server/src/main/java/org/opensearch/telemetry/tracing/handler/TraceableTransportResponseHandler.java +++ b/server/src/main/java/org/opensearch/telemetry/tracing/handler/TraceableTransportResponseHandler.java @@ -104,4 +104,10 @@ public void handleRejection(Exception exp) { span.endSpan(); } } + + @Override + public T read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } diff --git a/server/src/main/java/org/opensearch/transport/EmptyTransportResponseHandler.java b/server/src/main/java/org/opensearch/transport/EmptyTransportResponseHandler.java index 1691b427ffca1..7e4113bc77509 100644 --- a/server/src/main/java/org/opensearch/transport/EmptyTransportResponseHandler.java +++ b/server/src/main/java/org/opensearch/transport/EmptyTransportResponseHandler.java @@ -34,8 +34,11 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.transport.TransportResponse; +import org.opensearch.core.transport.TransportResponse.Empty; import org.opensearch.threadpool.ThreadPool; +import java.io.IOException; + /** * Handler for empty transport response * @@ -66,4 +69,10 @@ public void handleException(TransportException exp) {} public String executor() { return executor; } + + @Override + public Empty read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } diff --git a/server/src/main/java/org/opensearch/transport/PlainTransportFuture.java b/server/src/main/java/org/opensearch/transport/PlainTransportFuture.java index ff9ca8b189904..859b4c9f46945 100644 --- a/server/src/main/java/org/opensearch/transport/PlainTransportFuture.java +++ b/server/src/main/java/org/opensearch/transport/PlainTransportFuture.java @@ -125,4 +125,10 @@ public void handleException(TransportException exp) { public String toString() { return "future(" + handler.toString() + ")"; } + + @Override + public V read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } diff --git a/server/src/main/java/org/opensearch/transport/RemoteClusterConnection.java b/server/src/main/java/org/opensearch/transport/RemoteClusterConnection.java index 8a5f6dfffb036..050efc064b8ad 100644 --- a/server/src/main/java/org/opensearch/transport/RemoteClusterConnection.java +++ b/server/src/main/java/org/opensearch/transport/RemoteClusterConnection.java @@ -170,6 +170,12 @@ public void handleException(TransportException exp) { public String executor() { return ThreadPool.Names.SAME; } + + @Override + public ClusterStateResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } ); } diff --git a/server/src/main/java/org/opensearch/transport/SniffConnectionStrategy.java b/server/src/main/java/org/opensearch/transport/SniffConnectionStrategy.java index 07ba96b135189..a0cc34f44e5e9 100644 --- a/server/src/main/java/org/opensearch/transport/SniffConnectionStrategy.java +++ b/server/src/main/java/org/opensearch/transport/SniffConnectionStrategy.java @@ -471,6 +471,12 @@ public void handleException(TransportException exp) { public String executor() { return ThreadPool.Names.MANAGEMENT; } + + @Override + public ClusterStateResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } private Predicate getRemoteClusterNamePredicate() { diff --git a/server/src/main/java/org/opensearch/transport/TransportActionProxy.java b/server/src/main/java/org/opensearch/transport/TransportActionProxy.java index a61aec8a34e20..463e729739e93 100644 --- a/server/src/main/java/org/opensearch/transport/TransportActionProxy.java +++ b/server/src/main/java/org/opensearch/transport/TransportActionProxy.java @@ -130,6 +130,12 @@ public void handleException(TransportException exp) { public String executor() { return ThreadPool.Names.SAME; } + + @Override + public T read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } /** diff --git a/server/src/main/java/org/opensearch/transport/TransportHandshaker.java b/server/src/main/java/org/opensearch/transport/TransportHandshaker.java index d0b00ec9c59db..cc0cdd18890d0 100644 --- a/server/src/main/java/org/opensearch/transport/TransportHandshaker.java +++ b/server/src/main/java/org/opensearch/transport/TransportHandshaker.java @@ -187,6 +187,12 @@ void handleLocalException(TransportException e) { public String executor() { return ThreadPool.Names.SAME; } + + @Override + public HandshakeResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } static final class HandshakeRequest extends TransportRequest { diff --git a/server/src/main/java/org/opensearch/transport/TransportResponseHandler.java b/server/src/main/java/org/opensearch/transport/TransportResponseHandler.java index 748d2a4d867ec..630d917ca9662 100644 --- a/server/src/main/java/org/opensearch/transport/TransportResponseHandler.java +++ b/server/src/main/java/org/opensearch/transport/TransportResponseHandler.java @@ -33,6 +33,7 @@ package org.opensearch.transport; import org.opensearch.common.annotation.PublicApi; +import org.opensearch.core.common.io.stream.ProtobufWriteable; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.transport.TransportResponse; @@ -83,6 +84,12 @@ public String executor() { public Q read(StreamInput in) throws IOException { return reader.read(in); } + + @Override + public Q read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } }; } } diff --git a/server/src/main/java/org/opensearch/transport/TransportService.java b/server/src/main/java/org/opensearch/transport/TransportService.java index d08b28730d417..1cfe30e9c2e1c 100644 --- a/server/src/main/java/org/opensearch/transport/TransportService.java +++ b/server/src/main/java/org/opensearch/transport/TransportService.java @@ -1508,6 +1508,12 @@ void setTimeoutHandler(TimeoutHandler handler) { this.handler = handler; } + @Override + public T read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } + } /** @@ -1721,6 +1727,12 @@ public T read(StreamInput in) throws IOException { public String toString() { return getClass().getName() + "/[" + action + "]:" + handler.toString(); } + + @Override + public T read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } }; } else { delegate = handler; diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java index a015e671f4872..511865688cd6d 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java @@ -685,6 +685,12 @@ public String executor() { public AddVotingConfigExclusionsResponse read(StreamInput in) throws IOException { return new AddVotingConfigExclusionsResponse(in); } + + @Override + public AddVotingConfigExclusionsResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } }; } diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/configuration/TransportClearVotingConfigExclusionsActionTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/configuration/TransportClearVotingConfigExclusionsActionTests.java index 10e4ab6388be4..35673205519a7 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/configuration/TransportClearVotingConfigExclusionsActionTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/configuration/TransportClearVotingConfigExclusionsActionTests.java @@ -252,6 +252,12 @@ public String executor() { public ClearVotingConfigExclusionsResponse read(StreamInput in) throws IOException { return new ClearVotingConfigExclusionsResponse(in); } + + @Override + public ClearVotingConfigExclusionsResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } }; } } diff --git a/server/src/test/java/org/opensearch/action/search/DfsQueryPhaseTests.java b/server/src/test/java/org/opensearch/action/search/DfsQueryPhaseTests.java index 6952841c295e2..68ae014c9d562 100644 --- a/server/src/test/java/org/opensearch/action/search/DfsQueryPhaseTests.java +++ b/server/src/test/java/org/opensearch/action/search/DfsQueryPhaseTests.java @@ -39,6 +39,7 @@ import org.apache.lucene.tests.store.MockDirectoryWrapper; import org.opensearch.action.OriginalIndices; import org.opensearch.common.lucene.search.TopDocsAndMaxScore; +import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.AtomicArray; import org.opensearch.common.util.concurrent.OpenSearchExecutors; import org.opensearch.core.common.breaker.CircuitBreaker; @@ -101,7 +102,8 @@ public void sendExecuteQuery( QuerySearchResult queryResult = new QuerySearchResult( new ShardSearchContextId("", 123), new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE), - null + null, + Settings.EMPTY ); queryResult.topDocs( new TopDocsAndMaxScore( @@ -116,7 +118,8 @@ public void sendExecuteQuery( QuerySearchResult queryResult = new QuerySearchResult( new ShardSearchContextId("", 123), new SearchShardTarget("node2", new ShardId("test", "na", 0), null, OriginalIndices.NONE), - null + null, + Settings.EMPTY ); queryResult.topDocs( new TopDocsAndMaxScore( @@ -199,7 +202,8 @@ public void sendExecuteQuery( QuerySearchResult queryResult = new QuerySearchResult( new ShardSearchContextId("", 123), new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE), - null + null, + Settings.EMPTY ); queryResult.topDocs( new TopDocsAndMaxScore( @@ -286,7 +290,8 @@ public void sendExecuteQuery( QuerySearchResult queryResult = new QuerySearchResult( new ShardSearchContextId("", 123), new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE), - null + null, + Settings.EMPTY ); queryResult.topDocs( new TopDocsAndMaxScore( diff --git a/server/src/test/java/org/opensearch/action/search/FetchSearchPhaseTests.java b/server/src/test/java/org/opensearch/action/search/FetchSearchPhaseTests.java index 1eb3a44642806..09f03c5ba2e18 100644 --- a/server/src/test/java/org/opensearch/action/search/FetchSearchPhaseTests.java +++ b/server/src/test/java/org/opensearch/action/search/FetchSearchPhaseTests.java @@ -38,6 +38,7 @@ import org.opensearch.action.OriginalIndices; import org.opensearch.common.UUIDs; import org.opensearch.common.lucene.search.TopDocsAndMaxScore; +import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.OpenSearchExecutors; import org.opensearch.core.common.breaker.CircuitBreaker; import org.opensearch.core.common.breaker.NoopCircuitBreaker; @@ -140,7 +141,8 @@ public void testFetchTwoDocument() { QuerySearchResult queryResult = new QuerySearchResult( ctx1, new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE), - null + null, + Settings.EMPTY ); queryResult.topDocs( new TopDocsAndMaxScore( @@ -157,7 +159,8 @@ public void testFetchTwoDocument() { queryResult = new QuerySearchResult( ctx2, new SearchShardTarget("node2", new ShardId("test", "na", 1), null, OriginalIndices.NONE), - null + null, + Settings.EMPTY ); queryResult.topDocs( new TopDocsAndMaxScore( @@ -236,7 +239,8 @@ public void testFailFetchOneDoc() { QuerySearchResult queryResult = new QuerySearchResult( ctx, new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE), - null + null, + Settings.EMPTY ); queryResult.topDocs( new TopDocsAndMaxScore( @@ -252,7 +256,8 @@ public void testFailFetchOneDoc() { queryResult = new QuerySearchResult( new ShardSearchContextId("", 321), new SearchShardTarget("node2", new ShardId("test", "na", 1), null, OriginalIndices.NONE), - null + null, + Settings.EMPTY ); queryResult.topDocs( new TopDocsAndMaxScore( @@ -333,7 +338,8 @@ public void testFetchDocsConcurrently() throws InterruptedException { QuerySearchResult queryResult = new QuerySearchResult( new ShardSearchContextId("", i), new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE), - null + null, + Settings.EMPTY ); queryResult.topDocs( new TopDocsAndMaxScore( @@ -423,7 +429,8 @@ public void testExceptionFailsPhase() { QuerySearchResult queryResult = new QuerySearchResult( new ShardSearchContextId("", 123), new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE), - null + null, + Settings.EMPTY ); queryResult.topDocs( new TopDocsAndMaxScore( @@ -439,7 +446,8 @@ public void testExceptionFailsPhase() { queryResult = new QuerySearchResult( new ShardSearchContextId("", 321), new SearchShardTarget("node2", new ShardId("test", "na", 1), null, OriginalIndices.NONE), - null + null, + Settings.EMPTY ); queryResult.topDocs( new TopDocsAndMaxScore( @@ -516,7 +524,8 @@ public void testCleanupIrrelevantContexts() { // contexts that are not fetched s QuerySearchResult queryResult = new QuerySearchResult( ctx1, new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE), - null + null, + Settings.EMPTY ); queryResult.topDocs( new TopDocsAndMaxScore( @@ -533,7 +542,8 @@ public void testCleanupIrrelevantContexts() { // contexts that are not fetched s queryResult = new QuerySearchResult( ctx2, new SearchShardTarget("node2", new ShardId("test", "na", 1), null, OriginalIndices.NONE), - null + null, + Settings.EMPTY ); queryResult.topDocs( new TopDocsAndMaxScore( diff --git a/server/src/test/java/org/opensearch/action/search/SearchPhaseControllerTests.java b/server/src/test/java/org/opensearch/action/search/SearchPhaseControllerTests.java index a927f733cc504..eeb72969990fc 100644 --- a/server/src/test/java/org/opensearch/action/search/SearchPhaseControllerTests.java +++ b/server/src/test/java/org/opensearch/action/search/SearchPhaseControllerTests.java @@ -410,7 +410,12 @@ private static AtomicArray generateQueryResults( clusterAlias, OriginalIndices.NONE ); - QuerySearchResult querySearchResult = new QuerySearchResult(new ShardSearchContextId("", shardIndex), searchShardTarget, null); + QuerySearchResult querySearchResult = new QuerySearchResult( + new ShardSearchContextId("", shardIndex), + searchShardTarget, + null, + Settings.EMPTY + ); final TopDocs topDocs; float maxScore = 0; if (searchHitsSize == 0) { @@ -685,7 +690,8 @@ private void consumerTestCase(int numEmptyResponses) throws Exception { QuerySearchResult result = new QuerySearchResult( new ShardSearchContextId("", 0), new SearchShardTarget("node", new ShardId("a", "b", 0), null, OriginalIndices.NONE), - null + null, + Settings.EMPTY ); result.topDocs( new TopDocsAndMaxScore(new TopDocs(new TotalHits(0, TotalHits.Relation.EQUAL_TO), new ScoreDoc[0]), Float.NaN), @@ -699,7 +705,8 @@ private void consumerTestCase(int numEmptyResponses) throws Exception { result = new QuerySearchResult( new ShardSearchContextId("", 1), new SearchShardTarget("node", new ShardId("a", "b", 0), null, OriginalIndices.NONE), - null + null, + Settings.EMPTY ); result.topDocs( new TopDocsAndMaxScore(new TopDocs(new TotalHits(0, TotalHits.Relation.EQUAL_TO), new ScoreDoc[0]), Float.NaN), @@ -713,7 +720,8 @@ private void consumerTestCase(int numEmptyResponses) throws Exception { result = new QuerySearchResult( new ShardSearchContextId("", 1), new SearchShardTarget("node", new ShardId("a", "b", 0), null, OriginalIndices.NONE), - null + null, + Settings.EMPTY ); result.topDocs( new TopDocsAndMaxScore(new TopDocs(new TotalHits(0, TotalHits.Relation.EQUAL_TO), new ScoreDoc[0]), Float.NaN), @@ -789,7 +797,8 @@ public void testConsumerConcurrently() throws Exception { QuerySearchResult result = new QuerySearchResult( new ShardSearchContextId("", id), new SearchShardTarget("node", new ShardId("a", "b", id), null, OriginalIndices.NONE), - null + null, + Settings.EMPTY ); result.topDocs( new TopDocsAndMaxScore( @@ -850,7 +859,8 @@ public void testConsumerOnlyAggs() throws Exception { QuerySearchResult result = new QuerySearchResult( new ShardSearchContextId("", i), new SearchShardTarget("node", new ShardId("a", "b", i), null, OriginalIndices.NONE), - null + null, + Settings.EMPTY ); result.topDocs( new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO), new ScoreDoc[0]), number), @@ -903,7 +913,8 @@ public void testConsumerOnlyHits() throws Exception { QuerySearchResult result = new QuerySearchResult( new ShardSearchContextId("", i), new SearchShardTarget("node", new ShardId("a", "b", i), null, OriginalIndices.NONE), - null + null, + Settings.EMPTY ); result.topDocs( new TopDocsAndMaxScore( @@ -959,7 +970,8 @@ public void testReduceTopNWithFromOffset() throws Exception { QuerySearchResult result = new QuerySearchResult( new ShardSearchContextId("", i), new SearchShardTarget("node", new ShardId("a", "b", i), null, OriginalIndices.NONE), - null + null, + Settings.EMPTY ); ScoreDoc[] docs = new ScoreDoc[3]; for (int j = 0; j < docs.length; j++) { @@ -1014,7 +1026,8 @@ public void testConsumerSortByField() throws Exception { QuerySearchResult result = new QuerySearchResult( new ShardSearchContextId("", i), new SearchShardTarget("node", new ShardId("a", "b", i), null, OriginalIndices.NONE), - null + null, + Settings.EMPTY ); result.topDocs(new TopDocsAndMaxScore(topDocs, Float.NaN), docValueFormats); result.setShardIndex(i); @@ -1063,7 +1076,8 @@ public void testConsumerFieldCollapsing() throws Exception { QuerySearchResult result = new QuerySearchResult( new ShardSearchContextId("", i), new SearchShardTarget("node", new ShardId("a", "b", i), null, OriginalIndices.NONE), - null + null, + Settings.EMPTY ); result.topDocs(new TopDocsAndMaxScore(topDocs, Float.NaN), docValueFormats); result.setShardIndex(i); @@ -1107,7 +1121,8 @@ public void testConsumerSuggestions() throws Exception { QuerySearchResult result = new QuerySearchResult( new ShardSearchContextId("", i), new SearchShardTarget("node", new ShardId("a", "b", i), null, OriginalIndices.NONE), - null + null, + Settings.EMPTY ); List>> suggestions = new ArrayList<>(); @@ -1248,7 +1263,8 @@ public void onFinalReduce(List shards, TotalHits totalHits, Interna QuerySearchResult result = new QuerySearchResult( new ShardSearchContextId("", id), new SearchShardTarget("node", new ShardId("a", "b", id), null, OriginalIndices.NONE), - null + null, + Settings.EMPTY ); result.topDocs( new TopDocsAndMaxScore( @@ -1334,7 +1350,8 @@ private void testReduceCase(boolean shouldFail) throws Exception { QuerySearchResult result = new QuerySearchResult( new ShardSearchContextId(UUIDs.randomBase64UUID(), index), new SearchShardTarget("node", new ShardId("a", "b", index), null, OriginalIndices.NONE), - null + null, + Settings.EMPTY ); result.topDocs( new TopDocsAndMaxScore(new TopDocs(new TotalHits(0, TotalHits.Relation.EQUAL_TO), Lucene.EMPTY_SCORE_DOCS), Float.NaN), diff --git a/server/src/test/java/org/opensearch/action/search/SearchQueryThenFetchAsyncActionTests.java b/server/src/test/java/org/opensearch/action/search/SearchQueryThenFetchAsyncActionTests.java index aefbbe80d5fa1..a1f4c5922f10f 100644 --- a/server/src/test/java/org/opensearch/action/search/SearchQueryThenFetchAsyncActionTests.java +++ b/server/src/test/java/org/opensearch/action/search/SearchQueryThenFetchAsyncActionTests.java @@ -43,6 +43,7 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.routing.GroupShardsIterator; import org.opensearch.common.lucene.search.TopDocsAndMaxScore; +import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.OpenSearchExecutors; import org.opensearch.core.common.Strings; @@ -147,7 +148,8 @@ public void sendExecuteQuery( QuerySearchResult queryResult = new QuerySearchResult( new ShardSearchContextId("N/A", 123), new SearchShardTarget("node1", new ShardId("idx", "na", shardId), null, OriginalIndices.NONE), - null + null, + Settings.EMPTY ); SortField sortField = new SortField("timestamp", SortField.Type.LONG); if (withCollapse) { diff --git a/server/src/test/java/org/opensearch/action/support/replication/TransportReplicationAllPermitsAcquisitionTests.java b/server/src/test/java/org/opensearch/action/support/replication/TransportReplicationAllPermitsAcquisitionTests.java index cce8758ef1014..33fca739861d7 100644 --- a/server/src/test/java/org/opensearch/action/support/replication/TransportReplicationAllPermitsAcquisitionTests.java +++ b/server/src/test/java/org/opensearch/action/support/replication/TransportReplicationAllPermitsAcquisitionTests.java @@ -34,6 +34,7 @@ import org.opensearch.Version; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.PlainActionFuture; +import org.opensearch.action.support.replication.TransportReplicationAction.ReplicaResponse; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.action.shard.ShardStateAction; import org.opensearch.cluster.block.ClusterBlock; @@ -223,6 +224,12 @@ public void handleException(TransportException exp) { public String executor() { return ThreadPool.Names.SAME; } + + @Override + public ReplicaResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } ); } diff --git a/server/src/test/java/org/opensearch/cluster/coordination/FollowersCheckerTests.java b/server/src/test/java/org/opensearch/cluster/coordination/FollowersCheckerTests.java index a106706c00732..ab844bb2a2d9c 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/FollowersCheckerTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/FollowersCheckerTests.java @@ -60,6 +60,7 @@ import org.opensearch.transport.TransportResponseHandler; import org.opensearch.transport.TransportService; +import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; @@ -532,6 +533,12 @@ public void handleException(TransportException exp) { public String executor() { return Names.SAME; } + + @Override + public Empty read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } ); deterministicTaskQueue.runAllTasks(); @@ -620,6 +627,12 @@ public void handleException(TransportException exp) { public String executor() { return Names.SAME; } + + @Override + public Empty read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } ); deterministicTaskQueue.runAllTasks(); @@ -692,6 +705,12 @@ public void handleException(TransportException exp) { public String executor() { return Names.SAME; } + + @Override + public Empty read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } ); deterministicTaskQueue.runAllTasks(); @@ -789,5 +808,11 @@ public TransportResponse.Empty read(StreamInput in) { return TransportResponse.Empty.INSTANCE; } + @Override + public Empty read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } + } } diff --git a/server/src/test/java/org/opensearch/cluster/coordination/LeaderCheckerTests.java b/server/src/test/java/org/opensearch/cluster/coordination/LeaderCheckerTests.java index fe65058333116..d18703de20ccd 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/LeaderCheckerTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/LeaderCheckerTests.java @@ -57,6 +57,7 @@ import org.opensearch.transport.TransportResponseHandler; import org.opensearch.transport.TransportService; +import java.io.IOException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; @@ -549,6 +550,12 @@ public String executor() { public TransportResponse.Empty read(StreamInput in) { return TransportResponse.Empty.INSTANCE; } + + @Override + public Empty read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } public void testLeaderCheckRequestEqualsHashcodeSerialization() { diff --git a/server/src/test/java/org/opensearch/cluster/coordination/PreVoteCollectorTests.java b/server/src/test/java/org/opensearch/cluster/coordination/PreVoteCollectorTests.java index 5ddf614db3334..47ec3333dbefe 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/PreVoteCollectorTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/PreVoteCollectorTests.java @@ -358,6 +358,12 @@ public void handleException(TransportException exp) { public String executor() { return SAME; } + + @Override + public PreVoteResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } ); diff --git a/server/src/test/java/org/opensearch/discovery/PeerFinderTests.java b/server/src/test/java/org/opensearch/discovery/PeerFinderTests.java index f861ab90896db..275bd96e5ad07 100644 --- a/server/src/test/java/org/opensearch/discovery/PeerFinderTests.java +++ b/server/src/test/java/org/opensearch/discovery/PeerFinderTests.java @@ -547,6 +547,12 @@ public void handleException(TransportException exp) { public String executor() { return Names.SAME; } + + @Override + public PeersResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } ); diff --git a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationSourceServiceTests.java b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationSourceServiceTests.java index 8f84053f2618e..f8ad1484b55cd 100644 --- a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationSourceServiceTests.java +++ b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationSourceServiceTests.java @@ -220,6 +220,12 @@ public String executor() { public CheckpointInfoResponse read(StreamInput in) throws IOException { return new CheckpointInfoResponse(in); } + + @Override + public CheckpointInfoResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } ); } @@ -249,6 +255,12 @@ public String executor() { public GetSegmentFilesResponse read(StreamInput in) throws IOException { return new GetSegmentFilesResponse(in); } + + @Override + public GetSegmentFilesResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } ); } @@ -289,6 +301,12 @@ public String executor() { public CheckpointInfoResponse read(StreamInput in) throws IOException { return new CheckpointInfoResponse(in); } + + @Override + public TransportResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } ); } diff --git a/server/src/test/java/org/opensearch/search/query/QuerySearchResultTests.java b/server/src/test/java/org/opensearch/search/query/QuerySearchResultTests.java index 41e4e1ae45a73..81e9f4433cdc9 100644 --- a/server/src/test/java/org/opensearch/search/query/QuerySearchResultTests.java +++ b/server/src/test/java/org/opensearch/search/query/QuerySearchResultTests.java @@ -84,7 +84,8 @@ private static QuerySearchResult createTestInstance() throws Exception { QuerySearchResult result = new QuerySearchResult( new ShardSearchContextId(UUIDs.base64UUID(), randomLong()), new SearchShardTarget("node", shardId, null, OriginalIndices.NONE), - shardSearchRequest + shardSearchRequest, + Settings.EMPTY ); if (randomBoolean()) { result.terminatedEarly(randomBoolean()); diff --git a/server/src/test/java/org/opensearch/transport/InboundHandlerTests.java b/server/src/test/java/org/opensearch/transport/InboundHandlerTests.java index 0d171e17e70e1..be1ac47b55857 100644 --- a/server/src/test/java/org/opensearch/transport/InboundHandlerTests.java +++ b/server/src/test/java/org/opensearch/transport/InboundHandlerTests.java @@ -179,6 +179,12 @@ public String executor() { public TestResponse read(StreamInput in) throws IOException { return new TestResponse(in); } + + @Override + public TestResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } }, null, action)); RequestHandlerRegistry registry = new RequestHandlerRegistry<>( action, @@ -366,6 +372,12 @@ public String executor() { public TestResponse read(StreamInput in) throws IOException { return new TestResponse(in); } + + @Override + public TestResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } }, null, action)); RequestHandlerRegistry registry = new RequestHandlerRegistry<>( @@ -438,6 +450,12 @@ public String executor() { public TestResponse read(StreamInput in) throws IOException { return new TestResponse(in); } + + @Override + public TestResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } }, null, action)); RequestHandlerRegistry registry = new RequestHandlerRegistry<>( @@ -511,6 +529,12 @@ public String executor() { public TestResponse read(StreamInput in) throws IOException { return new TestResponse(in); } + + @Override + public TestResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } }, null, action)); RequestHandlerRegistry registry = new RequestHandlerRegistry<>( action, @@ -599,6 +623,12 @@ public String executor() { public TestResponse read(StreamInput in) throws IOException { return new TestResponse(in); } + + @Override + public TestResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } }, null, action)); RequestHandlerRegistry registry = new RequestHandlerRegistry<>( action, diff --git a/server/src/test/java/org/opensearch/transport/TransportActionProxyTests.java b/server/src/test/java/org/opensearch/transport/TransportActionProxyTests.java index dd2aefd2318f7..df3be09a22dfe 100644 --- a/server/src/test/java/org/opensearch/transport/TransportActionProxyTests.java +++ b/server/src/test/java/org/opensearch/transport/TransportActionProxyTests.java @@ -149,6 +149,12 @@ public void handleException(TransportException exp) { public String executor() { return ThreadPool.Names.SAME; } + + @Override + public SimpleTestResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } ); latch.await(); @@ -209,6 +215,12 @@ public void handleException(TransportException exp) { public String executor() { return ThreadPool.Names.SAME; } + + @Override + public SimpleTestResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } ); latch.await(); diff --git a/server/src/test/java/org/opensearch/transport/TransportServiceDeserializationFailureTests.java b/server/src/test/java/org/opensearch/transport/TransportServiceDeserializationFailureTests.java index d10b4f26100cc..b9c434b14b7d9 100644 --- a/server/src/test/java/org/opensearch/transport/TransportServiceDeserializationFailureTests.java +++ b/server/src/test/java/org/opensearch/transport/TransportServiceDeserializationFailureTests.java @@ -41,6 +41,7 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.tasks.TaskId; import org.opensearch.core.transport.TransportResponse; +import org.opensearch.core.transport.TransportResponse.Empty; import org.opensearch.tasks.Task; import org.opensearch.tasks.TaskAwareRequest; import org.opensearch.telemetry.tracing.noop.NoopTracer; @@ -48,6 +49,7 @@ import org.opensearch.test.transport.MockTransport; import org.opensearch.threadpool.ThreadPool; +import java.io.IOException; import java.util.Collections; import java.util.List; @@ -134,6 +136,12 @@ public TransportResponse.Empty read(StreamInput in) { public String toString() { return "test handler without parent"; } + + @Override + public Empty read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } ); @@ -190,6 +198,12 @@ public TransportResponse.Empty read(StreamInput in) { public String toString() { return "test handler with parent"; } + + @Override + public Empty read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } ); diff --git a/test/framework/src/main/java/org/opensearch/transport/AbstractSimpleTransportTestCase.java b/test/framework/src/main/java/org/opensearch/transport/AbstractSimpleTransportTestCase.java index e43b0756e2f2b..da4c00a3ac773 100644 --- a/test/framework/src/main/java/org/opensearch/transport/AbstractSimpleTransportTestCase.java +++ b/test/framework/src/main/java/org/opensearch/transport/AbstractSimpleTransportTestCase.java @@ -65,6 +65,7 @@ import org.opensearch.core.common.transport.BoundTransportAddress; import org.opensearch.core.common.transport.TransportAddress; import org.opensearch.core.transport.TransportResponse; +import org.opensearch.core.transport.TransportResponse.Empty; import org.opensearch.node.Node; import org.opensearch.tasks.Task; import org.opensearch.telemetry.tracing.noop.NoopTracer; @@ -327,6 +328,12 @@ public void handleException(TransportException exp) { logger.error("Unexpected failure", exp); fail("got exception instead of a response: " + exp.getMessage()); } + + @Override + public StringMessageResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } ); @@ -363,6 +370,12 @@ public void handleException(TransportException exp) { logger.error("Unexpected failure", exp); fail("got exception instead of a response: " + exp.getMessage()); } + + @Override + public StringMessageResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } ); @@ -420,6 +433,12 @@ public void handleException(TransportException exp) { logger.error("Unexpected failure", exp); fail("got exception instead of a response: " + exp.getMessage()); } + + @Override + public StringMessageResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } }; StringMessageRequest ping = new StringMessageRequest("ping"); threadPool.getThreadContext().putHeader("test.ping.user", "ping_user"); @@ -479,6 +498,12 @@ public void handleException(TransportException exp) { public String executor() { return ThreadPool.Names.GENERIC; } + + @Override + public StringMessageResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } ); responseLatch.await(); @@ -662,6 +687,12 @@ public void handleException(TransportException exp) { logger.error("Unexpected failure", exp); fail("got exception instead of a response: " + exp.getMessage()); } + + @Override + public Empty read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } ); @@ -724,6 +755,12 @@ public void handleException(TransportException exp) { logger.error("Unexpected failure", exp); fail("got exception instead of a response: " + exp.getMessage()); } + + @Override + public StringMessageResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } ); @@ -771,6 +808,12 @@ public void handleResponse(StringMessageResponse response) { public void handleException(TransportException exp) { assertThat("runtime_exception: bad message !!!", equalTo(exp.getCause().getMessage())); } + + @Override + public StringMessageResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } ); @@ -1020,6 +1063,12 @@ public void handleResponse(StringMessageResponse response) { public void handleException(TransportException exp) { assertThat(exp, instanceOf(ReceiveTimeoutTransportException.class)); } + + @Override + public StringMessageResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } ); @@ -1094,6 +1143,12 @@ public void handleException(TransportException exp) { latch.countDown(); assertThat(exp, instanceOf(ReceiveTimeoutTransportException.class)); } + + @Override + public StringMessageResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } ); @@ -1135,6 +1190,12 @@ public void handleException(TransportException exp) { logger.error("Unexpected failure", exp); fail("got exception instead of a response for " + counter + ": " + exp.getDetailedMessage()); } + + @Override + public StringMessageResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } ); @@ -1179,6 +1240,12 @@ public void handleException(TransportException exp) {} public String executor() { return ThreadPool.Names.SAME; } + + @Override + public StringMessageResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } }; serviceA.registerRequestHandler("internal:test", ThreadPool.Names.SAME, StringMessageRequest::new, handler); @@ -1482,6 +1549,12 @@ public void handleException(TransportException exp) { public String executor() { return ThreadPool.Names.SAME; } + + @Override + public Version0Response read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } ).txGet(); @@ -1533,6 +1606,12 @@ public void handleException(TransportException exp) { public String executor() { return ThreadPool.Names.SAME; } + + @Override + public Version1Response read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } ).txGet(); @@ -1578,6 +1657,12 @@ public void handleException(TransportException exp) { public String executor() { return ThreadPool.Names.SAME; } + + @Override + public Version1Response read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } ).txGet(); @@ -1620,6 +1705,12 @@ public void handleException(TransportException exp) { public String executor() { return ThreadPool.Names.SAME; } + + @Override + public Version0Response read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } ).txGet(); @@ -1665,6 +1756,12 @@ public void handleException(TransportException exp) { assertThat(cause, instanceOf(ConnectTransportException.class)); assertThat(((ConnectTransportException) cause).node(), equalTo(nodeA)); } + + @Override + public StringMessageResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } ); @@ -1729,6 +1826,12 @@ public void handleResponse(StringMessageResponse response) { public void handleException(TransportException exp) { assertThat(exp, instanceOf(ReceiveTimeoutTransportException.class)); } + + @Override + public StringMessageResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } ); @@ -1780,6 +1883,12 @@ public void handleException(TransportException exp) { public String executor() { return ThreadPool.Names.SAME; } + + @Override + public TestResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } }); if (!latch.await(10, TimeUnit.SECONDS)) { @@ -1828,6 +1937,12 @@ public void handleException(TransportException exp) { public String executor() { return ThreadPool.Names.SAME; } + + @Override + public TestResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } ); @@ -1863,6 +1978,12 @@ public void handleException(TransportException exp) { public String executor() { return ThreadPool.Names.SAME; } + + @Override + public TestResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } ); @@ -2021,6 +2142,12 @@ public void handleException(TransportException exp) { public String executor() { return randomBoolean() ? ThreadPool.Names.SAME : ThreadPool.Names.GENERIC; } + + @Override + public TestResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } ); } else { @@ -2083,6 +2210,12 @@ public void handleException(TransportException exp) { public String executor() { return randomBoolean() ? ThreadPool.Names.SAME : ThreadPool.Names.GENERIC; } + + @Override + public TestResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } for (int i = 0; i < iters; i++) { @@ -2418,6 +2551,12 @@ public void handleException(TransportException exp) { public String executor() { return randomFrom(executors); } + + @Override + public TransportResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } }; serviceB.sendRequest(nodeA, "internal:action", new TestRequest(randomFrom("fail", "pass")), transportResponseHandler); @@ -2468,6 +2607,12 @@ public void handleException(TransportException exp) { public String executor() { return randomFrom(executors); } + + @Override + public TransportResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } }; ConnectionProfile.Builder builder = new ConnectionProfile.Builder(); builder.addConnections( @@ -2538,6 +2683,12 @@ public void handleException(TransportException exp) { public String executor() { return ThreadPool.Names.SAME; } + + @Override + public TransportResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } }; ConnectionProfile.Builder builder = new ConnectionProfile.Builder(); @@ -2612,6 +2763,12 @@ public void handleException(TransportException exp) { public String executor() { return ThreadPool.Names.SAME; } + + @Override + public TransportResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } }; TransportStats stats = serviceC.transport.getStats(); // nothing transmitted / read yet @@ -2733,6 +2890,12 @@ public void handleException(TransportException exp) { public String executor() { return ThreadPool.Names.SAME; } + + @Override + public TransportResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } }; TransportStats stats = serviceC.transport.getStats(); // nothing transmitted / read yet @@ -3120,6 +3283,12 @@ public String executor() { public TransportResponse read(final StreamInput in) { return TransportResponse.Empty.INSTANCE; } + + @Override + public TransportResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } ); assertThat(te.get(), not(nullValue())); diff --git a/test/framework/src/test/java/org/opensearch/test/disruption/DisruptableMockTransportTests.java b/test/framework/src/test/java/org/opensearch/test/disruption/DisruptableMockTransportTests.java index 6b64270ca68e1..a925d9abd43b5 100644 --- a/test/framework/src/test/java/org/opensearch/test/disruption/DisruptableMockTransportTests.java +++ b/test/framework/src/test/java/org/opensearch/test/disruption/DisruptableMockTransportTests.java @@ -233,6 +233,12 @@ public void handleException(TransportException exp) { public String executor() { return ThreadPool.Names.SAME; } + + @Override + public TransportResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } }; } @@ -257,6 +263,12 @@ public void handleException(TransportException exp) { public String executor() { return ThreadPool.Names.SAME; } + + @Override + public TransportResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } }; } @@ -281,6 +293,12 @@ public void handleException(TransportException exp) { public String executor() { return ThreadPool.Names.SAME; } + + @Override + public TransportResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } }; } diff --git a/test/framework/src/test/java/org/opensearch/test/disruption/NetworkDisruptionIT.java b/test/framework/src/test/java/org/opensearch/test/disruption/NetworkDisruptionIT.java index 362ecd692360d..f9ea3e68b5da8 100644 --- a/test/framework/src/test/java/org/opensearch/test/disruption/NetworkDisruptionIT.java +++ b/test/framework/src/test/java/org/opensearch/test/disruption/NetworkDisruptionIT.java @@ -214,6 +214,12 @@ public String executor() { public TransportResponse read(StreamInput in) throws IOException { return ClusterHealthResponse.readResponseFrom(in); } + + @Override + public TransportResponse read(byte[] in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } } ); } From 540846e3c3b9a23ad61f6eac85afacc18f630f88 Mon Sep 17 00:00:00 2001 From: Vacha Shah Date: Wed, 17 Jan 2024 08:22:04 +0000 Subject: [PATCH 11/36] Protobuf support for node-to-node communication Signed-off-by: Vacha Shah --- .../core/transport/TransportMessage.java | 6 + .../netty4/Netty4MessageChannelHandler.java | 3 +- .../transport/nio/TcpReadWriteHandler.java | 3 +- .../search/SearchExecutionStatsCollector.java | 6 +- .../action/search/SearchTransportService.java | 3 +- .../java/org/opensearch/search/SearchHit.java | 60 ++++++++- .../org/opensearch/search/SearchHits.java | 28 ++++- .../search/fetch/FetchSearchResult.java | 28 +++++ .../search/fetch/QueryFetchSearchResult.java | 20 +++ .../search/query/QuerySearchResult.java | 73 +++++++---- .../opensearch/transport/InboundPipeline.java | 5 +- .../transport/NodeToNodeMessage.java | 118 ++++++++++++++++++ .../opensearch/transport/OutboundHandler.java | 77 ++++++++++-- .../opensearch/transport/TcpTransport.java | 8 ++ .../transport/TransportService.java | 6 +- .../proto/server/NodeToNodeMessageProto.proto | 43 +++++++ .../search/QuerySearchResultProto.proto | 2 +- .../transport/InboundPipelineTests.java | 25 +++- .../transport/OutboundHandlerTests.java | 3 +- .../transport/nio/MockNioTransport.java | 3 +- 20 files changed, 464 insertions(+), 56 deletions(-) create mode 100644 server/src/main/java/org/opensearch/transport/NodeToNodeMessage.java create mode 100644 server/src/main/proto/server/NodeToNodeMessageProto.proto diff --git a/libs/core/src/main/java/org/opensearch/core/transport/TransportMessage.java b/libs/core/src/main/java/org/opensearch/core/transport/TransportMessage.java index 69be6cbecc96a..37bfcef434582 100644 --- a/libs/core/src/main/java/org/opensearch/core/transport/TransportMessage.java +++ b/libs/core/src/main/java/org/opensearch/core/transport/TransportMessage.java @@ -46,6 +46,8 @@ public abstract class TransportMessage implements Writeable, ProtobufWriteable { private TransportAddress remoteAddress; + private boolean isProtobuf; + public void remoteAddress(TransportAddress remoteAddress) { this.remoteAddress = remoteAddress; } @@ -54,6 +56,10 @@ public TransportAddress remoteAddress() { return remoteAddress; } + public boolean isMessageProtobuf() { + return isProtobuf; + } + /** * Constructs a new empty transport message */ diff --git a/modules/transport-netty4/src/main/java/org/opensearch/transport/netty4/Netty4MessageChannelHandler.java b/modules/transport-netty4/src/main/java/org/opensearch/transport/netty4/Netty4MessageChannelHandler.java index 7b9999ce5b20e..a05c15d689b22 100644 --- a/modules/transport-netty4/src/main/java/org/opensearch/transport/netty4/Netty4MessageChannelHandler.java +++ b/modules/transport-netty4/src/main/java/org/opensearch/transport/netty4/Netty4MessageChannelHandler.java @@ -78,7 +78,8 @@ final class Netty4MessageChannelHandler extends ChannelDuplexHandler { threadPool::relativeTimeInMillis, transport.getInflightBreaker(), requestHandlers::getHandler, - transport::inboundMessage + transport::inboundMessage, + transport::inboundMessageProtobuf ); } diff --git a/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/TcpReadWriteHandler.java b/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/TcpReadWriteHandler.java index 0c90deed6411c..545f03e03761e 100644 --- a/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/TcpReadWriteHandler.java +++ b/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/TcpReadWriteHandler.java @@ -68,7 +68,8 @@ public TcpReadWriteHandler(NioTcpChannel channel, PageCacheRecycler recycler, Tc threadPool::relativeTimeInMillis, breaker, requestHandlers::getHandler, - transport::inboundMessage + transport::inboundMessage, + transport::inboundMessageProtobuf ); } diff --git a/server/src/main/java/org/opensearch/action/search/SearchExecutionStatsCollector.java b/server/src/main/java/org/opensearch/action/search/SearchExecutionStatsCollector.java index 842e87b3eb635..5fb838cb6c70d 100644 --- a/server/src/main/java/org/opensearch/action/search/SearchExecutionStatsCollector.java +++ b/server/src/main/java/org/opensearch/action/search/SearchExecutionStatsCollector.java @@ -70,8 +70,10 @@ public static BiFunction reader = fetchDocuments ? QueryFetchSearchResult::new : QuerySearchResult::new; final ActionListener handler = responseWrapper.apply(connection, listener); diff --git a/server/src/main/java/org/opensearch/search/SearchHit.java b/server/src/main/java/org/opensearch/search/SearchHit.java index 6391353cfe5b1..378a23b92d63d 100644 --- a/server/src/main/java/org/opensearch/search/SearchHit.java +++ b/server/src/main/java/org/opensearch/search/SearchHit.java @@ -44,6 +44,7 @@ import org.opensearch.core.common.ParsingException; import org.opensearch.core.common.Strings; import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.ProtobufWriteable; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; @@ -67,9 +68,12 @@ import org.opensearch.rest.action.search.RestSearchAction; import org.opensearch.search.fetch.subphase.highlight.HighlightField; import org.opensearch.search.lookup.SourceLookup; +import org.opensearch.server.proto.FetchSearchResultProto; import org.opensearch.transport.RemoteClusterAware; import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -98,7 +102,7 @@ * @opensearch.api */ @PublicApi(since = "1.0.0") -public final class SearchHit implements Writeable, ToXContentObject, Iterable { +public final class SearchHit implements Writeable, ToXContentObject, Iterable, ProtobufWriteable { private final transient int docId; @@ -139,6 +143,8 @@ public final class SearchHit implements Writeable, ToXContentObject, Iterable innerHits; + private FetchSearchResultProto.SearchHit searchHitProto; + // used only in tests public SearchHit(int docId) { this(docId, null, null, null); @@ -236,6 +242,53 @@ public SearchHit(StreamInput in) throws IOException { } } + public SearchHit(byte[] in) throws IOException { + this.searchHitProto = FetchSearchResultProto.SearchHit.parseFrom(in); + docId = -1; + score = this.searchHitProto.getScore(); + id = new Text(this.searchHitProto.getId()); + // Support for nestedIdentity to be added in the future + nestedIdentity = null; + version = this.searchHitProto.getVersion(); + seqNo = this.searchHitProto.getSeqNo(); + primaryTerm = this.searchHitProto.getPrimaryTerm(); + source = BytesReference.fromByteBuffer(ByteBuffer.wrap(this.searchHitProto.getSource().toByteArray())); + if (source.length() == 0) { + source = null; + } + metaFields = new HashMap<>(); + } + + private Map readFields(StreamInput in) throws IOException { + Map fields; + int size = in.readVInt(); + if (size == 0) { + fields = emptyMap(); + } else if (size == 1) { + DocumentField hitField = new DocumentField(in); + fields = singletonMap(hitField.getName(), hitField); + } else { + fields = new HashMap<>(size); + for (int i = 0; i < size; i++) { + DocumentField field = new DocumentField(in); + fields.put(field.getName(), field); + } + fields = unmodifiableMap(fields); + } + return fields; + } + + private void writeFields(StreamOutput out, Map fields) throws IOException { + if (fields == null) { + out.writeVInt(0); + } else { + out.writeVInt(fields.size()); + for (DocumentField field : fields.values()) { + field.writeTo(out); + } + } + } + private static final Text SINGLE_MAPPING_TYPE = new Text(MapperService.SINGLE_MAPPING_NAME); @Override @@ -290,6 +343,11 @@ public void writeTo(StreamOutput out) throws IOException { } } + @Override + public void writeTo(OutputStream out) throws IOException { + out.write(this.searchHitProto.toByteArray()); + } + public int docId() { return this.docId; } diff --git a/server/src/main/java/org/opensearch/search/SearchHits.java b/server/src/main/java/org/opensearch/search/SearchHits.java index 8232643b353f5..2df927318b6c7 100644 --- a/server/src/main/java/org/opensearch/search/SearchHits.java +++ b/server/src/main/java/org/opensearch/search/SearchHits.java @@ -38,6 +38,7 @@ import org.opensearch.common.Nullable; import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.lucene.Lucene; +import org.opensearch.core.common.io.stream.ProtobufWriteable; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; @@ -47,6 +48,7 @@ import org.opensearch.rest.action.search.RestSearchAction; import java.io.IOException; +import java.io.OutputStream; import java.util.ArrayList; import java.util.Arrays; import java.util.Iterator; @@ -61,7 +63,7 @@ * @opensearch.api */ @PublicApi(since = "1.0.0") -public final class SearchHits implements Writeable, ToXContentFragment, Iterable { +public final class SearchHits implements Writeable, ToXContentFragment, Iterable, ProtobufWriteable { public static SearchHits empty() { return empty(true); } @@ -82,6 +84,8 @@ public static SearchHits empty(boolean withTotalHits) { @Nullable private final Object[] collapseValues; + private org.opensearch.server.proto.FetchSearchResultProto.SearchHits searchHitsProto; + public SearchHits(SearchHit[] hits, @Nullable TotalHits totalHits, float maxScore) { this(hits, totalHits, maxScore, null, null, null); } @@ -124,6 +128,23 @@ public SearchHits(StreamInput in) throws IOException { collapseValues = in.readOptionalArray(Lucene::readSortValue, Object[]::new); } + public SearchHits(byte[] in) throws IOException { + this.searchHitsProto = org.opensearch.server.proto.FetchSearchResultProto.SearchHits.parseFrom(in); + this.hits = new SearchHit[this.searchHitsProto.getHitsCount()]; + for (int i = 0; i < this.searchHitsProto.getHitsCount(); i++) { + this.hits[i] = new SearchHit(this.searchHitsProto.getHits(i).toByteArray()); + } + this.totalHits = new TotalHits( + this.searchHitsProto.getTotalHits().getValue(), + Relation.valueOf(this.searchHitsProto.getTotalHits().getRelation().toString()) + ); + this.maxScore = this.searchHitsProto.getMaxScore(); + this.collapseField = this.searchHitsProto.getCollapseField(); + // Below fields are set to null currently, support to be added in the future + this.collapseValues = null; + this.sortFields = null; + } + @Override public void writeTo(StreamOutput out) throws IOException { final boolean hasTotalHits = totalHits != null; @@ -342,4 +363,9 @@ private static Relation parseRelation(String relation) { throw new IllegalArgumentException("invalid total hits relation: " + relation); } } + + @Override + public void writeTo(OutputStream out) throws IOException { + out.write(searchHitsProto.toByteArray()); + } } diff --git a/server/src/main/java/org/opensearch/search/fetch/FetchSearchResult.java b/server/src/main/java/org/opensearch/search/fetch/FetchSearchResult.java index 667b022091a85..76368f3f2de07 100644 --- a/server/src/main/java/org/opensearch/search/fetch/FetchSearchResult.java +++ b/server/src/main/java/org/opensearch/search/fetch/FetchSearchResult.java @@ -32,7 +32,10 @@ package org.opensearch.search.fetch; +import com.google.protobuf.ByteString; +import org.apache.lucene.search.TotalHits.Relation; import org.opensearch.common.annotation.PublicApi; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.search.SearchHit; @@ -76,6 +79,7 @@ public FetchSearchResult(byte[] in) throws IOException { this.fetchSearchResultProto.getContextId().getSessionId(), this.fetchSearchResultProto.getContextId().getId() ); + hits = new SearchHits(this.fetchSearchResultProto.getHits().toByteArray()); } public FetchSearchResult(ShardSearchContextId id, SearchShardTarget shardTarget) { @@ -101,6 +105,30 @@ public FetchSearchResult fetchResult() { public void hits(SearchHits hits) { assert assertNoSearchTarget(hits); this.hits = hits; + if (this.fetchSearchResultProto != null) { + QuerySearchResultProto.TotalHits.Builder totalHitsBuilder = QuerySearchResultProto.TotalHits.newBuilder(); + totalHitsBuilder.setValue(hits.getTotalHits().value); + totalHitsBuilder.setRelation( + hits.getTotalHits().relation == Relation.EQUAL_TO + ? QuerySearchResultProto.TotalHits.Relation.EQUAL_TO + : QuerySearchResultProto.TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO + ); + FetchSearchResultProto.SearchHits.Builder searchHitsBuilder = FetchSearchResultProto.SearchHits.newBuilder(); + searchHitsBuilder.setMaxScore(hits.getMaxScore()); + searchHitsBuilder.setTotalHits(totalHitsBuilder.build()); + for (SearchHit hit : hits.getHits()) { + FetchSearchResultProto.SearchHit.Builder searchHitBuilder = FetchSearchResultProto.SearchHit.newBuilder(); + searchHitBuilder.setIndex(hit.getIndex()); + searchHitBuilder.setId(hit.getId()); + searchHitBuilder.setScore(hit.getScore()); + searchHitBuilder.setSeqNo(hit.getSeqNo()); + searchHitBuilder.setPrimaryTerm(hit.getPrimaryTerm()); + searchHitBuilder.setVersion(hit.getVersion()); + searchHitBuilder.setSource(ByteString.copyFrom(BytesReference.toBytes(hit.getSourceRef()))); + searchHitsBuilder.addHits(searchHitBuilder.build()); + } + this.fetchSearchResultProto = this.fetchSearchResultProto.toBuilder().setHits(searchHitsBuilder.build()).build(); + } } private boolean assertNoSearchTarget(SearchHits hits) { diff --git a/server/src/main/java/org/opensearch/search/fetch/QueryFetchSearchResult.java b/server/src/main/java/org/opensearch/search/fetch/QueryFetchSearchResult.java index b57434a99edb4..6e0c0bf20e7ac 100644 --- a/server/src/main/java/org/opensearch/search/fetch/QueryFetchSearchResult.java +++ b/server/src/main/java/org/opensearch/search/fetch/QueryFetchSearchResult.java @@ -32,6 +32,7 @@ package org.opensearch.search.fetch; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.search.SearchPhaseResult; @@ -117,4 +118,23 @@ public void writeTo(StreamOutput out) throws IOException { queryResult.writeTo(out); fetchResult.writeTo(out); } + + @Override + public boolean isMessageProtobuf() { + // System.setProperty(FeatureFlags.PROTOBUF, "true"); + if (FeatureFlags.isEnabled(FeatureFlags.PROTOBUF_SETTING)) { + return true; + } + return false; + } + + public QueryFetchSearchResultProto.QueryFetchSearchResult response() { + return this.queryFetchSearchResultProto; + } + + public QueryFetchSearchResult(QueryFetchSearchResultProto.QueryFetchSearchResult queryFetchSearchResult) { + this.queryFetchSearchResultProto = queryFetchSearchResult; + this.queryResult = new QuerySearchResult(queryFetchSearchResult.getQueryResult()); + this.fetchResult = new FetchSearchResult(queryFetchSearchResult.getFetchResult()); + } } diff --git a/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java b/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java index b181d6c2462fb..f786c00d05377 100644 --- a/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java +++ b/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java @@ -34,7 +34,9 @@ import org.apache.lucene.search.FieldDoc; import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.TotalHits; +import org.apache.lucene.search.TotalHits.Relation; import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.io.stream.DelayableWriteable; import org.opensearch.common.lucene.search.TopDocsAndMaxScore; @@ -128,28 +130,27 @@ public QuerySearchResult( isNull = false; setShardSearchRequest(shardSearchRequest); - if (FeatureFlags.PROTOBUF_SETTING.get(settings)) { - QuerySearchResultProto.ShardId shardIdProto = QuerySearchResultProto.ShardId.newBuilder() - .setShardId(shardTarget.getShardId().getId()) - .setHashCode(shardTarget.getShardId().hashCode()) - .setIndexName(shardTarget.getShardId().getIndexName()) - .setIndexUUID(shardTarget.getShardId().getIndex().getUUID()) - .build(); - QuerySearchResultProto.SearchShardTarget searchShardTarget = QuerySearchResultProto.SearchShardTarget.newBuilder() - .setNodeId(shardTarget.getNodeId()) - .setShardId(shardIdProto) - .setClusterAlias(shardTarget.getClusterAlias()) - .build(); - this.querySearchResultProto = QuerySearchResultProto.QuerySearchResult.newBuilder() - .setContextId( - QuerySearchResultProto.ShardSearchContextId.newBuilder() - .setSessionId(contextId.getSessionId()) - .setId(contextId.getId()) - .build() - ) - .setSearchShardTarget(searchShardTarget) - .build(); + QuerySearchResultProto.ShardId shardIdProto = QuerySearchResultProto.ShardId.newBuilder() + .setShardId(shardTarget.getShardId().getId()) + .setHashCode(shardTarget.getShardId().hashCode()) + .setIndexName(shardTarget.getShardId().getIndexName()) + .setIndexUUID(shardTarget.getShardId().getIndex().getUUID()) + .build(); + QuerySearchResultProto.SearchShardTarget.Builder searchShardTarget = QuerySearchResultProto.SearchShardTarget.newBuilder() + .setNodeId(shardTarget.getNodeId()) + .setShardId(shardIdProto); + if (shardTarget.getClusterAlias() != null) { + searchShardTarget.setClusterAlias(shardTarget.getClusterAlias()); } + this.querySearchResultProto = QuerySearchResultProto.QuerySearchResult.newBuilder() + .setContextId( + QuerySearchResultProto.ShardSearchContextId.newBuilder() + .setSessionId(contextId.getSessionId()) + .setId(contextId.getId()) + .build() + ) + .setSearchShardTarget(searchShardTarget.build()) + .build(); } private QuerySearchResult(boolean isNull) { @@ -199,9 +200,33 @@ public Boolean terminatedEarly() { } public TopDocsAndMaxScore topDocs() { - if (topDocsAndMaxScore == null) { + if (topDocsAndMaxScore == null && this.querySearchResultProto.getTopDocsAndMaxScore() == null) { throw new IllegalStateException("topDocs already consumed"); } + if (FeatureFlags.isEnabled(FeatureFlags.PROTOBUF_SETTING)) { + ScoreDoc[] scoreDocs = new ScoreDoc[this.querySearchResultProto.getTopDocsAndMaxScore().getTopDocs().getScoreDocsCount()]; + for (int i = 0; i < scoreDocs.length; i++) { + org.opensearch.server.proto.QuerySearchResultProto.QuerySearchResult.TopDocs.ScoreDoc scoreDoc = this.querySearchResultProto + .getTopDocsAndMaxScore() + .getTopDocs() + .getScoreDocsList() + .get(i); + scoreDocs[i] = new ScoreDoc(scoreDoc.getDoc(), scoreDoc.getScore(), scoreDoc.getShardIndex()); + } + TopDocs topDocsFromProtobuf = new TopDocs( + new TotalHits( + this.querySearchResultProto.getTotalHits().getValue(), + Relation.valueOf(this.querySearchResultProto.getTotalHits().getRelation().toString()) + ), + scoreDocs + ); + + TopDocsAndMaxScore topDocsFromProtobufAndMaxScore = new TopDocsAndMaxScore( + topDocsFromProtobuf, + this.querySearchResultProto.getMaxScore() + ); + return topDocsFromProtobufAndMaxScore; + } return topDocsAndMaxScore; } @@ -217,7 +242,7 @@ public boolean hasConsumedTopDocs() { * @throws IllegalStateException if the top docs have already been consumed. */ public TopDocsAndMaxScore consumeTopDocs() { - TopDocsAndMaxScore topDocsAndMaxScore = this.topDocsAndMaxScore; + TopDocsAndMaxScore topDocsAndMaxScore = this.topDocsAndMaxScore == null ? topDocs() : this.topDocsAndMaxScore; if (topDocsAndMaxScore == null) { throw new IllegalStateException("topDocs already consumed"); } @@ -273,7 +298,7 @@ private void setTopDocs(TopDocsAndMaxScore topDocsAndMaxScore) { .setMaxScore(topDocsAndMaxScore.maxScore) .setTopDocs(topDocsBuilder) .build(); - this.querySearchResultProto.toBuilder() + this.querySearchResultProto = this.querySearchResultProto.toBuilder() .setTopDocsAndMaxScore(topDocsAndMaxScoreBuilder) .setMaxScore(this.maxScore) .setTotalHits(topDocsBuilder.getTotalHits()) diff --git a/server/src/main/java/org/opensearch/transport/InboundPipeline.java b/server/src/main/java/org/opensearch/transport/InboundPipeline.java index 5cee3bb975223..fca941fc08721 100644 --- a/server/src/main/java/org/opensearch/transport/InboundPipeline.java +++ b/server/src/main/java/org/opensearch/transport/InboundPipeline.java @@ -38,6 +38,7 @@ import org.opensearch.common.lease.Releasables; import org.opensearch.common.util.PageCacheRecycler; import org.opensearch.core.common.breaker.CircuitBreaker; +import org.opensearch.core.common.bytes.CompositeBytesReference; import org.opensearch.transport.nativeprotocol.NativeInboundBytesHandler; import java.io.IOException; @@ -124,10 +125,6 @@ public void handleBytes(TcpChannel channel, ReleasableBytesReference reference) } public void doHandleBytes(TcpChannel channel, ReleasableBytesReference reference) throws IOException { - channel.getChannelStats().markAccessed(relativeTimeInMillis.getAsLong()); - statsTracker.markBytesRead(reference.length()); - pending.add(reference.retain()); - // If we don't have a current handler, we should try to find one based on the protocol of the incoming bytes. if (currentHandler == null) { for (InboundBytesHandler handler : protocolBytesHandlers) { diff --git a/server/src/main/java/org/opensearch/transport/NodeToNodeMessage.java b/server/src/main/java/org/opensearch/transport/NodeToNodeMessage.java new file mode 100644 index 0000000000000..468a9327b86e5 --- /dev/null +++ b/server/src/main/java/org/opensearch/transport/NodeToNodeMessage.java @@ -0,0 +1,118 @@ +/* +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +*/ + +package org.opensearch.transport; + +import com.google.protobuf.ByteString; +import com.google.protobuf.InvalidProtocolBufferException; +import org.opensearch.Version; +import org.opensearch.common.util.concurrent.ThreadContext; +import org.opensearch.server.proto.NodeToNodeMessageProto; +import org.opensearch.server.proto.NodeToNodeMessageProto.NodeToNodeMessage.Header; +import org.opensearch.server.proto.NodeToNodeMessageProto.NodeToNodeMessage.ResponseHandlersList; +import org.opensearch.server.proto.QueryFetchSearchResultProto.QueryFetchSearchResult; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Outbound data as a message +* +* @opensearch.internal +*/ +public class NodeToNodeMessage { + + private final NodeToNodeMessageProto.NodeToNodeMessage message; + private static final byte[] PREFIX = { (byte) 'E', (byte) 'S' }; + + public NodeToNodeMessage( + long requestId, + byte[] status, + Version version, + ThreadContext threadContext, + QueryFetchSearchResult queryFetchSearchResult, + Set features, + String action + ) { + Header header = Header.newBuilder() + .addAllPrefix(Arrays.asList(ByteString.copyFrom(PREFIX))) + .setRequestId(requestId) + .setStatus(ByteString.copyFrom(status)) + .setVersionId(version.id) + .build(); + Map requestHeaders = threadContext.getHeaders(); + Map> responseHeaders = threadContext.getResponseHeaders(); + Map responseHandlers = new HashMap<>(); + for (Map.Entry> entry : responseHeaders.entrySet()) { + String key = entry.getKey(); + List value = entry.getValue(); + ResponseHandlersList responseHandlersList = ResponseHandlersList.newBuilder().addAllSetOfResponseHandlers(value).build(); + responseHandlers.put(key, responseHandlersList); + } + this.message = NodeToNodeMessageProto.NodeToNodeMessage.newBuilder() + .setHeader(header) + .putAllRequestHeaders(requestHeaders) + .putAllResponseHandlers(responseHandlers) + .setVersion(version.toString()) + .setStatus(ByteString.copyFrom(status)) + .setRequestId(requestId) + .setQueryFetchSearchResult(queryFetchSearchResult) + .setAction(action) + .addAllFeatures(features) + .setIsProtobuf(true) + .build(); + + } + + public NodeToNodeMessage(byte[] data) throws InvalidProtocolBufferException { + this.message = NodeToNodeMessageProto.NodeToNodeMessage.parseFrom(data); + } + + public void writeTo(OutputStream out) throws IOException { + out.write(this.message.toByteArray()); + } + + public NodeToNodeMessageProto.NodeToNodeMessage getMessage() { + return this.message; + } + + @Override + public String toString() { + return "NodeToNodeMessage [message=" + message + "]"; + } + + public org.opensearch.server.proto.NodeToNodeMessageProto.NodeToNodeMessage.Header getHeader() { + return this.message.getHeader(); + } + + public Map getRequestHeaders() { + return this.message.getRequestHeadersMap(); + } + + public Map> getResponseHandlers() { + Map responseHandlers = this.message.getResponseHandlersMap(); + Map> responseHandlersMap = new HashMap<>(); + for (Map.Entry entry : responseHandlers.entrySet()) { + String key = entry.getKey(); + ResponseHandlersList value = entry.getValue(); + Set setOfResponseHandlers = value.getSetOfResponseHandlersList().stream().collect(Collectors.toSet()); + responseHandlersMap.put(key, setOfResponseHandlers); + } + return responseHandlersMap; + } + + public boolean isProtobuf() { + return this.message.getIsProtobuf(); + } +} diff --git a/server/src/main/java/org/opensearch/transport/OutboundHandler.java b/server/src/main/java/org/opensearch/transport/OutboundHandler.java index b83dbdd0effe4..bb102c47c3085 100644 --- a/server/src/main/java/org/opensearch/transport/OutboundHandler.java +++ b/server/src/main/java/org/opensearch/transport/OutboundHandler.java @@ -38,6 +38,7 @@ import org.opensearch.Version; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.CheckedSupplier; +import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.io.stream.ReleasableBytesStreamOutput; import org.opensearch.common.lease.Releasable; import org.opensearch.common.lease.Releasables; @@ -51,9 +52,11 @@ import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.transport.TransportAddress; import org.opensearch.core.transport.TransportResponse; +import org.opensearch.search.fetch.QueryFetchSearchResult; import org.opensearch.threadpool.ThreadPool; import java.io.IOException; +import java.nio.ByteBuffer; import java.util.Set; /** @@ -146,17 +149,35 @@ void sendResponse( final boolean isHandshake ) throws IOException { Version version = Version.min(this.version, nodeVersion); - OutboundMessage.Response message = new OutboundMessage.Response( - threadPool.getThreadContext(), - features, - response, - version, - requestId, - isHandshake, - compress - ); ActionListener listener = ActionListener.wrap(() -> messageListener.onResponseSent(requestId, action, response)); - sendMessage(channel, message, listener); + if (response.isMessageProtobuf()) { + QueryFetchSearchResult queryFetchSearchResult = (QueryFetchSearchResult) response; + if (queryFetchSearchResult.response() != null) { + byte[] bytes = new byte[1]; + bytes[0] = 1; + NodeToNodeMessage protobufMessage = new NodeToNodeMessage( + requestId, + bytes, + Version.CURRENT, + threadPool.getThreadContext(), + queryFetchSearchResult.response(), + features, + action + ); + sendProtobufMessage(channel, protobufMessage, listener); + } + } else { + OutboundMessage.Response message = new OutboundMessage.Response( + threadPool.getThreadContext(), + features, + response, + version, + requestId, + isHandshake, + compress + ); + sendMessage(channel, message, listener); + } } /** @@ -192,6 +213,12 @@ private void sendMessage(TcpChannel channel, OutboundMessage networkMessage, Act internalSend(channel, sendContext); } + private void sendProtobufMessage(TcpChannel channel, NodeToNodeMessage message, ActionListener listener) throws IOException { + ProtobufMessageSerializer serializer = new ProtobufMessageSerializer(message, bigArrays); + SendContext sendContext = new SendContext(channel, serializer, listener, serializer); + internalSend(channel, sendContext); + } + private void internalSend(TcpChannel channel, SendContext sendContext) throws IOException { channel.getChannelStats().markAccessed(threadPool.relativeTimeInMillis()); BytesReference reference = sendContext.get(); @@ -241,6 +268,36 @@ public void close() { } } + private static class ProtobufMessageSerializer implements CheckedSupplier, Releasable { + + private final NodeToNodeMessage message; + private final BigArrays bigArrays; + private volatile ReleasableBytesStreamOutput bytesStreamOutput; + + private ProtobufMessageSerializer(NodeToNodeMessage message, BigArrays bigArrays) { + this.message = message; + this.bigArrays = bigArrays; + } + + @Override + public BytesReference get() throws IOException { + bytesStreamOutput = new ReleasableBytesStreamOutput(bigArrays); + BytesReference reference = serialize(bytesStreamOutput); + return reference; + } + + private BytesReference serialize(BytesStreamOutput bytesStream) throws IOException { + ByteBuffer byteBuffers = ByteBuffer.wrap(message.getMessage().toByteArray()); + message.getMessage().writeTo(bytesStream); + return BytesReference.fromByteBuffer(byteBuffers); + } + + @Override + public void close() { + IOUtils.closeWhileHandlingException(bytesStreamOutput); + } + } + private class SendContext extends NotifyOnceListener implements CheckedSupplier { private final TcpChannel channel; diff --git a/server/src/main/java/org/opensearch/transport/TcpTransport.java b/server/src/main/java/org/opensearch/transport/TcpTransport.java index e32bba5e836d3..7be50851c67c5 100644 --- a/server/src/main/java/org/opensearch/transport/TcpTransport.java +++ b/server/src/main/java/org/opensearch/transport/TcpTransport.java @@ -787,6 +787,14 @@ public void inboundMessage(TcpChannel channel, ProtocolInboundMessage message) { } } + public void inboundMessageProtobuf(TcpChannel channel, BytesReference message) { + try { + inboundHandler.inboundMessageProtobuf(channel, message); + } catch (Exception e) { + onException(channel, e); + } + } + /** * Validates the first 6 bytes of the message header and returns the length of the message. If 6 bytes * are not available, it returns -1. diff --git a/server/src/main/java/org/opensearch/transport/TransportService.java b/server/src/main/java/org/opensearch/transport/TransportService.java index 1cfe30e9c2e1c..1925fbfc03252 100644 --- a/server/src/main/java/org/opensearch/transport/TransportService.java +++ b/server/src/main/java/org/opensearch/transport/TransportService.java @@ -1510,8 +1510,7 @@ void setTimeoutHandler(TimeoutHandler handler) { @Override public T read(byte[] in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); + return delegate.read(in); } } @@ -1730,8 +1729,7 @@ public String toString() { @Override public T read(byte[] in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); + return handler.read(in); } }; } else { diff --git a/server/src/main/proto/server/NodeToNodeMessageProto.proto b/server/src/main/proto/server/NodeToNodeMessageProto.proto new file mode 100644 index 0000000000000..9348941cd2345 --- /dev/null +++ b/server/src/main/proto/server/NodeToNodeMessageProto.proto @@ -0,0 +1,43 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +syntax = "proto3"; +package org.opensearch.server.proto; + +import "server/search/QueryFetchSearchResultProto.proto"; + +option java_outer_classname = "NodeToNodeMessageProto"; + +message NodeToNodeMessage { + Header header = 1; + map requestHeaders = 2; + map responseHandlers = 3; + string version = 4; + repeated string features = 5; + bytes status = 6; + string action = 7; + int64 requestId = 8; + oneof message { + QueryFetchSearchResult queryFetchSearchResult = 16; + } + bool isProtobuf = 17; + + message Header { + repeated bytes prefix = 1; + int64 requestId = 2; + bytes status = 3; + int32 versionId = 4; + } + + message ResponseHandlersList { + repeated string setOfResponseHandlers = 1; + } +} diff --git a/server/src/main/proto/server/search/QuerySearchResultProto.proto b/server/src/main/proto/server/search/QuerySearchResultProto.proto index 8caf561bf2bdd..f61e144501a71 100644 --- a/server/src/main/proto/server/search/QuerySearchResultProto.proto +++ b/server/src/main/proto/server/search/QuerySearchResultProto.proto @@ -62,7 +62,7 @@ message QuerySearchResult { message SearchShardTarget { string nodeId = 1; ShardId shardId = 2; - string clusterAlias = 3; + optional string clusterAlias = 3; } message TotalHits { diff --git a/server/src/test/java/org/opensearch/transport/InboundPipelineTests.java b/server/src/test/java/org/opensearch/transport/InboundPipelineTests.java index 2dfe8a0dd8590..4dca1cfcf07aa 100644 --- a/server/src/test/java/org/opensearch/transport/InboundPipelineTests.java +++ b/server/src/test/java/org/opensearch/transport/InboundPipelineTests.java @@ -95,6 +95,7 @@ public void testPipelineHandling() throws IOException { throw new AssertionError(e); } }; + final BiConsumer messageHandlerProtobuf = (c, m) -> {}; final StatsTracker statsTracker = new StatsTracker(); final LongSupplier millisSupplier = () -> TimeValue.nsecToMSec(System.nanoTime()); @@ -105,7 +106,13 @@ public void testPipelineHandling() throws IOException { final TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); circuitBreaker.startBreaking(); final InboundAggregator aggregator = new InboundAggregator(() -> circuitBreaker, canTripBreaker); - final InboundPipeline pipeline = new InboundPipeline(statsTracker, millisSupplier, decoder, aggregator, messageHandler); + final InboundPipeline pipeline = new InboundPipeline( + statsTracker, + millisSupplier, + decoder, + aggregator, + messageHandler + ); final FakeTcpChannel channel = new FakeTcpChannel(); final int iterations = randomIntBetween(5, 10); @@ -221,7 +228,13 @@ public void testDecodeExceptionIsPropagated() throws IOException { final InboundDecoder decoder = new InboundDecoder(Version.CURRENT, PageCacheRecycler.NON_RECYCLING_INSTANCE); final Supplier breaker = () -> new NoopCircuitBreaker("test"); final InboundAggregator aggregator = new InboundAggregator(breaker, (Predicate) action -> true); - final InboundPipeline pipeline = new InboundPipeline(statsTracker, millisSupplier, decoder, aggregator, messageHandler); + final InboundPipeline pipeline = new InboundPipeline( + statsTracker, + millisSupplier, + decoder, + aggregator, + messageHandler + ); try (BytesStreamOutput streamOutput = new BytesStreamOutput()) { String actionName = "actionName"; @@ -275,7 +288,13 @@ public void testEnsureBodyIsNotPrematurelyReleased() throws IOException { final InboundDecoder decoder = new InboundDecoder(Version.CURRENT, PageCacheRecycler.NON_RECYCLING_INSTANCE); final Supplier breaker = () -> new NoopCircuitBreaker("test"); final InboundAggregator aggregator = new InboundAggregator(breaker, (Predicate) action -> true); - final InboundPipeline pipeline = new InboundPipeline(statsTracker, millisSupplier, decoder, aggregator, messageHandler); + final InboundPipeline pipeline = new InboundPipeline( + statsTracker, + millisSupplier, + decoder, + aggregator, + messageHandler + ); try (BytesStreamOutput streamOutput = new BytesStreamOutput()) { String actionName = "actionName"; diff --git a/server/src/test/java/org/opensearch/transport/OutboundHandlerTests.java b/server/src/test/java/org/opensearch/transport/OutboundHandlerTests.java index 36ba409a2de03..b1e5cf73eff61 100644 --- a/server/src/test/java/org/opensearch/transport/OutboundHandlerTests.java +++ b/server/src/test/java/org/opensearch/transport/OutboundHandlerTests.java @@ -76,6 +76,7 @@ public class OutboundHandlerTests extends OpenSearchTestCase { private final TestThreadPool threadPool = new TestThreadPool(getClass().getName()); private final TransportRequestOptions options = TransportRequestOptions.EMPTY; private final AtomicReference> message = new AtomicReference<>(); + private final AtomicReference protobufMessage = new AtomicReference<>(); private InboundPipeline pipeline; private OutboundHandler handler; private FakeTcpChannel channel; @@ -103,7 +104,7 @@ public void setUp() throws Exception { } catch (IOException e) { throw new AssertionError(e); } - }); + }, (c, m) -> { protobufMessage.set(m); }); } @After diff --git a/test/framework/src/main/java/org/opensearch/transport/nio/MockNioTransport.java b/test/framework/src/main/java/org/opensearch/transport/nio/MockNioTransport.java index cd6bf02efef6f..76eac48bc1a6d 100644 --- a/test/framework/src/main/java/org/opensearch/transport/nio/MockNioTransport.java +++ b/test/framework/src/main/java/org/opensearch/transport/nio/MockNioTransport.java @@ -327,7 +327,8 @@ private MockTcpReadWriteHandler(MockSocketChannel channel, PageCacheRecycler rec threadPool::relativeTimeInMillis, breaker, requestHandlers::getHandler, - transport::inboundMessage + transport::inboundMessage, + transport::inboundMessageProtobuf ); } From e7024db129f9dfa36fdf5de4d70becc42607c6f1 Mon Sep 17 00:00:00 2001 From: Vacha Shah Date: Wed, 17 Jan 2024 23:52:37 +0000 Subject: [PATCH 12/36] Fixing precommit Signed-off-by: Vacha Shah --- server/build.gradle | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/server/build.gradle b/server/build.gradle index 120aa7ffa45ea..09e90141a670b 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -350,6 +350,29 @@ tasks.named("dependencyLicenses").configure { } } +tasks.named("missingJavadoc").configure { + /* + * annotate_code in L210 does not add the Generated annotation to nested code generated using protobuf. + * TODO: Add support to missingJavadoc task to ignore all such nested classes. + * https://github.com/opensearch-project/OpenSearch/issues/11913 + */ + dependsOn("generateProto") + javadocMissingIgnore = [ + "org.opensearch.server.proto.QuerySearchResultProto.QuerySearchResult.RescoreDocIds.setIntegerOrBuilder", + "org.opensearch.server.proto.QuerySearchResultProto.QuerySearchResult.RescoreDocIdsOrBuilder", + "org.opensearch.server.proto.QuerySearchResultProto.QuerySearchResult.TopDocs.ScoreDocOrBuilder", + "org.opensearch.server.proto.QuerySearchResultProto.QuerySearchResult.TopDocsOrBuilder", + "org.opensearch.server.proto.QuerySearchResultProto.QuerySearchResult.TopDocsAndMaxScoreOrBuilder", + "org.opensearch.server.proto.FetchSearchResultProto.SearchHit.SearchSortValuesOrBuilder", + "org.opensearch.server.proto.FetchSearchResultProto.SearchHit.HighlightFieldOrBuilder", + "org.opensearch.server.proto.FetchSearchResultProto.SearchHit.DocumentFieldOrBuilder", + "org.opensearch.server.proto.FetchSearchResultProto.SearchHit.NestedIdentityOrBuilder", + "org.opensearch.server.proto.NodeToNodeMessageProto.NodeToNodeMessage.MessageCase", + "org.opensearch.server.proto.NodeToNodeMessageProto.NodeToNodeMessage.ResponseHandlersListOrBuilder", + "org.opensearch.server.proto.NodeToNodeMessageProto.NodeToNodeMessage.HeaderOrBuilder" + ] +} + tasks.named("filepermissions").configure { mustRunAfter("generateProto") } From 3d49a4b16b1abc9e595381b0e471024bad09006a Mon Sep 17 00:00:00 2001 From: Vacha Shah Date: Thu, 18 Jan 2024 00:50:07 +0000 Subject: [PATCH 13/36] Fixing tests Signed-off-by: Vacha Shah --- .../search/DefaultSearchContext.java | 2 +- .../search/fetch/FetchSearchResult.java | 16 +------ .../search/query/QuerySearchResult.java | 8 +--- .../action/search/DfsQueryPhaseTests.java | 13 ++---- .../action/search/FetchSearchPhaseTests.java | 28 ++++-------- .../search/SearchPhaseControllerTests.java | 43 ++++++------------- .../SearchQueryThenFetchAsyncActionTests.java | 4 +- .../search/query/QuerySearchResultTests.java | 3 +- 8 files changed, 32 insertions(+), 85 deletions(-) diff --git a/server/src/main/java/org/opensearch/search/DefaultSearchContext.java b/server/src/main/java/org/opensearch/search/DefaultSearchContext.java index 36be1fb2936ec..c76ea71c0a094 100644 --- a/server/src/main/java/org/opensearch/search/DefaultSearchContext.java +++ b/server/src/main/java/org/opensearch/search/DefaultSearchContext.java @@ -211,7 +211,7 @@ final class DefaultSearchContext extends SearchContext { // SearchContexts use a BigArrays that can circuit break this.bigArrays = bigArrays.withCircuitBreaking(); this.dfsResult = new DfsSearchResult(readerContext.id(), shardTarget, request); - this.queryResult = new QuerySearchResult(readerContext.id(), shardTarget, request, clusterService.getSettings()); + this.queryResult = new QuerySearchResult(readerContext.id(), shardTarget, request); this.fetchResult = new FetchSearchResult(readerContext.id(), shardTarget); this.indexService = readerContext.indexService(); this.indexShard = readerContext.indexShard(); diff --git a/server/src/main/java/org/opensearch/search/fetch/FetchSearchResult.java b/server/src/main/java/org/opensearch/search/fetch/FetchSearchResult.java index 76368f3f2de07..29dc3d5889d50 100644 --- a/server/src/main/java/org/opensearch/search/fetch/FetchSearchResult.java +++ b/server/src/main/java/org/opensearch/search/fetch/FetchSearchResult.java @@ -32,10 +32,9 @@ package org.opensearch.search.fetch; -import com.google.protobuf.ByteString; import org.apache.lucene.search.TotalHits.Relation; import org.opensearch.common.annotation.PublicApi; -import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.search.SearchHit; @@ -105,7 +104,7 @@ public FetchSearchResult fetchResult() { public void hits(SearchHits hits) { assert assertNoSearchTarget(hits); this.hits = hits; - if (this.fetchSearchResultProto != null) { + if (FeatureFlags.isEnabled(FeatureFlags.PROTOBUF_SETTING) && this.fetchSearchResultProto != null) { QuerySearchResultProto.TotalHits.Builder totalHitsBuilder = QuerySearchResultProto.TotalHits.newBuilder(); totalHitsBuilder.setValue(hits.getTotalHits().value); totalHitsBuilder.setRelation( @@ -116,17 +115,6 @@ public void hits(SearchHits hits) { FetchSearchResultProto.SearchHits.Builder searchHitsBuilder = FetchSearchResultProto.SearchHits.newBuilder(); searchHitsBuilder.setMaxScore(hits.getMaxScore()); searchHitsBuilder.setTotalHits(totalHitsBuilder.build()); - for (SearchHit hit : hits.getHits()) { - FetchSearchResultProto.SearchHit.Builder searchHitBuilder = FetchSearchResultProto.SearchHit.newBuilder(); - searchHitBuilder.setIndex(hit.getIndex()); - searchHitBuilder.setId(hit.getId()); - searchHitBuilder.setScore(hit.getScore()); - searchHitBuilder.setSeqNo(hit.getSeqNo()); - searchHitBuilder.setPrimaryTerm(hit.getPrimaryTerm()); - searchHitBuilder.setVersion(hit.getVersion()); - searchHitBuilder.setSource(ByteString.copyFrom(BytesReference.toBytes(hit.getSourceRef()))); - searchHitsBuilder.addHits(searchHitBuilder.build()); - } this.fetchSearchResultProto = this.fetchSearchResultProto.toBuilder().setHits(searchHitsBuilder.build()).build(); } } diff --git a/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java b/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java index f786c00d05377..599ba66850409 100644 --- a/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java +++ b/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java @@ -40,7 +40,6 @@ import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.io.stream.DelayableWriteable; import org.opensearch.common.lucene.search.TopDocsAndMaxScore; -import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; @@ -119,12 +118,7 @@ public QuerySearchResult(byte[] in) throws IOException { isNull = false; } - public QuerySearchResult( - ShardSearchContextId contextId, - SearchShardTarget shardTarget, - ShardSearchRequest shardSearchRequest, - Settings settings - ) { + public QuerySearchResult(ShardSearchContextId contextId, SearchShardTarget shardTarget, ShardSearchRequest shardSearchRequest) { this.contextId = contextId; setSearchShardTarget(shardTarget); isNull = false; diff --git a/server/src/test/java/org/opensearch/action/search/DfsQueryPhaseTests.java b/server/src/test/java/org/opensearch/action/search/DfsQueryPhaseTests.java index 68ae014c9d562..6952841c295e2 100644 --- a/server/src/test/java/org/opensearch/action/search/DfsQueryPhaseTests.java +++ b/server/src/test/java/org/opensearch/action/search/DfsQueryPhaseTests.java @@ -39,7 +39,6 @@ import org.apache.lucene.tests.store.MockDirectoryWrapper; import org.opensearch.action.OriginalIndices; import org.opensearch.common.lucene.search.TopDocsAndMaxScore; -import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.AtomicArray; import org.opensearch.common.util.concurrent.OpenSearchExecutors; import org.opensearch.core.common.breaker.CircuitBreaker; @@ -102,8 +101,7 @@ public void sendExecuteQuery( QuerySearchResult queryResult = new QuerySearchResult( new ShardSearchContextId("", 123), new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE), - null, - Settings.EMPTY + null ); queryResult.topDocs( new TopDocsAndMaxScore( @@ -118,8 +116,7 @@ public void sendExecuteQuery( QuerySearchResult queryResult = new QuerySearchResult( new ShardSearchContextId("", 123), new SearchShardTarget("node2", new ShardId("test", "na", 0), null, OriginalIndices.NONE), - null, - Settings.EMPTY + null ); queryResult.topDocs( new TopDocsAndMaxScore( @@ -202,8 +199,7 @@ public void sendExecuteQuery( QuerySearchResult queryResult = new QuerySearchResult( new ShardSearchContextId("", 123), new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE), - null, - Settings.EMPTY + null ); queryResult.topDocs( new TopDocsAndMaxScore( @@ -290,8 +286,7 @@ public void sendExecuteQuery( QuerySearchResult queryResult = new QuerySearchResult( new ShardSearchContextId("", 123), new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE), - null, - Settings.EMPTY + null ); queryResult.topDocs( new TopDocsAndMaxScore( diff --git a/server/src/test/java/org/opensearch/action/search/FetchSearchPhaseTests.java b/server/src/test/java/org/opensearch/action/search/FetchSearchPhaseTests.java index 09f03c5ba2e18..1eb3a44642806 100644 --- a/server/src/test/java/org/opensearch/action/search/FetchSearchPhaseTests.java +++ b/server/src/test/java/org/opensearch/action/search/FetchSearchPhaseTests.java @@ -38,7 +38,6 @@ import org.opensearch.action.OriginalIndices; import org.opensearch.common.UUIDs; import org.opensearch.common.lucene.search.TopDocsAndMaxScore; -import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.OpenSearchExecutors; import org.opensearch.core.common.breaker.CircuitBreaker; import org.opensearch.core.common.breaker.NoopCircuitBreaker; @@ -141,8 +140,7 @@ public void testFetchTwoDocument() { QuerySearchResult queryResult = new QuerySearchResult( ctx1, new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE), - null, - Settings.EMPTY + null ); queryResult.topDocs( new TopDocsAndMaxScore( @@ -159,8 +157,7 @@ public void testFetchTwoDocument() { queryResult = new QuerySearchResult( ctx2, new SearchShardTarget("node2", new ShardId("test", "na", 1), null, OriginalIndices.NONE), - null, - Settings.EMPTY + null ); queryResult.topDocs( new TopDocsAndMaxScore( @@ -239,8 +236,7 @@ public void testFailFetchOneDoc() { QuerySearchResult queryResult = new QuerySearchResult( ctx, new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE), - null, - Settings.EMPTY + null ); queryResult.topDocs( new TopDocsAndMaxScore( @@ -256,8 +252,7 @@ public void testFailFetchOneDoc() { queryResult = new QuerySearchResult( new ShardSearchContextId("", 321), new SearchShardTarget("node2", new ShardId("test", "na", 1), null, OriginalIndices.NONE), - null, - Settings.EMPTY + null ); queryResult.topDocs( new TopDocsAndMaxScore( @@ -338,8 +333,7 @@ public void testFetchDocsConcurrently() throws InterruptedException { QuerySearchResult queryResult = new QuerySearchResult( new ShardSearchContextId("", i), new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE), - null, - Settings.EMPTY + null ); queryResult.topDocs( new TopDocsAndMaxScore( @@ -429,8 +423,7 @@ public void testExceptionFailsPhase() { QuerySearchResult queryResult = new QuerySearchResult( new ShardSearchContextId("", 123), new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE), - null, - Settings.EMPTY + null ); queryResult.topDocs( new TopDocsAndMaxScore( @@ -446,8 +439,7 @@ public void testExceptionFailsPhase() { queryResult = new QuerySearchResult( new ShardSearchContextId("", 321), new SearchShardTarget("node2", new ShardId("test", "na", 1), null, OriginalIndices.NONE), - null, - Settings.EMPTY + null ); queryResult.topDocs( new TopDocsAndMaxScore( @@ -524,8 +516,7 @@ public void testCleanupIrrelevantContexts() { // contexts that are not fetched s QuerySearchResult queryResult = new QuerySearchResult( ctx1, new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE), - null, - Settings.EMPTY + null ); queryResult.topDocs( new TopDocsAndMaxScore( @@ -542,8 +533,7 @@ public void testCleanupIrrelevantContexts() { // contexts that are not fetched s queryResult = new QuerySearchResult( ctx2, new SearchShardTarget("node2", new ShardId("test", "na", 1), null, OriginalIndices.NONE), - null, - Settings.EMPTY + null ); queryResult.topDocs( new TopDocsAndMaxScore( diff --git a/server/src/test/java/org/opensearch/action/search/SearchPhaseControllerTests.java b/server/src/test/java/org/opensearch/action/search/SearchPhaseControllerTests.java index eeb72969990fc..a927f733cc504 100644 --- a/server/src/test/java/org/opensearch/action/search/SearchPhaseControllerTests.java +++ b/server/src/test/java/org/opensearch/action/search/SearchPhaseControllerTests.java @@ -410,12 +410,7 @@ private static AtomicArray generateQueryResults( clusterAlias, OriginalIndices.NONE ); - QuerySearchResult querySearchResult = new QuerySearchResult( - new ShardSearchContextId("", shardIndex), - searchShardTarget, - null, - Settings.EMPTY - ); + QuerySearchResult querySearchResult = new QuerySearchResult(new ShardSearchContextId("", shardIndex), searchShardTarget, null); final TopDocs topDocs; float maxScore = 0; if (searchHitsSize == 0) { @@ -690,8 +685,7 @@ private void consumerTestCase(int numEmptyResponses) throws Exception { QuerySearchResult result = new QuerySearchResult( new ShardSearchContextId("", 0), new SearchShardTarget("node", new ShardId("a", "b", 0), null, OriginalIndices.NONE), - null, - Settings.EMPTY + null ); result.topDocs( new TopDocsAndMaxScore(new TopDocs(new TotalHits(0, TotalHits.Relation.EQUAL_TO), new ScoreDoc[0]), Float.NaN), @@ -705,8 +699,7 @@ private void consumerTestCase(int numEmptyResponses) throws Exception { result = new QuerySearchResult( new ShardSearchContextId("", 1), new SearchShardTarget("node", new ShardId("a", "b", 0), null, OriginalIndices.NONE), - null, - Settings.EMPTY + null ); result.topDocs( new TopDocsAndMaxScore(new TopDocs(new TotalHits(0, TotalHits.Relation.EQUAL_TO), new ScoreDoc[0]), Float.NaN), @@ -720,8 +713,7 @@ private void consumerTestCase(int numEmptyResponses) throws Exception { result = new QuerySearchResult( new ShardSearchContextId("", 1), new SearchShardTarget("node", new ShardId("a", "b", 0), null, OriginalIndices.NONE), - null, - Settings.EMPTY + null ); result.topDocs( new TopDocsAndMaxScore(new TopDocs(new TotalHits(0, TotalHits.Relation.EQUAL_TO), new ScoreDoc[0]), Float.NaN), @@ -797,8 +789,7 @@ public void testConsumerConcurrently() throws Exception { QuerySearchResult result = new QuerySearchResult( new ShardSearchContextId("", id), new SearchShardTarget("node", new ShardId("a", "b", id), null, OriginalIndices.NONE), - null, - Settings.EMPTY + null ); result.topDocs( new TopDocsAndMaxScore( @@ -859,8 +850,7 @@ public void testConsumerOnlyAggs() throws Exception { QuerySearchResult result = new QuerySearchResult( new ShardSearchContextId("", i), new SearchShardTarget("node", new ShardId("a", "b", i), null, OriginalIndices.NONE), - null, - Settings.EMPTY + null ); result.topDocs( new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO), new ScoreDoc[0]), number), @@ -913,8 +903,7 @@ public void testConsumerOnlyHits() throws Exception { QuerySearchResult result = new QuerySearchResult( new ShardSearchContextId("", i), new SearchShardTarget("node", new ShardId("a", "b", i), null, OriginalIndices.NONE), - null, - Settings.EMPTY + null ); result.topDocs( new TopDocsAndMaxScore( @@ -970,8 +959,7 @@ public void testReduceTopNWithFromOffset() throws Exception { QuerySearchResult result = new QuerySearchResult( new ShardSearchContextId("", i), new SearchShardTarget("node", new ShardId("a", "b", i), null, OriginalIndices.NONE), - null, - Settings.EMPTY + null ); ScoreDoc[] docs = new ScoreDoc[3]; for (int j = 0; j < docs.length; j++) { @@ -1026,8 +1014,7 @@ public void testConsumerSortByField() throws Exception { QuerySearchResult result = new QuerySearchResult( new ShardSearchContextId("", i), new SearchShardTarget("node", new ShardId("a", "b", i), null, OriginalIndices.NONE), - null, - Settings.EMPTY + null ); result.topDocs(new TopDocsAndMaxScore(topDocs, Float.NaN), docValueFormats); result.setShardIndex(i); @@ -1076,8 +1063,7 @@ public void testConsumerFieldCollapsing() throws Exception { QuerySearchResult result = new QuerySearchResult( new ShardSearchContextId("", i), new SearchShardTarget("node", new ShardId("a", "b", i), null, OriginalIndices.NONE), - null, - Settings.EMPTY + null ); result.topDocs(new TopDocsAndMaxScore(topDocs, Float.NaN), docValueFormats); result.setShardIndex(i); @@ -1121,8 +1107,7 @@ public void testConsumerSuggestions() throws Exception { QuerySearchResult result = new QuerySearchResult( new ShardSearchContextId("", i), new SearchShardTarget("node", new ShardId("a", "b", i), null, OriginalIndices.NONE), - null, - Settings.EMPTY + null ); List>> suggestions = new ArrayList<>(); @@ -1263,8 +1248,7 @@ public void onFinalReduce(List shards, TotalHits totalHits, Interna QuerySearchResult result = new QuerySearchResult( new ShardSearchContextId("", id), new SearchShardTarget("node", new ShardId("a", "b", id), null, OriginalIndices.NONE), - null, - Settings.EMPTY + null ); result.topDocs( new TopDocsAndMaxScore( @@ -1350,8 +1334,7 @@ private void testReduceCase(boolean shouldFail) throws Exception { QuerySearchResult result = new QuerySearchResult( new ShardSearchContextId(UUIDs.randomBase64UUID(), index), new SearchShardTarget("node", new ShardId("a", "b", index), null, OriginalIndices.NONE), - null, - Settings.EMPTY + null ); result.topDocs( new TopDocsAndMaxScore(new TopDocs(new TotalHits(0, TotalHits.Relation.EQUAL_TO), Lucene.EMPTY_SCORE_DOCS), Float.NaN), diff --git a/server/src/test/java/org/opensearch/action/search/SearchQueryThenFetchAsyncActionTests.java b/server/src/test/java/org/opensearch/action/search/SearchQueryThenFetchAsyncActionTests.java index a1f4c5922f10f..aefbbe80d5fa1 100644 --- a/server/src/test/java/org/opensearch/action/search/SearchQueryThenFetchAsyncActionTests.java +++ b/server/src/test/java/org/opensearch/action/search/SearchQueryThenFetchAsyncActionTests.java @@ -43,7 +43,6 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.routing.GroupShardsIterator; import org.opensearch.common.lucene.search.TopDocsAndMaxScore; -import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.OpenSearchExecutors; import org.opensearch.core.common.Strings; @@ -148,8 +147,7 @@ public void sendExecuteQuery( QuerySearchResult queryResult = new QuerySearchResult( new ShardSearchContextId("N/A", 123), new SearchShardTarget("node1", new ShardId("idx", "na", shardId), null, OriginalIndices.NONE), - null, - Settings.EMPTY + null ); SortField sortField = new SortField("timestamp", SortField.Type.LONG); if (withCollapse) { diff --git a/server/src/test/java/org/opensearch/search/query/QuerySearchResultTests.java b/server/src/test/java/org/opensearch/search/query/QuerySearchResultTests.java index 81e9f4433cdc9..41e4e1ae45a73 100644 --- a/server/src/test/java/org/opensearch/search/query/QuerySearchResultTests.java +++ b/server/src/test/java/org/opensearch/search/query/QuerySearchResultTests.java @@ -84,8 +84,7 @@ private static QuerySearchResult createTestInstance() throws Exception { QuerySearchResult result = new QuerySearchResult( new ShardSearchContextId(UUIDs.base64UUID(), randomLong()), new SearchShardTarget("node", shardId, null, OriginalIndices.NONE), - shardSearchRequest, - Settings.EMPTY + shardSearchRequest ); if (randomBoolean()) { result.terminatedEarly(randomBoolean()); From a2cf25a42aeb43237ee3597a0cd98703b28537bc Mon Sep 17 00:00:00 2001 From: Vacha Shah Date: Mon, 22 Jan 2024 20:40:19 +0000 Subject: [PATCH 14/36] Addressing comments Signed-off-by: Vacha Shah --- .../java/org/opensearch/common/settings/ClusterSettings.java | 4 +--- server/src/main/java/org/opensearch/search/SearchHit.java | 4 +--- server/src/main/java/org/opensearch/search/SearchHits.java | 4 +--- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index 83b2186fa6cee..9d763c970c3e7 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -750,8 +750,6 @@ public void apply(Settings value, Settings current, Settings previous) { TelemetrySettings.METRICS_FEATURE_ENABLED_SETTING ), List.of(FeatureFlags.PLUGGABLE_CACHE), - List.of(CacheSettings.getConcreteStoreNameSettingForCacheType(CacheType.INDICES_REQUEST_CACHE)), - List.of(FeatureFlags.PROTOBUF), - List.of(FeatureFlags.PROTOBUF_SETTING) + List.of(CacheSettings.getConcreteStoreNameSettingForCacheType(CacheType.INDICES_REQUEST_CACHE)) ); } diff --git a/server/src/main/java/org/opensearch/search/SearchHit.java b/server/src/main/java/org/opensearch/search/SearchHit.java index 378a23b92d63d..804dadbf608db 100644 --- a/server/src/main/java/org/opensearch/search/SearchHit.java +++ b/server/src/main/java/org/opensearch/search/SearchHit.java @@ -44,7 +44,6 @@ import org.opensearch.core.common.ParsingException; import org.opensearch.core.common.Strings; import org.opensearch.core.common.bytes.BytesReference; -import org.opensearch.core.common.io.stream.ProtobufWriteable; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; @@ -102,7 +101,7 @@ * @opensearch.api */ @PublicApi(since = "1.0.0") -public final class SearchHit implements Writeable, ToXContentObject, Iterable, ProtobufWriteable { +public final class SearchHit implements Writeable, ToXContentObject, Iterable { private final transient int docId; @@ -343,7 +342,6 @@ public void writeTo(StreamOutput out) throws IOException { } } - @Override public void writeTo(OutputStream out) throws IOException { out.write(this.searchHitProto.toByteArray()); } diff --git a/server/src/main/java/org/opensearch/search/SearchHits.java b/server/src/main/java/org/opensearch/search/SearchHits.java index 2df927318b6c7..24037fc8aaf4d 100644 --- a/server/src/main/java/org/opensearch/search/SearchHits.java +++ b/server/src/main/java/org/opensearch/search/SearchHits.java @@ -38,7 +38,6 @@ import org.opensearch.common.Nullable; import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.lucene.Lucene; -import org.opensearch.core.common.io.stream.ProtobufWriteable; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; @@ -63,7 +62,7 @@ * @opensearch.api */ @PublicApi(since = "1.0.0") -public final class SearchHits implements Writeable, ToXContentFragment, Iterable, ProtobufWriteable { +public final class SearchHits implements Writeable, ToXContentFragment, Iterable { public static SearchHits empty() { return empty(true); } @@ -364,7 +363,6 @@ private static Relation parseRelation(String relation) { } } - @Override public void writeTo(OutputStream out) throws IOException { out.write(searchHitsProto.toByteArray()); } From 0cf2fcb272aa513605796041e5323d9cf99f989a Mon Sep 17 00:00:00 2001 From: Vacha Shah Date: Tue, 23 Jan 2024 20:07:24 +0000 Subject: [PATCH 15/36] Adding abstraction for InboundMessage - comment Signed-off-by: Vacha Shah --- .../netty4/Netty4MessageChannelHandler.java | 4 +-- .../transport/nio/TcpReadWriteHandler.java | 4 +-- .../transport/BaseInboundMessage.java | 27 +++++++++++++++++++ .../transport/NodeToNodeMessage.java | 13 ++++++++- .../opensearch/transport/TcpTransport.java | 8 ------ .../transport/OutboundHandlerTests.java | 6 +++-- .../transport/nio/MockNioTransport.java | 4 +-- 7 files changed, 49 insertions(+), 17 deletions(-) create mode 100644 server/src/main/java/org/opensearch/transport/BaseInboundMessage.java diff --git a/modules/transport-netty4/src/main/java/org/opensearch/transport/netty4/Netty4MessageChannelHandler.java b/modules/transport-netty4/src/main/java/org/opensearch/transport/netty4/Netty4MessageChannelHandler.java index a05c15d689b22..a772a9d9b533f 100644 --- a/modules/transport-netty4/src/main/java/org/opensearch/transport/netty4/Netty4MessageChannelHandler.java +++ b/modules/transport-netty4/src/main/java/org/opensearch/transport/netty4/Netty4MessageChannelHandler.java @@ -78,8 +78,8 @@ final class Netty4MessageChannelHandler extends ChannelDuplexHandler { threadPool::relativeTimeInMillis, transport.getInflightBreaker(), requestHandlers::getHandler, - transport::inboundMessage, - transport::inboundMessageProtobuf + transport::inboundMessage + // transport::inboundMessageProtobuf ); } diff --git a/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/TcpReadWriteHandler.java b/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/TcpReadWriteHandler.java index 545f03e03761e..09e0889b489ee 100644 --- a/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/TcpReadWriteHandler.java +++ b/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/TcpReadWriteHandler.java @@ -68,8 +68,8 @@ public TcpReadWriteHandler(NioTcpChannel channel, PageCacheRecycler recycler, Tc threadPool::relativeTimeInMillis, breaker, requestHandlers::getHandler, - transport::inboundMessage, - transport::inboundMessageProtobuf + transport::inboundMessage + // transport::inboundMessageProtobuf ); } diff --git a/server/src/main/java/org/opensearch/transport/BaseInboundMessage.java b/server/src/main/java/org/opensearch/transport/BaseInboundMessage.java new file mode 100644 index 0000000000000..488d342b469b1 --- /dev/null +++ b/server/src/main/java/org/opensearch/transport/BaseInboundMessage.java @@ -0,0 +1,27 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.transport; + +public interface BaseInboundMessage { + + enum Protocol { + DEFAULT, + PROTOBUF, + } + + /** + * @return the protocol used to encode this message + */ + public Protocol getProtocol(); + + /** + * set the protocol used to encode this message + */ + public void setProtocol(); +} diff --git a/server/src/main/java/org/opensearch/transport/NodeToNodeMessage.java b/server/src/main/java/org/opensearch/transport/NodeToNodeMessage.java index 468a9327b86e5..672e447e025e3 100644 --- a/server/src/main/java/org/opensearch/transport/NodeToNodeMessage.java +++ b/server/src/main/java/org/opensearch/transport/NodeToNodeMessage.java @@ -31,10 +31,11 @@ * * @opensearch.internal */ -public class NodeToNodeMessage { +public class NodeToNodeMessage implements BaseInboundMessage { private final NodeToNodeMessageProto.NodeToNodeMessage message; private static final byte[] PREFIX = { (byte) 'E', (byte) 'S' }; + private Protocol protocol; public NodeToNodeMessage( long requestId, @@ -115,4 +116,14 @@ public Map> getResponseHandlers() { public boolean isProtobuf() { return this.message.getIsProtobuf(); } + + @Override + public Protocol getProtocol() { + return Protocol.PROTOBUF; + } + + @Override + public void setProtocol() { + this.protocol = Protocol.PROTOBUF; + } } diff --git a/server/src/main/java/org/opensearch/transport/TcpTransport.java b/server/src/main/java/org/opensearch/transport/TcpTransport.java index 7be50851c67c5..e32bba5e836d3 100644 --- a/server/src/main/java/org/opensearch/transport/TcpTransport.java +++ b/server/src/main/java/org/opensearch/transport/TcpTransport.java @@ -787,14 +787,6 @@ public void inboundMessage(TcpChannel channel, ProtocolInboundMessage message) { } } - public void inboundMessageProtobuf(TcpChannel channel, BytesReference message) { - try { - inboundHandler.inboundMessageProtobuf(channel, message); - } catch (Exception e) { - onException(channel, e); - } - } - /** * Validates the first 6 bytes of the message header and returns the length of the message. If 6 bytes * are not available, it returns -1. diff --git a/server/src/test/java/org/opensearch/transport/OutboundHandlerTests.java b/server/src/test/java/org/opensearch/transport/OutboundHandlerTests.java index b1e5cf73eff61..d71e4b8a0310a 100644 --- a/server/src/test/java/org/opensearch/transport/OutboundHandlerTests.java +++ b/server/src/test/java/org/opensearch/transport/OutboundHandlerTests.java @@ -76,7 +76,7 @@ public class OutboundHandlerTests extends OpenSearchTestCase { private final TestThreadPool threadPool = new TestThreadPool(getClass().getName()); private final TransportRequestOptions options = TransportRequestOptions.EMPTY; private final AtomicReference> message = new AtomicReference<>(); - private final AtomicReference protobufMessage = new AtomicReference<>(); + // private final AtomicReference protobufMessage = new AtomicReference<>(); private InboundPipeline pipeline; private OutboundHandler handler; private FakeTcpChannel channel; @@ -104,7 +104,9 @@ public void setUp() throws Exception { } catch (IOException e) { throw new AssertionError(e); } - }, (c, m) -> { protobufMessage.set(m); }); + } + // , (c, m) -> { protobufMessage.set(m); } + ); } @After diff --git a/test/framework/src/main/java/org/opensearch/transport/nio/MockNioTransport.java b/test/framework/src/main/java/org/opensearch/transport/nio/MockNioTransport.java index 76eac48bc1a6d..62b79b5f715ce 100644 --- a/test/framework/src/main/java/org/opensearch/transport/nio/MockNioTransport.java +++ b/test/framework/src/main/java/org/opensearch/transport/nio/MockNioTransport.java @@ -327,8 +327,8 @@ private MockTcpReadWriteHandler(MockSocketChannel channel, PageCacheRecycler rec threadPool::relativeTimeInMillis, breaker, requestHandlers::getHandler, - transport::inboundMessage, - transport::inboundMessageProtobuf + transport::inboundMessage + // transport::inboundMessageProtobuf ); } From f7a9d47e2e24396b691d967a0fdc1bd29b432858 Mon Sep 17 00:00:00 2001 From: Vacha Shah Date: Tue, 23 Jan 2024 20:27:35 +0000 Subject: [PATCH 16/36] Removing unused code Signed-off-by: Vacha Shah --- .../netty4/Netty4MessageChannelHandler.java | 1 - .../transport/nio/TcpReadWriteHandler.java | 1 - .../action/search/SearchTransportService.java | 27 +++++++------------ .../transport/BaseInboundMessage.java | 11 +++++++- .../transport/NodeToNodeMessage.java | 5 ---- .../proto/server/NodeToNodeMessageProto.proto | 1 - .../transport/nio/MockNioTransport.java | 1 - 7 files changed, 20 insertions(+), 27 deletions(-) diff --git a/modules/transport-netty4/src/main/java/org/opensearch/transport/netty4/Netty4MessageChannelHandler.java b/modules/transport-netty4/src/main/java/org/opensearch/transport/netty4/Netty4MessageChannelHandler.java index a772a9d9b533f..7b9999ce5b20e 100644 --- a/modules/transport-netty4/src/main/java/org/opensearch/transport/netty4/Netty4MessageChannelHandler.java +++ b/modules/transport-netty4/src/main/java/org/opensearch/transport/netty4/Netty4MessageChannelHandler.java @@ -79,7 +79,6 @@ final class Netty4MessageChannelHandler extends ChannelDuplexHandler { transport.getInflightBreaker(), requestHandlers::getHandler, transport::inboundMessage - // transport::inboundMessageProtobuf ); } diff --git a/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/TcpReadWriteHandler.java b/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/TcpReadWriteHandler.java index 09e0889b489ee..0c90deed6411c 100644 --- a/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/TcpReadWriteHandler.java +++ b/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/TcpReadWriteHandler.java @@ -69,7 +69,6 @@ public TcpReadWriteHandler(NioTcpChannel channel, PageCacheRecycler recycler, Tc breaker, requestHandlers::getHandler, transport::inboundMessage - // transport::inboundMessageProtobuf ); } diff --git a/server/src/main/java/org/opensearch/action/search/SearchTransportService.java b/server/src/main/java/org/opensearch/action/search/SearchTransportService.java index dded80aff56ba..548fbd953db25 100644 --- a/server/src/main/java/org/opensearch/action/search/SearchTransportService.java +++ b/server/src/main/java/org/opensearch/action/search/SearchTransportService.java @@ -70,6 +70,7 @@ import org.opensearch.transport.TransportException; import org.opensearch.transport.TransportRequest; import org.opensearch.transport.TransportRequestOptions; +import org.opensearch.transport.TransportResponseHandler; import org.opensearch.transport.TransportService; import java.io.IOException; @@ -244,30 +245,22 @@ public void sendExecuteQuery( // we optimize this and expect a QueryFetchSearchResult if we only have a single shard in the search request // this used to be the QUERY_AND_FETCH which doesn't exist anymore. final boolean fetchDocuments = request.numberOfShards() == 1; + final ActionListener handler = responseWrapper.apply(connection, listener); + TransportResponseHandler transportResponseHandler; // System.setProperty(FeatureFlags.PROTOBUF, "true"); if (FeatureFlags.isEnabled(FeatureFlags.PROTOBUF_SETTING)) { ProtobufWriteable.Reader reader = fetchDocuments ? QueryFetchSearchResult::new : QuerySearchResult::new; - - final ActionListener handler = responseWrapper.apply(connection, listener); - transportService.sendChildRequest( - connection, - QUERY_ACTION_NAME, - request, - task, - new ProtobufConnectionCountingHandler<>(handler, reader, clientConnections, connection.getNode().getId()) + transportResponseHandler = new ProtobufConnectionCountingHandler<>( + handler, + reader, + clientConnections, + connection.getNode().getId() ); } else { Writeable.Reader reader = fetchDocuments ? QueryFetchSearchResult::new : QuerySearchResult::new; - - final ActionListener handler = responseWrapper.apply(connection, listener); - transportService.sendChildRequest( - connection, - QUERY_ACTION_NAME, - request, - task, - new ConnectionCountingHandler<>(handler, reader, clientConnections, connection.getNode().getId()) - ); + transportResponseHandler = new ConnectionCountingHandler<>(handler, reader, clientConnections, connection.getNode().getId()); } + transportService.sendChildRequest(connection, QUERY_ACTION_NAME, request, task, transportResponseHandler); } public void sendExecuteQuery( diff --git a/server/src/main/java/org/opensearch/transport/BaseInboundMessage.java b/server/src/main/java/org/opensearch/transport/BaseInboundMessage.java index 488d342b469b1..bc8833d45be9e 100644 --- a/server/src/main/java/org/opensearch/transport/BaseInboundMessage.java +++ b/server/src/main/java/org/opensearch/transport/BaseInboundMessage.java @@ -8,8 +8,17 @@ package org.opensearch.transport; +/** + * Base class for inbound data as a message. + * Different implementations are used for different protocols. + * + * @opensearch.internal + */ public interface BaseInboundMessage { + /** + * The protocol used to encode this message + */ enum Protocol { DEFAULT, PROTOBUF, @@ -21,7 +30,7 @@ enum Protocol { public Protocol getProtocol(); /** - * set the protocol used to encode this message + * Set the protocol used to encode this message */ public void setProtocol(); } diff --git a/server/src/main/java/org/opensearch/transport/NodeToNodeMessage.java b/server/src/main/java/org/opensearch/transport/NodeToNodeMessage.java index 672e447e025e3..f2fbd571725df 100644 --- a/server/src/main/java/org/opensearch/transport/NodeToNodeMessage.java +++ b/server/src/main/java/org/opensearch/transport/NodeToNodeMessage.java @@ -71,7 +71,6 @@ public NodeToNodeMessage( .setQueryFetchSearchResult(queryFetchSearchResult) .setAction(action) .addAllFeatures(features) - .setIsProtobuf(true) .build(); } @@ -113,10 +112,6 @@ public Map> getResponseHandlers() { return responseHandlersMap; } - public boolean isProtobuf() { - return this.message.getIsProtobuf(); - } - @Override public Protocol getProtocol() { return Protocol.PROTOBUF; diff --git a/server/src/main/proto/server/NodeToNodeMessageProto.proto b/server/src/main/proto/server/NodeToNodeMessageProto.proto index 9348941cd2345..ae4f2698a8306 100644 --- a/server/src/main/proto/server/NodeToNodeMessageProto.proto +++ b/server/src/main/proto/server/NodeToNodeMessageProto.proto @@ -28,7 +28,6 @@ message NodeToNodeMessage { oneof message { QueryFetchSearchResult queryFetchSearchResult = 16; } - bool isProtobuf = 17; message Header { repeated bytes prefix = 1; diff --git a/test/framework/src/main/java/org/opensearch/transport/nio/MockNioTransport.java b/test/framework/src/main/java/org/opensearch/transport/nio/MockNioTransport.java index 62b79b5f715ce..cd6bf02efef6f 100644 --- a/test/framework/src/main/java/org/opensearch/transport/nio/MockNioTransport.java +++ b/test/framework/src/main/java/org/opensearch/transport/nio/MockNioTransport.java @@ -328,7 +328,6 @@ private MockTcpReadWriteHandler(MockSocketChannel channel, PageCacheRecycler rec breaker, requestHandlers::getHandler, transport::inboundMessage - // transport::inboundMessageProtobuf ); } From 58a8af8acbe03185970dc1bd101d7546f9508a05 Mon Sep 17 00:00:00 2001 From: Vacha Shah Date: Fri, 26 Jan 2024 07:31:34 +0000 Subject: [PATCH 17/36] Fixing empty hits Signed-off-by: Vacha Shah --- .../action/search/SearchTransportService.java | 1 - .../java/org/opensearch/search/SearchHit.java | 20 +++++----- .../org/opensearch/search/SearchHits.java | 37 +++++++++++++++++++ .../search/fetch/FetchSearchResult.java | 19 ++++++++++ .../search/fetch/QueryFetchSearchResult.java | 1 - .../search/query/QuerySearchResult.java | 12 ++++++ 6 files changed, 79 insertions(+), 11 deletions(-) diff --git a/server/src/main/java/org/opensearch/action/search/SearchTransportService.java b/server/src/main/java/org/opensearch/action/search/SearchTransportService.java index 548fbd953db25..a71d8c2a64f4e 100644 --- a/server/src/main/java/org/opensearch/action/search/SearchTransportService.java +++ b/server/src/main/java/org/opensearch/action/search/SearchTransportService.java @@ -247,7 +247,6 @@ public void sendExecuteQuery( final boolean fetchDocuments = request.numberOfShards() == 1; final ActionListener handler = responseWrapper.apply(connection, listener); TransportResponseHandler transportResponseHandler; - // System.setProperty(FeatureFlags.PROTOBUF, "true"); if (FeatureFlags.isEnabled(FeatureFlags.PROTOBUF_SETTING)) { ProtobufWriteable.Reader reader = fetchDocuments ? QueryFetchSearchResult::new : QuerySearchResult::new; transportResponseHandler = new ProtobufConnectionCountingHandler<>( diff --git a/server/src/main/java/org/opensearch/search/SearchHit.java b/server/src/main/java/org/opensearch/search/SearchHit.java index 804dadbf608db..989f9bb970c52 100644 --- a/server/src/main/java/org/opensearch/search/SearchHit.java +++ b/server/src/main/java/org/opensearch/search/SearchHit.java @@ -243,19 +243,21 @@ public SearchHit(StreamInput in) throws IOException { public SearchHit(byte[] in) throws IOException { this.searchHitProto = FetchSearchResultProto.SearchHit.parseFrom(in); - docId = -1; - score = this.searchHitProto.getScore(); - id = new Text(this.searchHitProto.getId()); + this.docId = -1; + this.score = this.searchHitProto.getScore(); + this.id = new Text(this.searchHitProto.getId()); // Support for nestedIdentity to be added in the future - nestedIdentity = null; - version = this.searchHitProto.getVersion(); - seqNo = this.searchHitProto.getSeqNo(); - primaryTerm = this.searchHitProto.getPrimaryTerm(); - source = BytesReference.fromByteBuffer(ByteBuffer.wrap(this.searchHitProto.getSource().toByteArray())); + this.nestedIdentity = null; + this.version = this.searchHitProto.getVersion(); + this.seqNo = this.searchHitProto.getSeqNo(); + this.primaryTerm = this.searchHitProto.getPrimaryTerm(); + this.source = BytesReference.fromByteBuffer(ByteBuffer.wrap(this.searchHitProto.getSource().toByteArray())); if (source.length() == 0) { source = null; } - metaFields = new HashMap<>(); + // add support for metaFields and documentFields + this.metaFields = new HashMap<>(); + this.documentFields = new HashMap<>(); } private Map readFields(StreamInput in) throws IOException { diff --git a/server/src/main/java/org/opensearch/search/SearchHits.java b/server/src/main/java/org/opensearch/search/SearchHits.java index 24037fc8aaf4d..9b8fde9f831c2 100644 --- a/server/src/main/java/org/opensearch/search/SearchHits.java +++ b/server/src/main/java/org/opensearch/search/SearchHits.java @@ -32,12 +32,14 @@ package org.opensearch.search; +import com.google.protobuf.ByteString; import org.apache.lucene.search.SortField; import org.apache.lucene.search.TotalHits; import org.apache.lucene.search.TotalHits.Relation; import org.opensearch.common.Nullable; import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.lucene.Lucene; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; @@ -45,6 +47,8 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.rest.action.search.RestSearchAction; +import org.opensearch.server.proto.FetchSearchResultProto; +import org.opensearch.server.proto.QuerySearchResultProto; import java.io.IOException; import java.io.OutputStream; @@ -103,6 +107,39 @@ public SearchHits( this.sortFields = sortFields; this.collapseField = collapseField; this.collapseValues = collapseValues; + if (FeatureFlags.isEnabled(FeatureFlags.PROTOBUF_SETTING)) { + List searchHitList = new ArrayList<>(); + for (int i = 0; i < hits.length; i++) { + FetchSearchResultProto.SearchHit.Builder searchHitBuilder = FetchSearchResultProto.SearchHit.newBuilder(); + if (hits[i].getIndex() != null) { + searchHitBuilder.setIndex(hits[i].getIndex()); + } + searchHitBuilder.setId(hits[i].getId()); + searchHitBuilder.setScore(hits[i].getScore()); + searchHitBuilder.setSeqNo(hits[i].getSeqNo()); + searchHitBuilder.setPrimaryTerm(hits[i].getPrimaryTerm()); + searchHitBuilder.setVersion(hits[i].getVersion()); + if (hits[i].getSourceRef() != null) { + searchHitBuilder.setSource(ByteString.copyFrom(hits[i].getSourceRef().toBytesRef().bytes)); + } + searchHitList.add(searchHitBuilder.build()); + } + QuerySearchResultProto.TotalHits.Builder totalHitsBuilder = QuerySearchResultProto.TotalHits.newBuilder(); + totalHitsBuilder.setValue(totalHits.value); + totalHitsBuilder.setRelation( + totalHits.relation == Relation.EQUAL_TO + ? QuerySearchResultProto.TotalHits.Relation.EQUAL_TO + : QuerySearchResultProto.TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO + ); + FetchSearchResultProto.SearchHits.Builder searchHitsBuilder = FetchSearchResultProto.SearchHits.newBuilder(); + searchHitsBuilder.setMaxScore(maxScore); + searchHitsBuilder.addAllHits(searchHitList); + searchHitsBuilder.setTotalHits(totalHitsBuilder.build()); + if (collapseField != null) { + searchHitsBuilder.setCollapseField(collapseField); + } + this.searchHitsProto = searchHitsBuilder.build(); + } } public SearchHits(StreamInput in) throws IOException { diff --git a/server/src/main/java/org/opensearch/search/fetch/FetchSearchResult.java b/server/src/main/java/org/opensearch/search/fetch/FetchSearchResult.java index 29dc3d5889d50..867ee347f605d 100644 --- a/server/src/main/java/org/opensearch/search/fetch/FetchSearchResult.java +++ b/server/src/main/java/org/opensearch/search/fetch/FetchSearchResult.java @@ -32,6 +32,7 @@ package org.opensearch.search.fetch; +import com.google.protobuf.ByteString; import org.apache.lucene.search.TotalHits.Relation; import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.util.FeatureFlags; @@ -115,6 +116,15 @@ public void hits(SearchHits hits) { FetchSearchResultProto.SearchHits.Builder searchHitsBuilder = FetchSearchResultProto.SearchHits.newBuilder(); searchHitsBuilder.setMaxScore(hits.getMaxScore()); searchHitsBuilder.setTotalHits(totalHitsBuilder.build()); + for (SearchHit hit : hits.getHits()) { + FetchSearchResultProto.SearchHit.Builder searchHitBuilder = FetchSearchResultProto.SearchHit.newBuilder(); + searchHitBuilder.setId(hit.getId()); + searchHitBuilder.setSource(ByteString.copyFrom(hit.getSourceRef().toBytesRef().bytes)); + searchHitBuilder.setVersion(hit.getVersion()); + searchHitBuilder.setSeqNo(hit.getSeqNo()); + searchHitBuilder.setPrimaryTerm(hit.getPrimaryTerm()); + searchHitsBuilder.addHits(searchHitBuilder.build()); + } this.fetchSearchResultProto = this.fetchSearchResultProto.toBuilder().setHits(searchHitsBuilder.build()).build(); } } @@ -127,6 +137,15 @@ private boolean assertNoSearchTarget(SearchHits hits) { } public SearchHits hits() { + if (FeatureFlags.isEnabled(FeatureFlags.PROTOBUF_SETTING)) { + SearchHits hits; + try { + hits = new SearchHits(this.fetchSearchResultProto.getHits().toByteArray()); + return hits; + } catch (IOException e) { + throw new RuntimeException(e); + } + } return hits; } diff --git a/server/src/main/java/org/opensearch/search/fetch/QueryFetchSearchResult.java b/server/src/main/java/org/opensearch/search/fetch/QueryFetchSearchResult.java index 6e0c0bf20e7ac..469d7472981d5 100644 --- a/server/src/main/java/org/opensearch/search/fetch/QueryFetchSearchResult.java +++ b/server/src/main/java/org/opensearch/search/fetch/QueryFetchSearchResult.java @@ -121,7 +121,6 @@ public void writeTo(StreamOutput out) throws IOException { @Override public boolean isMessageProtobuf() { - // System.setProperty(FeatureFlags.PROTOBUF, "true"); if (FeatureFlags.isEnabled(FeatureFlags.PROTOBUF_SETTING)) { return true; } diff --git a/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java b/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java index 599ba66850409..eed4955844ca0 100644 --- a/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java +++ b/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java @@ -387,11 +387,17 @@ public void suggest(Suggest suggest) { } public int from() { + if (FeatureFlags.isEnabled(FeatureFlags.PROTOBUF_SETTING)) { + return this.querySearchResultProto.getFrom(); + } return from; } public QuerySearchResult from(int from) { this.from = from; + if (this.querySearchResultProto != null) { + this.querySearchResultProto = this.querySearchResultProto.toBuilder().setFrom(from).build(); + } return this; } @@ -399,11 +405,17 @@ public QuerySearchResult from(int from) { * Returns the maximum size of this results top docs. */ public int size() { + if (FeatureFlags.isEnabled(FeatureFlags.PROTOBUF_SETTING)) { + return this.querySearchResultProto.getSize(); + } return size; } public QuerySearchResult size(int size) { this.size = size; + if (this.querySearchResultProto != null) { + this.querySearchResultProto = this.querySearchResultProto.toBuilder().setSize(size).build(); + } return this; } From 1ae993b0a9564e8dc8fd4686a30cfbeabd539901 Mon Sep 17 00:00:00 2001 From: Vacha Shah Date: Sun, 28 Jan 2024 08:01:13 +0000 Subject: [PATCH 18/36] Improving the logic for inboundpipeline Signed-off-by: Vacha Shah --- .../src/main/java/org/opensearch/transport/InboundPipeline.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/main/java/org/opensearch/transport/InboundPipeline.java b/server/src/main/java/org/opensearch/transport/InboundPipeline.java index fca941fc08721..ab9cc60482def 100644 --- a/server/src/main/java/org/opensearch/transport/InboundPipeline.java +++ b/server/src/main/java/org/opensearch/transport/InboundPipeline.java @@ -36,6 +36,7 @@ import org.opensearch.common.bytes.ReleasableBytesReference; import org.opensearch.common.lease.Releasable; import org.opensearch.common.lease.Releasables; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.util.PageCacheRecycler; import org.opensearch.core.common.breaker.CircuitBreaker; import org.opensearch.core.common.bytes.CompositeBytesReference; From bd83a3f2398667d41d1f3d795b4ffcc50337ad16 Mon Sep 17 00:00:00 2001 From: Vacha Shah Date: Fri, 2 Feb 2024 01:25:11 +0000 Subject: [PATCH 19/36] Added tests Signed-off-by: Vacha Shah --- .../search/fetch/QueryFetchSearchResult.java | 1 + .../search/query/QuerySearchResult.java | 6 + .../search/query/QuerySearchResultTests.java | 24 ++++ .../transport/InboundHandlerTests.java | 91 ++++++++++++++ .../transport/OutboundHandlerTests.java | 112 +++++++++++++++++- 5 files changed, 230 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/opensearch/search/fetch/QueryFetchSearchResult.java b/server/src/main/java/org/opensearch/search/fetch/QueryFetchSearchResult.java index 469d7472981d5..9d38204aa8869 100644 --- a/server/src/main/java/org/opensearch/search/fetch/QueryFetchSearchResult.java +++ b/server/src/main/java/org/opensearch/search/fetch/QueryFetchSearchResult.java @@ -136,4 +136,5 @@ public QueryFetchSearchResult(QueryFetchSearchResultProto.QueryFetchSearchResult this.queryResult = new QuerySearchResult(queryFetchSearchResult.getQueryResult()); this.fetchResult = new FetchSearchResult(queryFetchSearchResult.getFetchResult()); } + } diff --git a/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java b/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java index eed4955844ca0..7a0603bdf782c 100644 --- a/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java +++ b/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java @@ -57,6 +57,7 @@ import org.opensearch.server.proto.QuerySearchResultProto; import java.io.IOException; +import java.io.OutputStream; import java.util.ArrayList; import java.util.List; @@ -487,6 +488,11 @@ public void writeTo(StreamOutput out) throws IOException { } } + @Override + public void writeTo(OutputStream out) throws IOException { + out.write(this.querySearchResultProto.toByteArray()); + } + public void writeToNoId(StreamOutput out) throws IOException { out.writeVInt(from); out.writeVInt(size); diff --git a/server/src/test/java/org/opensearch/search/query/QuerySearchResultTests.java b/server/src/test/java/org/opensearch/search/query/QuerySearchResultTests.java index 41e4e1ae45a73..234ac8e807d8d 100644 --- a/server/src/test/java/org/opensearch/search/query/QuerySearchResultTests.java +++ b/server/src/test/java/org/opensearch/search/query/QuerySearchResultTests.java @@ -54,8 +54,11 @@ import org.opensearch.search.internal.ShardSearchContextId; import org.opensearch.search.internal.ShardSearchRequest; import org.opensearch.search.suggest.SuggestTests; +import org.opensearch.server.proto.QuerySearchResultProto; import org.opensearch.test.OpenSearchTestCase; +import java.io.ByteArrayOutputStream; + import static java.util.Collections.emptyList; public class QuerySearchResultTests extends OpenSearchTestCase { @@ -125,4 +128,25 @@ public void testNullResponse() throws Exception { QuerySearchResult deserialized = copyWriteable(querySearchResult, namedWriteableRegistry, QuerySearchResult::new, Version.CURRENT); assertEquals(querySearchResult.isNull(), deserialized.isNull()); } + + public void testProtobufSerialization() throws Exception { + QuerySearchResult querySearchResult = createTestInstance(); + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + querySearchResult.writeTo(stream); + QuerySearchResult deserialized = new QuerySearchResult(stream.toByteArray()); + QuerySearchResultProto.QuerySearchResult querySearchResultProto = deserialized.response(); + assertNotNull(querySearchResultProto); + assertEquals(querySearchResult.getContextId().getId(), querySearchResultProto.getContextId().getId()); + assertEquals( + querySearchResult.getSearchShardTarget().getShardId().getIndex().getUUID(), + querySearchResultProto.getSearchShardTarget().getShardId().getIndexUUID() + ); + assertEquals(querySearchResult.topDocs().maxScore, querySearchResultProto.getTopDocsAndMaxScore().getMaxScore(), 0f); + assertEquals( + querySearchResult.topDocs().topDocs.totalHits.value, + querySearchResultProto.getTopDocsAndMaxScore().getTopDocs().getTotalHits().getValue() + ); + assertEquals(querySearchResult.from(), querySearchResultProto.getFrom()); + assertEquals(querySearchResult.size(), querySearchResultProto.getSize()); + } } diff --git a/server/src/test/java/org/opensearch/transport/InboundHandlerTests.java b/server/src/test/java/org/opensearch/transport/InboundHandlerTests.java index be1ac47b55857..1ae357ac0cc41 100644 --- a/server/src/test/java/org/opensearch/transport/InboundHandlerTests.java +++ b/server/src/test/java/org/opensearch/transport/InboundHandlerTests.java @@ -37,18 +37,23 @@ import org.apache.lucene.util.BytesRef; import org.opensearch.OpenSearchException; import org.opensearch.Version; +import org.opensearch.common.SuppressForbidden; import org.opensearch.common.bytes.ReleasableBytesReference; import org.opensearch.common.collect.Tuple; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.BigArrays; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.InputStreamStreamInput; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.search.fetch.FetchSearchResult; +import org.opensearch.search.fetch.QueryFetchSearchResult; +import org.opensearch.search.query.QuerySearchResult; import org.opensearch.tasks.TaskManager; import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.MockLogAppender; @@ -248,6 +253,92 @@ public TestResponse read(byte[] in) throws IOException { } } + @SuppressForbidden(reason = "manipulates system properties for testing") + public void testProtobufResponse() throws Exception { + String action = "test-request"; + int headerSize = TcpHeader.headerSize(version); + AtomicReference requestCaptor = new AtomicReference<>(); + AtomicReference exceptionCaptor = new AtomicReference<>(); + AtomicReference responseCaptor = new AtomicReference<>(); + AtomicReference channelCaptor = new AtomicReference<>(); + + long requestId = responseHandlers.add(new Transport.ResponseContext<>(new TransportResponseHandler() { + @Override + public void handleResponse(QueryFetchSearchResult response) { + responseCaptor.set(response); + } + + @Override + public void handleException(TransportException exp) { + exceptionCaptor.set(exp); + } + + @Override + public String executor() { + return ThreadPool.Names.SAME; + } + + @Override + public QueryFetchSearchResult read(StreamInput in) throws IOException { + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } + + @Override + public QueryFetchSearchResult read(byte[] in) throws IOException { + return new QueryFetchSearchResult(in); + } + }, null, action)); + RequestHandlerRegistry registry = new RequestHandlerRegistry<>( + action, + TestRequest::new, + taskManager, + (request, channel, task) -> { + channelCaptor.set(channel); + requestCaptor.set(request); + }, + ThreadPool.Names.SAME, + false, + true + ); + requestHandlers.registerHandler(registry); + String requestValue = randomAlphaOfLength(10); + OutboundMessage.Request request = new OutboundMessage.Request( + threadPool.getThreadContext(), + new String[0], + new TestRequest(requestValue), + version, + action, + requestId, + false, + false + ); + + BytesReference fullRequestBytes = request.serialize(new BytesStreamOutput()); + BytesReference requestContent = fullRequestBytes.slice(headerSize, fullRequestBytes.length() - headerSize); + Header requestHeader = new Header(fullRequestBytes.length() - 6, requestId, TransportStatus.setRequest((byte) 0), version); + InboundMessage requestMessage = new InboundMessage(requestHeader, ReleasableBytesReference.wrap(requestContent), () -> {}); + requestHeader.finishParsingHeader(requestMessage.openOrGetStreamInput()); + handler.inboundMessage(channel, requestMessage); + + TransportChannel transportChannel = channelCaptor.get(); + assertEquals(Version.CURRENT, transportChannel.getVersion()); + assertEquals("transport", transportChannel.getChannelType()); + assertEquals(requestValue, requestCaptor.get().value); + + QuerySearchResult queryResult = OutboundHandlerTests.createQuerySearchResult(); + FetchSearchResult fetchResult = OutboundHandlerTests.createFetchSearchResult(); + QueryFetchSearchResult response = new QueryFetchSearchResult(queryResult, fetchResult); + System.setProperty(FeatureFlags.PROTOBUF, "true"); + transportChannel.sendResponse(response); + + BytesReference fullResponseBytes = channel.getMessageCaptor().get(); + NodeToNodeMessage nodeToNodeMessage = new NodeToNodeMessage(fullResponseBytes.toBytesRef().bytes); + handler.inboundMessage(channel, nodeToNodeMessage); + QueryFetchSearchResult result = responseCaptor.get(); + assertNotNull(result); + assertEquals(queryResult.getMaxScore(), result.queryResult().getMaxScore(), 0.0); + } + public void testSendsErrorResponseToHandshakeFromCompatibleVersion() throws Exception { // Nodes use their minimum compatibility version for the TCP handshake, so a node from v(major-1).x will report its version as // v(major-2).last in the TCP handshake, with which we are not really compatible. We put extra effort into making sure that if diff --git a/server/src/test/java/org/opensearch/transport/OutboundHandlerTests.java b/server/src/test/java/org/opensearch/transport/OutboundHandlerTests.java index d71e4b8a0310a..573a728cfae2d 100644 --- a/server/src/test/java/org/opensearch/transport/OutboundHandlerTests.java +++ b/server/src/test/java/org/opensearch/transport/OutboundHandlerTests.java @@ -34,22 +34,37 @@ import org.opensearch.OpenSearchException; import org.opensearch.Version; +import org.opensearch.action.OriginalIndices; +import org.opensearch.action.OriginalIndicesTests; +import org.opensearch.action.search.SearchRequest; import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.common.SuppressForbidden; +import org.opensearch.common.UUIDs; import org.opensearch.common.bytes.ReleasableBytesReference; import org.opensearch.common.collect.Tuple; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.BigArrays; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.util.PageCacheRecycler; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.common.util.io.Streams; import org.opensearch.core.action.ActionListener; +import org.opensearch.core.common.Strings; import org.opensearch.core.common.breaker.CircuitBreaker; import org.opensearch.core.common.breaker.NoopCircuitBreaker; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.transport.TransportAddress; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.core.transport.TransportResponse; +import org.opensearch.search.SearchShardTarget; +import org.opensearch.search.fetch.FetchSearchResult; +import org.opensearch.search.fetch.QueryFetchSearchResult; +import org.opensearch.search.internal.AliasFilter; +import org.opensearch.search.internal.ShardSearchContextId; +import org.opensearch.search.internal.ShardSearchRequest; +import org.opensearch.search.query.QuerySearchResult; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.TestThreadPool; import org.opensearch.threadpool.ThreadPool; @@ -57,6 +72,7 @@ import org.junit.Before; import java.io.IOException; +import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.concurrent.TimeUnit; @@ -76,7 +92,7 @@ public class OutboundHandlerTests extends OpenSearchTestCase { private final TestThreadPool threadPool = new TestThreadPool(getClass().getName()); private final TransportRequestOptions options = TransportRequestOptions.EMPTY; private final AtomicReference> message = new AtomicReference<>(); - // private final AtomicReference protobufMessage = new AtomicReference<>(); + private final AtomicReference protobufMessage = new AtomicReference<>(); private InboundPipeline pipeline; private OutboundHandler handler; private FakeTcpChannel channel; @@ -104,9 +120,7 @@ public void setUp() throws Exception { } catch (IOException e) { throw new AssertionError(e); } - } - // , (c, m) -> { protobufMessage.set(m); } - ); + }); } @After @@ -266,6 +280,65 @@ public void onResponseSent(long requestId, String action, TransportResponse resp assertEquals("header_value", header.getHeaders().v1().get("header")); } + @SuppressForbidden(reason = "manipulates system properties for testing") + public void testSendProtobufResponse() throws IOException { + ThreadContext threadContext = threadPool.getThreadContext(); + Version version = Version.CURRENT; + String action = "handshake"; + long requestId = randomLongBetween(0, 300); + boolean isHandshake = randomBoolean(); + boolean compress = randomBoolean(); + threadContext.putHeader("header", "header_value"); + QuerySearchResult queryResult = createQuerySearchResult(); + FetchSearchResult fetchResult = createFetchSearchResult(); + QueryFetchSearchResult response = new QueryFetchSearchResult(queryResult, fetchResult); + System.setProperty(FeatureFlags.PROTOBUF, "true"); + assertTrue(response.isMessageProtobuf()); + + AtomicLong requestIdRef = new AtomicLong(); + AtomicReference actionRef = new AtomicReference<>(); + AtomicReference responseRef = new AtomicReference<>(); + handler.setMessageListener(new TransportMessageListener() { + @Override + public void onResponseSent(long requestId, String action, TransportResponse response) { + requestIdRef.set(requestId); + actionRef.set(action); + responseRef.set(response); + } + }); + handler.sendResponse(version, Collections.emptySet(), channel, requestId, action, response, compress, isHandshake); + + StatsTracker statsTracker = new StatsTracker(); + final LongSupplier millisSupplier = () -> TimeValue.nsecToMSec(System.nanoTime()); + final InboundDecoder decoder = new InboundDecoder(Version.CURRENT, PageCacheRecycler.NON_RECYCLING_INSTANCE); + final Supplier breaker = () -> new NoopCircuitBreaker("test"); + final InboundAggregator aggregator = new InboundAggregator(breaker, (Predicate) requestCanTripBreaker -> true); + InboundPipeline inboundPipeline = new InboundPipeline(statsTracker, millisSupplier, decoder, aggregator, (c, m) -> { + NodeToNodeMessage m1 = (NodeToNodeMessage) m; + protobufMessage.set(BytesReference.fromByteBuffer(ByteBuffer.wrap(m1.getMessage().toByteArray()))); + }); + BytesReference reference = channel.getMessageCaptor().get(); + ActionListener sendListener = channel.getListenerCaptor().get(); + if (randomBoolean()) { + sendListener.onResponse(null); + } else { + sendListener.onFailure(new IOException("failed")); + } + assertEquals(requestId, requestIdRef.get()); + assertEquals(action, actionRef.get()); + assertEquals(response, responseRef.get()); + + inboundPipeline.handleBytes(channel, new ReleasableBytesReference(reference, () -> {})); + final BytesReference responseBytes = protobufMessage.get(); + final NodeToNodeMessage message = new NodeToNodeMessage(responseBytes.toBytesRef().bytes); + assertEquals(version.toString(), message.getMessage().getVersion()); + assertEquals(requestId, message.getHeader().getRequestId()); + assertNotNull(message.getRequestHeaders()); + assertNotNull(message.getResponseHandlers()); + assertNotNull(message.getMessage()); + assertTrue(message.getMessage().hasQueryFetchSearchResult()); + } + public void testErrorResponse() throws IOException { ThreadContext threadContext = threadPool.getThreadContext(); Version version = randomFrom(Version.CURRENT, Version.CURRENT.minimumCompatibilityVersion()); @@ -317,4 +390,35 @@ public void onResponseSent(long requestId, String action, Exception error) { assertEquals("header_value", header.getHeaders().v1().get("header")); } + + public static QuerySearchResult createQuerySearchResult() { + ShardId shardId = new ShardId("index", "uuid", randomInt()); + SearchRequest searchRequest = new SearchRequest().allowPartialSearchResults(randomBoolean()); + ShardSearchRequest shardSearchRequest = new ShardSearchRequest( + OriginalIndicesTests.randomOriginalIndices(), + searchRequest, + shardId, + 1, + new AliasFilter(null, Strings.EMPTY_ARRAY), + 1.0f, + randomNonNegativeLong(), + null, + new String[0] + ); + QuerySearchResult result = new QuerySearchResult( + new ShardSearchContextId(UUIDs.base64UUID(), randomLong()), + new SearchShardTarget("node", shardId, null, OriginalIndices.NONE), + shardSearchRequest + ); + return result; + } + + public static FetchSearchResult createFetchSearchResult() { + ShardId shardId = new ShardId("index", "uuid", randomInt()); + FetchSearchResult result = new FetchSearchResult( + new ShardSearchContextId(UUIDs.base64UUID(), randomLong()), + new SearchShardTarget("node", shardId, null, OriginalIndices.NONE) + ); + return result; + } } From fb24d35af53935834d45460ea13f9cbd174f73a5 Mon Sep 17 00:00:00 2001 From: Vacha Shah Date: Thu, 8 Feb 2024 18:59:58 +0000 Subject: [PATCH 20/36] Adding protos for the remaining structures Signed-off-by: Vacha Shah --- server/build.gradle | 4 +- .../common/document/DocumentField.java | 23 ++++ .../org/opensearch/common/lucene/Lucene.java | 24 ++++ .../java/org/opensearch/search/SearchHit.java | 106 +++++++++++++----- .../org/opensearch/search/SearchHits.java | 63 ++++++++--- .../opensearch/search/SearchSortValues.java | 34 ++++++ .../subphase/highlight/HighlightField.java | 15 +++ .../search/FetchSearchResultProto.proto | 55 +++++++-- 8 files changed, 267 insertions(+), 57 deletions(-) diff --git a/server/build.gradle b/server/build.gradle index 09e90141a670b..f846258bfb8a6 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -369,7 +369,9 @@ tasks.named("missingJavadoc").configure { "org.opensearch.server.proto.FetchSearchResultProto.SearchHit.NestedIdentityOrBuilder", "org.opensearch.server.proto.NodeToNodeMessageProto.NodeToNodeMessage.MessageCase", "org.opensearch.server.proto.NodeToNodeMessageProto.NodeToNodeMessage.ResponseHandlersListOrBuilder", - "org.opensearch.server.proto.NodeToNodeMessageProto.NodeToNodeMessage.HeaderOrBuilder" + "org.opensearch.server.proto.NodeToNodeMessageProto.NodeToNodeMessage.HeaderOrBuilder", + "org.opensearch.server.proto.FetchSearchResultProto.SearchHit.Explanation.ExplanationValueCase", + "org.opensearch.server.proto.FetchSearchResultProto.SearchHit.ExplanationOrBuilder" ] } diff --git a/server/src/main/java/org/opensearch/common/document/DocumentField.java b/server/src/main/java/org/opensearch/common/document/DocumentField.java index 5cdc2bba8be16..63121f19d1b9a 100644 --- a/server/src/main/java/org/opensearch/common/document/DocumentField.java +++ b/server/src/main/java/org/opensearch/common/document/DocumentField.java @@ -32,6 +32,9 @@ package org.opensearch.common.document; +import com.google.protobuf.ByteString; +import org.apache.lucene.util.SuppressForbidden; +import org.opensearch.OpenSearchException; import org.opensearch.common.annotation.PublicApi; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; @@ -41,8 +44,12 @@ import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.get.GetResult; import org.opensearch.search.SearchHit; +import org.opensearch.server.proto.FetchSearchResultProto; +import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.InputStream; +import java.io.ObjectInputStream; import java.util.ArrayList; import java.util.Iterator; import java.util.List; @@ -64,6 +71,7 @@ public class DocumentField implements Writeable, ToXContentFragment, Iterable values; + private FetchSearchResultProto.SearchHit.DocumentField documentField; public DocumentField(StreamInput in) throws IOException { name = in.readString(); @@ -75,6 +83,21 @@ public DocumentField(String name, List values) { this.values = Objects.requireNonNull(values, "values must not be null"); } + @SuppressForbidden(reason = "We need to read from a byte array") + public DocumentField(byte[] in) throws IOException { + documentField = FetchSearchResultProto.SearchHit.DocumentField.parseFrom(in); + name = documentField.getName(); + values = new ArrayList<>(); + for (ByteString value : documentField.getValuesList()) { + InputStream is = new ByteArrayInputStream(value.toByteArray()); + try (ObjectInputStream ois = new ObjectInputStream(is)) { + values.add(ois.readObject()); + } catch (ClassNotFoundException e) { + throw new OpenSearchException(e); + } + } + } + /** * The name of the field. */ diff --git a/server/src/main/java/org/opensearch/common/lucene/Lucene.java b/server/src/main/java/org/opensearch/common/lucene/Lucene.java index 2c7b6b552b43f..d195859b63fed 100644 --- a/server/src/main/java/org/opensearch/common/lucene/Lucene.java +++ b/server/src/main/java/org/opensearch/common/lucene/Lucene.java @@ -93,6 +93,7 @@ import org.opensearch.index.analysis.NamedAnalyzer; import org.opensearch.index.fielddata.IndexFieldData; import org.opensearch.search.sort.SortedWiderNumericSortField; +import org.opensearch.server.proto.FetchSearchResultProto; import java.io.IOException; import java.math.BigInteger; @@ -651,6 +652,29 @@ public static Explanation readExplanation(StreamInput in) throws IOException { } } + public static Explanation readExplanation(byte[] in) throws IOException { + FetchSearchResultProto.SearchHit.Explanation explanationProto = FetchSearchResultProto.SearchHit.Explanation.parseFrom(in); + boolean match = explanationProto.getMatch(); + String description = explanationProto.getDescription(); + final Explanation[] subExplanations = new Explanation[explanationProto.getSubExplanationsCount()]; + for (int i = 0; i < subExplanations.length; ++i) { + subExplanations[i] = readExplanation(explanationProto.getSubExplanations(i).toByteArray()); + } + Number explanationValue = null; + if (explanationProto.hasValue1()) { + explanationValue = explanationProto.getValue1(); + } else if (explanationProto.hasValue2()) { + explanationValue = explanationProto.getValue2(); + } else if (explanationProto.hasValue3()) { + explanationValue = explanationProto.getValue3(); + } + if (match) { + return Explanation.match(explanationValue, description, subExplanations); + } else { + return Explanation.noMatch(description, subExplanations); + } + } + private static void writeExplanationValue(StreamOutput out, Number value) throws IOException { if (value instanceof Float) { out.writeByte((byte) 0); diff --git a/server/src/main/java/org/opensearch/search/SearchHit.java b/server/src/main/java/org/opensearch/search/SearchHit.java index 989f9bb970c52..68b24a67d655d 100644 --- a/server/src/main/java/org/opensearch/search/SearchHit.java +++ b/server/src/main/java/org/opensearch/search/SearchHit.java @@ -246,8 +246,11 @@ public SearchHit(byte[] in) throws IOException { this.docId = -1; this.score = this.searchHitProto.getScore(); this.id = new Text(this.searchHitProto.getId()); - // Support for nestedIdentity to be added in the future - this.nestedIdentity = null; + if (!this.searchHitProto.hasNestedIdentity() && this.searchHitProto.getNestedIdentity().toByteArray().length > 0) { + this.nestedIdentity = new NestedIdentity(this.searchHitProto.getNestedIdentity().toByteArray()); + } else { + this.nestedIdentity = null; + } this.version = this.searchHitProto.getVersion(); this.seqNo = this.searchHitProto.getSeqNo(); this.primaryTerm = this.searchHitProto.getPrimaryTerm(); @@ -255,39 +258,61 @@ public SearchHit(byte[] in) throws IOException { if (source.length() == 0) { source = null; } - // add support for metaFields and documentFields - this.metaFields = new HashMap<>(); this.documentFields = new HashMap<>(); - } - - private Map readFields(StreamInput in) throws IOException { - Map fields; - int size = in.readVInt(); - if (size == 0) { - fields = emptyMap(); - } else if (size == 1) { - DocumentField hitField = new DocumentField(in); - fields = singletonMap(hitField.getName(), hitField); - } else { - fields = new HashMap<>(size); - for (int i = 0; i < size; i++) { - DocumentField field = new DocumentField(in); - fields.put(field.getName(), field); + this.searchHitProto.getDocumentFieldsMap().forEach((k, v) -> { + try { + this.documentFields.put(k, new DocumentField(v.toByteArray())); + } catch (IOException e) { + throw new OpenSearchParseException("failed to parse document field", e); } - fields = unmodifiableMap(fields); - } - return fields; - } - - private void writeFields(StreamOutput out, Map fields) throws IOException { - if (fields == null) { - out.writeVInt(0); - } else { - out.writeVInt(fields.size()); - for (DocumentField field : fields.values()) { - field.writeTo(out); + }); + this.metaFields = new HashMap<>(); + this.searchHitProto.getMetaFieldsMap().forEach((k, v) -> { + try { + this.metaFields.put(k, new DocumentField(v.toByteArray())); + } catch (IOException e) { + throw new OpenSearchParseException("failed to parse document field", e); } + }); + this.highlightFields = new HashMap<>(); + this.searchHitProto.getHighlightFieldsMap().forEach((k, v) -> { + try { + this.highlightFields.put(k, new HighlightField(v.toByteArray())); + } catch (IOException e) { + throw new OpenSearchParseException("failed to parse highlight field", e); + } + }); + this.sortValues = new SearchSortValues(this.searchHitProto.getSortValues().toByteArray()); + if (this.searchHitProto.getMatchedQueriesCount() > 0) { + this.matchedQueries = this.searchHitProto.getMatchedQueriesList().toArray(new String[0]); + } + if (this.searchHitProto.hasExplanation()) { + this.explanation = readExplanation(this.searchHitProto.getExplanation().toByteArray()); + } + SearchShardTarget searchShardTarget = new SearchShardTarget( + this.searchHitProto.getShard().getNodeId(), + new ShardId( + this.searchHitProto.getShard().getShardId().getIndexName(), + this.searchHitProto.getShard().getShardId().getIndexUUID(), + this.searchHitProto.getShard().getShardId().getShardId() + ), + this.searchHitProto.getShard().getClusterAlias(), + OriginalIndices.NONE + ); + shard(searchShardTarget); + if (this.searchHitProto.getInnerHitsCount() > 0) { + this.innerHits = new HashMap<>(); + this.searchHitProto.getInnerHitsMap().forEach((k, v) -> { + try { + this.innerHits.put(k, new SearchHits(v.toByteArray())); + } catch (IOException e) { + throw new OpenSearchParseException("failed to parse inner hits", e); + } + }); + } else { + this.innerHits = null; } + } private static final Text SINGLE_MAPPING_TYPE = new Text(MapperService.SINGLE_MAPPING_NAME); @@ -1121,6 +1146,25 @@ public NestedIdentity(String field, int offset, NestedIdentity child) { child = in.readOptionalWriteable(NestedIdentity::new); } + NestedIdentity(byte[] in) throws IOException { + FetchSearchResultProto.SearchHit.NestedIdentity proto = FetchSearchResultProto.SearchHit.NestedIdentity.parseFrom(in); + if (proto.hasField()) { + field = new Text(proto.getField()); + } else { + field = null; + } + if (proto.hasOffset()) { + offset = proto.getOffset(); + } else { + offset = -1; + } + if (proto.hasChild()) { + child = new NestedIdentity(proto.getChild().toByteArray()); + } else { + child = null; + } + } + /** * Returns the nested field in the source this hit originates from */ diff --git a/server/src/main/java/org/opensearch/search/SearchHits.java b/server/src/main/java/org/opensearch/search/SearchHits.java index 9b8fde9f831c2..3e4fb76fe0371 100644 --- a/server/src/main/java/org/opensearch/search/SearchHits.java +++ b/server/src/main/java/org/opensearch/search/SearchHits.java @@ -36,6 +36,8 @@ import org.apache.lucene.search.SortField; import org.apache.lucene.search.TotalHits; import org.apache.lucene.search.TotalHits.Relation; +import org.apache.lucene.util.SuppressForbidden; +import org.opensearch.OpenSearchException; import org.opensearch.common.Nullable; import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.lucene.Lucene; @@ -50,7 +52,12 @@ import org.opensearch.server.proto.FetchSearchResultProto; import org.opensearch.server.proto.QuerySearchResultProto; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.InputStream; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; import java.io.OutputStream; import java.util.ArrayList; import java.util.Arrays; @@ -93,6 +100,7 @@ public SearchHits(SearchHit[] hits, @Nullable TotalHits totalHits, float maxScor this(hits, totalHits, maxScore, null, null, null); } + @SuppressForbidden(reason = "serialization of object to protobuf") public SearchHits( SearchHit[] hits, @Nullable TotalHits totalHits, @@ -109,18 +117,18 @@ public SearchHits( this.collapseValues = collapseValues; if (FeatureFlags.isEnabled(FeatureFlags.PROTOBUF_SETTING)) { List searchHitList = new ArrayList<>(); - for (int i = 0; i < hits.length; i++) { + for (SearchHit hit : hits) { FetchSearchResultProto.SearchHit.Builder searchHitBuilder = FetchSearchResultProto.SearchHit.newBuilder(); - if (hits[i].getIndex() != null) { - searchHitBuilder.setIndex(hits[i].getIndex()); + if (hit.getIndex() != null) { + searchHitBuilder.setIndex(hit.getIndex()); } - searchHitBuilder.setId(hits[i].getId()); - searchHitBuilder.setScore(hits[i].getScore()); - searchHitBuilder.setSeqNo(hits[i].getSeqNo()); - searchHitBuilder.setPrimaryTerm(hits[i].getPrimaryTerm()); - searchHitBuilder.setVersion(hits[i].getVersion()); - if (hits[i].getSourceRef() != null) { - searchHitBuilder.setSource(ByteString.copyFrom(hits[i].getSourceRef().toBytesRef().bytes)); + searchHitBuilder.setId(hit.getId()); + searchHitBuilder.setScore(hit.getScore()); + searchHitBuilder.setSeqNo(hit.getSeqNo()); + searchHitBuilder.setPrimaryTerm(hit.getPrimaryTerm()); + searchHitBuilder.setVersion(hit.getVersion()); + if (hit.getSourceRef() != null) { + searchHitBuilder.setSource(ByteString.copyFrom(hit.getSourceRef().toBytesRef().bytes)); } searchHitList.add(searchHitBuilder.build()); } @@ -135,8 +143,25 @@ public SearchHits( searchHitsBuilder.setMaxScore(maxScore); searchHitsBuilder.addAllHits(searchHitList); searchHitsBuilder.setTotalHits(totalHitsBuilder.build()); + if (sortFields != null && sortFields.length > 0) { + for (SortField sortField : sortFields) { + FetchSearchResultProto.SortField.Builder sortFieldBuilder = FetchSearchResultProto.SortField.newBuilder(); + sortFieldBuilder.setField(sortField.getField()); + sortFieldBuilder.setType(FetchSearchResultProto.SortField.Type.valueOf(sortField.getType().name())); + searchHitsBuilder.addSortFields(sortFieldBuilder.build()); + } + } if (collapseField != null) { searchHitsBuilder.setCollapseField(collapseField); + for (Object value : collapseValues) { + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + try (ObjectOutputStream stream = new ObjectOutputStream(bos)) { + stream.writeObject(value); + searchHitsBuilder.addCollapseValues(ByteString.copyFrom(bos.toByteArray())); + } catch (IOException e) { + throw new OpenSearchException(e); + } + } } this.searchHitsProto = searchHitsBuilder.build(); } @@ -164,6 +189,7 @@ public SearchHits(StreamInput in) throws IOException { collapseValues = in.readOptionalArray(Lucene::readSortValue, Object[]::new); } + @SuppressForbidden(reason = "serialization of object to protobuf") public SearchHits(byte[] in) throws IOException { this.searchHitsProto = org.opensearch.server.proto.FetchSearchResultProto.SearchHits.parseFrom(in); this.hits = new SearchHit[this.searchHitsProto.getHitsCount()]; @@ -175,10 +201,21 @@ public SearchHits(byte[] in) throws IOException { Relation.valueOf(this.searchHitsProto.getTotalHits().getRelation().toString()) ); this.maxScore = this.searchHitsProto.getMaxScore(); + this.sortFields = this.searchHitsProto.getSortFieldsList() + .stream() + .map(sortField -> new SortField(sortField.getField(), SortField.Type.valueOf(sortField.getType().toString()))) + .toArray(SortField[]::new); this.collapseField = this.searchHitsProto.getCollapseField(); - // Below fields are set to null currently, support to be added in the future - this.collapseValues = null; - this.sortFields = null; + this.collapseValues = new Object[this.searchHitsProto.getCollapseValuesCount()]; + for (int i = 0; i < this.searchHitsProto.getCollapseValuesCount(); i++) { + ByteString collapseValue = this.searchHitsProto.getCollapseValues(i); + InputStream is = new ByteArrayInputStream(collapseValue.toByteArray()); + try (ObjectInputStream ois = new ObjectInputStream(is)) { + this.collapseValues[i] = ois.readObject(); + } catch (ClassNotFoundException e) { + throw new OpenSearchException(e); + } + } } @Override diff --git a/server/src/main/java/org/opensearch/search/SearchSortValues.java b/server/src/main/java/org/opensearch/search/SearchSortValues.java index cbc3900f72f79..7a0c77296bea2 100644 --- a/server/src/main/java/org/opensearch/search/SearchSortValues.java +++ b/server/src/main/java/org/opensearch/search/SearchSortValues.java @@ -32,7 +32,10 @@ package org.opensearch.search; +import com.google.protobuf.ByteString; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.SuppressForbidden; +import org.opensearch.OpenSearchException; import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.lucene.Lucene; import org.opensearch.core.common.io.stream.StreamInput; @@ -43,8 +46,12 @@ import org.opensearch.core.xcontent.XContentParser; import org.opensearch.core.xcontent.XContentParserUtils; import org.opensearch.search.SearchHit.Fields; +import org.opensearch.server.proto.FetchSearchResultProto; +import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.InputStream; +import java.io.ObjectInputStream; import java.util.Arrays; import java.util.Objects; @@ -94,6 +101,33 @@ public SearchSortValues(Object[] rawSortValues, DocValueFormat[] sortValueFormat this.rawSortValues = in.readArray(Lucene::readSortValue, Object[]::new); } + @SuppressForbidden(reason = "We need to read from a byte array") + SearchSortValues(byte[] in) throws IOException { + FetchSearchResultProto.SearchHit.SearchSortValues searchSortValues = FetchSearchResultProto.SearchHit.SearchSortValues.parseFrom( + in + ); + this.formattedSortValues = new Object[searchSortValues.getFormattedSortValuesCount()]; + for (int i = 0; i < searchSortValues.getFormattedSortValuesCount(); i++) { + ByteString formattedSortValue = searchSortValues.getFormattedSortValues(i); + InputStream is = new ByteArrayInputStream(formattedSortValue.toByteArray()); + try (ObjectInputStream ois = new ObjectInputStream(is)) { + this.formattedSortValues[i] = ois.readObject(); + } catch (ClassNotFoundException e) { + throw new OpenSearchException(e); + } + } + this.rawSortValues = new Object[searchSortValues.getRawSortValuesCount()]; + for (int i = 0; i < searchSortValues.getRawSortValuesCount(); i++) { + ByteString rawSortValue = searchSortValues.getRawSortValues(i); + InputStream is = new ByteArrayInputStream(rawSortValue.toByteArray()); + try (ObjectInputStream ois = new ObjectInputStream(is)) { + this.rawSortValues[i] = ois.readObject(); + } catch (ClassNotFoundException e) { + throw new OpenSearchException(e); + } + } + } + @Override public void writeTo(StreamOutput out) throws IOException { out.writeArray(Lucene::writeSortValue, this.formattedSortValues); diff --git a/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/HighlightField.java b/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/HighlightField.java index 30effe2826d76..173af3957b8d5 100644 --- a/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/HighlightField.java +++ b/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/HighlightField.java @@ -41,6 +41,7 @@ import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.server.proto.FetchSearchResultProto; import java.io.IOException; import java.util.ArrayList; @@ -77,6 +78,20 @@ public HighlightField(StreamInput in) throws IOException { } } + public HighlightField(byte[] in) throws IOException { + FetchSearchResultProto.SearchHit.HighlightField highlightField = FetchSearchResultProto.SearchHit.HighlightField.parseFrom(in); + name = highlightField.getName(); + if (highlightField.getFragmentsCount() == 0) { + fragments = Text.EMPTY_ARRAY; + } else { + List values = new ArrayList<>(); + for (String fragment : highlightField.getFragmentsList()) { + values.add(new Text(fragment)); + } + fragments = values.toArray(new Text[0]); + } + } + public HighlightField(String name, Text[] fragments) { this.name = Objects.requireNonNull(name, "missing highlight field name"); this.fragments = fragments; diff --git a/server/src/main/proto/server/search/FetchSearchResultProto.proto b/server/src/main/proto/server/search/FetchSearchResultProto.proto index 1074725c9654c..f7c08d77dd553 100644 --- a/server/src/main/proto/server/search/FetchSearchResultProto.proto +++ b/server/src/main/proto/server/search/FetchSearchResultProto.proto @@ -12,6 +12,7 @@ syntax = "proto3"; package org.opensearch.server.proto; +import "google/protobuf/any.proto"; import "server/search/QuerySearchResultProto.proto"; option java_outer_classname = "FetchSearchResultProto"; @@ -26,16 +27,16 @@ message SearchHits { float maxScore = 2; int32 size = 3; repeated SearchHit hits = 4; - /* optional repeated SortField sortFields = 5; */ - optional string collapseField = 5; - repeated bytes collapseValues = 6; + repeated SortField sortFields = 5; + optional string collapseField = 6; + repeated bytes collapseValues = 7; } message SearchHit { int32 docId = 1; float score = 2; string id = 3; - NestedIdentity nestedIdentity = 4; + optional NestedIdentity nestedIdentity = 4; int64 version = 5; int64 seqNo = 6; int64 primaryTerm = 7; @@ -45,16 +46,16 @@ message SearchHit { map highlightFields = 11; SearchSortValues sortValues = 12; repeated string matchedQueries = 13; - /* Explanation explanation = 14;*/ - SearchShardTarget shard = 14; - string index = 15; - string clusterAlias = 16; - map sourceAsMap = 17; + optional Explanation explanation = 14; + SearchShardTarget shard = 15; + optional string index = 16; + optional string clusterAlias = 17; + map innerHits = 18; message NestedIdentity { - string field = 1; - int32 offset = 2; - NestedIdentity child = 3; + optional string field = 1; + optional int32 offset = 2; + optional NestedIdentity child = 3; } message DocumentField { @@ -71,4 +72,34 @@ message SearchHit { repeated bytes formattedSortValues = 1; repeated bytes rawSortValues = 2; } + + message Explanation { + bool match = 1; + string description = 2; + repeated Explanation subExplanations = 3; + oneof explanationValue { + float value1 = 4; + double value2 = 5; + int64 value3 = 6; + } + } } + +message SortField { + Type type = 1; + string field = 2; + + enum Type { + SCORE = 0; + DOC = 1; + STRING = 2; + INT = 3; + FLOAT = 4; + LONG = 5; + DOUBLE = 6; + CUSTOM = 7; + STRING_VAL = 8; + REWRITEABLE = 9; + } +} + From 67a3262113204de9f9cea7bc4ba66ea01731b8a8 Mon Sep 17 00:00:00 2001 From: Vacha Shah Date: Thu, 8 Feb 2024 20:08:59 +0000 Subject: [PATCH 21/36] Incorporating changes from main Signed-off-by: Vacha Shah --- .../org/opensearch/common/util/FeatureFlags.java | 2 +- .../java/org/opensearch/search/SearchHit.java | 15 ++++++++++++++- .../server/search/FetchSearchResultProto.proto | 1 + 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java index 7b009772b6acc..e43dfd3b3b063 100644 --- a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java +++ b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java @@ -65,7 +65,7 @@ public class FeatureFlags { * Gates the optimization to enable bloom filters for doc id lookup. */ public static final String DOC_ID_FUZZY_SET = "opensearch.experimental.optimize_doc_id_lookup.fuzzy_set.enabled"; - + /** * Gates the functionality of integrating protobuf within search API and node-to-node communication. */ diff --git a/server/src/main/java/org/opensearch/search/SearchHit.java b/server/src/main/java/org/opensearch/search/SearchHit.java index 68b24a67d655d..027e96b78e454 100644 --- a/server/src/main/java/org/opensearch/search/SearchHit.java +++ b/server/src/main/java/org/opensearch/search/SearchHit.java @@ -284,7 +284,20 @@ public SearchHit(byte[] in) throws IOException { }); this.sortValues = new SearchSortValues(this.searchHitProto.getSortValues().toByteArray()); if (this.searchHitProto.getMatchedQueriesCount() > 0) { - this.matchedQueries = this.searchHitProto.getMatchedQueriesList().toArray(new String[0]); + this.matchedQueries = new LinkedHashMap<>(this.searchHitProto.getMatchedQueriesCount()); + for (String query : this.searchHitProto.getMatchedQueriesList()) { + matchedQueries.put(query, Float.NaN); + } + } + if (this.searchHitProto.getMatchedQueriesWithScoresCount() > 0) { + Map tempMap = this.searchHitProto.getMatchedQueriesWithScoresMap() + .entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue())); + this.matchedQueries = tempMap.entrySet() + .stream() + .sorted(Map.Entry.comparingByKey()) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (oldValue, newValue) -> oldValue, LinkedHashMap::new)); } if (this.searchHitProto.hasExplanation()) { this.explanation = readExplanation(this.searchHitProto.getExplanation().toByteArray()); diff --git a/server/src/main/proto/server/search/FetchSearchResultProto.proto b/server/src/main/proto/server/search/FetchSearchResultProto.proto index f7c08d77dd553..85346c5394a77 100644 --- a/server/src/main/proto/server/search/FetchSearchResultProto.proto +++ b/server/src/main/proto/server/search/FetchSearchResultProto.proto @@ -51,6 +51,7 @@ message SearchHit { optional string index = 16; optional string clusterAlias = 17; map innerHits = 18; + map matchedQueriesWithScores = 19; message NestedIdentity { optional string field = 1; From 40df9aca96975e542e923f2c308598ecf5d9c210 Mon Sep 17 00:00:00 2001 From: Vacha Shah Date: Wed, 14 Feb 2024 06:43:05 +0000 Subject: [PATCH 22/36] Improving protocol detection for node to node message with protobuf Signed-off-by: Vacha Shah --- .../org/opensearch/transport/InboundPipeline.java | 1 - .../org/opensearch/transport/NodeToNodeMessage.java | 9 +++++++++ .../org/opensearch/transport/OutboundHandler.java | 11 +---------- .../main/java/org/opensearch/transport/TcpHeader.java | 5 +++++ .../java/org/opensearch/transport/TcpTransport.java | 8 ++++++++ .../main/proto/server/NodeToNodeMessageProto.proto | 2 +- 6 files changed, 24 insertions(+), 12 deletions(-) diff --git a/server/src/main/java/org/opensearch/transport/InboundPipeline.java b/server/src/main/java/org/opensearch/transport/InboundPipeline.java index ab9cc60482def..fca941fc08721 100644 --- a/server/src/main/java/org/opensearch/transport/InboundPipeline.java +++ b/server/src/main/java/org/opensearch/transport/InboundPipeline.java @@ -36,7 +36,6 @@ import org.opensearch.common.bytes.ReleasableBytesReference; import org.opensearch.common.lease.Releasable; import org.opensearch.common.lease.Releasables; -import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.util.PageCacheRecycler; import org.opensearch.core.common.breaker.CircuitBreaker; import org.opensearch.core.common.bytes.CompositeBytesReference; diff --git a/server/src/main/java/org/opensearch/transport/NodeToNodeMessage.java b/server/src/main/java/org/opensearch/transport/NodeToNodeMessage.java index f2fbd571725df..aa7b4539c3b1e 100644 --- a/server/src/main/java/org/opensearch/transport/NodeToNodeMessage.java +++ b/server/src/main/java/org/opensearch/transport/NodeToNodeMessage.java @@ -11,7 +11,9 @@ import com.google.protobuf.ByteString; import com.google.protobuf.InvalidProtocolBufferException; import org.opensearch.Version; +import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.util.concurrent.ThreadContext; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.server.proto.NodeToNodeMessageProto; import org.opensearch.server.proto.NodeToNodeMessageProto.NodeToNodeMessage.Header; import org.opensearch.server.proto.NodeToNodeMessageProto.NodeToNodeMessage.ResponseHandlersList; @@ -83,6 +85,13 @@ public void writeTo(OutputStream out) throws IOException { out.write(this.message.toByteArray()); } + BytesReference serialize(BytesStreamOutput bytesStream) throws IOException { + NodeToNodeMessageProto.NodeToNodeMessage message = getMessage(); + TcpHeader.writeHeaderForProtobuf(bytesStream); + message.writeTo(bytesStream); + return bytesStream.bytes(); + } + public NodeToNodeMessageProto.NodeToNodeMessage getMessage() { return this.message; } diff --git a/server/src/main/java/org/opensearch/transport/OutboundHandler.java b/server/src/main/java/org/opensearch/transport/OutboundHandler.java index bb102c47c3085..8a38dc4e0a303 100644 --- a/server/src/main/java/org/opensearch/transport/OutboundHandler.java +++ b/server/src/main/java/org/opensearch/transport/OutboundHandler.java @@ -38,7 +38,6 @@ import org.opensearch.Version; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.CheckedSupplier; -import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.io.stream.ReleasableBytesStreamOutput; import org.opensearch.common.lease.Releasable; import org.opensearch.common.lease.Releasables; @@ -56,7 +55,6 @@ import org.opensearch.threadpool.ThreadPool; import java.io.IOException; -import java.nio.ByteBuffer; import java.util.Set; /** @@ -282,14 +280,7 @@ private ProtobufMessageSerializer(NodeToNodeMessage message, BigArrays bigArrays @Override public BytesReference get() throws IOException { bytesStreamOutput = new ReleasableBytesStreamOutput(bigArrays); - BytesReference reference = serialize(bytesStreamOutput); - return reference; - } - - private BytesReference serialize(BytesStreamOutput bytesStream) throws IOException { - ByteBuffer byteBuffers = ByteBuffer.wrap(message.getMessage().toByteArray()); - message.getMessage().writeTo(bytesStream); - return BytesReference.fromByteBuffer(byteBuffers); + return message.serialize(bytesStreamOutput); } @Override diff --git a/server/src/main/java/org/opensearch/transport/TcpHeader.java b/server/src/main/java/org/opensearch/transport/TcpHeader.java index 78353a9a80403..1dd6e89ca9bee 100644 --- a/server/src/main/java/org/opensearch/transport/TcpHeader.java +++ b/server/src/main/java/org/opensearch/transport/TcpHeader.java @@ -73,6 +73,7 @@ public static int headerSize(Version version) { } private static final byte[] PREFIX = { (byte) 'E', (byte) 'S' }; + private static final byte[] PROTOBUF_PREFIX = { (byte) 'O', (byte) 'S', (byte) 'P' }; public static void writeHeader( StreamOutput output, @@ -91,4 +92,8 @@ public static void writeHeader( assert variableHeaderSize != -1 : "Variable header size not set"; output.writeInt(variableHeaderSize); } + + public static void writeHeaderForProtobuf(StreamOutput output) throws IOException { + output.writeBytes(PROTOBUF_PREFIX); + } } diff --git a/server/src/main/java/org/opensearch/transport/TcpTransport.java b/server/src/main/java/org/opensearch/transport/TcpTransport.java index e32bba5e836d3..8b0ecf10b16f3 100644 --- a/server/src/main/java/org/opensearch/transport/TcpTransport.java +++ b/server/src/main/java/org/opensearch/transport/TcpTransport.java @@ -806,6 +806,14 @@ public static int readMessageLength(BytesReference networkBytes) throws IOExcept } } + public static BaseInboundMessage.Protocol determineTransportProtocol(BytesReference headerBuffer) { + if (headerBuffer.get(0) == 'O' && headerBuffer.get(1) == 'S' && headerBuffer.get(2) == 'P'){ + return BaseInboundMessage.Protocol.PROTOBUF; + } else { + return BaseInboundMessage.Protocol.DEFAULT; + } + } + private static int readHeaderBuffer(BytesReference headerBuffer) throws IOException { if (headerBuffer.get(0) != 'E' || headerBuffer.get(1) != 'S') { if (appearsToBeHTTPRequest(headerBuffer)) { diff --git a/server/src/main/proto/server/NodeToNodeMessageProto.proto b/server/src/main/proto/server/NodeToNodeMessageProto.proto index ae4f2698a8306..05332016b4181 100644 --- a/server/src/main/proto/server/NodeToNodeMessageProto.proto +++ b/server/src/main/proto/server/NodeToNodeMessageProto.proto @@ -26,7 +26,7 @@ message NodeToNodeMessage { string action = 7; int64 requestId = 8; oneof message { - QueryFetchSearchResult queryFetchSearchResult = 16; + QueryFetchSearchResult queryFetchSearchResult = 9; } message Header { From d6431ed7218a0e4a29173430b6776e0ae623e1d9 Mon Sep 17 00:00:00 2001 From: Vacha Shah Date: Fri, 23 Feb 2024 01:01:30 +0000 Subject: [PATCH 23/36] Fixing reading for collapse values object Signed-off-by: Vacha Shah --- .../org/opensearch/search/SearchHits.java | 89 +++++++++++++++---- .../opensearch/transport/TcpTransport.java | 2 +- .../search/FetchSearchResultProto.proto | 12 ++- .../transport/InboundHandlerTests.java | 3 +- .../transport/InboundPipelineTests.java | 2 - 5 files changed, 86 insertions(+), 22 deletions(-) diff --git a/server/src/main/java/org/opensearch/search/SearchHits.java b/server/src/main/java/org/opensearch/search/SearchHits.java index 3e4fb76fe0371..3a0cf8e5a403e 100644 --- a/server/src/main/java/org/opensearch/search/SearchHits.java +++ b/server/src/main/java/org/opensearch/search/SearchHits.java @@ -36,6 +36,7 @@ import org.apache.lucene.search.SortField; import org.apache.lucene.search.TotalHits; import org.apache.lucene.search.TotalHits.Relation; +import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.SuppressForbidden; import org.opensearch.OpenSearchException; import org.opensearch.common.Nullable; @@ -52,13 +53,9 @@ import org.opensearch.server.proto.FetchSearchResultProto; import org.opensearch.server.proto.QuerySearchResultProto; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.InputStream; -import java.io.ObjectInputStream; -import java.io.ObjectOutputStream; import java.io.OutputStream; +import java.math.BigInteger; import java.util.ArrayList; import java.util.Arrays; import java.util.Iterator; @@ -100,7 +97,6 @@ public SearchHits(SearchHit[] hits, @Nullable TotalHits totalHits, float maxScor this(hits, totalHits, maxScore, null, null, null); } - @SuppressForbidden(reason = "serialization of object to protobuf") public SearchHits( SearchHit[] hits, @Nullable TotalHits totalHits, @@ -154,19 +150,57 @@ public SearchHits( if (collapseField != null) { searchHitsBuilder.setCollapseField(collapseField); for (Object value : collapseValues) { - ByteArrayOutputStream bos = new ByteArrayOutputStream(); - try (ObjectOutputStream stream = new ObjectOutputStream(bos)) { - stream.writeObject(value); - searchHitsBuilder.addCollapseValues(ByteString.copyFrom(bos.toByteArray())); + FetchSearchResultProto.CollapseValue.Builder collapseValueBuilder = FetchSearchResultProto.CollapseValue.newBuilder(); + try { + collapseValueBuilder = readCollapseValueForProtobuf(value, collapseValueBuilder); } catch (IOException e) { throw new OpenSearchException(e); } + // ByteArrayOutputStream bos = new ByteArrayOutputStream(); + // try (ObjectOutputStream stream = new ObjectOutputStream(bos)) { + // stream.writeObject(value); + // searchHitsBuilder.addCollapseValues(ByteString.copyFrom(bos.toByteArray())); + // } catch (IOException e) { + // throw new OpenSearchException(e); + // } + searchHitsBuilder.addCollapseValues(collapseValueBuilder.build()); } } this.searchHitsProto = searchHitsBuilder.build(); } } + private FetchSearchResultProto.CollapseValue.Builder readCollapseValueForProtobuf( + Object collapseValue, + FetchSearchResultProto.CollapseValue.Builder collapseValueBuilder + ) throws IOException { + Class type = collapseValue.getClass(); + if (type == String.class) { + collapseValueBuilder.setCollapseString((String) collapseValue); + } else if (type == Integer.class || type == Short.class) { + collapseValueBuilder.setCollapseInt((Integer) collapseValue); + } else if (type == Long.class) { + collapseValueBuilder.setCollapseLong((Long) collapseValue); + } else if (type == Float.class) { + collapseValueBuilder.setCollapseFloat((Float) collapseValue); + } else if (type == Double.class) { + collapseValueBuilder.setCollapseDouble((Double) collapseValue); + } else if (type == Byte.class) { + byte b = (Byte) collapseValue; + collapseValueBuilder.setCollapseBytes(ByteString.copyFrom(new byte[] { b })); + } else if (type == Boolean.class) { + collapseValueBuilder.setCollapseBool((Boolean) collapseValue); + } else if (type == BytesRef.class) { + collapseValueBuilder.setCollapseBytes(ByteString.copyFrom(((BytesRef) collapseValue).bytes)); + } else if (type == BigInteger.class) { + BigInteger bigInt = (BigInteger) collapseValue; + collapseValueBuilder.setCollapseBytes(ByteString.copyFrom(bigInt.toByteArray())); + } else { + throw new IOException("Can't handle sort field value of type [" + type + "]"); + } + return collapseValueBuilder; + } + public SearchHits(StreamInput in) throws IOException { if (in.readBoolean()) { totalHits = Lucene.readTotalHits(in); @@ -208,13 +242,34 @@ public SearchHits(byte[] in) throws IOException { this.collapseField = this.searchHitsProto.getCollapseField(); this.collapseValues = new Object[this.searchHitsProto.getCollapseValuesCount()]; for (int i = 0; i < this.searchHitsProto.getCollapseValuesCount(); i++) { - ByteString collapseValue = this.searchHitsProto.getCollapseValues(i); - InputStream is = new ByteArrayInputStream(collapseValue.toByteArray()); - try (ObjectInputStream ois = new ObjectInputStream(is)) { - this.collapseValues[i] = ois.readObject(); - } catch (ClassNotFoundException e) { - throw new OpenSearchException(e); - } + this.collapseValues[i] = readCollapseValueFromProtobuf(this.searchHitsProto.getCollapseValues(i)); + // ByteString collapseValue = this.searchHitsProto.getCollapseValues(i); + // InputStream is = new ByteArrayInputStream(collapseValue.toByteArray()); + // try (ObjectInputStream ois = new ObjectInputStream(is)) { + // this.collapseValues[i] = ois.readObject(); + // } catch (ClassNotFoundException e) { + // throw new OpenSearchException(e); + // } + } + } + + private Object readCollapseValueFromProtobuf(FetchSearchResultProto.CollapseValue collapseValue) throws IOException { + if (collapseValue.hasCollapseString()) { + return collapseValue.getCollapseString(); + } else if (collapseValue.hasCollapseInt()) { + return collapseValue.getCollapseInt(); + } else if (collapseValue.hasCollapseLong()) { + return collapseValue.getCollapseLong(); + } else if (collapseValue.hasCollapseFloat()) { + return collapseValue.getCollapseFloat(); + } else if (collapseValue.hasCollapseDouble()) { + return collapseValue.getCollapseDouble(); + } else if (collapseValue.hasCollapseBytes()) { + return new BytesRef(collapseValue.getCollapseBytes().toByteArray()); + } else if (collapseValue.hasCollapseBool()) { + return collapseValue.getCollapseBool(); + } else { + throw new IOException("Can't handle sort field value of type [" + collapseValue + "]"); } } diff --git a/server/src/main/java/org/opensearch/transport/TcpTransport.java b/server/src/main/java/org/opensearch/transport/TcpTransport.java index 8b0ecf10b16f3..feaa2042fa258 100644 --- a/server/src/main/java/org/opensearch/transport/TcpTransport.java +++ b/server/src/main/java/org/opensearch/transport/TcpTransport.java @@ -807,7 +807,7 @@ public static int readMessageLength(BytesReference networkBytes) throws IOExcept } public static BaseInboundMessage.Protocol determineTransportProtocol(BytesReference headerBuffer) { - if (headerBuffer.get(0) == 'O' && headerBuffer.get(1) == 'S' && headerBuffer.get(2) == 'P'){ + if (headerBuffer.get(0) == 'O' && headerBuffer.get(1) == 'S' && headerBuffer.get(2) == 'P') { return BaseInboundMessage.Protocol.PROTOBUF; } else { return BaseInboundMessage.Protocol.DEFAULT; diff --git a/server/src/main/proto/server/search/FetchSearchResultProto.proto b/server/src/main/proto/server/search/FetchSearchResultProto.proto index 85346c5394a77..6b07e849b16d5 100644 --- a/server/src/main/proto/server/search/FetchSearchResultProto.proto +++ b/server/src/main/proto/server/search/FetchSearchResultProto.proto @@ -29,7 +29,7 @@ message SearchHits { repeated SearchHit hits = 4; repeated SortField sortFields = 5; optional string collapseField = 6; - repeated bytes collapseValues = 7; + repeated CollapseValue collapseValues = 7; } message SearchHit { @@ -104,3 +104,13 @@ message SortField { } } +message CollapseValue { + optional string collapseString = 1; + optional int32 collapseInt = 2; + optional int64 collapseLong = 3; + optional float collapseFloat = 4; + optional double collapseDouble = 5; + optional bytes collapseBytes = 6; + optional bool collapseBool = 7; +} + diff --git a/server/src/test/java/org/opensearch/transport/InboundHandlerTests.java b/server/src/test/java/org/opensearch/transport/InboundHandlerTests.java index 1ae357ac0cc41..32ceb1932ccf7 100644 --- a/server/src/test/java/org/opensearch/transport/InboundHandlerTests.java +++ b/server/src/test/java/org/opensearch/transport/InboundHandlerTests.java @@ -332,7 +332,8 @@ public QueryFetchSearchResult read(byte[] in) throws IOException { transportChannel.sendResponse(response); BytesReference fullResponseBytes = channel.getMessageCaptor().get(); - NodeToNodeMessage nodeToNodeMessage = new NodeToNodeMessage(fullResponseBytes.toBytesRef().bytes); + byte[] incomingBytes = BytesReference.toBytes(fullResponseBytes.slice(3, fullResponseBytes.length() - 3)); + NodeToNodeMessage nodeToNodeMessage = new NodeToNodeMessage(incomingBytes); handler.inboundMessage(channel, nodeToNodeMessage); QueryFetchSearchResult result = responseCaptor.get(); assertNotNull(result); diff --git a/server/src/test/java/org/opensearch/transport/InboundPipelineTests.java b/server/src/test/java/org/opensearch/transport/InboundPipelineTests.java index 4dca1cfcf07aa..8e0880ea85326 100644 --- a/server/src/test/java/org/opensearch/transport/InboundPipelineTests.java +++ b/server/src/test/java/org/opensearch/transport/InboundPipelineTests.java @@ -95,8 +95,6 @@ public void testPipelineHandling() throws IOException { throw new AssertionError(e); } }; - final BiConsumer messageHandlerProtobuf = (c, m) -> {}; - final StatsTracker statsTracker = new StatsTracker(); final LongSupplier millisSupplier = () -> TimeValue.nsecToMSec(System.nanoTime()); final InboundDecoder decoder = new InboundDecoder(Version.CURRENT, PageCacheRecycler.NON_RECYCLING_INSTANCE); From 69831a8f852ce88acd8dc5ecffbe4bf4f1f5dedc Mon Sep 17 00:00:00 2001 From: Vacha Shah Date: Mon, 26 Feb 2024 23:32:51 +0000 Subject: [PATCH 24/36] Converting from byte[] to InputStream for ProtobufWriteable Signed-off-by: Vacha Shah --- .../common/io/stream/ProtobufWriteable.java | 3 +- .../core/transport/TransportMessage.java | 4 +- .../core/transport/TransportResponse.java | 3 +- .../node/tasks/CancellableTasksIT.java | 3 +- .../action/ActionListenerResponseHandler.java | 3 +- ...ProtobufActionListenerResponseHandler.java | 3 +- ...TransportFieldCapabilitiesIndexAction.java | 3 +- .../TransportResyncReplicationAction.java | 3 +- .../opensearch/action/search/PitService.java | 3 +- .../broadcast/TransportBroadcastAction.java | 3 +- .../node/TransportBroadcastByNodeAction.java | 3 +- .../support/nodes/TransportNodesAction.java | 3 +- .../TransportReplicationAction.java | 3 +- ...ransportInstanceSingleOperationAction.java | 3 +- .../shard/TransportSingleShardAction.java | 5 +- .../support/tasks/TransportTasksAction.java | 3 +- .../coordination/FollowersChecker.java | 3 +- .../cluster/coordination/JoinHelper.java | 5 +- .../cluster/coordination/LeaderChecker.java | 3 +- .../coordination/PreVoteCollector.java | 3 +- .../PublicationTransportHandler.java | 5 +- .../decommission/DecommissionController.java | 3 +- .../org/opensearch/discovery/PeerFinder.java | 3 +- .../extensions/ExtensionsManager.java | 3 +- .../UpdateSettingsResponseHandler.java | 3 +- .../ExtensionTransportActionsHandler.java | 5 +- .../rest/RestSendToExtensionAction.java | 3 +- .../RetentionLeaseBackgroundSyncAction.java | 3 +- .../index/seqno/RetentionLeaseSyncAction.java | 3 +- .../recovery/PeerRecoveryTargetService.java | 3 +- .../checkpoint/PublishCheckpointAction.java | 3 +- .../indices/store/IndicesStore.java | 3 +- .../opensearch/search/SearchPhaseResult.java | 3 +- .../search/fetch/FetchSearchResult.java | 3 +- .../search/fetch/QueryFetchSearchResult.java | 3 +- .../search/query/QuerySearchResult.java | 5 +- .../snapshots/SnapshotShardsService.java | 3 +- .../TraceableTransportResponseHandler.java | 3 +- .../EmptyTransportResponseHandler.java | 3 +- .../transport/PlainTransportFuture.java | 3 +- .../transport/RemoteClusterConnection.java | 3 +- .../transport/SniffConnectionStrategy.java | 3 +- .../transport/TransportActionProxy.java | 3 +- .../transport/TransportHandshaker.java | 3 +- .../transport/TransportRequest.java | 5 +- .../transport/TransportResponseHandler.java | 3 +- .../transport/TransportService.java | 5 +- ...tAddVotingConfigExclusionsActionTests.java | 3 +- ...learVotingConfigExclusionsActionTests.java | 3 +- ...ReplicationAllPermitsAcquisitionTests.java | 3 +- .../coordination/FollowersCheckerTests.java | 9 +-- .../coordination/LeaderCheckerTests.java | 3 +- .../coordination/PreVoteCollectorTests.java | 3 +- .../opensearch/discovery/PeerFinderTests.java | 3 +- .../SegmentReplicationSourceServiceTests.java | 7 ++- .../search/query/QuerySearchResultTests.java | 6 +- .../transport/InboundHandlerTests.java | 12 ++-- .../transport/TransportActionProxyTests.java | 5 +- ...ortServiceDeserializationFailureTests.java | 5 +- .../AbstractSimpleTransportTestCase.java | 57 ++++++++++--------- .../DisruptableMockTransportTests.java | 7 ++- .../test/disruption/NetworkDisruptionIT.java | 3 +- 62 files changed, 175 insertions(+), 110 deletions(-) diff --git a/libs/core/src/main/java/org/opensearch/core/common/io/stream/ProtobufWriteable.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/ProtobufWriteable.java index da8ba03ccaaa8..7338c025921c3 100644 --- a/libs/core/src/main/java/org/opensearch/core/common/io/stream/ProtobufWriteable.java +++ b/libs/core/src/main/java/org/opensearch/core/common/io/stream/ProtobufWriteable.java @@ -16,6 +16,7 @@ import org.opensearch.common.annotation.ExperimentalApi; import java.io.IOException; +import java.io.InputStream; import java.io.OutputStream; /** @@ -64,6 +65,6 @@ interface Reader { * * @param in byte array to read the value from */ - V read(final byte[] in) throws IOException; + V read(final InputStream in) throws IOException; } } diff --git a/libs/core/src/main/java/org/opensearch/core/transport/TransportMessage.java b/libs/core/src/main/java/org/opensearch/core/transport/TransportMessage.java index 37bfcef434582..4ef2e0deed069 100644 --- a/libs/core/src/main/java/org/opensearch/core/transport/TransportMessage.java +++ b/libs/core/src/main/java/org/opensearch/core/transport/TransportMessage.java @@ -37,6 +37,8 @@ import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.common.transport.TransportAddress; +import java.io.InputStream; + /** * Message over the transport interface * @@ -75,5 +77,5 @@ public TransportMessage(StreamInput in) {} * Constructs a new transport message with the data from the byte array. This is * currently a no-op */ - public TransportMessage(byte[] in) {} + public TransportMessage(InputStream in) {} } diff --git a/libs/core/src/main/java/org/opensearch/core/transport/TransportResponse.java b/libs/core/src/main/java/org/opensearch/core/transport/TransportResponse.java index 2537997c239b3..3f0d2595365c7 100644 --- a/libs/core/src/main/java/org/opensearch/core/transport/TransportResponse.java +++ b/libs/core/src/main/java/org/opensearch/core/transport/TransportResponse.java @@ -37,6 +37,7 @@ import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; +import java.io.InputStream; import java.io.OutputStream; /** @@ -66,7 +67,7 @@ public TransportResponse(StreamInput in) throws IOException { * currently a no-op. However, this exists to allow extenders to call super(in) * so that reading can mirror writing where we often call super.writeTo(out). */ - public TransportResponse(byte[] in) throws IOException { + public TransportResponse(InputStream in) throws IOException { super(in); } diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/node/tasks/CancellableTasksIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/node/tasks/CancellableTasksIT.java index 30128ac73ba44..de98f144aa99d 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/node/tasks/CancellableTasksIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/node/tasks/CancellableTasksIT.java @@ -76,6 +76,7 @@ import org.junit.Before; import java.io.IOException; +import java.io.InputStream; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -562,7 +563,7 @@ public TestResponse read(StreamInput in) throws IOException { } @Override - public TestResponse read(byte[] in) throws IOException { + public TestResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } diff --git a/server/src/main/java/org/opensearch/action/ActionListenerResponseHandler.java b/server/src/main/java/org/opensearch/action/ActionListenerResponseHandler.java index af431619f894c..b58d67f826b34 100644 --- a/server/src/main/java/org/opensearch/action/ActionListenerResponseHandler.java +++ b/server/src/main/java/org/opensearch/action/ActionListenerResponseHandler.java @@ -41,6 +41,7 @@ import org.opensearch.transport.TransportResponseHandler; import java.io.IOException; +import java.io.InputStream; import java.util.Objects; /** @@ -91,7 +92,7 @@ public String toString() { } @Override - public Response read(byte[] in) throws IOException { + public Response read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } diff --git a/server/src/main/java/org/opensearch/action/ProtobufActionListenerResponseHandler.java b/server/src/main/java/org/opensearch/action/ProtobufActionListenerResponseHandler.java index 10272ca797517..4dc67273f3eed 100644 --- a/server/src/main/java/org/opensearch/action/ProtobufActionListenerResponseHandler.java +++ b/server/src/main/java/org/opensearch/action/ProtobufActionListenerResponseHandler.java @@ -17,6 +17,7 @@ import org.opensearch.transport.TransportResponseHandler; import java.io.IOException; +import java.io.InputStream; import java.util.Objects; /** @@ -72,7 +73,7 @@ public Response read(StreamInput in) throws IOException { } @Override - public Response read(byte[] in) throws IOException { + public Response read(InputStream in) throws IOException { return reader.read(in); } } diff --git a/server/src/main/java/org/opensearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java b/server/src/main/java/org/opensearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java index 491473116d843..ad42f20fc3dfa 100644 --- a/server/src/main/java/org/opensearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java +++ b/server/src/main/java/org/opensearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java @@ -76,6 +76,7 @@ import org.opensearch.transport.TransportService; import java.io.IOException; +import java.io.InputStream; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -341,7 +342,7 @@ public void handleException(TransportException exp) { } @Override - public FieldCapabilitiesIndexResponse read(byte[] in) throws IOException { + public FieldCapabilitiesIndexResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } diff --git a/server/src/main/java/org/opensearch/action/resync/TransportResyncReplicationAction.java b/server/src/main/java/org/opensearch/action/resync/TransportResyncReplicationAction.java index e79053e2a19e4..95b13616ab048 100644 --- a/server/src/main/java/org/opensearch/action/resync/TransportResyncReplicationAction.java +++ b/server/src/main/java/org/opensearch/action/resync/TransportResyncReplicationAction.java @@ -62,6 +62,7 @@ import org.opensearch.transport.TransportService; import java.io.IOException; +import java.io.InputStream; import java.util.function.Function; import java.util.stream.Stream; @@ -255,7 +256,7 @@ public void handleException(TransportException exp) { } @Override - public ResyncReplicationResponse read(byte[] in) throws IOException { + public ResyncReplicationResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } diff --git a/server/src/main/java/org/opensearch/action/search/PitService.java b/server/src/main/java/org/opensearch/action/search/PitService.java index 4f67b3414b3a4..c1bf5ffbd4a45 100644 --- a/server/src/main/java/org/opensearch/action/search/PitService.java +++ b/server/src/main/java/org/opensearch/action/search/PitService.java @@ -27,6 +27,7 @@ import org.opensearch.transport.TransportService; import java.io.IOException; +import java.io.InputStream; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -204,7 +205,7 @@ public GetAllPitNodesResponse read(StreamInput in) throws IOException { } @Override - public GetAllPitNodesResponse read(byte[] in) throws IOException { + public GetAllPitNodesResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } diff --git a/server/src/main/java/org/opensearch/action/support/broadcast/TransportBroadcastAction.java b/server/src/main/java/org/opensearch/action/support/broadcast/TransportBroadcastAction.java index 4cd26bd739f1b..b41384854c122 100644 --- a/server/src/main/java/org/opensearch/action/support/broadcast/TransportBroadcastAction.java +++ b/server/src/main/java/org/opensearch/action/support/broadcast/TransportBroadcastAction.java @@ -61,6 +61,7 @@ import org.opensearch.transport.TransportService; import java.io.IOException; +import java.io.InputStream; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReferenceArray; @@ -231,7 +232,7 @@ public void handleException(TransportException e) { } @Override - public ShardResponse read(byte[] in) throws IOException { + public ShardResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } diff --git a/server/src/main/java/org/opensearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java b/server/src/main/java/org/opensearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java index 7bab85d3b68cd..89e18773e1079 100644 --- a/server/src/main/java/org/opensearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java +++ b/server/src/main/java/org/opensearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java @@ -69,6 +69,7 @@ import org.opensearch.transport.TransportService; import java.io.IOException; +import java.io.InputStream; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -394,7 +395,7 @@ public String executor() { } @Override - public TransportBroadcastByNodeAction.NodeResponse read(byte[] in) + public TransportBroadcastByNodeAction.NodeResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); diff --git a/server/src/main/java/org/opensearch/action/support/nodes/TransportNodesAction.java b/server/src/main/java/org/opensearch/action/support/nodes/TransportNodesAction.java index 3d88bd623135b..bb7b848e2b69d 100644 --- a/server/src/main/java/org/opensearch/action/support/nodes/TransportNodesAction.java +++ b/server/src/main/java/org/opensearch/action/support/nodes/TransportNodesAction.java @@ -55,6 +55,7 @@ import org.opensearch.transport.TransportService; import java.io.IOException; +import java.io.InputStream; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -288,7 +289,7 @@ public String executor() { } @Override - public NodeResponse read(byte[] in) throws IOException { + public NodeResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } diff --git a/server/src/main/java/org/opensearch/action/support/replication/TransportReplicationAction.java b/server/src/main/java/org/opensearch/action/support/replication/TransportReplicationAction.java index 98e75944fc2ff..ec88b60b36f1a 100644 --- a/server/src/main/java/org/opensearch/action/support/replication/TransportReplicationAction.java +++ b/server/src/main/java/org/opensearch/action/support/replication/TransportReplicationAction.java @@ -94,6 +94,7 @@ import org.opensearch.transport.TransportService; import java.io.IOException; +import java.io.InputStream; import java.util.Map; import java.util.Objects; import java.util.concurrent.atomic.AtomicBoolean; @@ -1177,7 +1178,7 @@ public void handleException(TransportException exp) { } @Override - public Response read(byte[] in) throws IOException { + public Response read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } diff --git a/server/src/main/java/org/opensearch/action/support/single/instance/TransportInstanceSingleOperationAction.java b/server/src/main/java/org/opensearch/action/support/single/instance/TransportInstanceSingleOperationAction.java index 8600613dbe54c..ee642f96aedd4 100644 --- a/server/src/main/java/org/opensearch/action/support/single/instance/TransportInstanceSingleOperationAction.java +++ b/server/src/main/java/org/opensearch/action/support/single/instance/TransportInstanceSingleOperationAction.java @@ -66,6 +66,7 @@ import org.opensearch.transport.TransportService; import java.io.IOException; +import java.io.InputStream; import static org.opensearch.cluster.metadata.IndexNameExpressionResolver.EXCLUDED_DATA_STREAMS_KEY; @@ -247,7 +248,7 @@ public void handleException(TransportException exp) { } @Override - public Response read(byte[] in) throws IOException { + public Response read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } diff --git a/server/src/main/java/org/opensearch/action/support/single/shard/TransportSingleShardAction.java b/server/src/main/java/org/opensearch/action/support/single/shard/TransportSingleShardAction.java index 3a80dbabb535f..0a5fe78c5dbf9 100644 --- a/server/src/main/java/org/opensearch/action/support/single/shard/TransportSingleShardAction.java +++ b/server/src/main/java/org/opensearch/action/support/single/shard/TransportSingleShardAction.java @@ -65,6 +65,7 @@ import org.opensearch.transport.TransportService; import java.io.IOException; +import java.io.InputStream; import static org.opensearch.action.support.TransportActions.isShardNotAvailableException; @@ -227,7 +228,7 @@ public void handleException(TransportException exp) { } @Override - public Response read(byte[] in) throws IOException { + public Response read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } @@ -310,7 +311,7 @@ public void handleException(TransportException exp) { } @Override - public Response read(byte[] in) throws IOException { + public Response read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } diff --git a/server/src/main/java/org/opensearch/action/support/tasks/TransportTasksAction.java b/server/src/main/java/org/opensearch/action/support/tasks/TransportTasksAction.java index fd960df77d492..338750506c735 100644 --- a/server/src/main/java/org/opensearch/action/support/tasks/TransportTasksAction.java +++ b/server/src/main/java/org/opensearch/action/support/tasks/TransportTasksAction.java @@ -62,6 +62,7 @@ import org.opensearch.transport.TransportService; import java.io.IOException; +import java.io.InputStream; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -320,7 +321,7 @@ public String executor() { @Override public TransportTasksAction.NodeTasksResponse - read(byte[] in) throws IOException { + read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } diff --git a/server/src/main/java/org/opensearch/cluster/coordination/FollowersChecker.java b/server/src/main/java/org/opensearch/cluster/coordination/FollowersChecker.java index 08c3c58dacc2d..5118173ea301b 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/FollowersChecker.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/FollowersChecker.java @@ -61,6 +61,7 @@ import org.opensearch.transport.TransportService; import java.io.IOException; +import java.io.InputStream; import java.util.HashSet; import java.util.Map; import java.util.Objects; @@ -410,7 +411,7 @@ public String executor() { } @Override - public Empty read(byte[] in) throws IOException { + public Empty read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } diff --git a/server/src/main/java/org/opensearch/cluster/coordination/JoinHelper.java b/server/src/main/java/org/opensearch/cluster/coordination/JoinHelper.java index f6f2bd029ca2e..3d675335ae867 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/JoinHelper.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/JoinHelper.java @@ -74,6 +74,7 @@ import org.opensearch.transport.TransportService; import java.io.IOException; +import java.io.InputStream; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -423,7 +424,7 @@ public String executor() { } @Override - public Empty read(byte[] in) throws IOException { + public Empty read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } @@ -459,7 +460,7 @@ public String executor() { } @Override - public Empty read(byte[] in) throws IOException { + public Empty read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } diff --git a/server/src/main/java/org/opensearch/cluster/coordination/LeaderChecker.java b/server/src/main/java/org/opensearch/cluster/coordination/LeaderChecker.java index dee8619f3b213..4aad1fa3fc4a9 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/LeaderChecker.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/LeaderChecker.java @@ -63,6 +63,7 @@ import org.opensearch.transport.TransportService; import java.io.IOException; +import java.io.InputStream; import java.util.Objects; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; @@ -351,7 +352,7 @@ public String executor() { } @Override - public Empty read(byte[] in) throws IOException { + public Empty read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } diff --git a/server/src/main/java/org/opensearch/cluster/coordination/PreVoteCollector.java b/server/src/main/java/org/opensearch/cluster/coordination/PreVoteCollector.java index b6bfe54973718..2b2f14db31ed9 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/PreVoteCollector.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/PreVoteCollector.java @@ -50,6 +50,7 @@ import org.opensearch.transport.TransportService; import java.io.IOException; +import java.io.InputStream; import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.LongConsumer; @@ -219,7 +220,7 @@ public String toString() { } @Override - public PreVoteResponse read(byte[] in) throws IOException { + public PreVoteResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } diff --git a/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java b/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java index 359ee0f91482f..50db5d1ba8342 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java @@ -57,6 +57,7 @@ import org.opensearch.transport.TransportService; import java.io.IOException; +import java.io.InputStream; import java.util.HashMap; import java.util.Map; import java.util.concurrent.atomic.AtomicLong; @@ -383,7 +384,7 @@ public String executor() { } @Override - public Empty read(byte[] in) throws IOException { + public Empty read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } @@ -460,7 +461,7 @@ public String executor() { } @Override - public PublishWithJoinResponse read(byte[] in) throws IOException { + public PublishWithJoinResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } diff --git a/server/src/main/java/org/opensearch/cluster/decommission/DecommissionController.java b/server/src/main/java/org/opensearch/cluster/decommission/DecommissionController.java index 307155f779742..722a836c2dc21 100644 --- a/server/src/main/java/org/opensearch/cluster/decommission/DecommissionController.java +++ b/server/src/main/java/org/opensearch/cluster/decommission/DecommissionController.java @@ -36,6 +36,7 @@ import org.opensearch.transport.TransportService; import java.io.IOException; +import java.io.InputStream; import java.util.Arrays; import java.util.HashMap; import java.util.LinkedHashMap; @@ -259,7 +260,7 @@ public NodesStatsResponse read(StreamInput in) throws IOException { } @Override - public NodesStatsResponse read(byte[] in) throws IOException { + public NodesStatsResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } diff --git a/server/src/main/java/org/opensearch/discovery/PeerFinder.java b/server/src/main/java/org/opensearch/discovery/PeerFinder.java index 8eb65cfe12b24..a233409232c42 100644 --- a/server/src/main/java/org/opensearch/discovery/PeerFinder.java +++ b/server/src/main/java/org/opensearch/discovery/PeerFinder.java @@ -54,6 +54,7 @@ import org.opensearch.transport.TransportService; import java.io.IOException; +import java.io.InputStream; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -506,7 +507,7 @@ public String executor() { } @Override - public PeersResponse read(byte[] in) throws IOException { + public PeersResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } diff --git a/server/src/main/java/org/opensearch/extensions/ExtensionsManager.java b/server/src/main/java/org/opensearch/extensions/ExtensionsManager.java index d2fecaf53d5b4..1ac274524be28 100644 --- a/server/src/main/java/org/opensearch/extensions/ExtensionsManager.java +++ b/server/src/main/java/org/opensearch/extensions/ExtensionsManager.java @@ -49,6 +49,7 @@ import org.opensearch.transport.TransportService; import java.io.IOException; +import java.io.InputStream; import java.net.InetAddress; import java.util.HashMap; import java.util.HashSet; @@ -386,7 +387,7 @@ public String executor() { } @Override - public InitializeExtensionResponse read(byte[] in) throws IOException { + public InitializeExtensionResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } diff --git a/server/src/main/java/org/opensearch/extensions/UpdateSettingsResponseHandler.java b/server/src/main/java/org/opensearch/extensions/UpdateSettingsResponseHandler.java index c8e7d8e28e86c..d824b35845cad 100644 --- a/server/src/main/java/org/opensearch/extensions/UpdateSettingsResponseHandler.java +++ b/server/src/main/java/org/opensearch/extensions/UpdateSettingsResponseHandler.java @@ -17,6 +17,7 @@ import org.opensearch.transport.TransportResponseHandler; import java.io.IOException; +import java.io.InputStream; /** * Response handler for {@link UpdateSettingsRequest} @@ -50,7 +51,7 @@ public String executor() { } @Override - public AcknowledgedResponse read(byte[] in) throws IOException { + public AcknowledgedResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } diff --git a/server/src/main/java/org/opensearch/extensions/action/ExtensionTransportActionsHandler.java b/server/src/main/java/org/opensearch/extensions/action/ExtensionTransportActionsHandler.java index 700a66c2babce..699b99d442b2c 100644 --- a/server/src/main/java/org/opensearch/extensions/action/ExtensionTransportActionsHandler.java +++ b/server/src/main/java/org/opensearch/extensions/action/ExtensionTransportActionsHandler.java @@ -27,6 +27,7 @@ import org.opensearch.transport.TransportService; import java.io.IOException; +import java.io.InputStream; import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; @@ -221,7 +222,7 @@ public String executor() { } @Override - public ExtensionActionResponse read(byte[] in) throws IOException { + public ExtensionActionResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } @@ -291,7 +292,7 @@ public String executor() { } @Override - public RemoteExtensionActionResponse read(byte[] in) throws IOException { + public RemoteExtensionActionResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } diff --git a/server/src/main/java/org/opensearch/extensions/rest/RestSendToExtensionAction.java b/server/src/main/java/org/opensearch/extensions/rest/RestSendToExtensionAction.java index ec03c9015e695..63db8ad04eea4 100644 --- a/server/src/main/java/org/opensearch/extensions/rest/RestSendToExtensionAction.java +++ b/server/src/main/java/org/opensearch/extensions/rest/RestSendToExtensionAction.java @@ -35,6 +35,7 @@ import org.opensearch.transport.TransportService; import java.io.IOException; +import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashSet; @@ -242,7 +243,7 @@ public String executor() { } @Override - public RestExecuteOnExtensionResponse read(byte[] in) throws IOException { + public RestExecuteOnExtensionResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } diff --git a/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseBackgroundSyncAction.java b/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseBackgroundSyncAction.java index c00931bf4eb15..1e6732407d98c 100644 --- a/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseBackgroundSyncAction.java +++ b/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseBackgroundSyncAction.java @@ -65,6 +65,7 @@ import org.opensearch.transport.TransportService; import java.io.IOException; +import java.io.InputStream; import java.util.Map; import java.util.Objects; @@ -169,7 +170,7 @@ public void handleException(TransportException e) { } @Override - public ReplicationResponse read(byte[] in) throws IOException { + public ReplicationResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } diff --git a/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseSyncAction.java b/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseSyncAction.java index 88d415af0cd68..1d89e5bc47604 100644 --- a/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseSyncAction.java +++ b/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseSyncAction.java @@ -69,6 +69,7 @@ import org.opensearch.transport.TransportService; import java.io.IOException; +import java.io.InputStream; import java.util.Map; import java.util.Objects; @@ -180,7 +181,7 @@ public void handleException(TransportException e) { } @Override - public ReplicationResponse read(byte[] in) throws IOException { + public ReplicationResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } diff --git a/server/src/main/java/org/opensearch/indices/recovery/PeerRecoveryTargetService.java b/server/src/main/java/org/opensearch/indices/recovery/PeerRecoveryTargetService.java index fa8c8b24d86d4..5a9b9c88318fa 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/PeerRecoveryTargetService.java +++ b/server/src/main/java/org/opensearch/indices/recovery/PeerRecoveryTargetService.java @@ -80,6 +80,7 @@ import org.opensearch.transport.TransportService; import java.io.IOException; +import java.io.InputStream; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Consumer; @@ -764,7 +765,7 @@ public RecoveryResponse read(StreamInput in) throws IOException { } @Override - public RecoveryResponse read(byte[] in) throws IOException { + public RecoveryResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } diff --git a/server/src/main/java/org/opensearch/indices/replication/checkpoint/PublishCheckpointAction.java b/server/src/main/java/org/opensearch/indices/replication/checkpoint/PublishCheckpointAction.java index 600063b218eae..6b32155be072b 100644 --- a/server/src/main/java/org/opensearch/indices/replication/checkpoint/PublishCheckpointAction.java +++ b/server/src/main/java/org/opensearch/indices/replication/checkpoint/PublishCheckpointAction.java @@ -41,6 +41,7 @@ import org.opensearch.transport.TransportService; import java.io.IOException; +import java.io.InputStream; import java.util.Objects; /** @@ -174,7 +175,7 @@ public void handleException(TransportException e) { } @Override - public ReplicationResponse read(byte[] in) throws IOException { + public ReplicationResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } diff --git a/server/src/main/java/org/opensearch/indices/store/IndicesStore.java b/server/src/main/java/org/opensearch/indices/store/IndicesStore.java index 67f44332c40c4..33e825cda34d1 100644 --- a/server/src/main/java/org/opensearch/indices/store/IndicesStore.java +++ b/server/src/main/java/org/opensearch/indices/store/IndicesStore.java @@ -75,6 +75,7 @@ import java.io.Closeable; import java.io.IOException; +import java.io.InputStream; import java.util.ArrayList; import java.util.EnumSet; import java.util.HashSet; @@ -353,7 +354,7 @@ private void allNodesResponded() { } @Override - public ShardActiveResponse read(byte[] in) throws IOException { + public ShardActiveResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } diff --git a/server/src/main/java/org/opensearch/search/SearchPhaseResult.java b/server/src/main/java/org/opensearch/search/SearchPhaseResult.java index bac518079da63..ac411e1bdb362 100644 --- a/server/src/main/java/org/opensearch/search/SearchPhaseResult.java +++ b/server/src/main/java/org/opensearch/search/SearchPhaseResult.java @@ -43,6 +43,7 @@ import org.opensearch.search.query.QuerySearchResult; import java.io.IOException; +import java.io.InputStream; /** * This class is a base class for all search related results. It contains the shard target it @@ -71,7 +72,7 @@ protected SearchPhaseResult(StreamInput in) throws IOException { super(in); } - protected SearchPhaseResult(byte[] in) throws IOException { + protected SearchPhaseResult(InputStream in) throws IOException { super(in); } diff --git a/server/src/main/java/org/opensearch/search/fetch/FetchSearchResult.java b/server/src/main/java/org/opensearch/search/fetch/FetchSearchResult.java index 867ee347f605d..ac930cc6c8e08 100644 --- a/server/src/main/java/org/opensearch/search/fetch/FetchSearchResult.java +++ b/server/src/main/java/org/opensearch/search/fetch/FetchSearchResult.java @@ -48,6 +48,7 @@ import org.opensearch.server.proto.QuerySearchResultProto; import java.io.IOException; +import java.io.InputStream; import java.io.OutputStream; /** @@ -72,7 +73,7 @@ public FetchSearchResult(StreamInput in) throws IOException { hits = new SearchHits(in); } - public FetchSearchResult(byte[] in) throws IOException { + public FetchSearchResult(InputStream in) throws IOException { super(in); this.fetchSearchResultProto = FetchSearchResultProto.FetchSearchResult.parseFrom(in); contextId = new ShardSearchContextId( diff --git a/server/src/main/java/org/opensearch/search/fetch/QueryFetchSearchResult.java b/server/src/main/java/org/opensearch/search/fetch/QueryFetchSearchResult.java index 9d38204aa8869..3ba566645a40b 100644 --- a/server/src/main/java/org/opensearch/search/fetch/QueryFetchSearchResult.java +++ b/server/src/main/java/org/opensearch/search/fetch/QueryFetchSearchResult.java @@ -42,6 +42,7 @@ import org.opensearch.server.proto.QueryFetchSearchResultProto; import java.io.IOException; +import java.io.InputStream; /** * Query fetch result @@ -61,7 +62,7 @@ public QueryFetchSearchResult(StreamInput in) throws IOException { fetchResult = new FetchSearchResult(in); } - public QueryFetchSearchResult(byte[] in) throws IOException { + public QueryFetchSearchResult(InputStream in) throws IOException { super(in); this.queryFetchSearchResultProto = QueryFetchSearchResultProto.QueryFetchSearchResult.parseFrom(in); queryResult = new QuerySearchResult(in); diff --git a/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java b/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java index 7a0603bdf782c..6a74d275a51bc 100644 --- a/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java +++ b/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java @@ -57,6 +57,7 @@ import org.opensearch.server.proto.QuerySearchResultProto; import java.io.IOException; +import java.io.InputStream; import java.io.OutputStream; import java.util.ArrayList; import java.util.List; @@ -113,8 +114,8 @@ public QuerySearchResult(StreamInput in) throws IOException { } } - public QuerySearchResult(byte[] in) throws IOException { - super(in); + public QuerySearchResult(InputStream in) throws IOException { + // super(in); this.querySearchResultProto = QuerySearchResultProto.QuerySearchResult.parseFrom(in); isNull = false; } diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotShardsService.java b/server/src/main/java/org/opensearch/snapshots/SnapshotShardsService.java index 7528e3a09142e..b6650a763cf01 100644 --- a/server/src/main/java/org/opensearch/snapshots/SnapshotShardsService.java +++ b/server/src/main/java/org/opensearch/snapshots/SnapshotShardsService.java @@ -74,6 +74,7 @@ import org.opensearch.transport.TransportService; import java.io.IOException; +import java.io.InputStream; import java.nio.file.NoSuchFileException; import java.util.HashMap; import java.util.Iterator; @@ -625,7 +626,7 @@ public String executor() { } @Override - public UpdateIndexShardSnapshotStatusResponse read(byte[] in) throws IOException { + public UpdateIndexShardSnapshotStatusResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } diff --git a/server/src/main/java/org/opensearch/telemetry/tracing/handler/TraceableTransportResponseHandler.java b/server/src/main/java/org/opensearch/telemetry/tracing/handler/TraceableTransportResponseHandler.java index f938fe6cd1799..e09a7213903c0 100644 --- a/server/src/main/java/org/opensearch/telemetry/tracing/handler/TraceableTransportResponseHandler.java +++ b/server/src/main/java/org/opensearch/telemetry/tracing/handler/TraceableTransportResponseHandler.java @@ -17,6 +17,7 @@ import org.opensearch.transport.TransportResponseHandler; import java.io.IOException; +import java.io.InputStream; import java.util.Objects; /** @@ -106,7 +107,7 @@ public void handleRejection(Exception exp) { } @Override - public T read(byte[] in) throws IOException { + public T read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } diff --git a/server/src/main/java/org/opensearch/transport/EmptyTransportResponseHandler.java b/server/src/main/java/org/opensearch/transport/EmptyTransportResponseHandler.java index 7e4113bc77509..62c9ab738be21 100644 --- a/server/src/main/java/org/opensearch/transport/EmptyTransportResponseHandler.java +++ b/server/src/main/java/org/opensearch/transport/EmptyTransportResponseHandler.java @@ -38,6 +38,7 @@ import org.opensearch.threadpool.ThreadPool; import java.io.IOException; +import java.io.InputStream; /** * Handler for empty transport response @@ -71,7 +72,7 @@ public String executor() { } @Override - public Empty read(byte[] in) throws IOException { + public Empty read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } diff --git a/server/src/main/java/org/opensearch/transport/PlainTransportFuture.java b/server/src/main/java/org/opensearch/transport/PlainTransportFuture.java index 859b4c9f46945..73d8fbd4d528d 100644 --- a/server/src/main/java/org/opensearch/transport/PlainTransportFuture.java +++ b/server/src/main/java/org/opensearch/transport/PlainTransportFuture.java @@ -39,6 +39,7 @@ import org.opensearch.core.transport.TransportResponse; import java.io.IOException; +import java.io.InputStream; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; @@ -127,7 +128,7 @@ public String toString() { } @Override - public V read(byte[] in) throws IOException { + public V read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } diff --git a/server/src/main/java/org/opensearch/transport/RemoteClusterConnection.java b/server/src/main/java/org/opensearch/transport/RemoteClusterConnection.java index 050efc064b8ad..4b2f8c2014206 100644 --- a/server/src/main/java/org/opensearch/transport/RemoteClusterConnection.java +++ b/server/src/main/java/org/opensearch/transport/RemoteClusterConnection.java @@ -47,6 +47,7 @@ import java.io.Closeable; import java.io.IOException; +import java.io.InputStream; import java.util.function.Function; /** @@ -172,7 +173,7 @@ public String executor() { } @Override - public ClusterStateResponse read(byte[] in) throws IOException { + public ClusterStateResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } diff --git a/server/src/main/java/org/opensearch/transport/SniffConnectionStrategy.java b/server/src/main/java/org/opensearch/transport/SniffConnectionStrategy.java index a0cc34f44e5e9..3313ab7103db6 100644 --- a/server/src/main/java/org/opensearch/transport/SniffConnectionStrategy.java +++ b/server/src/main/java/org/opensearch/transport/SniffConnectionStrategy.java @@ -58,6 +58,7 @@ import org.opensearch.threadpool.ThreadPool; import java.io.IOException; +import java.io.InputStream; import java.net.InetSocketAddress; import java.util.Arrays; import java.util.Collections; @@ -473,7 +474,7 @@ public String executor() { } @Override - public ClusterStateResponse read(byte[] in) throws IOException { + public ClusterStateResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } diff --git a/server/src/main/java/org/opensearch/transport/TransportActionProxy.java b/server/src/main/java/org/opensearch/transport/TransportActionProxy.java index 463e729739e93..830c320679cb6 100644 --- a/server/src/main/java/org/opensearch/transport/TransportActionProxy.java +++ b/server/src/main/java/org/opensearch/transport/TransportActionProxy.java @@ -40,6 +40,7 @@ import org.opensearch.threadpool.ThreadPool; import java.io.IOException; +import java.io.InputStream; import java.io.UncheckedIOException; import java.util.function.Function; @@ -132,7 +133,7 @@ public String executor() { } @Override - public T read(byte[] in) throws IOException { + public T read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } diff --git a/server/src/main/java/org/opensearch/transport/TransportHandshaker.java b/server/src/main/java/org/opensearch/transport/TransportHandshaker.java index cc0cdd18890d0..a6dddc430086a 100644 --- a/server/src/main/java/org/opensearch/transport/TransportHandshaker.java +++ b/server/src/main/java/org/opensearch/transport/TransportHandshaker.java @@ -45,6 +45,7 @@ import java.io.EOFException; import java.io.IOException; +import java.io.InputStream; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicBoolean; @@ -189,7 +190,7 @@ public String executor() { } @Override - public HandshakeResponse read(byte[] in) throws IOException { + public HandshakeResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } diff --git a/server/src/main/java/org/opensearch/transport/TransportRequest.java b/server/src/main/java/org/opensearch/transport/TransportRequest.java index fb5d6fd858eb4..8cc53c8ca43a4 100644 --- a/server/src/main/java/org/opensearch/transport/TransportRequest.java +++ b/server/src/main/java/org/opensearch/transport/TransportRequest.java @@ -40,6 +40,7 @@ import org.opensearch.tasks.TaskAwareRequest; import java.io.IOException; +import java.io.InputStream; import java.io.OutputStream; /** @@ -63,7 +64,7 @@ public Empty(StreamInput in) throws IOException { super(in); } - public Empty(byte[] in) throws IOException { + public Empty(InputStream in) throws IOException { super(in); } } @@ -82,7 +83,7 @@ public TransportRequest(StreamInput in) throws IOException { /** * This is added here so that classes don't have to implement since it is an experimental feature and only being added for search apis incrementally. */ - public TransportRequest(byte[] in) throws IOException {} + public TransportRequest(InputStream in) throws IOException {} /** * Set a reference to task that created this request. diff --git a/server/src/main/java/org/opensearch/transport/TransportResponseHandler.java b/server/src/main/java/org/opensearch/transport/TransportResponseHandler.java index 630d917ca9662..b11892b3d6985 100644 --- a/server/src/main/java/org/opensearch/transport/TransportResponseHandler.java +++ b/server/src/main/java/org/opensearch/transport/TransportResponseHandler.java @@ -39,6 +39,7 @@ import org.opensearch.core.transport.TransportResponse; import java.io.IOException; +import java.io.InputStream; import java.util.function.Function; /** @@ -86,7 +87,7 @@ public Q read(StreamInput in) throws IOException { } @Override - public Q read(byte[] in) throws IOException { + public Q read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } diff --git a/server/src/main/java/org/opensearch/transport/TransportService.java b/server/src/main/java/org/opensearch/transport/TransportService.java index 1925fbfc03252..a716a6efb97f7 100644 --- a/server/src/main/java/org/opensearch/transport/TransportService.java +++ b/server/src/main/java/org/opensearch/transport/TransportService.java @@ -76,6 +76,7 @@ import org.opensearch.threadpool.ThreadPool; import java.io.IOException; +import java.io.InputStream; import java.io.UncheckedIOException; import java.net.UnknownHostException; import java.util.Arrays; @@ -1509,7 +1510,7 @@ void setTimeoutHandler(TimeoutHandler handler) { } @Override - public T read(byte[] in) throws IOException { + public T read(InputStream in) throws IOException { return delegate.read(in); } @@ -1728,7 +1729,7 @@ public String toString() { } @Override - public T read(byte[] in) throws IOException { + public T read(InputStream in) throws IOException { return handler.read(in); } }; diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java index 511865688cd6d..0e9298639a145 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java @@ -69,6 +69,7 @@ import org.junit.BeforeClass; import java.io.IOException; +import java.io.InputStream; import java.util.HashSet; import java.util.Set; import java.util.concurrent.CountDownLatch; @@ -687,7 +688,7 @@ public AddVotingConfigExclusionsResponse read(StreamInput in) throws IOException } @Override - public AddVotingConfigExclusionsResponse read(byte[] in) throws IOException { + public AddVotingConfigExclusionsResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/configuration/TransportClearVotingConfigExclusionsActionTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/configuration/TransportClearVotingConfigExclusionsActionTests.java index 35673205519a7..78975e19c4d2b 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/configuration/TransportClearVotingConfigExclusionsActionTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/configuration/TransportClearVotingConfigExclusionsActionTests.java @@ -63,6 +63,7 @@ import org.junit.BeforeClass; import java.io.IOException; +import java.io.InputStream; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; @@ -254,7 +255,7 @@ public ClearVotingConfigExclusionsResponse read(StreamInput in) throws IOExcepti } @Override - public ClearVotingConfigExclusionsResponse read(byte[] in) throws IOException { + public ClearVotingConfigExclusionsResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } diff --git a/server/src/test/java/org/opensearch/action/support/replication/TransportReplicationAllPermitsAcquisitionTests.java b/server/src/test/java/org/opensearch/action/support/replication/TransportReplicationAllPermitsAcquisitionTests.java index 33fca739861d7..91caed7a8c898 100644 --- a/server/src/test/java/org/opensearch/action/support/replication/TransportReplicationAllPermitsAcquisitionTests.java +++ b/server/src/test/java/org/opensearch/action/support/replication/TransportReplicationAllPermitsAcquisitionTests.java @@ -78,6 +78,7 @@ import org.junit.Before; import java.io.IOException; +import java.io.InputStream; import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -226,7 +227,7 @@ public String executor() { } @Override - public ReplicaResponse read(byte[] in) throws IOException { + public ReplicaResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } diff --git a/server/src/test/java/org/opensearch/cluster/coordination/FollowersCheckerTests.java b/server/src/test/java/org/opensearch/cluster/coordination/FollowersCheckerTests.java index ab844bb2a2d9c..fb495de053b4d 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/FollowersCheckerTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/FollowersCheckerTests.java @@ -61,6 +61,7 @@ import org.opensearch.transport.TransportService; import java.io.IOException; +import java.io.InputStream; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; @@ -535,7 +536,7 @@ public String executor() { } @Override - public Empty read(byte[] in) throws IOException { + public Empty read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } @@ -629,7 +630,7 @@ public String executor() { } @Override - public Empty read(byte[] in) throws IOException { + public Empty read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } @@ -707,7 +708,7 @@ public String executor() { } @Override - public Empty read(byte[] in) throws IOException { + public Empty read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } @@ -809,7 +810,7 @@ public TransportResponse.Empty read(StreamInput in) { } @Override - public Empty read(byte[] in) throws IOException { + public Empty read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } diff --git a/server/src/test/java/org/opensearch/cluster/coordination/LeaderCheckerTests.java b/server/src/test/java/org/opensearch/cluster/coordination/LeaderCheckerTests.java index d18703de20ccd..956b0887d1faa 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/LeaderCheckerTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/LeaderCheckerTests.java @@ -58,6 +58,7 @@ import org.opensearch.transport.TransportService; import java.io.IOException; +import java.io.InputStream; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; @@ -552,7 +553,7 @@ public TransportResponse.Empty read(StreamInput in) { } @Override - public Empty read(byte[] in) throws IOException { + public Empty read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } diff --git a/server/src/test/java/org/opensearch/cluster/coordination/PreVoteCollectorTests.java b/server/src/test/java/org/opensearch/cluster/coordination/PreVoteCollectorTests.java index 47ec3333dbefe..b4cf0c77270b3 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/PreVoteCollectorTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/PreVoteCollectorTests.java @@ -53,6 +53,7 @@ import org.junit.Before; import java.io.IOException; +import java.io.InputStream; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -360,7 +361,7 @@ public String executor() { } @Override - public PreVoteResponse read(byte[] in) throws IOException { + public PreVoteResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } diff --git a/server/src/test/java/org/opensearch/discovery/PeerFinderTests.java b/server/src/test/java/org/opensearch/discovery/PeerFinderTests.java index 275bd96e5ad07..e64696b00311d 100644 --- a/server/src/test/java/org/opensearch/discovery/PeerFinderTests.java +++ b/server/src/test/java/org/opensearch/discovery/PeerFinderTests.java @@ -59,6 +59,7 @@ import org.junit.Before; import java.io.IOException; +import java.io.InputStream; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -549,7 +550,7 @@ public String executor() { } @Override - public PeersResponse read(byte[] in) throws IOException { + public PeersResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } diff --git a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationSourceServiceTests.java b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationSourceServiceTests.java index f8ad1484b55cd..19acb8ef938d5 100644 --- a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationSourceServiceTests.java +++ b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationSourceServiceTests.java @@ -40,6 +40,7 @@ import org.opensearch.transport.TransportService; import java.io.IOException; +import java.io.InputStream; import java.util.Collections; import java.util.List; import java.util.concurrent.TimeUnit; @@ -222,7 +223,7 @@ public CheckpointInfoResponse read(StreamInput in) throws IOException { } @Override - public CheckpointInfoResponse read(byte[] in) throws IOException { + public CheckpointInfoResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } @@ -257,7 +258,7 @@ public GetSegmentFilesResponse read(StreamInput in) throws IOException { } @Override - public GetSegmentFilesResponse read(byte[] in) throws IOException { + public GetSegmentFilesResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } @@ -303,7 +304,7 @@ public CheckpointInfoResponse read(StreamInput in) throws IOException { } @Override - public TransportResponse read(byte[] in) throws IOException { + public TransportResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } diff --git a/server/src/test/java/org/opensearch/search/query/QuerySearchResultTests.java b/server/src/test/java/org/opensearch/search/query/QuerySearchResultTests.java index 234ac8e807d8d..8821ce141168d 100644 --- a/server/src/test/java/org/opensearch/search/query/QuerySearchResultTests.java +++ b/server/src/test/java/org/opensearch/search/query/QuerySearchResultTests.java @@ -57,7 +57,9 @@ import org.opensearch.server.proto.QuerySearchResultProto; import org.opensearch.test.OpenSearchTestCase; +import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.io.InputStream; import static java.util.Collections.emptyList; @@ -133,7 +135,9 @@ public void testProtobufSerialization() throws Exception { QuerySearchResult querySearchResult = createTestInstance(); ByteArrayOutputStream stream = new ByteArrayOutputStream(); querySearchResult.writeTo(stream); - QuerySearchResult deserialized = new QuerySearchResult(stream.toByteArray()); + + InputStream inputStream = new ByteArrayInputStream(stream.toByteArray()); + QuerySearchResult deserialized = new QuerySearchResult(inputStream); QuerySearchResultProto.QuerySearchResult querySearchResultProto = deserialized.response(); assertNotNull(querySearchResultProto); assertEquals(querySearchResult.getContextId().getId(), querySearchResultProto.getContextId().getId()); diff --git a/server/src/test/java/org/opensearch/transport/InboundHandlerTests.java b/server/src/test/java/org/opensearch/transport/InboundHandlerTests.java index 32ceb1932ccf7..92ec27c23d23c 100644 --- a/server/src/test/java/org/opensearch/transport/InboundHandlerTests.java +++ b/server/src/test/java/org/opensearch/transport/InboundHandlerTests.java @@ -186,7 +186,7 @@ public TestResponse read(StreamInput in) throws IOException { } @Override - public TestResponse read(byte[] in) throws IOException { + public TestResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } @@ -284,7 +284,7 @@ public QueryFetchSearchResult read(StreamInput in) throws IOException { } @Override - public QueryFetchSearchResult read(byte[] in) throws IOException { + public QueryFetchSearchResult read(InputStream in) throws IOException { return new QueryFetchSearchResult(in); } }, null, action)); @@ -466,7 +466,7 @@ public TestResponse read(StreamInput in) throws IOException { } @Override - public TestResponse read(byte[] in) throws IOException { + public TestResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } @@ -544,7 +544,7 @@ public TestResponse read(StreamInput in) throws IOException { } @Override - public TestResponse read(byte[] in) throws IOException { + public TestResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } @@ -623,7 +623,7 @@ public TestResponse read(StreamInput in) throws IOException { } @Override - public TestResponse read(byte[] in) throws IOException { + public TestResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } @@ -717,7 +717,7 @@ public TestResponse read(StreamInput in) throws IOException { } @Override - public TestResponse read(byte[] in) throws IOException { + public TestResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } diff --git a/server/src/test/java/org/opensearch/transport/TransportActionProxyTests.java b/server/src/test/java/org/opensearch/transport/TransportActionProxyTests.java index df3be09a22dfe..77c6022bfbfbe 100644 --- a/server/src/test/java/org/opensearch/transport/TransportActionProxyTests.java +++ b/server/src/test/java/org/opensearch/transport/TransportActionProxyTests.java @@ -48,6 +48,7 @@ import org.junit.Before; import java.io.IOException; +import java.io.InputStream; import java.util.concurrent.CountDownLatch; public class TransportActionProxyTests extends OpenSearchTestCase { @@ -151,7 +152,7 @@ public String executor() { } @Override - public SimpleTestResponse read(byte[] in) throws IOException { + public SimpleTestResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } @@ -217,7 +218,7 @@ public String executor() { } @Override - public SimpleTestResponse read(byte[] in) throws IOException { + public SimpleTestResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } diff --git a/server/src/test/java/org/opensearch/transport/TransportServiceDeserializationFailureTests.java b/server/src/test/java/org/opensearch/transport/TransportServiceDeserializationFailureTests.java index b9c434b14b7d9..443178a9610d0 100644 --- a/server/src/test/java/org/opensearch/transport/TransportServiceDeserializationFailureTests.java +++ b/server/src/test/java/org/opensearch/transport/TransportServiceDeserializationFailureTests.java @@ -50,6 +50,7 @@ import org.opensearch.threadpool.ThreadPool; import java.io.IOException; +import java.io.InputStream; import java.util.Collections; import java.util.List; @@ -138,7 +139,7 @@ public String toString() { } @Override - public Empty read(byte[] in) throws IOException { + public Empty read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } @@ -200,7 +201,7 @@ public String toString() { } @Override - public Empty read(byte[] in) throws IOException { + public Empty read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } diff --git a/test/framework/src/main/java/org/opensearch/transport/AbstractSimpleTransportTestCase.java b/test/framework/src/main/java/org/opensearch/transport/AbstractSimpleTransportTestCase.java index da4c00a3ac773..93500d3c300cd 100644 --- a/test/framework/src/main/java/org/opensearch/transport/AbstractSimpleTransportTestCase.java +++ b/test/framework/src/main/java/org/opensearch/transport/AbstractSimpleTransportTestCase.java @@ -81,6 +81,7 @@ import org.junit.Before; import java.io.IOException; +import java.io.InputStream; import java.io.UncheckedIOException; import java.net.Inet4Address; import java.net.Inet6Address; @@ -330,7 +331,7 @@ public void handleException(TransportException exp) { } @Override - public StringMessageResponse read(byte[] in) throws IOException { + public StringMessageResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } @@ -372,7 +373,7 @@ public void handleException(TransportException exp) { } @Override - public StringMessageResponse read(byte[] in) throws IOException { + public StringMessageResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } @@ -435,7 +436,7 @@ public void handleException(TransportException exp) { } @Override - public StringMessageResponse read(byte[] in) throws IOException { + public StringMessageResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } @@ -500,7 +501,7 @@ public String executor() { } @Override - public StringMessageResponse read(byte[] in) throws IOException { + public StringMessageResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } @@ -689,7 +690,7 @@ public void handleException(TransportException exp) { } @Override - public Empty read(byte[] in) throws IOException { + public Empty read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } @@ -757,7 +758,7 @@ public void handleException(TransportException exp) { } @Override - public StringMessageResponse read(byte[] in) throws IOException { + public StringMessageResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } @@ -810,7 +811,7 @@ public void handleException(TransportException exp) { } @Override - public StringMessageResponse read(byte[] in) throws IOException { + public StringMessageResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } @@ -1065,7 +1066,7 @@ public void handleException(TransportException exp) { } @Override - public StringMessageResponse read(byte[] in) throws IOException { + public StringMessageResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } @@ -1145,7 +1146,7 @@ public void handleException(TransportException exp) { } @Override - public StringMessageResponse read(byte[] in) throws IOException { + public StringMessageResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } @@ -1192,7 +1193,7 @@ public void handleException(TransportException exp) { } @Override - public StringMessageResponse read(byte[] in) throws IOException { + public StringMessageResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } @@ -1242,7 +1243,7 @@ public String executor() { } @Override - public StringMessageResponse read(byte[] in) throws IOException { + public StringMessageResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } @@ -1551,7 +1552,7 @@ public String executor() { } @Override - public Version0Response read(byte[] in) throws IOException { + public Version0Response read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } @@ -1608,7 +1609,7 @@ public String executor() { } @Override - public Version1Response read(byte[] in) throws IOException { + public Version1Response read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } @@ -1659,7 +1660,7 @@ public String executor() { } @Override - public Version1Response read(byte[] in) throws IOException { + public Version1Response read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } @@ -1707,7 +1708,7 @@ public String executor() { } @Override - public Version0Response read(byte[] in) throws IOException { + public Version0Response read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } @@ -1758,7 +1759,7 @@ public void handleException(TransportException exp) { } @Override - public StringMessageResponse read(byte[] in) throws IOException { + public StringMessageResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } @@ -1828,7 +1829,7 @@ public void handleException(TransportException exp) { } @Override - public StringMessageResponse read(byte[] in) throws IOException { + public StringMessageResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } @@ -1885,7 +1886,7 @@ public String executor() { } @Override - public TestResponse read(byte[] in) throws IOException { + public TestResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } @@ -1939,7 +1940,7 @@ public String executor() { } @Override - public TestResponse read(byte[] in) throws IOException { + public TestResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } @@ -1980,7 +1981,7 @@ public String executor() { } @Override - public TestResponse read(byte[] in) throws IOException { + public TestResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } @@ -2144,7 +2145,7 @@ public String executor() { } @Override - public TestResponse read(byte[] in) throws IOException { + public TestResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } @@ -2212,7 +2213,7 @@ public String executor() { } @Override - public TestResponse read(byte[] in) throws IOException { + public TestResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } @@ -2553,7 +2554,7 @@ public String executor() { } @Override - public TransportResponse read(byte[] in) throws IOException { + public TransportResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } @@ -2609,7 +2610,7 @@ public String executor() { } @Override - public TransportResponse read(byte[] in) throws IOException { + public TransportResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } @@ -2685,7 +2686,7 @@ public String executor() { } @Override - public TransportResponse read(byte[] in) throws IOException { + public TransportResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } @@ -2765,7 +2766,7 @@ public String executor() { } @Override - public TransportResponse read(byte[] in) throws IOException { + public TransportResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } @@ -2892,7 +2893,7 @@ public String executor() { } @Override - public TransportResponse read(byte[] in) throws IOException { + public TransportResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } @@ -3285,7 +3286,7 @@ public TransportResponse read(final StreamInput in) { } @Override - public TransportResponse read(byte[] in) throws IOException { + public TransportResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } diff --git a/test/framework/src/test/java/org/opensearch/test/disruption/DisruptableMockTransportTests.java b/test/framework/src/test/java/org/opensearch/test/disruption/DisruptableMockTransportTests.java index a925d9abd43b5..d4583b6c29419 100644 --- a/test/framework/src/test/java/org/opensearch/test/disruption/DisruptableMockTransportTests.java +++ b/test/framework/src/test/java/org/opensearch/test/disruption/DisruptableMockTransportTests.java @@ -57,6 +57,7 @@ import org.junit.Before; import java.io.IOException; +import java.io.InputStream; import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; @@ -235,7 +236,7 @@ public String executor() { } @Override - public TransportResponse read(byte[] in) throws IOException { + public TransportResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } @@ -265,7 +266,7 @@ public String executor() { } @Override - public TransportResponse read(byte[] in) throws IOException { + public TransportResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } @@ -295,7 +296,7 @@ public String executor() { } @Override - public TransportResponse read(byte[] in) throws IOException { + public TransportResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } diff --git a/test/framework/src/test/java/org/opensearch/test/disruption/NetworkDisruptionIT.java b/test/framework/src/test/java/org/opensearch/test/disruption/NetworkDisruptionIT.java index f9ea3e68b5da8..f36b7bde14083 100644 --- a/test/framework/src/test/java/org/opensearch/test/disruption/NetworkDisruptionIT.java +++ b/test/framework/src/test/java/org/opensearch/test/disruption/NetworkDisruptionIT.java @@ -51,6 +51,7 @@ import org.opensearch.transport.TransportService; import java.io.IOException; +import java.io.InputStream; import java.util.Arrays; import java.util.Collection; import java.util.HashSet; @@ -216,7 +217,7 @@ public TransportResponse read(StreamInput in) throws IOException { } @Override - public TransportResponse read(byte[] in) throws IOException { + public TransportResponse read(InputStream in) throws IOException { // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } From 0cb9018b43782db624d33f714059c43d5b73b21d Mon Sep 17 00:00:00 2001 From: Vacha Shah Date: Tue, 27 Feb 2024 22:39:36 +0000 Subject: [PATCH 25/36] Adding shardSearchRequest to QuerySearchResult proto Signed-off-by: Vacha Shah --- .../search/SearchExecutionStatsCollector.java | 6 +- .../search/fetch/FetchSearchResult.java | 3 +- .../search/internal/ShardSearchRequest.java | 46 ++++++++++++ .../search/query/QuerySearchResult.java | 51 +++++++++++-- .../search/FetchSearchResultProto.proto | 1 + .../search/QuerySearchResultProto.proto | 16 +--- .../search/ShardSearchRequestProto.proto | 75 +++++++++++++++++++ 7 files changed, 172 insertions(+), 26 deletions(-) create mode 100644 server/src/main/proto/server/search/ShardSearchRequestProto.proto diff --git a/server/src/main/java/org/opensearch/action/search/SearchExecutionStatsCollector.java b/server/src/main/java/org/opensearch/action/search/SearchExecutionStatsCollector.java index 5fb838cb6c70d..842e87b3eb635 100644 --- a/server/src/main/java/org/opensearch/action/search/SearchExecutionStatsCollector.java +++ b/server/src/main/java/org/opensearch/action/search/SearchExecutionStatsCollector.java @@ -70,10 +70,8 @@ public static BiFunction Date: Wed, 28 Feb 2024 01:07:07 +0000 Subject: [PATCH 26/36] Addressing minor comments and cleaning up code Signed-off-by: Vacha Shah --- server/build.gradle | 3 +- .../common/document/DocumentField.java | 2 + .../org/opensearch/common/lucene/Lucene.java | 2 + .../java/org/opensearch/search/SearchHit.java | 20 ++++ .../org/opensearch/search/SearchHits.java | 99 ++++++++----------- .../opensearch/search/SearchSortValues.java | 2 + .../search/fetch/FetchSearchResult.java | 25 +---- .../search/fetch/QueryFetchSearchResult.java | 1 + .../subphase/highlight/HighlightField.java | 2 + .../search/internal/ShardSearchRequest.java | 2 + .../search/query/QuerySearchResult.java | 1 + .../transport/NodeToNodeMessage.java | 4 +- .../search/query/QuerySearchResultTests.java | 4 + 13 files changed, 81 insertions(+), 86 deletions(-) diff --git a/server/build.gradle b/server/build.gradle index f846258bfb8a6..9ea9f6a81425f 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -371,7 +371,8 @@ tasks.named("missingJavadoc").configure { "org.opensearch.server.proto.NodeToNodeMessageProto.NodeToNodeMessage.ResponseHandlersListOrBuilder", "org.opensearch.server.proto.NodeToNodeMessageProto.NodeToNodeMessage.HeaderOrBuilder", "org.opensearch.server.proto.FetchSearchResultProto.SearchHit.Explanation.ExplanationValueCase", - "org.opensearch.server.proto.FetchSearchResultProto.SearchHit.ExplanationOrBuilder" + "org.opensearch.server.proto.FetchSearchResultProto.SearchHit.ExplanationOrBuilder", + "org.opensearch.server.proto.ShardSearchRequestProto.OriginalIndices.IndicesOptionsOrBuilder", ] } diff --git a/server/src/main/java/org/opensearch/common/document/DocumentField.java b/server/src/main/java/org/opensearch/common/document/DocumentField.java index 63121f19d1b9a..a3153c296d0c4 100644 --- a/server/src/main/java/org/opensearch/common/document/DocumentField.java +++ b/server/src/main/java/org/opensearch/common/document/DocumentField.java @@ -36,6 +36,7 @@ import org.apache.lucene.util.SuppressForbidden; import org.opensearch.OpenSearchException; import org.opensearch.common.annotation.PublicApi; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; @@ -85,6 +86,7 @@ public DocumentField(String name, List values) { @SuppressForbidden(reason = "We need to read from a byte array") public DocumentField(byte[] in) throws IOException { + assert FeatureFlags.isEnabled(FeatureFlags.PROTOBUF) : "protobuf feature flag is not enabled"; documentField = FetchSearchResultProto.SearchHit.DocumentField.parseFrom(in); name = documentField.getName(); values = new ArrayList<>(); diff --git a/server/src/main/java/org/opensearch/common/lucene/Lucene.java b/server/src/main/java/org/opensearch/common/lucene/Lucene.java index d195859b63fed..51025209a5fd1 100644 --- a/server/src/main/java/org/opensearch/common/lucene/Lucene.java +++ b/server/src/main/java/org/opensearch/common/lucene/Lucene.java @@ -85,6 +85,7 @@ import org.opensearch.common.Nullable; import org.opensearch.common.SuppressForbidden; import org.opensearch.common.lucene.search.TopDocsAndMaxScore; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.util.iterable.Iterables; import org.opensearch.core.common.Strings; import org.opensearch.core.common.io.stream.StreamInput; @@ -653,6 +654,7 @@ public static Explanation readExplanation(StreamInput in) throws IOException { } public static Explanation readExplanation(byte[] in) throws IOException { + assert FeatureFlags.isEnabled(FeatureFlags.PROTOBUF) : "protobuf feature flag is not enabled"; FetchSearchResultProto.SearchHit.Explanation explanationProto = FetchSearchResultProto.SearchHit.Explanation.parseFrom(in); boolean match = explanationProto.getMatch(); String description = explanationProto.getDescription(); diff --git a/server/src/main/java/org/opensearch/search/SearchHit.java b/server/src/main/java/org/opensearch/search/SearchHit.java index 027e96b78e454..c0c21be704ec3 100644 --- a/server/src/main/java/org/opensearch/search/SearchHit.java +++ b/server/src/main/java/org/opensearch/search/SearchHit.java @@ -32,6 +32,7 @@ package org.opensearch.search; +import com.google.protobuf.ByteString; import org.apache.lucene.search.Explanation; import org.opensearch.OpenSearchParseException; import org.opensearch.Version; @@ -39,6 +40,7 @@ import org.opensearch.common.Nullable; import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.document.DocumentField; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.ParseField; import org.opensearch.core.common.ParsingException; @@ -242,6 +244,7 @@ public SearchHit(StreamInput in) throws IOException { } public SearchHit(byte[] in) throws IOException { + assert FeatureFlags.isEnabled(FeatureFlags.PROTOBUF) : "protobuf feature flag is not enabled"; this.searchHitProto = FetchSearchResultProto.SearchHit.parseFrom(in); this.docId = -1; this.score = this.searchHitProto.getScore(); @@ -328,6 +331,22 @@ public SearchHit(byte[] in) throws IOException { } + public static FetchSearchResultProto.SearchHit convertHitToProto(SearchHit hit) { + FetchSearchResultProto.SearchHit.Builder searchHitBuilder = FetchSearchResultProto.SearchHit.newBuilder(); + if (hit.getIndex() != null) { + searchHitBuilder.setIndex(hit.getIndex()); + } + searchHitBuilder.setId(hit.getId()); + searchHitBuilder.setScore(hit.getScore()); + searchHitBuilder.setSeqNo(hit.getSeqNo()); + searchHitBuilder.setPrimaryTerm(hit.getPrimaryTerm()); + searchHitBuilder.setVersion(hit.getVersion()); + if (hit.getSourceRef() != null) { + searchHitBuilder.setSource(ByteString.copyFrom(hit.getSourceRef().toBytesRef().bytes)); + } + return searchHitBuilder.build(); + } + private static final Text SINGLE_MAPPING_TYPE = new Text(MapperService.SINGLE_MAPPING_NAME); @Override @@ -1160,6 +1179,7 @@ public NestedIdentity(String field, int offset, NestedIdentity child) { } NestedIdentity(byte[] in) throws IOException { + assert FeatureFlags.isEnabled(FeatureFlags.PROTOBUF) : "protobuf feature flag is not enabled"; FetchSearchResultProto.SearchHit.NestedIdentity proto = FetchSearchResultProto.SearchHit.NestedIdentity.parseFrom(in); if (proto.hasField()) { field = new Text(proto.getField()); diff --git a/server/src/main/java/org/opensearch/search/SearchHits.java b/server/src/main/java/org/opensearch/search/SearchHits.java index 3a0cf8e5a403e..417e85399e3f1 100644 --- a/server/src/main/java/org/opensearch/search/SearchHits.java +++ b/server/src/main/java/org/opensearch/search/SearchHits.java @@ -112,65 +112,50 @@ public SearchHits( this.collapseField = collapseField; this.collapseValues = collapseValues; if (FeatureFlags.isEnabled(FeatureFlags.PROTOBUF_SETTING)) { - List searchHitList = new ArrayList<>(); - for (SearchHit hit : hits) { - FetchSearchResultProto.SearchHit.Builder searchHitBuilder = FetchSearchResultProto.SearchHit.newBuilder(); - if (hit.getIndex() != null) { - searchHitBuilder.setIndex(hit.getIndex()); - } - searchHitBuilder.setId(hit.getId()); - searchHitBuilder.setScore(hit.getScore()); - searchHitBuilder.setSeqNo(hit.getSeqNo()); - searchHitBuilder.setPrimaryTerm(hit.getPrimaryTerm()); - searchHitBuilder.setVersion(hit.getVersion()); - if (hit.getSourceRef() != null) { - searchHitBuilder.setSource(ByteString.copyFrom(hit.getSourceRef().toBytesRef().bytes)); - } - searchHitList.add(searchHitBuilder.build()); - } - QuerySearchResultProto.TotalHits.Builder totalHitsBuilder = QuerySearchResultProto.TotalHits.newBuilder(); - totalHitsBuilder.setValue(totalHits.value); - totalHitsBuilder.setRelation( - totalHits.relation == Relation.EQUAL_TO - ? QuerySearchResultProto.TotalHits.Relation.EQUAL_TO - : QuerySearchResultProto.TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO - ); - FetchSearchResultProto.SearchHits.Builder searchHitsBuilder = FetchSearchResultProto.SearchHits.newBuilder(); - searchHitsBuilder.setMaxScore(maxScore); - searchHitsBuilder.addAllHits(searchHitList); - searchHitsBuilder.setTotalHits(totalHitsBuilder.build()); - if (sortFields != null && sortFields.length > 0) { - for (SortField sortField : sortFields) { - FetchSearchResultProto.SortField.Builder sortFieldBuilder = FetchSearchResultProto.SortField.newBuilder(); - sortFieldBuilder.setField(sortField.getField()); - sortFieldBuilder.setType(FetchSearchResultProto.SortField.Type.valueOf(sortField.getType().name())); - searchHitsBuilder.addSortFields(sortFieldBuilder.build()); - } + this.searchHitsProto = convertHitsToProto(this); + } + } + + public static FetchSearchResultProto.SearchHits convertHitsToProto(SearchHits hits) { + List searchHitList = new ArrayList<>(); + for (SearchHit hit : hits) { + searchHitList.add(SearchHit.convertHitToProto(hit)); + } + QuerySearchResultProto.TotalHits.Builder totalHitsBuilder = QuerySearchResultProto.TotalHits.newBuilder(); + totalHitsBuilder.setValue(hits.getTotalHits().value); + totalHitsBuilder.setRelation( + hits.getTotalHits().relation == Relation.EQUAL_TO + ? QuerySearchResultProto.TotalHits.Relation.EQUAL_TO + : QuerySearchResultProto.TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO + ); + FetchSearchResultProto.SearchHits.Builder searchHitsBuilder = FetchSearchResultProto.SearchHits.newBuilder(); + searchHitsBuilder.setMaxScore(hits.getMaxScore()); + searchHitsBuilder.addAllHits(searchHitList); + searchHitsBuilder.setTotalHits(totalHitsBuilder.build()); + if (hits.getSortFields() != null && hits.getSortFields().length > 0) { + for (SortField sortField : hits.getSortFields()) { + FetchSearchResultProto.SortField.Builder sortFieldBuilder = FetchSearchResultProto.SortField.newBuilder(); + sortFieldBuilder.setField(sortField.getField()); + sortFieldBuilder.setType(FetchSearchResultProto.SortField.Type.valueOf(sortField.getType().name())); + searchHitsBuilder.addSortFields(sortFieldBuilder.build()); } - if (collapseField != null) { - searchHitsBuilder.setCollapseField(collapseField); - for (Object value : collapseValues) { - FetchSearchResultProto.CollapseValue.Builder collapseValueBuilder = FetchSearchResultProto.CollapseValue.newBuilder(); - try { - collapseValueBuilder = readCollapseValueForProtobuf(value, collapseValueBuilder); - } catch (IOException e) { - throw new OpenSearchException(e); - } - // ByteArrayOutputStream bos = new ByteArrayOutputStream(); - // try (ObjectOutputStream stream = new ObjectOutputStream(bos)) { - // stream.writeObject(value); - // searchHitsBuilder.addCollapseValues(ByteString.copyFrom(bos.toByteArray())); - // } catch (IOException e) { - // throw new OpenSearchException(e); - // } - searchHitsBuilder.addCollapseValues(collapseValueBuilder.build()); + } + if (hits.getCollapseField() != null) { + searchHitsBuilder.setCollapseField(hits.getCollapseField()); + for (Object value : hits.getCollapseValues()) { + FetchSearchResultProto.CollapseValue.Builder collapseValueBuilder = FetchSearchResultProto.CollapseValue.newBuilder(); + try { + collapseValueBuilder = readCollapseValueForProtobuf(value, collapseValueBuilder); + } catch (IOException e) { + throw new OpenSearchException(e); } + searchHitsBuilder.addCollapseValues(collapseValueBuilder.build()); } - this.searchHitsProto = searchHitsBuilder.build(); } + return searchHitsBuilder.build(); } - private FetchSearchResultProto.CollapseValue.Builder readCollapseValueForProtobuf( + private static FetchSearchResultProto.CollapseValue.Builder readCollapseValueForProtobuf( Object collapseValue, FetchSearchResultProto.CollapseValue.Builder collapseValueBuilder ) throws IOException { @@ -225,6 +210,7 @@ public SearchHits(StreamInput in) throws IOException { @SuppressForbidden(reason = "serialization of object to protobuf") public SearchHits(byte[] in) throws IOException { + assert FeatureFlags.isEnabled(FeatureFlags.PROTOBUF) : "protobuf feature flag is not enabled"; this.searchHitsProto = org.opensearch.server.proto.FetchSearchResultProto.SearchHits.parseFrom(in); this.hits = new SearchHit[this.searchHitsProto.getHitsCount()]; for (int i = 0; i < this.searchHitsProto.getHitsCount(); i++) { @@ -243,13 +229,6 @@ public SearchHits(byte[] in) throws IOException { this.collapseValues = new Object[this.searchHitsProto.getCollapseValuesCount()]; for (int i = 0; i < this.searchHitsProto.getCollapseValuesCount(); i++) { this.collapseValues[i] = readCollapseValueFromProtobuf(this.searchHitsProto.getCollapseValues(i)); - // ByteString collapseValue = this.searchHitsProto.getCollapseValues(i); - // InputStream is = new ByteArrayInputStream(collapseValue.toByteArray()); - // try (ObjectInputStream ois = new ObjectInputStream(is)) { - // this.collapseValues[i] = ois.readObject(); - // } catch (ClassNotFoundException e) { - // throw new OpenSearchException(e); - // } } } diff --git a/server/src/main/java/org/opensearch/search/SearchSortValues.java b/server/src/main/java/org/opensearch/search/SearchSortValues.java index 7a0c77296bea2..1eef677ae6863 100644 --- a/server/src/main/java/org/opensearch/search/SearchSortValues.java +++ b/server/src/main/java/org/opensearch/search/SearchSortValues.java @@ -38,6 +38,7 @@ import org.opensearch.OpenSearchException; import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.lucene.Lucene; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; @@ -103,6 +104,7 @@ public SearchSortValues(Object[] rawSortValues, DocValueFormat[] sortValueFormat @SuppressForbidden(reason = "We need to read from a byte array") SearchSortValues(byte[] in) throws IOException { + assert FeatureFlags.isEnabled(FeatureFlags.PROTOBUF) : "protobuf feature flag is not enabled"; FetchSearchResultProto.SearchHit.SearchSortValues searchSortValues = FetchSearchResultProto.SearchHit.SearchSortValues.parseFrom( in ); diff --git a/server/src/main/java/org/opensearch/search/fetch/FetchSearchResult.java b/server/src/main/java/org/opensearch/search/fetch/FetchSearchResult.java index f5aa94fb12e55..f966f0a0fdccc 100644 --- a/server/src/main/java/org/opensearch/search/fetch/FetchSearchResult.java +++ b/server/src/main/java/org/opensearch/search/fetch/FetchSearchResult.java @@ -32,8 +32,6 @@ package org.opensearch.search.fetch; -import com.google.protobuf.ByteString; -import org.apache.lucene.search.TotalHits.Relation; import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.common.io.stream.StreamInput; @@ -45,7 +43,6 @@ import org.opensearch.search.internal.ShardSearchContextId; import org.opensearch.search.query.QuerySearchResult; import org.opensearch.server.proto.FetchSearchResultProto; -import org.opensearch.server.proto.QuerySearchResultProto; import org.opensearch.server.proto.ShardSearchRequestProto; import java.io.IOException; @@ -76,6 +73,7 @@ public FetchSearchResult(StreamInput in) throws IOException { public FetchSearchResult(InputStream in) throws IOException { super(in); + assert FeatureFlags.isEnabled(FeatureFlags.PROTOBUF) : "protobuf feature flag is not enabled"; this.fetchSearchResultProto = FetchSearchResultProto.FetchSearchResult.parseFrom(in); contextId = new ShardSearchContextId( this.fetchSearchResultProto.getContextId().getSessionId(), @@ -108,26 +106,7 @@ public void hits(SearchHits hits) { assert assertNoSearchTarget(hits); this.hits = hits; if (FeatureFlags.isEnabled(FeatureFlags.PROTOBUF_SETTING) && this.fetchSearchResultProto != null) { - QuerySearchResultProto.TotalHits.Builder totalHitsBuilder = QuerySearchResultProto.TotalHits.newBuilder(); - totalHitsBuilder.setValue(hits.getTotalHits().value); - totalHitsBuilder.setRelation( - hits.getTotalHits().relation == Relation.EQUAL_TO - ? QuerySearchResultProto.TotalHits.Relation.EQUAL_TO - : QuerySearchResultProto.TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO - ); - FetchSearchResultProto.SearchHits.Builder searchHitsBuilder = FetchSearchResultProto.SearchHits.newBuilder(); - searchHitsBuilder.setMaxScore(hits.getMaxScore()); - searchHitsBuilder.setTotalHits(totalHitsBuilder.build()); - for (SearchHit hit : hits.getHits()) { - FetchSearchResultProto.SearchHit.Builder searchHitBuilder = FetchSearchResultProto.SearchHit.newBuilder(); - searchHitBuilder.setId(hit.getId()); - searchHitBuilder.setSource(ByteString.copyFrom(hit.getSourceRef().toBytesRef().bytes)); - searchHitBuilder.setVersion(hit.getVersion()); - searchHitBuilder.setSeqNo(hit.getSeqNo()); - searchHitBuilder.setPrimaryTerm(hit.getPrimaryTerm()); - searchHitsBuilder.addHits(searchHitBuilder.build()); - } - this.fetchSearchResultProto = this.fetchSearchResultProto.toBuilder().setHits(searchHitsBuilder.build()).build(); + this.fetchSearchResultProto = this.fetchSearchResultProto.toBuilder().setHits(SearchHits.convertHitsToProto(hits)).build(); } } diff --git a/server/src/main/java/org/opensearch/search/fetch/QueryFetchSearchResult.java b/server/src/main/java/org/opensearch/search/fetch/QueryFetchSearchResult.java index 3ba566645a40b..8f8697f18765b 100644 --- a/server/src/main/java/org/opensearch/search/fetch/QueryFetchSearchResult.java +++ b/server/src/main/java/org/opensearch/search/fetch/QueryFetchSearchResult.java @@ -64,6 +64,7 @@ public QueryFetchSearchResult(StreamInput in) throws IOException { public QueryFetchSearchResult(InputStream in) throws IOException { super(in); + assert FeatureFlags.isEnabled(FeatureFlags.PROTOBUF) : "protobuf feature flag is not enabled"; this.queryFetchSearchResultProto = QueryFetchSearchResultProto.QueryFetchSearchResult.parseFrom(in); queryResult = new QuerySearchResult(in); fetchResult = new FetchSearchResult(in); diff --git a/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/HighlightField.java b/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/HighlightField.java index 173af3957b8d5..7ea5f5fdca7d0 100644 --- a/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/HighlightField.java +++ b/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/HighlightField.java @@ -33,6 +33,7 @@ package org.opensearch.search.fetch.subphase.highlight; import org.opensearch.common.annotation.PublicApi; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.common.ParsingException; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; @@ -79,6 +80,7 @@ public HighlightField(StreamInput in) throws IOException { } public HighlightField(byte[] in) throws IOException { + assert FeatureFlags.isEnabled(FeatureFlags.PROTOBUF) : "protobuf feature flag is not enabled"; FetchSearchResultProto.SearchHit.HighlightField highlightField = FetchSearchResultProto.SearchHit.HighlightField.parseFrom(in); name = highlightField.getName(); if (highlightField.getFragmentsCount() == 0) { diff --git a/server/src/main/java/org/opensearch/search/internal/ShardSearchRequest.java b/server/src/main/java/org/opensearch/search/internal/ShardSearchRequest.java index 947018f2b1c85..9ab84dda119f2 100644 --- a/server/src/main/java/org/opensearch/search/internal/ShardSearchRequest.java +++ b/server/src/main/java/org/opensearch/search/internal/ShardSearchRequest.java @@ -46,6 +46,7 @@ import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.common.Strings; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.StreamInput; @@ -271,6 +272,7 @@ public ShardSearchRequest(StreamInput in) throws IOException { } public ShardSearchRequest(byte[] in) throws IOException { + assert FeatureFlags.isEnabled(FeatureFlags.PROTOBUF) : "protobuf feature flag is not enabled"; ShardSearchRequestProto.ShardSearchRequest searchRequestProto = ShardSearchRequestProto.ShardSearchRequest.parseFrom(in); this.clusterAlias = searchRequestProto.getClusterAlias(); shardId = new ShardId( diff --git a/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java b/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java index f46880e02f8c5..822628e960c3c 100644 --- a/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java +++ b/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java @@ -121,6 +121,7 @@ public QuerySearchResult(StreamInput in) throws IOException { public QuerySearchResult(InputStream in) throws IOException { super(in); + assert FeatureFlags.isEnabled(FeatureFlags.PROTOBUF) : "protobuf feature flag is not enabled"; this.querySearchResultProto = QuerySearchResultProto.QuerySearchResult.parseFrom(in); isNull = false; ShardSearchRequest shardSearchRequest; diff --git a/server/src/main/java/org/opensearch/transport/NodeToNodeMessage.java b/server/src/main/java/org/opensearch/transport/NodeToNodeMessage.java index aa7b4539c3b1e..e96dafb55265c 100644 --- a/server/src/main/java/org/opensearch/transport/NodeToNodeMessage.java +++ b/server/src/main/java/org/opensearch/transport/NodeToNodeMessage.java @@ -77,8 +77,8 @@ public NodeToNodeMessage( } - public NodeToNodeMessage(byte[] data) throws InvalidProtocolBufferException { - this.message = NodeToNodeMessageProto.NodeToNodeMessage.parseFrom(data); + public NodeToNodeMessage(byte[] in) throws InvalidProtocolBufferException { + this.message = NodeToNodeMessageProto.NodeToNodeMessage.parseFrom(in); } public void writeTo(OutputStream out) throws IOException { diff --git a/server/src/test/java/org/opensearch/search/query/QuerySearchResultTests.java b/server/src/test/java/org/opensearch/search/query/QuerySearchResultTests.java index 8821ce141168d..df7c38752b97d 100644 --- a/server/src/test/java/org/opensearch/search/query/QuerySearchResultTests.java +++ b/server/src/test/java/org/opensearch/search/query/QuerySearchResultTests.java @@ -39,9 +39,11 @@ import org.opensearch.action.OriginalIndices; import org.opensearch.action.OriginalIndicesTests; import org.opensearch.action.search.SearchRequest; +import org.opensearch.common.SuppressForbidden; import org.opensearch.common.UUIDs; import org.opensearch.common.lucene.search.TopDocsAndMaxScore; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.common.Strings; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.index.shard.ShardId; @@ -131,7 +133,9 @@ public void testNullResponse() throws Exception { assertEquals(querySearchResult.isNull(), deserialized.isNull()); } + @SuppressForbidden(reason = "manipulates system properties for testing") public void testProtobufSerialization() throws Exception { + System.setProperty(FeatureFlags.PROTOBUF, "true"); QuerySearchResult querySearchResult = createTestInstance(); ByteArrayOutputStream stream = new ByteArrayOutputStream(); querySearchResult.writeTo(stream); From 5ae83ab6444a1b560957c8a182f29238d53070e5 Mon Sep 17 00:00:00 2001 From: Vacha Shah Date: Thu, 29 Feb 2024 02:16:39 +0000 Subject: [PATCH 27/36] Extract native and protobuf message handler from InboundHandler Signed-off-by: Vacha Shah --- .../transport/ProtobufMessageHandler.java | 152 ++++++++++++++++++ .../search/query/QuerySearchResultTests.java | 1 + 2 files changed, 153 insertions(+) create mode 100644 server/src/main/java/org/opensearch/transport/ProtobufMessageHandler.java diff --git a/server/src/main/java/org/opensearch/transport/ProtobufMessageHandler.java b/server/src/main/java/org/opensearch/transport/ProtobufMessageHandler.java new file mode 100644 index 0000000000000..b46ff2c86c7b8 --- /dev/null +++ b/server/src/main/java/org/opensearch/transport/ProtobufMessageHandler.java @@ -0,0 +1,152 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.transport; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.opensearch.common.collect.Tuple; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.util.concurrent.ThreadContext; +import org.opensearch.core.common.transport.TransportAddress; +import org.opensearch.core.transport.TransportResponse; +import org.opensearch.server.proto.QueryFetchSearchResultProto.QueryFetchSearchResult; +import org.opensearch.threadpool.ThreadPool; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.Map; +import java.util.Set; + +/** + * Protobuf handler for inbound data + * + * @opensearch.internal + */ +public class ProtobufMessageHandler { + + private static final Logger logger = LogManager.getLogger(ProtobufMessageHandler.class); + + private final ThreadPool threadPool; + private final Transport.ResponseHandlers responseHandlers; + + private volatile TransportMessageListener messageListener = TransportMessageListener.NOOP_LISTENER; + + private volatile long slowLogThresholdMs = Long.MAX_VALUE; + + ProtobufMessageHandler(ThreadPool threadPool, Transport.ResponseHandlers responseHandlers) { + this.threadPool = threadPool; + this.responseHandlers = responseHandlers; + } + + void setMessageListener(TransportMessageListener listener) { + if (messageListener == TransportMessageListener.NOOP_LISTENER) { + messageListener = listener; + } else { + throw new IllegalStateException("Cannot set message listener twice"); + } + } + + void setSlowLogThreshold(TimeValue slowLogThreshold) { + this.slowLogThresholdMs = slowLogThreshold.getMillis(); + } + + public void messageReceivedProtobuf(TcpChannel channel, NodeToNodeMessage message, long startTime) throws IOException { + final InetSocketAddress remoteAddress = channel.getRemoteAddress(); + final org.opensearch.server.proto.NodeToNodeMessageProto.NodeToNodeMessage.Header header = message.getHeader(); + + ThreadContext threadContext = threadPool.getThreadContext(); + try (ThreadContext.StoredContext existing = threadContext.stashContext()) { + // Place the context with the headers from the message + final Tuple, Map>> headers = new Tuple, Map>>( + message.getRequestHeaders(), + message.getResponseHandlers() + ); + threadContext.setHeaders(headers); + threadContext.putTransient("_remote_address", remoteAddress); + + long requestId = header.getRequestId(); + TransportResponseHandler handler = responseHandlers.onResponseReceived(requestId, messageListener); + if (handler != null) { + handleProtobufResponse(requestId, remoteAddress, message, handler); + } + } finally { + final long took = threadPool.relativeTimeInMillis() - startTime; + final long logThreshold = slowLogThresholdMs; + if (logThreshold > 0 && took > logThreshold) { + logger.warn( + "handling inbound transport message [{}] took [{}ms] which is above the warn threshold of [{}ms]", + message, + took, + logThreshold + ); + } + } + } + + private void handleProtobufResponse( + final long requestId, + InetSocketAddress remoteAddress, + final NodeToNodeMessage message, + final TransportResponseHandler handler + ) throws IOException { + try { + org.opensearch.server.proto.NodeToNodeMessageProto.NodeToNodeMessage receivedMessage = message.getMessage(); + if (receivedMessage.hasQueryFetchSearchResult()) { + final QueryFetchSearchResult queryFetchSearchResult = receivedMessage.getQueryFetchSearchResult(); + org.opensearch.search.fetch.QueryFetchSearchResult queryFetchSearchResult2 = + new org.opensearch.search.fetch.QueryFetchSearchResult(queryFetchSearchResult); + final T response = (T) queryFetchSearchResult2; + response.remoteAddress(new TransportAddress(remoteAddress)); + + final String executor = handler.executor(); + if (ThreadPool.Names.SAME.equals(executor)) { + doHandleResponse(handler, response); + } else { + threadPool.executor(executor).execute(() -> doHandleResponse(handler, response)); + } + } + } catch (Exception e) { + final Exception serializationException = new TransportSerializationException( + "Failed to deserialize response from handler [" + handler + "]", + e + ); + logger.warn(new ParameterizedMessage("Failed to deserialize response from [{}]", remoteAddress), serializationException); + handleException(handler, serializationException); + return; + } + } + + private void doHandleResponse(TransportResponseHandler handler, T response) { + try { + handler.handleResponse(response); + } catch (Exception e) { + handleException(handler, new ResponseHandlerFailureTransportException(e)); + } + } + + private void handleException(final TransportResponseHandler handler, Throwable error) { + if (!(error instanceof RemoteTransportException)) { + error = new RemoteTransportException(error.getMessage(), error); + } + final RemoteTransportException rtx = (RemoteTransportException) error; + threadPool.executor(handler.executor()).execute(() -> { + try { + handler.handleException(rtx); + } catch (Exception e) { + logger.error(() -> new ParameterizedMessage("failed to handle exception response [{}]", handler), e); + } + }); + } +} diff --git a/server/src/test/java/org/opensearch/search/query/QuerySearchResultTests.java b/server/src/test/java/org/opensearch/search/query/QuerySearchResultTests.java index df7c38752b97d..5ab6714a69116 100644 --- a/server/src/test/java/org/opensearch/search/query/QuerySearchResultTests.java +++ b/server/src/test/java/org/opensearch/search/query/QuerySearchResultTests.java @@ -156,5 +156,6 @@ public void testProtobufSerialization() throws Exception { ); assertEquals(querySearchResult.from(), querySearchResultProto.getFrom()); assertEquals(querySearchResult.size(), querySearchResultProto.getSize()); + System.setProperty(FeatureFlags.PROTOBUF, "false"); } } From 243f039f3f5375e476a9cd5161622abb6efc7683 Mon Sep 17 00:00:00 2001 From: Vacha Shah Date: Thu, 29 Feb 2024 19:53:20 +0000 Subject: [PATCH 28/36] Changing protocol enum to string Signed-off-by: Vacha Shah --- .../org/opensearch/core/transport/TransportMessage.java | 9 ++++++--- .../opensearch/search/fetch/QueryFetchSearchResult.java | 7 ++++--- .../org/opensearch/transport/BaseInboundMessage.java | 8 +++----- .../java/org/opensearch/transport/NodeToNodeMessage.java | 8 ++++---- .../java/org/opensearch/transport/OutboundHandler.java | 2 +- .../main/java/org/opensearch/transport/TcpTransport.java | 6 +++--- .../org/opensearch/transport/OutboundHandlerTests.java | 2 +- 7 files changed, 22 insertions(+), 20 deletions(-) diff --git a/libs/core/src/main/java/org/opensearch/core/transport/TransportMessage.java b/libs/core/src/main/java/org/opensearch/core/transport/TransportMessage.java index 4ef2e0deed069..219326b44216c 100644 --- a/libs/core/src/main/java/org/opensearch/core/transport/TransportMessage.java +++ b/libs/core/src/main/java/org/opensearch/core/transport/TransportMessage.java @@ -48,7 +48,7 @@ public abstract class TransportMessage implements Writeable, ProtobufWriteable { private TransportAddress remoteAddress; - private boolean isProtobuf; + private String protocol; public void remoteAddress(TransportAddress remoteAddress) { this.remoteAddress = remoteAddress; @@ -58,8 +58,11 @@ public TransportAddress remoteAddress() { return remoteAddress; } - public boolean isMessageProtobuf() { - return isProtobuf; + public String getProtocol() { + if (protocol != null) { + return protocol; + } + return "native"; } /** diff --git a/server/src/main/java/org/opensearch/search/fetch/QueryFetchSearchResult.java b/server/src/main/java/org/opensearch/search/fetch/QueryFetchSearchResult.java index 8f8697f18765b..b5e0a820da9af 100644 --- a/server/src/main/java/org/opensearch/search/fetch/QueryFetchSearchResult.java +++ b/server/src/main/java/org/opensearch/search/fetch/QueryFetchSearchResult.java @@ -40,6 +40,7 @@ import org.opensearch.search.internal.ShardSearchContextId; import org.opensearch.search.query.QuerySearchResult; import org.opensearch.server.proto.QueryFetchSearchResultProto; +import org.opensearch.transport.BaseInboundMessage; import java.io.IOException; import java.io.InputStream; @@ -122,11 +123,11 @@ public void writeTo(StreamOutput out) throws IOException { } @Override - public boolean isMessageProtobuf() { + public String getProtocol() { if (FeatureFlags.isEnabled(FeatureFlags.PROTOBUF_SETTING)) { - return true; + return BaseInboundMessage.PROTOBUF_PROTOCOL; } - return false; + return BaseInboundMessage.NATIVE_PROTOCOL; } public QueryFetchSearchResultProto.QueryFetchSearchResult response() { diff --git a/server/src/main/java/org/opensearch/transport/BaseInboundMessage.java b/server/src/main/java/org/opensearch/transport/BaseInboundMessage.java index bc8833d45be9e..112b55fdf72be 100644 --- a/server/src/main/java/org/opensearch/transport/BaseInboundMessage.java +++ b/server/src/main/java/org/opensearch/transport/BaseInboundMessage.java @@ -19,15 +19,13 @@ public interface BaseInboundMessage { /** * The protocol used to encode this message */ - enum Protocol { - DEFAULT, - PROTOBUF, - } + static String NATIVE_PROTOCOL = "native"; + static String PROTOBUF_PROTOCOL = "protobuf"; /** * @return the protocol used to encode this message */ - public Protocol getProtocol(); + public String getProtocol(); /** * Set the protocol used to encode this message diff --git a/server/src/main/java/org/opensearch/transport/NodeToNodeMessage.java b/server/src/main/java/org/opensearch/transport/NodeToNodeMessage.java index e96dafb55265c..d1fe9775033dd 100644 --- a/server/src/main/java/org/opensearch/transport/NodeToNodeMessage.java +++ b/server/src/main/java/org/opensearch/transport/NodeToNodeMessage.java @@ -37,7 +37,7 @@ public class NodeToNodeMessage implements BaseInboundMessage { private final NodeToNodeMessageProto.NodeToNodeMessage message; private static final byte[] PREFIX = { (byte) 'E', (byte) 'S' }; - private Protocol protocol; + private String protocol; public NodeToNodeMessage( long requestId, @@ -122,12 +122,12 @@ public Map> getResponseHandlers() { } @Override - public Protocol getProtocol() { - return Protocol.PROTOBUF; + public String getProtocol() { + return PROTOBUF_PROTOCOL; } @Override public void setProtocol() { - this.protocol = Protocol.PROTOBUF; + this.protocol = PROTOBUF_PROTOCOL; } } diff --git a/server/src/main/java/org/opensearch/transport/OutboundHandler.java b/server/src/main/java/org/opensearch/transport/OutboundHandler.java index 8a38dc4e0a303..96dcfe2b2d327 100644 --- a/server/src/main/java/org/opensearch/transport/OutboundHandler.java +++ b/server/src/main/java/org/opensearch/transport/OutboundHandler.java @@ -148,7 +148,7 @@ void sendResponse( ) throws IOException { Version version = Version.min(this.version, nodeVersion); ActionListener listener = ActionListener.wrap(() -> messageListener.onResponseSent(requestId, action, response)); - if (response.isMessageProtobuf()) { + if ((response.getProtocol()).equals(BaseInboundMessage.PROTOBUF_PROTOCOL)) { QueryFetchSearchResult queryFetchSearchResult = (QueryFetchSearchResult) response; if (queryFetchSearchResult.response() != null) { byte[] bytes = new byte[1]; diff --git a/server/src/main/java/org/opensearch/transport/TcpTransport.java b/server/src/main/java/org/opensearch/transport/TcpTransport.java index feaa2042fa258..7fb600a484035 100644 --- a/server/src/main/java/org/opensearch/transport/TcpTransport.java +++ b/server/src/main/java/org/opensearch/transport/TcpTransport.java @@ -806,11 +806,11 @@ public static int readMessageLength(BytesReference networkBytes) throws IOExcept } } - public static BaseInboundMessage.Protocol determineTransportProtocol(BytesReference headerBuffer) { + public static String determineTransportProtocol(BytesReference headerBuffer) { if (headerBuffer.get(0) == 'O' && headerBuffer.get(1) == 'S' && headerBuffer.get(2) == 'P') { - return BaseInboundMessage.Protocol.PROTOBUF; + return BaseInboundMessage.PROTOBUF_PROTOCOL; } else { - return BaseInboundMessage.Protocol.DEFAULT; + return BaseInboundMessage.NATIVE_PROTOCOL; } } diff --git a/server/src/test/java/org/opensearch/transport/OutboundHandlerTests.java b/server/src/test/java/org/opensearch/transport/OutboundHandlerTests.java index 573a728cfae2d..f0d4785fe73da 100644 --- a/server/src/test/java/org/opensearch/transport/OutboundHandlerTests.java +++ b/server/src/test/java/org/opensearch/transport/OutboundHandlerTests.java @@ -293,7 +293,7 @@ public void testSendProtobufResponse() throws IOException { FetchSearchResult fetchResult = createFetchSearchResult(); QueryFetchSearchResult response = new QueryFetchSearchResult(queryResult, fetchResult); System.setProperty(FeatureFlags.PROTOBUF, "true"); - assertTrue(response.isMessageProtobuf()); + assertTrue((response.getProtocol()).equals(BaseInboundMessage.PROTOBUF_PROTOCOL)); AtomicLong requestIdRef = new AtomicLong(); AtomicReference actionRef = new AtomicReference<>(); From 64b79dcaafc8085bf55d821dfcb63ed5899d955d Mon Sep 17 00:00:00 2001 From: Vacha Shah Date: Fri, 1 Mar 2024 19:49:04 +0000 Subject: [PATCH 29/36] Fixing tests for message listener in inbound handler Signed-off-by: Vacha Shah --- .../test/java/org/opensearch/transport/InboundHandlerTests.java | 1 + .../test/java/org/opensearch/transport/OutboundHandlerTests.java | 1 + 2 files changed, 2 insertions(+) diff --git a/server/src/test/java/org/opensearch/transport/InboundHandlerTests.java b/server/src/test/java/org/opensearch/transport/InboundHandlerTests.java index 92ec27c23d23c..cfe009d21d944 100644 --- a/server/src/test/java/org/opensearch/transport/InboundHandlerTests.java +++ b/server/src/test/java/org/opensearch/transport/InboundHandlerTests.java @@ -338,6 +338,7 @@ public QueryFetchSearchResult read(InputStream in) throws IOException { QueryFetchSearchResult result = responseCaptor.get(); assertNotNull(result); assertEquals(queryResult.getMaxScore(), result.queryResult().getMaxScore(), 0.0); + System.setProperty(FeatureFlags.PROTOBUF, "false"); } public void testSendsErrorResponseToHandshakeFromCompatibleVersion() throws Exception { diff --git a/server/src/test/java/org/opensearch/transport/OutboundHandlerTests.java b/server/src/test/java/org/opensearch/transport/OutboundHandlerTests.java index f0d4785fe73da..4f52471bce938 100644 --- a/server/src/test/java/org/opensearch/transport/OutboundHandlerTests.java +++ b/server/src/test/java/org/opensearch/transport/OutboundHandlerTests.java @@ -337,6 +337,7 @@ public void onResponseSent(long requestId, String action, TransportResponse resp assertNotNull(message.getResponseHandlers()); assertNotNull(message.getMessage()); assertTrue(message.getMessage().hasQueryFetchSearchResult()); + System.setProperty(FeatureFlags.PROTOBUF, "false"); } public void testErrorResponse() throws IOException { From ff965e6b5be937c5cb3203e334b423b1077038ea Mon Sep 17 00:00:00 2001 From: Vacha Shah Date: Tue, 5 Mar 2024 02:21:57 +0000 Subject: [PATCH 30/36] Fixing mixed cluster tests Signed-off-by: Vacha Shah --- .../main/java/org/opensearch/transport/OutboundHandler.java | 2 +- .../java/org/opensearch/transport/OutboundHandlerTests.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/opensearch/transport/OutboundHandler.java b/server/src/main/java/org/opensearch/transport/OutboundHandler.java index 96dcfe2b2d327..a45a7e65116ba 100644 --- a/server/src/main/java/org/opensearch/transport/OutboundHandler.java +++ b/server/src/main/java/org/opensearch/transport/OutboundHandler.java @@ -148,7 +148,7 @@ void sendResponse( ) throws IOException { Version version = Version.min(this.version, nodeVersion); ActionListener listener = ActionListener.wrap(() -> messageListener.onResponseSent(requestId, action, response)); - if ((response.getProtocol()).equals(BaseInboundMessage.PROTOBUF_PROTOCOL)) { + if ((response.getProtocol()).equals(BaseInboundMessage.PROTOBUF_PROTOCOL) && version.onOrAfter(Version.V_3_0_0)) { QueryFetchSearchResult queryFetchSearchResult = (QueryFetchSearchResult) response; if (queryFetchSearchResult.response() != null) { byte[] bytes = new byte[1]; diff --git a/server/src/test/java/org/opensearch/transport/OutboundHandlerTests.java b/server/src/test/java/org/opensearch/transport/OutboundHandlerTests.java index 4f52471bce938..e03d550fb0266 100644 --- a/server/src/test/java/org/opensearch/transport/OutboundHandlerTests.java +++ b/server/src/test/java/org/opensearch/transport/OutboundHandlerTests.java @@ -120,7 +120,7 @@ public void setUp() throws Exception { } catch (IOException e) { throw new AssertionError(e); } - }); + }, Version.CURRENT); } @After @@ -316,7 +316,7 @@ public void onResponseSent(long requestId, String action, TransportResponse resp InboundPipeline inboundPipeline = new InboundPipeline(statsTracker, millisSupplier, decoder, aggregator, (c, m) -> { NodeToNodeMessage m1 = (NodeToNodeMessage) m; protobufMessage.set(BytesReference.fromByteBuffer(ByteBuffer.wrap(m1.getMessage().toByteArray()))); - }); + }, Version.CURRENT); BytesReference reference = channel.getMessageCaptor().get(); ActionListener sendListener = channel.getListenerCaptor().get(); if (randomBoolean()) { From de88e4655dbdab0d62fd384cb3b132b7ceb3d31a Mon Sep 17 00:00:00 2001 From: Vacha Shah Date: Wed, 13 Mar 2024 06:03:35 +0000 Subject: [PATCH 31/36] Addressing comments and fixing some proto messages Signed-off-by: Vacha Shah --- ...obufWriteable.java => BytesWriteable.java} | 2 +- .../core/transport/TransportMessage.java | 4 +- .../node/tasks/CancellableTasksIT.java | 7 - .../action/ActionListenerResponseHandler.java | 7 - ...ProtobufActionListenerResponseHandler.java | 8 +- ...TransportFieldCapabilitiesIndexAction.java | 7 - .../TransportResyncReplicationAction.java | 7 - .../opensearch/action/search/PitService.java | 7 - .../search/QueryPhaseResultConsumer.java | 11 +- .../action/search/SearchTransportService.java | 6 +- .../broadcast/TransportBroadcastAction.java | 7 - .../node/TransportBroadcastByNodeAction.java | 8 - .../support/nodes/TransportNodesAction.java | 7 - .../TransportReplicationAction.java | 7 - ...ransportInstanceSingleOperationAction.java | 7 - .../shard/TransportSingleShardAction.java | 13 -- .../support/tasks/TransportTasksAction.java | 9 - .../coordination/FollowersChecker.java | 7 - .../cluster/coordination/JoinHelper.java | 13 -- .../cluster/coordination/LeaderChecker.java | 7 - .../coordination/PreVoteCollector.java | 7 - .../PublicationTransportHandler.java | 14 -- .../decommission/DecommissionController.java | 7 - .../common/document/DocumentField.java | 116 ++++++++++-- .../org/opensearch/discovery/PeerFinder.java | 7 - .../extensions/ExtensionsManager.java | 7 - .../UpdateSettingsResponseHandler.java | 7 - .../ExtensionTransportActionsHandler.java | 13 -- .../rest/RestSendToExtensionAction.java | 7 - .../RetentionLeaseBackgroundSyncAction.java | 7 - .../index/seqno/RetentionLeaseSyncAction.java | 7 - .../recovery/PeerRecoveryTargetService.java | 7 - .../checkpoint/PublishCheckpointAction.java | 7 - .../indices/store/IndicesStore.java | 7 - .../java/org/opensearch/search/SearchHit.java | 10 ++ .../org/opensearch/search/SearchHits.java | 18 +- .../search/fetch/FetchSearchResult.java | 2 +- .../search/internal/ShardSearchRequest.java | 3 + .../search/query/QuerySearchResult.java | 75 ++++++-- .../snapshots/SnapshotShardsService.java | 7 - .../TraceableTransportResponseHandler.java | 7 - .../EmptyTransportResponseHandler.java | 10 -- .../opensearch/transport/InboundPipeline.java | 1 + .../transport/NodeToNodeMessage.java | 43 ++++- .../opensearch/transport/OutboundHandler.java | 48 +++-- .../transport/PlainTransportFuture.java | 7 - .../transport/ProtobufMessageHandler.java | 14 ++ .../transport/RemoteClusterConnection.java | 7 - .../transport/SniffConnectionStrategy.java | 7 - .../transport/TransportActionProxy.java | 7 - .../transport/TransportHandshaker.java | 7 - .../transport/TransportResponseHandler.java | 14 +- .../proto/server/NodeToNodeMessageProto.proto | 2 + .../search/FetchSearchResultProto.proto | 18 +- .../search/QuerySearchResultProto.proto | 1 + .../search/ShardSearchRequestProto.proto | 6 +- ...tAddVotingConfigExclusionsActionTests.java | 7 - ...learVotingConfigExclusionsActionTests.java | 7 - ...ReplicationAllPermitsAcquisitionTests.java | 8 - .../coordination/FollowersCheckerTests.java | 26 --- .../coordination/LeaderCheckerTests.java | 8 - .../coordination/PreVoteCollectorTests.java | 7 - .../opensearch/discovery/PeerFinderTests.java | 7 - .../SegmentReplicationSourceServiceTests.java | 19 -- .../transport/InboundHandlerTests.java | 33 +--- .../transport/OutboundHandlerTests.java | 3 +- .../transport/TransportActionProxyTests.java | 13 -- ...ortServiceDeserializationFailureTests.java | 15 -- .../AbstractSimpleTransportTestCase.java | 170 ------------------ .../DisruptableMockTransportTests.java | 19 -- .../test/disruption/NetworkDisruptionIT.java | 7 - 71 files changed, 342 insertions(+), 692 deletions(-) rename libs/core/src/main/java/org/opensearch/core/common/io/stream/{ProtobufWriteable.java => BytesWriteable.java} (97%) diff --git a/libs/core/src/main/java/org/opensearch/core/common/io/stream/ProtobufWriteable.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/BytesWriteable.java similarity index 97% rename from libs/core/src/main/java/org/opensearch/core/common/io/stream/ProtobufWriteable.java rename to libs/core/src/main/java/org/opensearch/core/common/io/stream/BytesWriteable.java index 7338c025921c3..001af4e672b1d 100644 --- a/libs/core/src/main/java/org/opensearch/core/common/io/stream/ProtobufWriteable.java +++ b/libs/core/src/main/java/org/opensearch/core/common/io/stream/BytesWriteable.java @@ -26,7 +26,7 @@ * @opensearch.api */ @ExperimentalApi -public interface ProtobufWriteable { +public interface BytesWriteable { /** * Write this into the {@linkplain OutputStream}. diff --git a/libs/core/src/main/java/org/opensearch/core/transport/TransportMessage.java b/libs/core/src/main/java/org/opensearch/core/transport/TransportMessage.java index 219326b44216c..a50fc51e8e964 100644 --- a/libs/core/src/main/java/org/opensearch/core/transport/TransportMessage.java +++ b/libs/core/src/main/java/org/opensearch/core/transport/TransportMessage.java @@ -32,7 +32,7 @@ package org.opensearch.core.transport; -import org.opensearch.core.common.io.stream.ProtobufWriteable; +import org.opensearch.core.common.io.stream.BytesWriteable; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.common.transport.TransportAddress; @@ -44,7 +44,7 @@ * * @opensearch.internal */ -public abstract class TransportMessage implements Writeable, ProtobufWriteable { +public abstract class TransportMessage implements Writeable, BytesWriteable { private TransportAddress remoteAddress; diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/node/tasks/CancellableTasksIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/node/tasks/CancellableTasksIT.java index de98f144aa99d..bdb36b62ada21 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/node/tasks/CancellableTasksIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/node/tasks/CancellableTasksIT.java @@ -76,7 +76,6 @@ import org.junit.Before; import java.io.IOException; -import java.io.InputStream; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -561,12 +560,6 @@ public String executor() { public TestResponse read(StreamInput in) throws IOException { return new TestResponse(in); } - - @Override - public TestResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } ); } diff --git a/server/src/main/java/org/opensearch/action/ActionListenerResponseHandler.java b/server/src/main/java/org/opensearch/action/ActionListenerResponseHandler.java index b58d67f826b34..86cbb8a6307be 100644 --- a/server/src/main/java/org/opensearch/action/ActionListenerResponseHandler.java +++ b/server/src/main/java/org/opensearch/action/ActionListenerResponseHandler.java @@ -41,7 +41,6 @@ import org.opensearch.transport.TransportResponseHandler; import java.io.IOException; -import java.io.InputStream; import java.util.Objects; /** @@ -90,10 +89,4 @@ public Response read(StreamInput in) throws IOException { public String toString() { return super.toString() + "/" + listener; } - - @Override - public Response read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } diff --git a/server/src/main/java/org/opensearch/action/ProtobufActionListenerResponseHandler.java b/server/src/main/java/org/opensearch/action/ProtobufActionListenerResponseHandler.java index 4dc67273f3eed..7e69bb9dc6cbd 100644 --- a/server/src/main/java/org/opensearch/action/ProtobufActionListenerResponseHandler.java +++ b/server/src/main/java/org/opensearch/action/ProtobufActionListenerResponseHandler.java @@ -9,7 +9,7 @@ package org.opensearch.action; import org.opensearch.core.action.ActionListener; -import org.opensearch.core.common.io.stream.ProtobufWriteable; +import org.opensearch.core.common.io.stream.BytesWriteable; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.transport.TransportResponse; import org.opensearch.threadpool.ThreadPool; @@ -29,12 +29,12 @@ public class ProtobufActionListenerResponseHandler implements TransportResponseHandler { private final ActionListener listener; - private final ProtobufWriteable.Reader reader; + private final BytesWriteable.Reader reader; private final String executor; public ProtobufActionListenerResponseHandler( ActionListener listener, - ProtobufWriteable.Reader reader, + BytesWriteable.Reader reader, String executor ) { this.listener = Objects.requireNonNull(listener); @@ -42,7 +42,7 @@ public ProtobufActionListenerResponseHandler( this.executor = Objects.requireNonNull(executor); } - public ProtobufActionListenerResponseHandler(ActionListener listener, ProtobufWriteable.Reader reader) { + public ProtobufActionListenerResponseHandler(ActionListener listener, BytesWriteable.Reader reader) { this(listener, reader, ThreadPool.Names.SAME); } diff --git a/server/src/main/java/org/opensearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java b/server/src/main/java/org/opensearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java index ad42f20fc3dfa..10bf4975311d6 100644 --- a/server/src/main/java/org/opensearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java +++ b/server/src/main/java/org/opensearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java @@ -76,7 +76,6 @@ import org.opensearch.transport.TransportService; import java.io.IOException; -import java.io.InputStream; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -340,12 +339,6 @@ public void handleResponse(final FieldCapabilitiesIndexResponse response) { public void handleException(TransportException exp) { onFailure(shardRouting, exp); } - - @Override - public FieldCapabilitiesIndexResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } ); } diff --git a/server/src/main/java/org/opensearch/action/resync/TransportResyncReplicationAction.java b/server/src/main/java/org/opensearch/action/resync/TransportResyncReplicationAction.java index 95b13616ab048..9d60706d1f100 100644 --- a/server/src/main/java/org/opensearch/action/resync/TransportResyncReplicationAction.java +++ b/server/src/main/java/org/opensearch/action/resync/TransportResyncReplicationAction.java @@ -62,7 +62,6 @@ import org.opensearch.transport.TransportService; import java.io.IOException; -import java.io.InputStream; import java.util.function.Function; import java.util.stream.Stream; @@ -254,12 +253,6 @@ public void handleResponse(ResyncReplicationResponse response) { public void handleException(TransportException exp) { listener.onFailure(exp); } - - @Override - public ResyncReplicationResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } ); } diff --git a/server/src/main/java/org/opensearch/action/search/PitService.java b/server/src/main/java/org/opensearch/action/search/PitService.java index c1bf5ffbd4a45..b6480ce63f827 100644 --- a/server/src/main/java/org/opensearch/action/search/PitService.java +++ b/server/src/main/java/org/opensearch/action/search/PitService.java @@ -27,7 +27,6 @@ import org.opensearch.transport.TransportService; import java.io.IOException; -import java.io.InputStream; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -203,12 +202,6 @@ public String executor() { public GetAllPitNodesResponse read(StreamInput in) throws IOException { return new GetAllPitNodesResponse(in); } - - @Override - public GetAllPitNodesResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } ); } diff --git a/server/src/main/java/org/opensearch/action/search/QueryPhaseResultConsumer.java b/server/src/main/java/org/opensearch/action/search/QueryPhaseResultConsumer.java index f1b06378bd579..3081e95dc5e26 100644 --- a/server/src/main/java/org/opensearch/action/search/QueryPhaseResultConsumer.java +++ b/server/src/main/java/org/opensearch/action/search/QueryPhaseResultConsumer.java @@ -38,6 +38,7 @@ import org.opensearch.common.lease.Releasable; import org.opensearch.common.lease.Releasables; import org.opensearch.common.lucene.search.TopDocsAndMaxScore; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.util.concurrent.AbstractRunnable; import org.opensearch.core.common.breaker.CircuitBreaker; import org.opensearch.core.common.breaker.CircuitBreakingException; @@ -114,7 +115,11 @@ public QueryPhaseResultConsumer( SearchSourceBuilder source = request.source(); this.hasTopDocs = source == null || source.size() != 0; - this.hasAggs = source != null && source.aggregations() != null; + if (FeatureFlags.isEnabled(FeatureFlags.PROTOBUF)) { + this.hasAggs = false; + } else { + this.hasAggs = source != null && source.aggregations() != null; + } int batchReduceSize = (hasAggs || hasTopDocs) ? Math.min(request.getBatchedReduceSize(), expectedResultSize) : expectedResultSize; this.pendingMerges = new PendingMerges(batchReduceSize, request.resolveTrackTotalHitsUpTo()); } @@ -320,7 +325,7 @@ synchronized long addEstimateAndMaybeBreak(long estimatedSize) { * provided {@link QuerySearchResult}. */ long ramBytesUsedQueryResult(QuerySearchResult result) { - if (hasAggs == false) { + if (hasAggs == false || FeatureFlags.isEnabled(FeatureFlags.PROTOBUF)) { return 0; } return result.aggregations().asSerialized(InternalAggregations::readFrom, namedWriteableRegistry).ramBytesUsed(); @@ -489,7 +494,7 @@ public synchronized List consumeTopDocs() { } public synchronized List consumeAggs() { - if (hasAggs == false) { + if (hasAggs == false || FeatureFlags.isEnabled(FeatureFlags.PROTOBUF)) { return Collections.emptyList(); } List aggsList = new ArrayList<>(); diff --git a/server/src/main/java/org/opensearch/action/search/SearchTransportService.java b/server/src/main/java/org/opensearch/action/search/SearchTransportService.java index a71d8c2a64f4e..ce244883d457c 100644 --- a/server/src/main/java/org/opensearch/action/search/SearchTransportService.java +++ b/server/src/main/java/org/opensearch/action/search/SearchTransportService.java @@ -43,7 +43,7 @@ import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.util.concurrent.ConcurrentCollections; import org.opensearch.core.action.ActionListener; -import org.opensearch.core.common.io.stream.ProtobufWriteable; +import org.opensearch.core.common.io.stream.BytesWriteable; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; @@ -248,7 +248,7 @@ public void sendExecuteQuery( final ActionListener handler = responseWrapper.apply(connection, listener); TransportResponseHandler transportResponseHandler; if (FeatureFlags.isEnabled(FeatureFlags.PROTOBUF_SETTING)) { - ProtobufWriteable.Reader reader = fetchDocuments ? QueryFetchSearchResult::new : QuerySearchResult::new; + BytesWriteable.Reader reader = fetchDocuments ? QueryFetchSearchResult::new : QuerySearchResult::new; transportResponseHandler = new ProtobufConnectionCountingHandler<>( handler, reader, @@ -797,7 +797,7 @@ final class ProtobufConnectionCountingHandler listener, - final ProtobufWriteable.Reader responseReader, + final BytesWriteable.Reader responseReader, final Map clientConnections, final String nodeId ) { diff --git a/server/src/main/java/org/opensearch/action/support/broadcast/TransportBroadcastAction.java b/server/src/main/java/org/opensearch/action/support/broadcast/TransportBroadcastAction.java index b41384854c122..8bf8555194976 100644 --- a/server/src/main/java/org/opensearch/action/support/broadcast/TransportBroadcastAction.java +++ b/server/src/main/java/org/opensearch/action/support/broadcast/TransportBroadcastAction.java @@ -61,7 +61,6 @@ import org.opensearch.transport.TransportService; import java.io.IOException; -import java.io.InputStream; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReferenceArray; @@ -230,12 +229,6 @@ public void handleResponse(ShardResponse response) { public void handleException(TransportException e) { onOperation(shard, shardIt, shardIndex, e); } - - @Override - public ShardResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } ); } diff --git a/server/src/main/java/org/opensearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java b/server/src/main/java/org/opensearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java index 89e18773e1079..c08cfb7af0e3d 100644 --- a/server/src/main/java/org/opensearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java +++ b/server/src/main/java/org/opensearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java @@ -69,7 +69,6 @@ import org.opensearch.transport.TransportService; import java.io.IOException; -import java.io.InputStream; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -393,13 +392,6 @@ public void handleException(TransportException exp) { public String executor() { return ThreadPool.Names.SAME; } - - @Override - public TransportBroadcastByNodeAction.NodeResponse read(InputStream in) - throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } ); } catch (Exception e) { diff --git a/server/src/main/java/org/opensearch/action/support/nodes/TransportNodesAction.java b/server/src/main/java/org/opensearch/action/support/nodes/TransportNodesAction.java index bb7b848e2b69d..9a1a28dd70636 100644 --- a/server/src/main/java/org/opensearch/action/support/nodes/TransportNodesAction.java +++ b/server/src/main/java/org/opensearch/action/support/nodes/TransportNodesAction.java @@ -55,7 +55,6 @@ import org.opensearch.transport.TransportService; import java.io.IOException; -import java.io.InputStream; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -287,12 +286,6 @@ public void handleException(TransportException exp) { public String executor() { return ThreadPool.Names.SAME; } - - @Override - public NodeResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } ); } catch (Exception e) { diff --git a/server/src/main/java/org/opensearch/action/support/replication/TransportReplicationAction.java b/server/src/main/java/org/opensearch/action/support/replication/TransportReplicationAction.java index ec88b60b36f1a..49a96603f6802 100644 --- a/server/src/main/java/org/opensearch/action/support/replication/TransportReplicationAction.java +++ b/server/src/main/java/org/opensearch/action/support/replication/TransportReplicationAction.java @@ -94,7 +94,6 @@ import org.opensearch.transport.TransportService; import java.io.IOException; -import java.io.InputStream; import java.util.Map; import java.util.Objects; import java.util.concurrent.atomic.AtomicBoolean; @@ -1176,12 +1175,6 @@ public void handleException(TransportException exp) { finishWithUnexpectedFailure(e); } } - - @Override - public Response read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } }); } diff --git a/server/src/main/java/org/opensearch/action/support/single/instance/TransportInstanceSingleOperationAction.java b/server/src/main/java/org/opensearch/action/support/single/instance/TransportInstanceSingleOperationAction.java index ee642f96aedd4..21d4ba726e86f 100644 --- a/server/src/main/java/org/opensearch/action/support/single/instance/TransportInstanceSingleOperationAction.java +++ b/server/src/main/java/org/opensearch/action/support/single/instance/TransportInstanceSingleOperationAction.java @@ -66,7 +66,6 @@ import org.opensearch.transport.TransportService; import java.io.IOException; -import java.io.InputStream; import static org.opensearch.cluster.metadata.IndexNameExpressionResolver.EXCLUDED_DATA_STREAMS_KEY; @@ -246,12 +245,6 @@ public void handleException(TransportException exp) { listener.onFailure(exp); } } - - @Override - public Response read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } }); } diff --git a/server/src/main/java/org/opensearch/action/support/single/shard/TransportSingleShardAction.java b/server/src/main/java/org/opensearch/action/support/single/shard/TransportSingleShardAction.java index 0a5fe78c5dbf9..df91559a2f8cb 100644 --- a/server/src/main/java/org/opensearch/action/support/single/shard/TransportSingleShardAction.java +++ b/server/src/main/java/org/opensearch/action/support/single/shard/TransportSingleShardAction.java @@ -65,7 +65,6 @@ import org.opensearch.transport.TransportService; import java.io.IOException; -import java.io.InputStream; import static org.opensearch.action.support.TransportActions.isShardNotAvailableException; @@ -226,12 +225,6 @@ public void handleResponse(final Response response) { public void handleException(TransportException exp) { listener.onFailure(exp); } - - @Override - public Response read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } ); } else { @@ -309,12 +302,6 @@ public void handleResponse(final Response response) { public void handleException(TransportException exp) { onFailure(finalShardRouting, exp); } - - @Override - public Response read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } ); } diff --git a/server/src/main/java/org/opensearch/action/support/tasks/TransportTasksAction.java b/server/src/main/java/org/opensearch/action/support/tasks/TransportTasksAction.java index 338750506c735..f33d7161660a3 100644 --- a/server/src/main/java/org/opensearch/action/support/tasks/TransportTasksAction.java +++ b/server/src/main/java/org/opensearch/action/support/tasks/TransportTasksAction.java @@ -62,7 +62,6 @@ import org.opensearch.transport.TransportService; import java.io.IOException; -import java.io.InputStream; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -317,14 +316,6 @@ public void handleException(TransportException exp) { public String executor() { return ThreadPool.Names.SAME; } - - @Override - public - TransportTasksAction.NodeTasksResponse - read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } ); } diff --git a/server/src/main/java/org/opensearch/cluster/coordination/FollowersChecker.java b/server/src/main/java/org/opensearch/cluster/coordination/FollowersChecker.java index 5118173ea301b..70bb0515bb022 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/FollowersChecker.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/FollowersChecker.java @@ -61,7 +61,6 @@ import org.opensearch.transport.TransportService; import java.io.IOException; -import java.io.InputStream; import java.util.HashSet; import java.util.Map; import java.util.Objects; @@ -409,12 +408,6 @@ public void handleException(TransportException exp) { public String executor() { return Names.SAME; } - - @Override - public Empty read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } ); } diff --git a/server/src/main/java/org/opensearch/cluster/coordination/JoinHelper.java b/server/src/main/java/org/opensearch/cluster/coordination/JoinHelper.java index 3d675335ae867..9bf6bac07da53 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/JoinHelper.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/JoinHelper.java @@ -74,7 +74,6 @@ import org.opensearch.transport.TransportService; import java.io.IOException; -import java.io.InputStream; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -422,12 +421,6 @@ public void handleException(TransportException exp) { public String executor() { return Names.SAME; } - - @Override - public Empty read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } ); } else { @@ -458,12 +451,6 @@ public void handleException(TransportException exp) { public String executor() { return ThreadPool.Names.SAME; } - - @Override - public Empty read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } }); } diff --git a/server/src/main/java/org/opensearch/cluster/coordination/LeaderChecker.java b/server/src/main/java/org/opensearch/cluster/coordination/LeaderChecker.java index 4aad1fa3fc4a9..8d4373b865f62 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/LeaderChecker.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/LeaderChecker.java @@ -63,7 +63,6 @@ import org.opensearch.transport.TransportService; import java.io.IOException; -import java.io.InputStream; import java.util.Objects; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; @@ -350,12 +349,6 @@ public void handleException(TransportException exp) { public String executor() { return Names.SAME; } - - @Override - public Empty read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } ); } diff --git a/server/src/main/java/org/opensearch/cluster/coordination/PreVoteCollector.java b/server/src/main/java/org/opensearch/cluster/coordination/PreVoteCollector.java index 2b2f14db31ed9..cc4d1ac156c53 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/PreVoteCollector.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/PreVoteCollector.java @@ -50,7 +50,6 @@ import org.opensearch.transport.TransportService; import java.io.IOException; -import java.io.InputStream; import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.LongConsumer; @@ -218,12 +217,6 @@ public String executor() { public String toString() { return "TransportResponseHandler{" + PreVoteCollector.this + ", node=" + n + '}'; } - - @Override - public PreVoteResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } ) ); diff --git a/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java b/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java index 50db5d1ba8342..1fdaeead0d28d 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java @@ -47,7 +47,6 @@ import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.transport.TransportResponse; -import org.opensearch.core.transport.TransportResponse.Empty; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.BytesTransportRequest; import org.opensearch.transport.TransportChannel; @@ -57,7 +56,6 @@ import org.opensearch.transport.TransportService; import java.io.IOException; -import java.io.InputStream; import java.util.HashMap; import java.util.Map; import java.util.concurrent.atomic.AtomicLong; @@ -382,12 +380,6 @@ public void handleException(TransportException exp) { public String executor() { return ThreadPool.Names.GENERIC; } - - @Override - public Empty read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } ); } @@ -459,12 +451,6 @@ public void handleException(TransportException exp) { public String executor() { return ThreadPool.Names.GENERIC; } - - @Override - public PublishWithJoinResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } }; transportService.sendRequest(destination, PUBLISH_STATE_ACTION_NAME, request, stateRequestOptions, responseHandler); } catch (Exception e) { diff --git a/server/src/main/java/org/opensearch/cluster/decommission/DecommissionController.java b/server/src/main/java/org/opensearch/cluster/decommission/DecommissionController.java index 722a836c2dc21..fec313b4b0b73 100644 --- a/server/src/main/java/org/opensearch/cluster/decommission/DecommissionController.java +++ b/server/src/main/java/org/opensearch/cluster/decommission/DecommissionController.java @@ -36,7 +36,6 @@ import org.opensearch.transport.TransportService; import java.io.IOException; -import java.io.InputStream; import java.util.Arrays; import java.util.HashMap; import java.util.LinkedHashMap; @@ -258,12 +257,6 @@ public String executor() { public NodesStatsResponse read(StreamInput in) throws IOException { return new NodesStatsResponse(in); } - - @Override - public NodesStatsResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } ); } diff --git a/server/src/main/java/org/opensearch/common/document/DocumentField.java b/server/src/main/java/org/opensearch/common/document/DocumentField.java index a3153c296d0c4..71b0c4488dd82 100644 --- a/server/src/main/java/org/opensearch/common/document/DocumentField.java +++ b/server/src/main/java/org/opensearch/common/document/DocumentField.java @@ -33,27 +33,33 @@ package org.opensearch.common.document; import com.google.protobuf.ByteString; -import org.apache.lucene.util.SuppressForbidden; import org.opensearch.OpenSearchException; import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.common.text.Text; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.get.GetResult; import org.opensearch.search.SearchHit; import org.opensearch.server.proto.FetchSearchResultProto; +import org.opensearch.server.proto.FetchSearchResultProto.DocumentFieldValue; +import org.opensearch.server.proto.FetchSearchResultProto.DocumentFieldValue.Builder; -import java.io.ByteArrayInputStream; import java.io.IOException; -import java.io.InputStream; -import java.io.ObjectInputStream; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZonedDateTime; import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.Objects; import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; @@ -84,19 +90,107 @@ public DocumentField(String name, List values) { this.values = Objects.requireNonNull(values, "values must not be null"); } - @SuppressForbidden(reason = "We need to read from a byte array") public DocumentField(byte[] in) throws IOException { assert FeatureFlags.isEnabled(FeatureFlags.PROTOBUF) : "protobuf feature flag is not enabled"; documentField = FetchSearchResultProto.SearchHit.DocumentField.parseFrom(in); name = documentField.getName(); values = new ArrayList<>(); - for (ByteString value : documentField.getValuesList()) { - InputStream is = new ByteArrayInputStream(value.toByteArray()); - try (ObjectInputStream ois = new ObjectInputStream(is)) { - values.add(ois.readObject()); - } catch (ClassNotFoundException e) { - throw new OpenSearchException(e); + for (FetchSearchResultProto.DocumentFieldValue value : documentField.getValuesList()) { + values.add(readDocumentFieldValueFromProtobuf(value)); + } + } + + public static FetchSearchResultProto.SearchHit.DocumentField convertDocumentFieldToProto(DocumentField documentField) { + FetchSearchResultProto.SearchHit.DocumentField.Builder builder = FetchSearchResultProto.SearchHit.DocumentField.newBuilder(); + builder.setName(documentField.getName()); + for (Object value : documentField.getValues()) { + FetchSearchResultProto.DocumentFieldValue.Builder valueBuilder = FetchSearchResultProto.DocumentFieldValue.newBuilder(); + builder.addValues(convertDocumentFieldValueToProto(value, valueBuilder)); + } + return builder.build(); + } + + private static DocumentFieldValue.Builder convertDocumentFieldValueToProto(Object value, Builder valueBuilder) { + if (value == null) { + // null is not allowed in protobuf, so we use a special string to represent null + return valueBuilder.setValueString("null"); + } + Class type = value.getClass(); + if (type == String.class) { + valueBuilder.setValueString((String) value); + } else if (type == Integer.class) { + valueBuilder.setValueInt((Integer) value); + } else if (type == Long.class) { + valueBuilder.setValueLong((Long) value); + } else if (type == Float.class) { + valueBuilder.setValueFloat((Float) value); + } else if (type == Double.class) { + valueBuilder.setValueDouble((Double) value); + } else if (type == Boolean.class) { + valueBuilder.setValueBool((Boolean) value); + } else if (type == byte[].class) { + valueBuilder.addValueByteArray(ByteString.copyFrom((byte[]) value)); + } else if (type == List.class) { + List list = (List) value; + for (Object listValue : list) { + valueBuilder.addValueArrayList(convertDocumentFieldValueToProto(listValue, valueBuilder)); + } + } else if (type == Map.class || type == HashMap.class || type == LinkedHashMap.class) { + Map map = (Map) value; + for (Map.Entry entry : map.entrySet()) { + valueBuilder.putValueMap(entry.getKey(), convertDocumentFieldValueToProto(entry.getValue(), valueBuilder).build()); + } + } else if (type == Date.class) { + valueBuilder.setValueDate(((Date) value).getTime()); + } else if (type == ZonedDateTime.class) { + valueBuilder.setValueZonedDate(((ZonedDateTime) value).getZone().getId()); + valueBuilder.setValueZonedTime(((ZonedDateTime) value).toInstant().toEpochMilli()); + } else if (type == Text.class) { + valueBuilder.setValueText(((Text) value).string()); + } else { + throw new OpenSearchException("Can't convert generic value of type [" + type + "] to protobuf"); + } + return valueBuilder; + } + + private Object readDocumentFieldValueFromProtobuf(FetchSearchResultProto.DocumentFieldValue documentFieldValue) throws IOException { + if (documentFieldValue.hasValueString()) { + return documentFieldValue.getValueString(); + } else if (documentFieldValue.hasValueInt()) { + return documentFieldValue.getValueInt(); + } else if (documentFieldValue.hasValueLong()) { + return documentFieldValue.getValueLong(); + } else if (documentFieldValue.hasValueFloat()) { + return documentFieldValue.getValueFloat(); + } else if (documentFieldValue.hasValueDouble()) { + return documentFieldValue.getValueDouble(); + } else if (documentFieldValue.hasValueBool()) { + return documentFieldValue.getValueBool(); + } else if (documentFieldValue.getValueByteArrayList().size() > 0) { + return documentFieldValue.getValueByteArrayList().toArray(); + } else if (documentFieldValue.getValueArrayListList().size() > 0) { + List list = new ArrayList<>(); + for (FetchSearchResultProto.DocumentFieldValue value : documentFieldValue.getValueArrayListList()) { + list.add(readDocumentFieldValueFromProtobuf(value)); + } + return list; + } else if (documentFieldValue.getValueMapMap().size() > 0) { + Map map = Map.of(); + for (Map.Entry entrySet : documentFieldValue.getValueMapMap().entrySet()) { + map.put(entrySet.getKey(), readDocumentFieldValueFromProtobuf(entrySet.getValue())); } + return map; + } else if (documentFieldValue.hasValueDate()) { + return new Date(documentFieldValue.getValueDate()); + } else if (documentFieldValue.hasValueZonedDate() && documentFieldValue.hasValueZonedTime()) { + return ZonedDateTime.ofInstant( + Instant.ofEpochMilli(documentFieldValue.getValueZonedTime()), + ZoneId.of(documentFieldValue.getValueZonedDate()) + ); + } else if (documentFieldValue.hasValueText()) { + return new Text(documentFieldValue.getValueText()); + } else { + throw new IOException("Can't read generic value of type [" + documentFieldValue + "]"); } } diff --git a/server/src/main/java/org/opensearch/discovery/PeerFinder.java b/server/src/main/java/org/opensearch/discovery/PeerFinder.java index a233409232c42..1d997c8cbabe8 100644 --- a/server/src/main/java/org/opensearch/discovery/PeerFinder.java +++ b/server/src/main/java/org/opensearch/discovery/PeerFinder.java @@ -54,7 +54,6 @@ import org.opensearch.transport.TransportService; import java.io.IOException; -import java.io.InputStream; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -505,12 +504,6 @@ public void handleException(TransportException exp) { public String executor() { return Names.GENERIC; } - - @Override - public PeersResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } }; transportService.sendRequest( discoveryNode, diff --git a/server/src/main/java/org/opensearch/extensions/ExtensionsManager.java b/server/src/main/java/org/opensearch/extensions/ExtensionsManager.java index 1ac274524be28..b531abcb845d7 100644 --- a/server/src/main/java/org/opensearch/extensions/ExtensionsManager.java +++ b/server/src/main/java/org/opensearch/extensions/ExtensionsManager.java @@ -49,7 +49,6 @@ import org.opensearch.transport.TransportService; import java.io.IOException; -import java.io.InputStream; import java.net.InetAddress; import java.util.HashMap; import java.util.HashSet; @@ -385,12 +384,6 @@ public void handleException(TransportException exp) { public String executor() { return ThreadPool.Names.GENERIC; } - - @Override - public InitializeExtensionResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } }; logger.info("Sending extension request type: " + REQUEST_EXTENSION_ACTION_NAME); diff --git a/server/src/main/java/org/opensearch/extensions/UpdateSettingsResponseHandler.java b/server/src/main/java/org/opensearch/extensions/UpdateSettingsResponseHandler.java index d824b35845cad..8c7c3c4cb9bd9 100644 --- a/server/src/main/java/org/opensearch/extensions/UpdateSettingsResponseHandler.java +++ b/server/src/main/java/org/opensearch/extensions/UpdateSettingsResponseHandler.java @@ -17,7 +17,6 @@ import org.opensearch.transport.TransportResponseHandler; import java.io.IOException; -import java.io.InputStream; /** * Response handler for {@link UpdateSettingsRequest} @@ -49,10 +48,4 @@ public void handleException(TransportException exp) { public String executor() { return ThreadPool.Names.GENERIC; } - - @Override - public AcknowledgedResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } diff --git a/server/src/main/java/org/opensearch/extensions/action/ExtensionTransportActionsHandler.java b/server/src/main/java/org/opensearch/extensions/action/ExtensionTransportActionsHandler.java index 699b99d442b2c..ac60df1b73764 100644 --- a/server/src/main/java/org/opensearch/extensions/action/ExtensionTransportActionsHandler.java +++ b/server/src/main/java/org/opensearch/extensions/action/ExtensionTransportActionsHandler.java @@ -27,7 +27,6 @@ import org.opensearch.transport.TransportService; import java.io.IOException; -import java.io.InputStream; import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; @@ -220,12 +219,6 @@ public void handleException(TransportException exp) { public String executor() { return ThreadPool.Names.GENERIC; } - - @Override - public ExtensionActionResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } }; try { transportService.sendRequest( @@ -290,12 +283,6 @@ public void handleException(TransportException exp) { public String executor() { return ThreadPool.Names.GENERIC; } - - @Override - public RemoteExtensionActionResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } }; try { transportService.sendRequest( diff --git a/server/src/main/java/org/opensearch/extensions/rest/RestSendToExtensionAction.java b/server/src/main/java/org/opensearch/extensions/rest/RestSendToExtensionAction.java index 63db8ad04eea4..f4503ce55e6bc 100644 --- a/server/src/main/java/org/opensearch/extensions/rest/RestSendToExtensionAction.java +++ b/server/src/main/java/org/opensearch/extensions/rest/RestSendToExtensionAction.java @@ -35,7 +35,6 @@ import org.opensearch.transport.TransportService; import java.io.IOException; -import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashSet; @@ -241,12 +240,6 @@ public void handleException(TransportException exp) { public String executor() { return ThreadPool.Names.GENERIC; } - - @Override - public RestExecuteOnExtensionResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } }; try { diff --git a/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseBackgroundSyncAction.java b/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseBackgroundSyncAction.java index 1e6732407d98c..5fa0a1a6459e7 100644 --- a/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseBackgroundSyncAction.java +++ b/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseBackgroundSyncAction.java @@ -65,7 +65,6 @@ import org.opensearch.transport.TransportService; import java.io.IOException; -import java.io.InputStream; import java.util.Map; import java.util.Objects; @@ -168,12 +167,6 @@ public void handleException(TransportException e) { } getLogger().warn(new ParameterizedMessage("{} retention lease background sync failed", shardId), e); } - - @Override - public ReplicationResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } ); } diff --git a/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseSyncAction.java b/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseSyncAction.java index 1d89e5bc47604..ca3c7e1d49700 100644 --- a/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseSyncAction.java +++ b/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseSyncAction.java @@ -69,7 +69,6 @@ import org.opensearch.transport.TransportService; import java.io.IOException; -import java.io.InputStream; import java.util.Map; import java.util.Objects; @@ -179,12 +178,6 @@ public void handleException(TransportException e) { taskManager.unregister(task); listener.onFailure(e); } - - @Override - public ReplicationResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } ); } diff --git a/server/src/main/java/org/opensearch/indices/recovery/PeerRecoveryTargetService.java b/server/src/main/java/org/opensearch/indices/recovery/PeerRecoveryTargetService.java index 5a9b9c88318fa..c24840d0c1333 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/PeerRecoveryTargetService.java +++ b/server/src/main/java/org/opensearch/indices/recovery/PeerRecoveryTargetService.java @@ -80,7 +80,6 @@ import org.opensearch.transport.TransportService; import java.io.IOException; -import java.io.InputStream; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Consumer; @@ -763,11 +762,5 @@ public String executor() { public RecoveryResponse read(StreamInput in) throws IOException { return new RecoveryResponse(in); } - - @Override - public RecoveryResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } } diff --git a/server/src/main/java/org/opensearch/indices/replication/checkpoint/PublishCheckpointAction.java b/server/src/main/java/org/opensearch/indices/replication/checkpoint/PublishCheckpointAction.java index 6b32155be072b..8f39aa194b06c 100644 --- a/server/src/main/java/org/opensearch/indices/replication/checkpoint/PublishCheckpointAction.java +++ b/server/src/main/java/org/opensearch/indices/replication/checkpoint/PublishCheckpointAction.java @@ -41,7 +41,6 @@ import org.opensearch.transport.TransportService; import java.io.IOException; -import java.io.InputStream; import java.util.Objects; /** @@ -173,12 +172,6 @@ public void handleException(TransportException e) { e ); } - - @Override - public ReplicationResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } ); logger.trace( diff --git a/server/src/main/java/org/opensearch/indices/store/IndicesStore.java b/server/src/main/java/org/opensearch/indices/store/IndicesStore.java index 33e825cda34d1..1efaca09204da 100644 --- a/server/src/main/java/org/opensearch/indices/store/IndicesStore.java +++ b/server/src/main/java/org/opensearch/indices/store/IndicesStore.java @@ -75,7 +75,6 @@ import java.io.Closeable; import java.io.IOException; -import java.io.InputStream; import java.util.ArrayList; import java.util.EnumSet; import java.util.HashSet; @@ -353,12 +352,6 @@ private void allNodesResponded() { ); } - @Override - public ShardActiveResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } - } private class ShardActiveRequestHandler implements TransportRequestHandler { diff --git a/server/src/main/java/org/opensearch/search/SearchHit.java b/server/src/main/java/org/opensearch/search/SearchHit.java index c0c21be704ec3..1e43e60611eb9 100644 --- a/server/src/main/java/org/opensearch/search/SearchHit.java +++ b/server/src/main/java/org/opensearch/search/SearchHit.java @@ -171,6 +171,9 @@ public SearchHit( this.nestedIdentity = nestedIdentity; this.documentFields = documentFields == null ? emptyMap() : documentFields; this.metaFields = metaFields == null ? emptyMap() : metaFields; + if (FeatureFlags.isEnabled(FeatureFlags.PROTOBUF)) { + this.searchHitProto = convertHitToProto(this); + } } public SearchHit(StreamInput in) throws IOException { @@ -341,9 +344,13 @@ public static FetchSearchResultProto.SearchHit convertHitToProto(SearchHit hit) searchHitBuilder.setSeqNo(hit.getSeqNo()); searchHitBuilder.setPrimaryTerm(hit.getPrimaryTerm()); searchHitBuilder.setVersion(hit.getVersion()); + searchHitBuilder.setDocId(hit.docId); if (hit.getSourceRef() != null) { searchHitBuilder.setSource(ByteString.copyFrom(hit.getSourceRef().toBytesRef().bytes)); } + for (Map.Entry entry : hit.getFields().entrySet()) { + searchHitBuilder.putDocumentFields(entry.getKey(), DocumentField.convertDocumentFieldToProto(entry.getValue())); + } return searchHitBuilder.build(); } @@ -563,6 +570,9 @@ public void setDocumentField(String fieldName, DocumentField field) { if (fieldName == null || field == null) return; if (documentFields.isEmpty()) this.documentFields = new HashMap<>(); this.documentFields.put(fieldName, field); + if (FeatureFlags.isEnabled(FeatureFlags.PROTOBUF)) { + this.searchHitProto = convertHitToProto(this); + } } public DocumentField removeDocumentField(String fieldName) { diff --git a/server/src/main/java/org/opensearch/search/SearchHits.java b/server/src/main/java/org/opensearch/search/SearchHits.java index 417e85399e3f1..630a7c90f8b3d 100644 --- a/server/src/main/java/org/opensearch/search/SearchHits.java +++ b/server/src/main/java/org/opensearch/search/SearchHits.java @@ -122,12 +122,14 @@ public static FetchSearchResultProto.SearchHits convertHitsToProto(SearchHits hi searchHitList.add(SearchHit.convertHitToProto(hit)); } QuerySearchResultProto.TotalHits.Builder totalHitsBuilder = QuerySearchResultProto.TotalHits.newBuilder(); - totalHitsBuilder.setValue(hits.getTotalHits().value); - totalHitsBuilder.setRelation( - hits.getTotalHits().relation == Relation.EQUAL_TO - ? QuerySearchResultProto.TotalHits.Relation.EQUAL_TO - : QuerySearchResultProto.TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO - ); + if (hits.getTotalHits() != null) { + totalHitsBuilder.setValue(hits.getTotalHits().value); + totalHitsBuilder.setRelation( + hits.getTotalHits().relation == Relation.EQUAL_TO + ? QuerySearchResultProto.TotalHits.Relation.EQUAL_TO + : QuerySearchResultProto.TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO + ); + } FetchSearchResultProto.SearchHits.Builder searchHitsBuilder = FetchSearchResultProto.SearchHits.newBuilder(); searchHitsBuilder.setMaxScore(hits.getMaxScore()); searchHitsBuilder.addAllHits(searchHitList); @@ -135,7 +137,9 @@ public static FetchSearchResultProto.SearchHits convertHitsToProto(SearchHits hi if (hits.getSortFields() != null && hits.getSortFields().length > 0) { for (SortField sortField : hits.getSortFields()) { FetchSearchResultProto.SortField.Builder sortFieldBuilder = FetchSearchResultProto.SortField.newBuilder(); - sortFieldBuilder.setField(sortField.getField()); + if (sortField.getField() != null) { + sortFieldBuilder.setField(sortField.getField()); + } sortFieldBuilder.setType(FetchSearchResultProto.SortField.Type.valueOf(sortField.getType().name())); searchHitsBuilder.addSortFields(sortFieldBuilder.build()); } diff --git a/server/src/main/java/org/opensearch/search/fetch/FetchSearchResult.java b/server/src/main/java/org/opensearch/search/fetch/FetchSearchResult.java index f966f0a0fdccc..a9c70d336225d 100644 --- a/server/src/main/java/org/opensearch/search/fetch/FetchSearchResult.java +++ b/server/src/main/java/org/opensearch/search/fetch/FetchSearchResult.java @@ -118,7 +118,7 @@ private boolean assertNoSearchTarget(SearchHits hits) { } public SearchHits hits() { - if (FeatureFlags.isEnabled(FeatureFlags.PROTOBUF_SETTING)) { + if (FeatureFlags.isEnabled(FeatureFlags.PROTOBUF_SETTING) && this.fetchSearchResultProto != null) { SearchHits hits; try { hits = new SearchHits(this.fetchSearchResultProto.getHits().toByteArray()); diff --git a/server/src/main/java/org/opensearch/search/internal/ShardSearchRequest.java b/server/src/main/java/org/opensearch/search/internal/ShardSearchRequest.java index 9ab84dda119f2..dde9f7130afa5 100644 --- a/server/src/main/java/org/opensearch/search/internal/ShardSearchRequest.java +++ b/server/src/main/java/org/opensearch/search/internal/ShardSearchRequest.java @@ -315,6 +315,9 @@ public ShardSearchRequest(byte[] in) throws IOException { this.keepAlive = searchRequestProto.hasTimeValue() ? TimeValue.parseTimeValue(searchRequestProto.getTimeValue(), "keepAlive") : null; + this.aliasFilter = searchRequestProto.hasAliasFilter() + ? new AliasFilter(null, searchRequestProto.getAliasFilter().getAliasesList().toArray(Strings.EMPTY_ARRAY)) + : AliasFilter.EMPTY; } public ShardSearchRequest(ShardSearchRequest clone) { diff --git a/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java b/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java index 822628e960c3c..7ae21e6667caf 100644 --- a/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java +++ b/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java @@ -58,6 +58,9 @@ import org.opensearch.search.suggest.Suggest; import org.opensearch.server.proto.QuerySearchResultProto; import org.opensearch.server.proto.ShardSearchRequestProto; +import org.opensearch.server.proto.ShardSearchRequestProto.AliasFilter; +import org.opensearch.server.proto.ShardSearchRequestProto.ShardSearchRequest.SearchType; +import org.opensearch.transport.BaseInboundMessage; import java.io.IOException; import java.io.InputStream; @@ -123,13 +126,20 @@ public QuerySearchResult(InputStream in) throws IOException { super(in); assert FeatureFlags.isEnabled(FeatureFlags.PROTOBUF) : "protobuf feature flag is not enabled"; this.querySearchResultProto = QuerySearchResultProto.QuerySearchResult.parseFrom(in); - isNull = false; - ShardSearchRequest shardSearchRequest; - try { - shardSearchRequest = new ShardSearchRequest(this.querySearchResultProto.getSearchShardRequest().toByteArray()); - setShardSearchRequest(shardSearchRequest); - } catch (IOException e) { - logger.error("Error while setting shard search request", e); + isNull = this.querySearchResultProto.getIsNull(); + if (!isNull) { + this.contextId = new ShardSearchContextId( + this.querySearchResultProto.getContextId().getSessionId(), + this.querySearchResultProto.getContextId().getId() + ); + ShardSearchRequest shardSearchRequest; + hasAggs = false; + try { + shardSearchRequest = new ShardSearchRequest(this.querySearchResultProto.getSearchShardRequest().toByteArray()); + setShardSearchRequest(shardSearchRequest); + } catch (IOException e) { + logger.error("Error while setting shard search request", e); + } } } @@ -155,13 +165,40 @@ public QuerySearchResult(ShardSearchContextId contextId, SearchShardTarget shard ShardSearchRequestProto.ShardSearchRequest.Builder shardSearchRequestProto = ShardSearchRequestProto.ShardSearchRequest .newBuilder(); if (shardSearchRequest != null) { + ShardSearchRequestProto.OriginalIndices.Builder originalIndices = ShardSearchRequestProto.OriginalIndices.newBuilder(); + if (shardSearchRequest.indices() != null) { + for (String index : shardSearchRequest.indices()) { + originalIndices.addIndices(index); + } + originalIndices.setIndicesOptions( + ShardSearchRequestProto.OriginalIndices.IndicesOptions.newBuilder() + .setIgnoreUnavailable(shardSearchRequest.indicesOptions().ignoreUnavailable()) + .setAllowNoIndices(shardSearchRequest.indicesOptions().allowNoIndices()) + .setExpandWildcardsOpen(shardSearchRequest.indicesOptions().expandWildcardsOpen()) + .setExpandWildcardsClosed(shardSearchRequest.indicesOptions().expandWildcardsClosed()) + .setExpandWildcardsHidden(shardSearchRequest.indicesOptions().expandWildcardsHidden()) + .setAllowAliasesToMultipleIndices(shardSearchRequest.indicesOptions().allowAliasesToMultipleIndices()) + .setForbidClosedIndices(shardSearchRequest.indicesOptions().forbidClosedIndices()) + .setIgnoreAliases(shardSearchRequest.indicesOptions().ignoreAliases()) + .setIgnoreThrottled(shardSearchRequest.indicesOptions().ignoreThrottled()) + .build() + ); + } + AliasFilter.Builder aliasFilter = AliasFilter.newBuilder(); + if (shardSearchRequest.getAliasFilter() != null) { + for (int i = 0; i < shardSearchRequest.getAliasFilter().getAliases().length; i++) { + aliasFilter.addAliases(shardSearchRequest.getAliasFilter().getAliases()[i]); + } + } shardSearchRequestProto.setInboundNetworkTime(shardSearchRequest.getInboundNetworkTime()) .setOutboundNetworkTime(shardSearchRequest.getOutboundNetworkTime()) .setShardId(shardIdProto) .setAllowPartialSearchResults(shardSearchRequest.allowPartialSearchResults()) .setNumberOfShards(shardSearchRequest.numberOfShards()) - .setReaderId(shardSearchContextId); - + .setReaderId(shardSearchContextId) + .setOriginalIndices(originalIndices) + .setSearchType(SearchType.QUERY_THEN_FETCH) + .setAliasFilter(aliasFilter); if (shardSearchRequest.keepAlive() != null) { shardSearchRequestProto.setTimeValue(shardSearchRequest.keepAlive().getStringRep()); } @@ -175,6 +212,8 @@ public QuerySearchResult(ShardSearchContextId contextId, SearchShardTarget shard .setContextId(shardSearchContextId) .setSearchShardTarget(searchShardTarget.build()) .setSearchShardRequest(shardSearchRequestProto.build()) + .setHasAggs(false) + .setIsNull(isNull) .build(); } @@ -520,7 +559,9 @@ public void writeTo(StreamOutput out) throws IOException { @Override public void writeTo(OutputStream out) throws IOException { - out.write(this.querySearchResultProto.toByteArray()); + if (!isNull) { + out.write(this.querySearchResultProto.toByteArray()); + } } public void writeToNoId(StreamOutput out) throws IOException { @@ -570,7 +611,11 @@ public QuerySearchResultProto.QuerySearchResult response() { public QuerySearchResult(QuerySearchResultProto.QuerySearchResult querySearchResult) { this.querySearchResultProto = querySearchResult; - this.isNull = false; + this.isNull = this.querySearchResultProto.getIsNull(); + this.contextId = new ShardSearchContextId( + this.querySearchResultProto.getContextId().getSessionId(), + this.querySearchResultProto.getContextId().getId() + ); ShardSearchRequest shardSearchRequest; try { shardSearchRequest = new ShardSearchRequest(this.querySearchResultProto.getSearchShardRequest().toByteArray()); @@ -579,4 +624,12 @@ public QuerySearchResult(QuerySearchResultProto.QuerySearchResult querySearchRes logger.error("Error while setting shard search request", e); } } + + @Override + public String getProtocol() { + if (FeatureFlags.isEnabled(FeatureFlags.PROTOBUF_SETTING)) { + return BaseInboundMessage.PROTOBUF_PROTOCOL; + } + return BaseInboundMessage.NATIVE_PROTOCOL; + } } diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotShardsService.java b/server/src/main/java/org/opensearch/snapshots/SnapshotShardsService.java index b6650a763cf01..89f1ea142336e 100644 --- a/server/src/main/java/org/opensearch/snapshots/SnapshotShardsService.java +++ b/server/src/main/java/org/opensearch/snapshots/SnapshotShardsService.java @@ -74,7 +74,6 @@ import org.opensearch.transport.TransportService; import java.io.IOException; -import java.io.InputStream; import java.nio.file.NoSuchFileException; import java.util.HashMap; import java.util.Iterator; @@ -624,12 +623,6 @@ public void handleException(TransportException exp) { public String executor() { return ThreadPool.Names.SAME; } - - @Override - public UpdateIndexShardSnapshotStatusResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } ) ); diff --git a/server/src/main/java/org/opensearch/telemetry/tracing/handler/TraceableTransportResponseHandler.java b/server/src/main/java/org/opensearch/telemetry/tracing/handler/TraceableTransportResponseHandler.java index e09a7213903c0..eb9d53d2df51b 100644 --- a/server/src/main/java/org/opensearch/telemetry/tracing/handler/TraceableTransportResponseHandler.java +++ b/server/src/main/java/org/opensearch/telemetry/tracing/handler/TraceableTransportResponseHandler.java @@ -17,7 +17,6 @@ import org.opensearch.transport.TransportResponseHandler; import java.io.IOException; -import java.io.InputStream; import java.util.Objects; /** @@ -105,10 +104,4 @@ public void handleRejection(Exception exp) { span.endSpan(); } } - - @Override - public T read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } diff --git a/server/src/main/java/org/opensearch/transport/EmptyTransportResponseHandler.java b/server/src/main/java/org/opensearch/transport/EmptyTransportResponseHandler.java index 62c9ab738be21..1691b427ffca1 100644 --- a/server/src/main/java/org/opensearch/transport/EmptyTransportResponseHandler.java +++ b/server/src/main/java/org/opensearch/transport/EmptyTransportResponseHandler.java @@ -34,12 +34,8 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.transport.TransportResponse; -import org.opensearch.core.transport.TransportResponse.Empty; import org.opensearch.threadpool.ThreadPool; -import java.io.IOException; -import java.io.InputStream; - /** * Handler for empty transport response * @@ -70,10 +66,4 @@ public void handleException(TransportException exp) {} public String executor() { return executor; } - - @Override - public Empty read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } diff --git a/server/src/main/java/org/opensearch/transport/InboundPipeline.java b/server/src/main/java/org/opensearch/transport/InboundPipeline.java index fca941fc08721..f4c410671b08a 100644 --- a/server/src/main/java/org/opensearch/transport/InboundPipeline.java +++ b/server/src/main/java/org/opensearch/transport/InboundPipeline.java @@ -41,6 +41,7 @@ import org.opensearch.core.common.bytes.CompositeBytesReference; import org.opensearch.transport.nativeprotocol.NativeInboundBytesHandler; +import java.io.ByteArrayInputStream; import java.io.IOException; import java.util.ArrayDeque; import java.util.List; diff --git a/server/src/main/java/org/opensearch/transport/NodeToNodeMessage.java b/server/src/main/java/org/opensearch/transport/NodeToNodeMessage.java index d1fe9775033dd..943007f7913c6 100644 --- a/server/src/main/java/org/opensearch/transport/NodeToNodeMessage.java +++ b/server/src/main/java/org/opensearch/transport/NodeToNodeMessage.java @@ -9,7 +9,6 @@ package org.opensearch.transport; import com.google.protobuf.ByteString; -import com.google.protobuf.InvalidProtocolBufferException; import org.opensearch.Version; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.util.concurrent.ThreadContext; @@ -18,8 +17,10 @@ import org.opensearch.server.proto.NodeToNodeMessageProto.NodeToNodeMessage.Header; import org.opensearch.server.proto.NodeToNodeMessageProto.NodeToNodeMessage.ResponseHandlersList; import org.opensearch.server.proto.QueryFetchSearchResultProto.QueryFetchSearchResult; +import org.opensearch.server.proto.QuerySearchResultProto.QuerySearchResult; import java.io.IOException; +import java.io.InputStream; import java.io.OutputStream; import java.util.Arrays; import java.util.HashMap; @@ -74,15 +75,51 @@ public NodeToNodeMessage( .setAction(action) .addAllFeatures(features) .build(); + } + public NodeToNodeMessage( + long requestId, + byte[] status, + Version version, + ThreadContext threadContext, + QuerySearchResult querySearchResult, + Set features, + String action + ) { + Header header = Header.newBuilder() + .addAllPrefix(Arrays.asList(ByteString.copyFrom(PREFIX))) + .setRequestId(requestId) + .setStatus(ByteString.copyFrom(status)) + .setVersionId(version.id) + .build(); + Map requestHeaders = threadContext.getHeaders(); + Map> responseHeaders = threadContext.getResponseHeaders(); + Map responseHandlers = new HashMap<>(); + for (Map.Entry> entry : responseHeaders.entrySet()) { + String key = entry.getKey(); + List value = entry.getValue(); + ResponseHandlersList responseHandlersList = ResponseHandlersList.newBuilder().addAllSetOfResponseHandlers(value).build(); + responseHandlers.put(key, responseHandlersList); + } + this.message = NodeToNodeMessageProto.NodeToNodeMessage.newBuilder() + .setHeader(header) + .putAllRequestHeaders(requestHeaders) + .putAllResponseHandlers(responseHandlers) + .setVersion(version.toString()) + .setStatus(ByteString.copyFrom(status)) + .setRequestId(requestId) + .setQuerySearchResult(querySearchResult) + .setAction(action) + .addAllFeatures(features) + .build(); } - public NodeToNodeMessage(byte[] in) throws InvalidProtocolBufferException { + public NodeToNodeMessage(InputStream in) throws IOException { this.message = NodeToNodeMessageProto.NodeToNodeMessage.parseFrom(in); } public void writeTo(OutputStream out) throws IOException { - out.write(this.message.toByteArray()); + this.message.writeTo(out); } BytesReference serialize(BytesStreamOutput bytesStream) throws IOException { diff --git a/server/src/main/java/org/opensearch/transport/OutboundHandler.java b/server/src/main/java/org/opensearch/transport/OutboundHandler.java index a45a7e65116ba..ec3a0efd3c834 100644 --- a/server/src/main/java/org/opensearch/transport/OutboundHandler.java +++ b/server/src/main/java/org/opensearch/transport/OutboundHandler.java @@ -52,6 +52,7 @@ import org.opensearch.core.common.transport.TransportAddress; import org.opensearch.core.transport.TransportResponse; import org.opensearch.search.fetch.QueryFetchSearchResult; +import org.opensearch.search.query.QuerySearchResult; import org.opensearch.threadpool.ThreadPool; import java.io.IOException; @@ -149,21 +150,40 @@ void sendResponse( Version version = Version.min(this.version, nodeVersion); ActionListener listener = ActionListener.wrap(() -> messageListener.onResponseSent(requestId, action, response)); if ((response.getProtocol()).equals(BaseInboundMessage.PROTOBUF_PROTOCOL) && version.onOrAfter(Version.V_3_0_0)) { - QueryFetchSearchResult queryFetchSearchResult = (QueryFetchSearchResult) response; - if (queryFetchSearchResult.response() != null) { - byte[] bytes = new byte[1]; - bytes[0] = 1; - NodeToNodeMessage protobufMessage = new NodeToNodeMessage( - requestId, - bytes, - Version.CURRENT, - threadPool.getThreadContext(), - queryFetchSearchResult.response(), - features, - action - ); - sendProtobufMessage(channel, protobufMessage, listener); + if (response instanceof QueryFetchSearchResult) { + QueryFetchSearchResult queryFetchSearchResult = (QueryFetchSearchResult) response; + if (queryFetchSearchResult.response() != null) { + byte[] bytes = new byte[1]; + bytes[0] = 1; + NodeToNodeMessage protobufMessage = new NodeToNodeMessage( + requestId, + bytes, + Version.CURRENT, + threadPool.getThreadContext(), + queryFetchSearchResult.response(), + features, + action + ); + sendProtobufMessage(channel, protobufMessage, listener); + } + } else if (response instanceof QuerySearchResult) { + QuerySearchResult querySearchResult = (QuerySearchResult) response; + if (querySearchResult.response() != null) { + byte[] bytes = new byte[1]; + bytes[0] = 1; + NodeToNodeMessage protobufMessage = new NodeToNodeMessage( + requestId, + bytes, + Version.CURRENT, + threadPool.getThreadContext(), + querySearchResult.response(), + features, + action + ); + sendProtobufMessage(channel, protobufMessage, listener); + } } + } else { OutboundMessage.Response message = new OutboundMessage.Response( threadPool.getThreadContext(), diff --git a/server/src/main/java/org/opensearch/transport/PlainTransportFuture.java b/server/src/main/java/org/opensearch/transport/PlainTransportFuture.java index 73d8fbd4d528d..ff9ca8b189904 100644 --- a/server/src/main/java/org/opensearch/transport/PlainTransportFuture.java +++ b/server/src/main/java/org/opensearch/transport/PlainTransportFuture.java @@ -39,7 +39,6 @@ import org.opensearch.core.transport.TransportResponse; import java.io.IOException; -import java.io.InputStream; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; @@ -126,10 +125,4 @@ public void handleException(TransportException exp) { public String toString() { return "future(" + handler.toString() + ")"; } - - @Override - public V read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } diff --git a/server/src/main/java/org/opensearch/transport/ProtobufMessageHandler.java b/server/src/main/java/org/opensearch/transport/ProtobufMessageHandler.java index b46ff2c86c7b8..d3262f4d3e5c4 100644 --- a/server/src/main/java/org/opensearch/transport/ProtobufMessageHandler.java +++ b/server/src/main/java/org/opensearch/transport/ProtobufMessageHandler.java @@ -21,6 +21,7 @@ import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.core.common.transport.TransportAddress; import org.opensearch.core.transport.TransportResponse; +import org.opensearch.search.query.QuerySearchResult; import org.opensearch.server.proto.QueryFetchSearchResultProto.QueryFetchSearchResult; import org.opensearch.threadpool.ThreadPool; @@ -110,6 +111,19 @@ private void handleProtobufResponse( final T response = (T) queryFetchSearchResult2; response.remoteAddress(new TransportAddress(remoteAddress)); + final String executor = handler.executor(); + if (ThreadPool.Names.SAME.equals(executor)) { + doHandleResponse(handler, response); + } else { + threadPool.executor(executor).execute(() -> doHandleResponse(handler, response)); + } + } else if (receivedMessage.hasQuerySearchResult()) { + final org.opensearch.server.proto.QuerySearchResultProto.QuerySearchResult querySearchResult = receivedMessage + .getQuerySearchResult(); + QuerySearchResult querySearchResult2 = new QuerySearchResult(querySearchResult); + final T response = (T) querySearchResult2; + response.remoteAddress(new TransportAddress(remoteAddress)); + final String executor = handler.executor(); if (ThreadPool.Names.SAME.equals(executor)) { doHandleResponse(handler, response); diff --git a/server/src/main/java/org/opensearch/transport/RemoteClusterConnection.java b/server/src/main/java/org/opensearch/transport/RemoteClusterConnection.java index 4b2f8c2014206..8a5f6dfffb036 100644 --- a/server/src/main/java/org/opensearch/transport/RemoteClusterConnection.java +++ b/server/src/main/java/org/opensearch/transport/RemoteClusterConnection.java @@ -47,7 +47,6 @@ import java.io.Closeable; import java.io.IOException; -import java.io.InputStream; import java.util.function.Function; /** @@ -171,12 +170,6 @@ public void handleException(TransportException exp) { public String executor() { return ThreadPool.Names.SAME; } - - @Override - public ClusterStateResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } ); } diff --git a/server/src/main/java/org/opensearch/transport/SniffConnectionStrategy.java b/server/src/main/java/org/opensearch/transport/SniffConnectionStrategy.java index 3313ab7103db6..07ba96b135189 100644 --- a/server/src/main/java/org/opensearch/transport/SniffConnectionStrategy.java +++ b/server/src/main/java/org/opensearch/transport/SniffConnectionStrategy.java @@ -58,7 +58,6 @@ import org.opensearch.threadpool.ThreadPool; import java.io.IOException; -import java.io.InputStream; import java.net.InetSocketAddress; import java.util.Arrays; import java.util.Collections; @@ -472,12 +471,6 @@ public void handleException(TransportException exp) { public String executor() { return ThreadPool.Names.MANAGEMENT; } - - @Override - public ClusterStateResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } private Predicate getRemoteClusterNamePredicate() { diff --git a/server/src/main/java/org/opensearch/transport/TransportActionProxy.java b/server/src/main/java/org/opensearch/transport/TransportActionProxy.java index 830c320679cb6..a61aec8a34e20 100644 --- a/server/src/main/java/org/opensearch/transport/TransportActionProxy.java +++ b/server/src/main/java/org/opensearch/transport/TransportActionProxy.java @@ -40,7 +40,6 @@ import org.opensearch.threadpool.ThreadPool; import java.io.IOException; -import java.io.InputStream; import java.io.UncheckedIOException; import java.util.function.Function; @@ -131,12 +130,6 @@ public void handleException(TransportException exp) { public String executor() { return ThreadPool.Names.SAME; } - - @Override - public T read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } /** diff --git a/server/src/main/java/org/opensearch/transport/TransportHandshaker.java b/server/src/main/java/org/opensearch/transport/TransportHandshaker.java index a6dddc430086a..d0b00ec9c59db 100644 --- a/server/src/main/java/org/opensearch/transport/TransportHandshaker.java +++ b/server/src/main/java/org/opensearch/transport/TransportHandshaker.java @@ -45,7 +45,6 @@ import java.io.EOFException; import java.io.IOException; -import java.io.InputStream; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicBoolean; @@ -188,12 +187,6 @@ void handleLocalException(TransportException e) { public String executor() { return ThreadPool.Names.SAME; } - - @Override - public HandshakeResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } static final class HandshakeRequest extends TransportRequest { diff --git a/server/src/main/java/org/opensearch/transport/TransportResponseHandler.java b/server/src/main/java/org/opensearch/transport/TransportResponseHandler.java index b11892b3d6985..42048258bb5c9 100644 --- a/server/src/main/java/org/opensearch/transport/TransportResponseHandler.java +++ b/server/src/main/java/org/opensearch/transport/TransportResponseHandler.java @@ -33,7 +33,7 @@ package org.opensearch.transport; import org.opensearch.common.annotation.PublicApi; -import org.opensearch.core.common.io.stream.ProtobufWriteable; +import org.opensearch.core.common.io.stream.BytesWriteable; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.transport.TransportResponse; @@ -48,7 +48,7 @@ * @opensearch.api */ @PublicApi(since = "1.0.0") -public interface TransportResponseHandler extends Writeable.Reader { +public interface TransportResponseHandler extends Writeable.Reader, BytesWriteable.Reader { void handleResponse(T response); @@ -56,6 +56,16 @@ public interface TransportResponseHandler extends W String executor(); + /** + * Read {@code V}-type value from a byte array. + * + * @param in byte array to read the value from + */ + default T read(final InputStream in) throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } + /** * This method should be handling the rejection/failure scenarios where connection to the node is rejected or failed. * It should be used to clear up the resources held by the {@link TransportResponseHandler}. diff --git a/server/src/main/proto/server/NodeToNodeMessageProto.proto b/server/src/main/proto/server/NodeToNodeMessageProto.proto index 05332016b4181..e0d46bac94216 100644 --- a/server/src/main/proto/server/NodeToNodeMessageProto.proto +++ b/server/src/main/proto/server/NodeToNodeMessageProto.proto @@ -13,6 +13,7 @@ syntax = "proto3"; package org.opensearch.server.proto; import "server/search/QueryFetchSearchResultProto.proto"; +import "server/search/QuerySearchResultProto.proto"; option java_outer_classname = "NodeToNodeMessageProto"; @@ -27,6 +28,7 @@ message NodeToNodeMessage { int64 requestId = 8; oneof message { QueryFetchSearchResult queryFetchSearchResult = 9; + QuerySearchResult querySearchResult = 10; } message Header { diff --git a/server/src/main/proto/server/search/FetchSearchResultProto.proto b/server/src/main/proto/server/search/FetchSearchResultProto.proto index c0ba7b4399c2b..b126ca439e9f2 100644 --- a/server/src/main/proto/server/search/FetchSearchResultProto.proto +++ b/server/src/main/proto/server/search/FetchSearchResultProto.proto @@ -62,7 +62,7 @@ message SearchHit { message DocumentField { string name = 1; - repeated bytes values = 2; + repeated DocumentFieldValue values = 2; } message HighlightField { @@ -115,3 +115,19 @@ message CollapseValue { optional bool collapseBool = 7; } +message DocumentFieldValue { + optional string valueString = 1; + optional int32 valueInt = 2; + optional int64 valueLong = 3; + optional float valueFloat = 4; + optional double valueDouble = 5; + optional bool valueBool = 6; + repeated bytes valueByteArray = 7; + repeated DocumentFieldValue valueArrayList = 8; + map valueMap = 9; + optional int64 valueDate = 10; + optional string valueZonedDate = 11; + optional int64 valueZonedTime = 12; + optional string valueText = 13; +} + diff --git a/server/src/main/proto/server/search/QuerySearchResultProto.proto b/server/src/main/proto/server/search/QuerySearchResultProto.proto index 2adc887826007..32ed57d4d9063 100644 --- a/server/src/main/proto/server/search/QuerySearchResultProto.proto +++ b/server/src/main/proto/server/search/QuerySearchResultProto.proto @@ -34,6 +34,7 @@ message QuerySearchResult { optional int32 nodeQueueSize = 15; SearchShardTarget searchShardTarget = 17; ShardSearchRequest searchShardRequest = 18; + bool isNull = 19; message TopDocsAndMaxScore { TopDocs topDocs = 1; diff --git a/server/src/main/proto/server/search/ShardSearchRequestProto.proto b/server/src/main/proto/server/search/ShardSearchRequestProto.proto index dad6ce044d69d..0975a96d77167 100644 --- a/server/src/main/proto/server/search/ShardSearchRequestProto.proto +++ b/server/src/main/proto/server/search/ShardSearchRequestProto.proto @@ -21,7 +21,7 @@ message ShardSearchRequest { SearchType searchType = 4; bytes source = 5; bool requestCache = 6; - bytes aliasFilter = 7; + AliasFilter aliasFilter = 7; float indexBoost = 8; bool allowPartialSearchResults = 9; repeated string indexRoutings = 10; @@ -72,4 +72,8 @@ message OriginalIndices { bool ignoreAliases = 8; bool ignoreThrottled = 9; } +} + +message AliasFilter { + repeated string aliases = 1; } \ No newline at end of file diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java index 0e9298639a145..a015e671f4872 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java @@ -69,7 +69,6 @@ import org.junit.BeforeClass; import java.io.IOException; -import java.io.InputStream; import java.util.HashSet; import java.util.Set; import java.util.concurrent.CountDownLatch; @@ -686,12 +685,6 @@ public String executor() { public AddVotingConfigExclusionsResponse read(StreamInput in) throws IOException { return new AddVotingConfigExclusionsResponse(in); } - - @Override - public AddVotingConfigExclusionsResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } }; } diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/configuration/TransportClearVotingConfigExclusionsActionTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/configuration/TransportClearVotingConfigExclusionsActionTests.java index 78975e19c4d2b..10e4ab6388be4 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/configuration/TransportClearVotingConfigExclusionsActionTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/configuration/TransportClearVotingConfigExclusionsActionTests.java @@ -63,7 +63,6 @@ import org.junit.BeforeClass; import java.io.IOException; -import java.io.InputStream; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; @@ -253,12 +252,6 @@ public String executor() { public ClearVotingConfigExclusionsResponse read(StreamInput in) throws IOException { return new ClearVotingConfigExclusionsResponse(in); } - - @Override - public ClearVotingConfigExclusionsResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } }; } } diff --git a/server/src/test/java/org/opensearch/action/support/replication/TransportReplicationAllPermitsAcquisitionTests.java b/server/src/test/java/org/opensearch/action/support/replication/TransportReplicationAllPermitsAcquisitionTests.java index 91caed7a8c898..cce8758ef1014 100644 --- a/server/src/test/java/org/opensearch/action/support/replication/TransportReplicationAllPermitsAcquisitionTests.java +++ b/server/src/test/java/org/opensearch/action/support/replication/TransportReplicationAllPermitsAcquisitionTests.java @@ -34,7 +34,6 @@ import org.opensearch.Version; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.PlainActionFuture; -import org.opensearch.action.support.replication.TransportReplicationAction.ReplicaResponse; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.action.shard.ShardStateAction; import org.opensearch.cluster.block.ClusterBlock; @@ -78,7 +77,6 @@ import org.junit.Before; import java.io.IOException; -import java.io.InputStream; import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -225,12 +223,6 @@ public void handleException(TransportException exp) { public String executor() { return ThreadPool.Names.SAME; } - - @Override - public ReplicaResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } ); } diff --git a/server/src/test/java/org/opensearch/cluster/coordination/FollowersCheckerTests.java b/server/src/test/java/org/opensearch/cluster/coordination/FollowersCheckerTests.java index fb495de053b4d..a106706c00732 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/FollowersCheckerTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/FollowersCheckerTests.java @@ -60,8 +60,6 @@ import org.opensearch.transport.TransportResponseHandler; import org.opensearch.transport.TransportService; -import java.io.IOException; -import java.io.InputStream; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; @@ -534,12 +532,6 @@ public void handleException(TransportException exp) { public String executor() { return Names.SAME; } - - @Override - public Empty read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } ); deterministicTaskQueue.runAllTasks(); @@ -628,12 +620,6 @@ public void handleException(TransportException exp) { public String executor() { return Names.SAME; } - - @Override - public Empty read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } ); deterministicTaskQueue.runAllTasks(); @@ -706,12 +692,6 @@ public void handleException(TransportException exp) { public String executor() { return Names.SAME; } - - @Override - public Empty read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } ); deterministicTaskQueue.runAllTasks(); @@ -809,11 +789,5 @@ public TransportResponse.Empty read(StreamInput in) { return TransportResponse.Empty.INSTANCE; } - @Override - public Empty read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } - } } diff --git a/server/src/test/java/org/opensearch/cluster/coordination/LeaderCheckerTests.java b/server/src/test/java/org/opensearch/cluster/coordination/LeaderCheckerTests.java index 956b0887d1faa..fe65058333116 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/LeaderCheckerTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/LeaderCheckerTests.java @@ -57,8 +57,6 @@ import org.opensearch.transport.TransportResponseHandler; import org.opensearch.transport.TransportService; -import java.io.IOException; -import java.io.InputStream; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; @@ -551,12 +549,6 @@ public String executor() { public TransportResponse.Empty read(StreamInput in) { return TransportResponse.Empty.INSTANCE; } - - @Override - public Empty read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } public void testLeaderCheckRequestEqualsHashcodeSerialization() { diff --git a/server/src/test/java/org/opensearch/cluster/coordination/PreVoteCollectorTests.java b/server/src/test/java/org/opensearch/cluster/coordination/PreVoteCollectorTests.java index b4cf0c77270b3..5ddf614db3334 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/PreVoteCollectorTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/PreVoteCollectorTests.java @@ -53,7 +53,6 @@ import org.junit.Before; import java.io.IOException; -import java.io.InputStream; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -359,12 +358,6 @@ public void handleException(TransportException exp) { public String executor() { return SAME; } - - @Override - public PreVoteResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } ); diff --git a/server/src/test/java/org/opensearch/discovery/PeerFinderTests.java b/server/src/test/java/org/opensearch/discovery/PeerFinderTests.java index e64696b00311d..f861ab90896db 100644 --- a/server/src/test/java/org/opensearch/discovery/PeerFinderTests.java +++ b/server/src/test/java/org/opensearch/discovery/PeerFinderTests.java @@ -59,7 +59,6 @@ import org.junit.Before; import java.io.IOException; -import java.io.InputStream; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -548,12 +547,6 @@ public void handleException(TransportException exp) { public String executor() { return Names.SAME; } - - @Override - public PeersResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } ); diff --git a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationSourceServiceTests.java b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationSourceServiceTests.java index 19acb8ef938d5..8f84053f2618e 100644 --- a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationSourceServiceTests.java +++ b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationSourceServiceTests.java @@ -40,7 +40,6 @@ import org.opensearch.transport.TransportService; import java.io.IOException; -import java.io.InputStream; import java.util.Collections; import java.util.List; import java.util.concurrent.TimeUnit; @@ -221,12 +220,6 @@ public String executor() { public CheckpointInfoResponse read(StreamInput in) throws IOException { return new CheckpointInfoResponse(in); } - - @Override - public CheckpointInfoResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } ); } @@ -256,12 +249,6 @@ public String executor() { public GetSegmentFilesResponse read(StreamInput in) throws IOException { return new GetSegmentFilesResponse(in); } - - @Override - public GetSegmentFilesResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } ); } @@ -302,12 +289,6 @@ public String executor() { public CheckpointInfoResponse read(StreamInput in) throws IOException { return new CheckpointInfoResponse(in); } - - @Override - public TransportResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } ); } diff --git a/server/src/test/java/org/opensearch/transport/InboundHandlerTests.java b/server/src/test/java/org/opensearch/transport/InboundHandlerTests.java index cfe009d21d944..892909b094eeb 100644 --- a/server/src/test/java/org/opensearch/transport/InboundHandlerTests.java +++ b/server/src/test/java/org/opensearch/transport/InboundHandlerTests.java @@ -64,6 +64,7 @@ import org.junit.After; import org.junit.Before; +import java.io.ByteArrayInputStream; import java.io.EOFException; import java.io.IOException; import java.io.InputStream; @@ -184,12 +185,6 @@ public String executor() { public TestResponse read(StreamInput in) throws IOException { return new TestResponse(in); } - - @Override - public TestResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } }, null, action)); RequestHandlerRegistry registry = new RequestHandlerRegistry<>( action, @@ -333,7 +328,7 @@ public QueryFetchSearchResult read(InputStream in) throws IOException { BytesReference fullResponseBytes = channel.getMessageCaptor().get(); byte[] incomingBytes = BytesReference.toBytes(fullResponseBytes.slice(3, fullResponseBytes.length() - 3)); - NodeToNodeMessage nodeToNodeMessage = new NodeToNodeMessage(incomingBytes); + NodeToNodeMessage nodeToNodeMessage = new NodeToNodeMessage(new ByteArrayInputStream(incomingBytes)); handler.inboundMessage(channel, nodeToNodeMessage); QueryFetchSearchResult result = responseCaptor.get(); assertNotNull(result); @@ -465,12 +460,6 @@ public String executor() { public TestResponse read(StreamInput in) throws IOException { return new TestResponse(in); } - - @Override - public TestResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } }, null, action)); RequestHandlerRegistry registry = new RequestHandlerRegistry<>( @@ -543,12 +532,6 @@ public String executor() { public TestResponse read(StreamInput in) throws IOException { return new TestResponse(in); } - - @Override - public TestResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } }, null, action)); RequestHandlerRegistry registry = new RequestHandlerRegistry<>( @@ -622,12 +605,6 @@ public String executor() { public TestResponse read(StreamInput in) throws IOException { return new TestResponse(in); } - - @Override - public TestResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } }, null, action)); RequestHandlerRegistry registry = new RequestHandlerRegistry<>( action, @@ -716,12 +693,6 @@ public String executor() { public TestResponse read(StreamInput in) throws IOException { return new TestResponse(in); } - - @Override - public TestResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } }, null, action)); RequestHandlerRegistry registry = new RequestHandlerRegistry<>( action, diff --git a/server/src/test/java/org/opensearch/transport/OutboundHandlerTests.java b/server/src/test/java/org/opensearch/transport/OutboundHandlerTests.java index e03d550fb0266..ec40e95fe45c1 100644 --- a/server/src/test/java/org/opensearch/transport/OutboundHandlerTests.java +++ b/server/src/test/java/org/opensearch/transport/OutboundHandlerTests.java @@ -71,6 +71,7 @@ import org.junit.After; import org.junit.Before; +import java.io.ByteArrayInputStream; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; @@ -330,7 +331,7 @@ public void onResponseSent(long requestId, String action, TransportResponse resp inboundPipeline.handleBytes(channel, new ReleasableBytesReference(reference, () -> {})); final BytesReference responseBytes = protobufMessage.get(); - final NodeToNodeMessage message = new NodeToNodeMessage(responseBytes.toBytesRef().bytes); + final NodeToNodeMessage message = new NodeToNodeMessage(new ByteArrayInputStream(responseBytes.toBytesRef().bytes)); assertEquals(version.toString(), message.getMessage().getVersion()); assertEquals(requestId, message.getHeader().getRequestId()); assertNotNull(message.getRequestHeaders()); diff --git a/server/src/test/java/org/opensearch/transport/TransportActionProxyTests.java b/server/src/test/java/org/opensearch/transport/TransportActionProxyTests.java index 77c6022bfbfbe..dd2aefd2318f7 100644 --- a/server/src/test/java/org/opensearch/transport/TransportActionProxyTests.java +++ b/server/src/test/java/org/opensearch/transport/TransportActionProxyTests.java @@ -48,7 +48,6 @@ import org.junit.Before; import java.io.IOException; -import java.io.InputStream; import java.util.concurrent.CountDownLatch; public class TransportActionProxyTests extends OpenSearchTestCase { @@ -150,12 +149,6 @@ public void handleException(TransportException exp) { public String executor() { return ThreadPool.Names.SAME; } - - @Override - public SimpleTestResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } ); latch.await(); @@ -216,12 +209,6 @@ public void handleException(TransportException exp) { public String executor() { return ThreadPool.Names.SAME; } - - @Override - public SimpleTestResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } ); latch.await(); diff --git a/server/src/test/java/org/opensearch/transport/TransportServiceDeserializationFailureTests.java b/server/src/test/java/org/opensearch/transport/TransportServiceDeserializationFailureTests.java index 443178a9610d0..d10b4f26100cc 100644 --- a/server/src/test/java/org/opensearch/transport/TransportServiceDeserializationFailureTests.java +++ b/server/src/test/java/org/opensearch/transport/TransportServiceDeserializationFailureTests.java @@ -41,7 +41,6 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.tasks.TaskId; import org.opensearch.core.transport.TransportResponse; -import org.opensearch.core.transport.TransportResponse.Empty; import org.opensearch.tasks.Task; import org.opensearch.tasks.TaskAwareRequest; import org.opensearch.telemetry.tracing.noop.NoopTracer; @@ -49,8 +48,6 @@ import org.opensearch.test.transport.MockTransport; import org.opensearch.threadpool.ThreadPool; -import java.io.IOException; -import java.io.InputStream; import java.util.Collections; import java.util.List; @@ -137,12 +134,6 @@ public TransportResponse.Empty read(StreamInput in) { public String toString() { return "test handler without parent"; } - - @Override - public Empty read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } ); @@ -199,12 +190,6 @@ public TransportResponse.Empty read(StreamInput in) { public String toString() { return "test handler with parent"; } - - @Override - public Empty read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } ); diff --git a/test/framework/src/main/java/org/opensearch/transport/AbstractSimpleTransportTestCase.java b/test/framework/src/main/java/org/opensearch/transport/AbstractSimpleTransportTestCase.java index 93500d3c300cd..e43b0756e2f2b 100644 --- a/test/framework/src/main/java/org/opensearch/transport/AbstractSimpleTransportTestCase.java +++ b/test/framework/src/main/java/org/opensearch/transport/AbstractSimpleTransportTestCase.java @@ -65,7 +65,6 @@ import org.opensearch.core.common.transport.BoundTransportAddress; import org.opensearch.core.common.transport.TransportAddress; import org.opensearch.core.transport.TransportResponse; -import org.opensearch.core.transport.TransportResponse.Empty; import org.opensearch.node.Node; import org.opensearch.tasks.Task; import org.opensearch.telemetry.tracing.noop.NoopTracer; @@ -81,7 +80,6 @@ import org.junit.Before; import java.io.IOException; -import java.io.InputStream; import java.io.UncheckedIOException; import java.net.Inet4Address; import java.net.Inet6Address; @@ -329,12 +327,6 @@ public void handleException(TransportException exp) { logger.error("Unexpected failure", exp); fail("got exception instead of a response: " + exp.getMessage()); } - - @Override - public StringMessageResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } ); @@ -371,12 +363,6 @@ public void handleException(TransportException exp) { logger.error("Unexpected failure", exp); fail("got exception instead of a response: " + exp.getMessage()); } - - @Override - public StringMessageResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } ); @@ -434,12 +420,6 @@ public void handleException(TransportException exp) { logger.error("Unexpected failure", exp); fail("got exception instead of a response: " + exp.getMessage()); } - - @Override - public StringMessageResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } }; StringMessageRequest ping = new StringMessageRequest("ping"); threadPool.getThreadContext().putHeader("test.ping.user", "ping_user"); @@ -499,12 +479,6 @@ public void handleException(TransportException exp) { public String executor() { return ThreadPool.Names.GENERIC; } - - @Override - public StringMessageResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } ); responseLatch.await(); @@ -688,12 +662,6 @@ public void handleException(TransportException exp) { logger.error("Unexpected failure", exp); fail("got exception instead of a response: " + exp.getMessage()); } - - @Override - public Empty read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } ); @@ -756,12 +724,6 @@ public void handleException(TransportException exp) { logger.error("Unexpected failure", exp); fail("got exception instead of a response: " + exp.getMessage()); } - - @Override - public StringMessageResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } ); @@ -809,12 +771,6 @@ public void handleResponse(StringMessageResponse response) { public void handleException(TransportException exp) { assertThat("runtime_exception: bad message !!!", equalTo(exp.getCause().getMessage())); } - - @Override - public StringMessageResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } ); @@ -1064,12 +1020,6 @@ public void handleResponse(StringMessageResponse response) { public void handleException(TransportException exp) { assertThat(exp, instanceOf(ReceiveTimeoutTransportException.class)); } - - @Override - public StringMessageResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } ); @@ -1144,12 +1094,6 @@ public void handleException(TransportException exp) { latch.countDown(); assertThat(exp, instanceOf(ReceiveTimeoutTransportException.class)); } - - @Override - public StringMessageResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } ); @@ -1191,12 +1135,6 @@ public void handleException(TransportException exp) { logger.error("Unexpected failure", exp); fail("got exception instead of a response for " + counter + ": " + exp.getDetailedMessage()); } - - @Override - public StringMessageResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } ); @@ -1241,12 +1179,6 @@ public void handleException(TransportException exp) {} public String executor() { return ThreadPool.Names.SAME; } - - @Override - public StringMessageResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } }; serviceA.registerRequestHandler("internal:test", ThreadPool.Names.SAME, StringMessageRequest::new, handler); @@ -1550,12 +1482,6 @@ public void handleException(TransportException exp) { public String executor() { return ThreadPool.Names.SAME; } - - @Override - public Version0Response read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } ).txGet(); @@ -1607,12 +1533,6 @@ public void handleException(TransportException exp) { public String executor() { return ThreadPool.Names.SAME; } - - @Override - public Version1Response read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } ).txGet(); @@ -1658,12 +1578,6 @@ public void handleException(TransportException exp) { public String executor() { return ThreadPool.Names.SAME; } - - @Override - public Version1Response read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } ).txGet(); @@ -1706,12 +1620,6 @@ public void handleException(TransportException exp) { public String executor() { return ThreadPool.Names.SAME; } - - @Override - public Version0Response read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } ).txGet(); @@ -1757,12 +1665,6 @@ public void handleException(TransportException exp) { assertThat(cause, instanceOf(ConnectTransportException.class)); assertThat(((ConnectTransportException) cause).node(), equalTo(nodeA)); } - - @Override - public StringMessageResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } ); @@ -1827,12 +1729,6 @@ public void handleResponse(StringMessageResponse response) { public void handleException(TransportException exp) { assertThat(exp, instanceOf(ReceiveTimeoutTransportException.class)); } - - @Override - public StringMessageResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } ); @@ -1884,12 +1780,6 @@ public void handleException(TransportException exp) { public String executor() { return ThreadPool.Names.SAME; } - - @Override - public TestResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } }); if (!latch.await(10, TimeUnit.SECONDS)) { @@ -1938,12 +1828,6 @@ public void handleException(TransportException exp) { public String executor() { return ThreadPool.Names.SAME; } - - @Override - public TestResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } ); @@ -1979,12 +1863,6 @@ public void handleException(TransportException exp) { public String executor() { return ThreadPool.Names.SAME; } - - @Override - public TestResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } ); @@ -2143,12 +2021,6 @@ public void handleException(TransportException exp) { public String executor() { return randomBoolean() ? ThreadPool.Names.SAME : ThreadPool.Names.GENERIC; } - - @Override - public TestResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } ); } else { @@ -2211,12 +2083,6 @@ public void handleException(TransportException exp) { public String executor() { return randomBoolean() ? ThreadPool.Names.SAME : ThreadPool.Names.GENERIC; } - - @Override - public TestResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } for (int i = 0; i < iters; i++) { @@ -2552,12 +2418,6 @@ public void handleException(TransportException exp) { public String executor() { return randomFrom(executors); } - - @Override - public TransportResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } }; serviceB.sendRequest(nodeA, "internal:action", new TestRequest(randomFrom("fail", "pass")), transportResponseHandler); @@ -2608,12 +2468,6 @@ public void handleException(TransportException exp) { public String executor() { return randomFrom(executors); } - - @Override - public TransportResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } }; ConnectionProfile.Builder builder = new ConnectionProfile.Builder(); builder.addConnections( @@ -2684,12 +2538,6 @@ public void handleException(TransportException exp) { public String executor() { return ThreadPool.Names.SAME; } - - @Override - public TransportResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } }; ConnectionProfile.Builder builder = new ConnectionProfile.Builder(); @@ -2764,12 +2612,6 @@ public void handleException(TransportException exp) { public String executor() { return ThreadPool.Names.SAME; } - - @Override - public TransportResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } }; TransportStats stats = serviceC.transport.getStats(); // nothing transmitted / read yet @@ -2891,12 +2733,6 @@ public void handleException(TransportException exp) { public String executor() { return ThreadPool.Names.SAME; } - - @Override - public TransportResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } }; TransportStats stats = serviceC.transport.getStats(); // nothing transmitted / read yet @@ -3284,12 +3120,6 @@ public String executor() { public TransportResponse read(final StreamInput in) { return TransportResponse.Empty.INSTANCE; } - - @Override - public TransportResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } ); assertThat(te.get(), not(nullValue())); diff --git a/test/framework/src/test/java/org/opensearch/test/disruption/DisruptableMockTransportTests.java b/test/framework/src/test/java/org/opensearch/test/disruption/DisruptableMockTransportTests.java index d4583b6c29419..6b64270ca68e1 100644 --- a/test/framework/src/test/java/org/opensearch/test/disruption/DisruptableMockTransportTests.java +++ b/test/framework/src/test/java/org/opensearch/test/disruption/DisruptableMockTransportTests.java @@ -57,7 +57,6 @@ import org.junit.Before; import java.io.IOException; -import java.io.InputStream; import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; @@ -234,12 +233,6 @@ public void handleException(TransportException exp) { public String executor() { return ThreadPool.Names.SAME; } - - @Override - public TransportResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } }; } @@ -264,12 +257,6 @@ public void handleException(TransportException exp) { public String executor() { return ThreadPool.Names.SAME; } - - @Override - public TransportResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } }; } @@ -294,12 +281,6 @@ public void handleException(TransportException exp) { public String executor() { return ThreadPool.Names.SAME; } - - @Override - public TransportResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } }; } diff --git a/test/framework/src/test/java/org/opensearch/test/disruption/NetworkDisruptionIT.java b/test/framework/src/test/java/org/opensearch/test/disruption/NetworkDisruptionIT.java index f36b7bde14083..362ecd692360d 100644 --- a/test/framework/src/test/java/org/opensearch/test/disruption/NetworkDisruptionIT.java +++ b/test/framework/src/test/java/org/opensearch/test/disruption/NetworkDisruptionIT.java @@ -51,7 +51,6 @@ import org.opensearch.transport.TransportService; import java.io.IOException; -import java.io.InputStream; import java.util.Arrays; import java.util.Collection; import java.util.HashSet; @@ -215,12 +214,6 @@ public String executor() { public TransportResponse read(StreamInput in) throws IOException { return ClusterHealthResponse.readResponseFrom(in); } - - @Override - public TransportResponse read(InputStream in) throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'read'"); - } } ); } From b7ee7896e2e8750bc12f74f303432a07e9d88773 Mon Sep 17 00:00:00 2001 From: Vacha Shah Date: Mon, 25 Mar 2024 19:33:15 +0000 Subject: [PATCH 32/36] Extracting protobuf serialization from model classes Signed-off-by: Vacha Shah --- distribution/src/config/opensearch.yml | 4 + .../processor/ApiAnnotationProcessor.java | 5 + ...ProtobufActionListenerResponseHandler.java | 1 - .../search/QueryPhaseResultConsumer.java | 11 +- .../common/document/DocumentField.java | 110 +---------- .../DocumentFieldProtobufSerializer.java | 135 +++++++++++++ .../serializer/DocumentFieldSerializer.java | 24 +++ .../org/opensearch/common/lucene/Lucene.java | 2 - .../java/org/opensearch/search/SearchHit.java | 140 +------------- .../org/opensearch/search/SearchHits.java | 132 +------------ .../opensearch/search/SearchSortValues.java | 41 +--- .../search/fetch/FetchSearchResult.java | 12 +- .../subphase/highlight/HighlightField.java | 17 -- .../HighlightFieldProtobufSerializer.java | 44 +++++ .../serializer/HighlightFieldSerializer.java | 24 +++ .../NestedIdentityProtobufSerializer.java | 49 +++++ .../serializer/NestedIdentitySerializer.java | 23 +++ .../SearchHitProtobufSerializer.java | 179 ++++++++++++++++++ .../serializer/SearchHitSerializer.java | 24 +++ .../SearchHitsProtobufSerializer.java | 157 +++++++++++++++ .../serializer/SearchHitsSerializer.java | 24 +++ .../SearchSortValuesProtobufSerializer.java | 40 ++++ .../SearchSortValuesSerializer.java | 24 +++ .../transport/BaseInboundMessage.java | 3 + .../search/FetchSearchResultProto.proto | 8 +- 25 files changed, 787 insertions(+), 446 deletions(-) create mode 100644 server/src/main/java/org/opensearch/common/document/serializer/DocumentFieldProtobufSerializer.java create mode 100644 server/src/main/java/org/opensearch/common/document/serializer/DocumentFieldSerializer.java create mode 100644 server/src/main/java/org/opensearch/search/fetch/subphase/highlight/serializer/HighlightFieldProtobufSerializer.java create mode 100644 server/src/main/java/org/opensearch/search/fetch/subphase/highlight/serializer/HighlightFieldSerializer.java create mode 100644 server/src/main/java/org/opensearch/search/serializer/NestedIdentityProtobufSerializer.java create mode 100644 server/src/main/java/org/opensearch/search/serializer/NestedIdentitySerializer.java create mode 100644 server/src/main/java/org/opensearch/search/serializer/SearchHitProtobufSerializer.java create mode 100644 server/src/main/java/org/opensearch/search/serializer/SearchHitSerializer.java create mode 100644 server/src/main/java/org/opensearch/search/serializer/SearchHitsProtobufSerializer.java create mode 100644 server/src/main/java/org/opensearch/search/serializer/SearchHitsSerializer.java create mode 100644 server/src/main/java/org/opensearch/search/serializer/SearchSortValuesProtobufSerializer.java create mode 100644 server/src/main/java/org/opensearch/search/serializer/SearchSortValuesSerializer.java diff --git a/distribution/src/config/opensearch.yml b/distribution/src/config/opensearch.yml index 10bab9b3fce92..e0e623652d015 100644 --- a/distribution/src/config/opensearch.yml +++ b/distribution/src/config/opensearch.yml @@ -125,3 +125,7 @@ ${path.logs} # Gates the functionality of enabling Opensearch to use pluggable caches with respective store names via setting. # #opensearch.experimental.feature.pluggable.caching.enabled: false +# +# Gates the functionality of enabling Opensearch to use protobuf with basic searches and for node-to-node communication. +# +#opensearch.experimental.feature.search_with_protobuf.enabled: false diff --git a/libs/common/src/main/java/org/opensearch/common/annotation/processor/ApiAnnotationProcessor.java b/libs/common/src/main/java/org/opensearch/common/annotation/processor/ApiAnnotationProcessor.java index 569f48a8465f3..0d4e342bd9dc6 100644 --- a/libs/common/src/main/java/org/opensearch/common/annotation/processor/ApiAnnotationProcessor.java +++ b/libs/common/src/main/java/org/opensearch/common/annotation/processor/ApiAnnotationProcessor.java @@ -179,6 +179,11 @@ private void process(ExecutableElement executable, ReferenceType ref) { return; } + // Skip protobuf generated classes used in public apis + if (ref.toString().contains("Proto")) { + return; + } + if (ref instanceof DeclaredType) { final DeclaredType declaredType = (DeclaredType) ref; diff --git a/server/src/main/java/org/opensearch/action/ProtobufActionListenerResponseHandler.java b/server/src/main/java/org/opensearch/action/ProtobufActionListenerResponseHandler.java index 7e69bb9dc6cbd..e62eaed1f51a4 100644 --- a/server/src/main/java/org/opensearch/action/ProtobufActionListenerResponseHandler.java +++ b/server/src/main/java/org/opensearch/action/ProtobufActionListenerResponseHandler.java @@ -68,7 +68,6 @@ public String toString() { @Override public Response read(StreamInput in) throws IOException { - // TODO Auto-generated method stub throw new UnsupportedOperationException("Unimplemented method 'read'"); } diff --git a/server/src/main/java/org/opensearch/action/search/QueryPhaseResultConsumer.java b/server/src/main/java/org/opensearch/action/search/QueryPhaseResultConsumer.java index 3081e95dc5e26..f1b06378bd579 100644 --- a/server/src/main/java/org/opensearch/action/search/QueryPhaseResultConsumer.java +++ b/server/src/main/java/org/opensearch/action/search/QueryPhaseResultConsumer.java @@ -38,7 +38,6 @@ import org.opensearch.common.lease.Releasable; import org.opensearch.common.lease.Releasables; import org.opensearch.common.lucene.search.TopDocsAndMaxScore; -import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.util.concurrent.AbstractRunnable; import org.opensearch.core.common.breaker.CircuitBreaker; import org.opensearch.core.common.breaker.CircuitBreakingException; @@ -115,11 +114,7 @@ public QueryPhaseResultConsumer( SearchSourceBuilder source = request.source(); this.hasTopDocs = source == null || source.size() != 0; - if (FeatureFlags.isEnabled(FeatureFlags.PROTOBUF)) { - this.hasAggs = false; - } else { - this.hasAggs = source != null && source.aggregations() != null; - } + this.hasAggs = source != null && source.aggregations() != null; int batchReduceSize = (hasAggs || hasTopDocs) ? Math.min(request.getBatchedReduceSize(), expectedResultSize) : expectedResultSize; this.pendingMerges = new PendingMerges(batchReduceSize, request.resolveTrackTotalHitsUpTo()); } @@ -325,7 +320,7 @@ synchronized long addEstimateAndMaybeBreak(long estimatedSize) { * provided {@link QuerySearchResult}. */ long ramBytesUsedQueryResult(QuerySearchResult result) { - if (hasAggs == false || FeatureFlags.isEnabled(FeatureFlags.PROTOBUF)) { + if (hasAggs == false) { return 0; } return result.aggregations().asSerialized(InternalAggregations::readFrom, namedWriteableRegistry).ramBytesUsed(); @@ -494,7 +489,7 @@ public synchronized List consumeTopDocs() { } public synchronized List consumeAggs() { - if (hasAggs == false || FeatureFlags.isEnabled(FeatureFlags.PROTOBUF)) { + if (hasAggs == false) { return Collections.emptyList(); } List aggsList = new ArrayList<>(); diff --git a/server/src/main/java/org/opensearch/common/document/DocumentField.java b/server/src/main/java/org/opensearch/common/document/DocumentField.java index 71b0c4488dd82..383648e175dc0 100644 --- a/server/src/main/java/org/opensearch/common/document/DocumentField.java +++ b/server/src/main/java/org/opensearch/common/document/DocumentField.java @@ -32,34 +32,22 @@ package org.opensearch.common.document; -import com.google.protobuf.ByteString; -import org.opensearch.OpenSearchException; import org.opensearch.common.annotation.PublicApi; -import org.opensearch.common.util.FeatureFlags; +import org.opensearch.common.document.serializer.DocumentFieldProtobufSerializer; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; -import org.opensearch.core.common.text.Text; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.get.GetResult; import org.opensearch.search.SearchHit; import org.opensearch.server.proto.FetchSearchResultProto; -import org.opensearch.server.proto.FetchSearchResultProto.DocumentFieldValue; -import org.opensearch.server.proto.FetchSearchResultProto.DocumentFieldValue.Builder; import java.io.IOException; -import java.time.Instant; -import java.time.ZoneId; -import java.time.ZonedDateTime; import java.util.ArrayList; -import java.util.Date; -import java.util.HashMap; import java.util.Iterator; -import java.util.LinkedHashMap; import java.util.List; -import java.util.Map; import java.util.Objects; import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; @@ -90,110 +78,16 @@ public DocumentField(String name, List values) { this.values = Objects.requireNonNull(values, "values must not be null"); } - public DocumentField(byte[] in) throws IOException { - assert FeatureFlags.isEnabled(FeatureFlags.PROTOBUF) : "protobuf feature flag is not enabled"; - documentField = FetchSearchResultProto.SearchHit.DocumentField.parseFrom(in); - name = documentField.getName(); - values = new ArrayList<>(); - for (FetchSearchResultProto.DocumentFieldValue value : documentField.getValuesList()) { - values.add(readDocumentFieldValueFromProtobuf(value)); - } - } - public static FetchSearchResultProto.SearchHit.DocumentField convertDocumentFieldToProto(DocumentField documentField) { FetchSearchResultProto.SearchHit.DocumentField.Builder builder = FetchSearchResultProto.SearchHit.DocumentField.newBuilder(); builder.setName(documentField.getName()); for (Object value : documentField.getValues()) { FetchSearchResultProto.DocumentFieldValue.Builder valueBuilder = FetchSearchResultProto.DocumentFieldValue.newBuilder(); - builder.addValues(convertDocumentFieldValueToProto(value, valueBuilder)); + builder.addValues(DocumentFieldProtobufSerializer.convertDocumentFieldValueToProto(value, valueBuilder)); } return builder.build(); } - private static DocumentFieldValue.Builder convertDocumentFieldValueToProto(Object value, Builder valueBuilder) { - if (value == null) { - // null is not allowed in protobuf, so we use a special string to represent null - return valueBuilder.setValueString("null"); - } - Class type = value.getClass(); - if (type == String.class) { - valueBuilder.setValueString((String) value); - } else if (type == Integer.class) { - valueBuilder.setValueInt((Integer) value); - } else if (type == Long.class) { - valueBuilder.setValueLong((Long) value); - } else if (type == Float.class) { - valueBuilder.setValueFloat((Float) value); - } else if (type == Double.class) { - valueBuilder.setValueDouble((Double) value); - } else if (type == Boolean.class) { - valueBuilder.setValueBool((Boolean) value); - } else if (type == byte[].class) { - valueBuilder.addValueByteArray(ByteString.copyFrom((byte[]) value)); - } else if (type == List.class) { - List list = (List) value; - for (Object listValue : list) { - valueBuilder.addValueArrayList(convertDocumentFieldValueToProto(listValue, valueBuilder)); - } - } else if (type == Map.class || type == HashMap.class || type == LinkedHashMap.class) { - Map map = (Map) value; - for (Map.Entry entry : map.entrySet()) { - valueBuilder.putValueMap(entry.getKey(), convertDocumentFieldValueToProto(entry.getValue(), valueBuilder).build()); - } - } else if (type == Date.class) { - valueBuilder.setValueDate(((Date) value).getTime()); - } else if (type == ZonedDateTime.class) { - valueBuilder.setValueZonedDate(((ZonedDateTime) value).getZone().getId()); - valueBuilder.setValueZonedTime(((ZonedDateTime) value).toInstant().toEpochMilli()); - } else if (type == Text.class) { - valueBuilder.setValueText(((Text) value).string()); - } else { - throw new OpenSearchException("Can't convert generic value of type [" + type + "] to protobuf"); - } - return valueBuilder; - } - - private Object readDocumentFieldValueFromProtobuf(FetchSearchResultProto.DocumentFieldValue documentFieldValue) throws IOException { - if (documentFieldValue.hasValueString()) { - return documentFieldValue.getValueString(); - } else if (documentFieldValue.hasValueInt()) { - return documentFieldValue.getValueInt(); - } else if (documentFieldValue.hasValueLong()) { - return documentFieldValue.getValueLong(); - } else if (documentFieldValue.hasValueFloat()) { - return documentFieldValue.getValueFloat(); - } else if (documentFieldValue.hasValueDouble()) { - return documentFieldValue.getValueDouble(); - } else if (documentFieldValue.hasValueBool()) { - return documentFieldValue.getValueBool(); - } else if (documentFieldValue.getValueByteArrayList().size() > 0) { - return documentFieldValue.getValueByteArrayList().toArray(); - } else if (documentFieldValue.getValueArrayListList().size() > 0) { - List list = new ArrayList<>(); - for (FetchSearchResultProto.DocumentFieldValue value : documentFieldValue.getValueArrayListList()) { - list.add(readDocumentFieldValueFromProtobuf(value)); - } - return list; - } else if (documentFieldValue.getValueMapMap().size() > 0) { - Map map = Map.of(); - for (Map.Entry entrySet : documentFieldValue.getValueMapMap().entrySet()) { - map.put(entrySet.getKey(), readDocumentFieldValueFromProtobuf(entrySet.getValue())); - } - return map; - } else if (documentFieldValue.hasValueDate()) { - return new Date(documentFieldValue.getValueDate()); - } else if (documentFieldValue.hasValueZonedDate() && documentFieldValue.hasValueZonedTime()) { - return ZonedDateTime.ofInstant( - Instant.ofEpochMilli(documentFieldValue.getValueZonedTime()), - ZoneId.of(documentFieldValue.getValueZonedDate()) - ); - } else if (documentFieldValue.hasValueText()) { - return new Text(documentFieldValue.getValueText()); - } else { - throw new IOException("Can't read generic value of type [" + documentFieldValue + "]"); - } - } - /** * The name of the field. */ diff --git a/server/src/main/java/org/opensearch/common/document/serializer/DocumentFieldProtobufSerializer.java b/server/src/main/java/org/opensearch/common/document/serializer/DocumentFieldProtobufSerializer.java new file mode 100644 index 0000000000000..614837ce7c9d3 --- /dev/null +++ b/server/src/main/java/org/opensearch/common/document/serializer/DocumentFieldProtobufSerializer.java @@ -0,0 +1,135 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.common.document.serializer; + +import com.google.protobuf.ByteString; +import org.opensearch.OpenSearchException; +import org.opensearch.common.document.DocumentField; +import org.opensearch.core.common.text.Text; +import org.opensearch.server.proto.FetchSearchResultProto; +import org.opensearch.server.proto.FetchSearchResultProto.DocumentFieldValue; +import org.opensearch.server.proto.FetchSearchResultProto.DocumentFieldValue.Builder; + +import java.io.IOException; +import java.io.InputStream; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +public class DocumentFieldProtobufSerializer implements DocumentFieldSerializer { + + private FetchSearchResultProto.SearchHit.DocumentField documentField; + + @Override + public DocumentField createDocumentField(InputStream inputStream) throws IOException { + documentField = FetchSearchResultProto.SearchHit.DocumentField.parseFrom(inputStream); + String name = documentField.getName(); + List values = new ArrayList<>(); + for (FetchSearchResultProto.DocumentFieldValue value : documentField.getValuesList()) { + values.add(readDocumentFieldValueFromProtobuf(value)); + } + return new DocumentField(name, values); + } + + private Object readDocumentFieldValueFromProtobuf(FetchSearchResultProto.DocumentFieldValue documentFieldValue) throws IOException { + if (documentFieldValue.hasValueString()) { + return documentFieldValue.getValueString(); + } else if (documentFieldValue.hasValueInt()) { + return documentFieldValue.getValueInt(); + } else if (documentFieldValue.hasValueLong()) { + return documentFieldValue.getValueLong(); + } else if (documentFieldValue.hasValueFloat()) { + return documentFieldValue.getValueFloat(); + } else if (documentFieldValue.hasValueDouble()) { + return documentFieldValue.getValueDouble(); + } else if (documentFieldValue.hasValueBool()) { + return documentFieldValue.getValueBool(); + } else if (documentFieldValue.getValueByteArrayList().size() > 0) { + return documentFieldValue.getValueByteArrayList().toArray(); + } else if (documentFieldValue.getValueArrayListList().size() > 0) { + List list = new ArrayList<>(); + for (FetchSearchResultProto.DocumentFieldValue value : documentFieldValue.getValueArrayListList()) { + list.add(readDocumentFieldValueFromProtobuf(value)); + } + return list; + } else if (documentFieldValue.getValueMapMap().size() > 0) { + Map map = Map.of(); + for (Map.Entry entrySet : documentFieldValue.getValueMapMap().entrySet()) { + map.put(entrySet.getKey(), readDocumentFieldValueFromProtobuf(entrySet.getValue())); + } + return map; + } else if (documentFieldValue.hasValueDate()) { + return new Date(documentFieldValue.getValueDate()); + } else if (documentFieldValue.hasValueZonedDate() && documentFieldValue.hasValueZonedTime()) { + return ZonedDateTime.ofInstant( + Instant.ofEpochMilli(documentFieldValue.getValueZonedTime()), + ZoneId.of(documentFieldValue.getValueZonedDate()) + ); + } else if (documentFieldValue.hasValueText()) { + return new Text(documentFieldValue.getValueText()); + } else { + throw new IOException("Can't read generic value of type [" + documentFieldValue + "]"); + } + } + + public static DocumentFieldValue.Builder convertDocumentFieldValueToProto(Object value, Builder valueBuilder) { + if (value == null) { + // null is not allowed in protobuf, so we use a special string to represent null + return valueBuilder.setValueString("null"); + } + Class type = value.getClass(); + if (type == String.class) { + valueBuilder.setValueString((String) value); + } else if (type == Integer.class) { + valueBuilder.setValueInt((Integer) value); + } else if (type == Long.class) { + valueBuilder.setValueLong((Long) value); + } else if (type == Float.class) { + valueBuilder.setValueFloat((Float) value); + } else if (type == Double.class) { + valueBuilder.setValueDouble((Double) value); + } else if (type == Boolean.class) { + valueBuilder.setValueBool((Boolean) value); + } else if (type == byte[].class) { + valueBuilder.addValueByteArray(ByteString.copyFrom((byte[]) value)); + } else if (type == List.class) { + List list = (List) value; + for (Object listValue : list) { + valueBuilder.addValueArrayList(convertDocumentFieldValueToProto(listValue, valueBuilder)); + } + } else if (type == Map.class || type == HashMap.class || type == LinkedHashMap.class) { + Map map = (Map) value; + for (Map.Entry entry : map.entrySet()) { + valueBuilder.putValueMap(entry.getKey(), convertDocumentFieldValueToProto(entry.getValue(), valueBuilder).build()); + } + } else if (type == Date.class) { + valueBuilder.setValueDate(((Date) value).getTime()); + } else if (type == ZonedDateTime.class) { + valueBuilder.setValueZonedDate(((ZonedDateTime) value).getZone().getId()); + valueBuilder.setValueZonedTime(((ZonedDateTime) value).toInstant().toEpochMilli()); + } else if (type == Text.class) { + valueBuilder.setValueText(((Text) value).string()); + } else { + throw new OpenSearchException("Can't convert generic value of type [" + type + "] to protobuf"); + } + return valueBuilder; + } + +} diff --git a/server/src/main/java/org/opensearch/common/document/serializer/DocumentFieldSerializer.java b/server/src/main/java/org/opensearch/common/document/serializer/DocumentFieldSerializer.java new file mode 100644 index 0000000000000..04d0b5a81bcf6 --- /dev/null +++ b/server/src/main/java/org/opensearch/common/document/serializer/DocumentFieldSerializer.java @@ -0,0 +1,24 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.common.document.serializer; + +import org.opensearch.common.document.DocumentField; + +import java.io.IOException; + +public interface DocumentFieldSerializer { + + DocumentField createDocumentField(T inputStream) throws IOException; + +} diff --git a/server/src/main/java/org/opensearch/common/lucene/Lucene.java b/server/src/main/java/org/opensearch/common/lucene/Lucene.java index 51025209a5fd1..d195859b63fed 100644 --- a/server/src/main/java/org/opensearch/common/lucene/Lucene.java +++ b/server/src/main/java/org/opensearch/common/lucene/Lucene.java @@ -85,7 +85,6 @@ import org.opensearch.common.Nullable; import org.opensearch.common.SuppressForbidden; import org.opensearch.common.lucene.search.TopDocsAndMaxScore; -import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.util.iterable.Iterables; import org.opensearch.core.common.Strings; import org.opensearch.core.common.io.stream.StreamInput; @@ -654,7 +653,6 @@ public static Explanation readExplanation(StreamInput in) throws IOException { } public static Explanation readExplanation(byte[] in) throws IOException { - assert FeatureFlags.isEnabled(FeatureFlags.PROTOBUF) : "protobuf feature flag is not enabled"; FetchSearchResultProto.SearchHit.Explanation explanationProto = FetchSearchResultProto.SearchHit.Explanation.parseFrom(in); boolean match = explanationProto.getMatch(); String description = explanationProto.getDescription(); diff --git a/server/src/main/java/org/opensearch/search/SearchHit.java b/server/src/main/java/org/opensearch/search/SearchHit.java index 1e43e60611eb9..3cd85488bd0c6 100644 --- a/server/src/main/java/org/opensearch/search/SearchHit.java +++ b/server/src/main/java/org/opensearch/search/SearchHit.java @@ -32,7 +32,6 @@ package org.opensearch.search; -import com.google.protobuf.ByteString; import org.apache.lucene.search.Explanation; import org.opensearch.OpenSearchParseException; import org.opensearch.Version; @@ -69,12 +68,11 @@ import org.opensearch.rest.action.search.RestSearchAction; import org.opensearch.search.fetch.subphase.highlight.HighlightField; import org.opensearch.search.lookup.SourceLookup; +import org.opensearch.search.serializer.SearchHitProtobufSerializer; import org.opensearch.server.proto.FetchSearchResultProto; import org.opensearch.transport.RemoteClusterAware; import java.io.IOException; -import java.io.OutputStream; -import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -172,7 +170,7 @@ public SearchHit( this.documentFields = documentFields == null ? emptyMap() : documentFields; this.metaFields = metaFields == null ? emptyMap() : metaFields; if (FeatureFlags.isEnabled(FeatureFlags.PROTOBUF)) { - this.searchHitProto = convertHitToProto(this); + this.searchHitProto = SearchHitProtobufSerializer.convertHitToProto(this); } } @@ -246,114 +244,6 @@ public SearchHit(StreamInput in) throws IOException { } } - public SearchHit(byte[] in) throws IOException { - assert FeatureFlags.isEnabled(FeatureFlags.PROTOBUF) : "protobuf feature flag is not enabled"; - this.searchHitProto = FetchSearchResultProto.SearchHit.parseFrom(in); - this.docId = -1; - this.score = this.searchHitProto.getScore(); - this.id = new Text(this.searchHitProto.getId()); - if (!this.searchHitProto.hasNestedIdentity() && this.searchHitProto.getNestedIdentity().toByteArray().length > 0) { - this.nestedIdentity = new NestedIdentity(this.searchHitProto.getNestedIdentity().toByteArray()); - } else { - this.nestedIdentity = null; - } - this.version = this.searchHitProto.getVersion(); - this.seqNo = this.searchHitProto.getSeqNo(); - this.primaryTerm = this.searchHitProto.getPrimaryTerm(); - this.source = BytesReference.fromByteBuffer(ByteBuffer.wrap(this.searchHitProto.getSource().toByteArray())); - if (source.length() == 0) { - source = null; - } - this.documentFields = new HashMap<>(); - this.searchHitProto.getDocumentFieldsMap().forEach((k, v) -> { - try { - this.documentFields.put(k, new DocumentField(v.toByteArray())); - } catch (IOException e) { - throw new OpenSearchParseException("failed to parse document field", e); - } - }); - this.metaFields = new HashMap<>(); - this.searchHitProto.getMetaFieldsMap().forEach((k, v) -> { - try { - this.metaFields.put(k, new DocumentField(v.toByteArray())); - } catch (IOException e) { - throw new OpenSearchParseException("failed to parse document field", e); - } - }); - this.highlightFields = new HashMap<>(); - this.searchHitProto.getHighlightFieldsMap().forEach((k, v) -> { - try { - this.highlightFields.put(k, new HighlightField(v.toByteArray())); - } catch (IOException e) { - throw new OpenSearchParseException("failed to parse highlight field", e); - } - }); - this.sortValues = new SearchSortValues(this.searchHitProto.getSortValues().toByteArray()); - if (this.searchHitProto.getMatchedQueriesCount() > 0) { - this.matchedQueries = new LinkedHashMap<>(this.searchHitProto.getMatchedQueriesCount()); - for (String query : this.searchHitProto.getMatchedQueriesList()) { - matchedQueries.put(query, Float.NaN); - } - } - if (this.searchHitProto.getMatchedQueriesWithScoresCount() > 0) { - Map tempMap = this.searchHitProto.getMatchedQueriesWithScoresMap() - .entrySet() - .stream() - .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue())); - this.matchedQueries = tempMap.entrySet() - .stream() - .sorted(Map.Entry.comparingByKey()) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (oldValue, newValue) -> oldValue, LinkedHashMap::new)); - } - if (this.searchHitProto.hasExplanation()) { - this.explanation = readExplanation(this.searchHitProto.getExplanation().toByteArray()); - } - SearchShardTarget searchShardTarget = new SearchShardTarget( - this.searchHitProto.getShard().getNodeId(), - new ShardId( - this.searchHitProto.getShard().getShardId().getIndexName(), - this.searchHitProto.getShard().getShardId().getIndexUUID(), - this.searchHitProto.getShard().getShardId().getShardId() - ), - this.searchHitProto.getShard().getClusterAlias(), - OriginalIndices.NONE - ); - shard(searchShardTarget); - if (this.searchHitProto.getInnerHitsCount() > 0) { - this.innerHits = new HashMap<>(); - this.searchHitProto.getInnerHitsMap().forEach((k, v) -> { - try { - this.innerHits.put(k, new SearchHits(v.toByteArray())); - } catch (IOException e) { - throw new OpenSearchParseException("failed to parse inner hits", e); - } - }); - } else { - this.innerHits = null; - } - - } - - public static FetchSearchResultProto.SearchHit convertHitToProto(SearchHit hit) { - FetchSearchResultProto.SearchHit.Builder searchHitBuilder = FetchSearchResultProto.SearchHit.newBuilder(); - if (hit.getIndex() != null) { - searchHitBuilder.setIndex(hit.getIndex()); - } - searchHitBuilder.setId(hit.getId()); - searchHitBuilder.setScore(hit.getScore()); - searchHitBuilder.setSeqNo(hit.getSeqNo()); - searchHitBuilder.setPrimaryTerm(hit.getPrimaryTerm()); - searchHitBuilder.setVersion(hit.getVersion()); - searchHitBuilder.setDocId(hit.docId); - if (hit.getSourceRef() != null) { - searchHitBuilder.setSource(ByteString.copyFrom(hit.getSourceRef().toBytesRef().bytes)); - } - for (Map.Entry entry : hit.getFields().entrySet()) { - searchHitBuilder.putDocumentFields(entry.getKey(), DocumentField.convertDocumentFieldToProto(entry.getValue())); - } - return searchHitBuilder.build(); - } - private static final Text SINGLE_MAPPING_TYPE = new Text(MapperService.SINGLE_MAPPING_NAME); @Override @@ -408,10 +298,6 @@ public void writeTo(StreamOutput out) throws IOException { } } - public void writeTo(OutputStream out) throws IOException { - out.write(this.searchHitProto.toByteArray()); - } - public int docId() { return this.docId; } @@ -571,7 +457,7 @@ public void setDocumentField(String fieldName, DocumentField field) { if (documentFields.isEmpty()) this.documentFields = new HashMap<>(); this.documentFields.put(fieldName, field); if (FeatureFlags.isEnabled(FeatureFlags.PROTOBUF)) { - this.searchHitProto = convertHitToProto(this); + this.searchHitProto = SearchHitProtobufSerializer.convertHitToProto(this); } } @@ -1188,26 +1074,6 @@ public NestedIdentity(String field, int offset, NestedIdentity child) { child = in.readOptionalWriteable(NestedIdentity::new); } - NestedIdentity(byte[] in) throws IOException { - assert FeatureFlags.isEnabled(FeatureFlags.PROTOBUF) : "protobuf feature flag is not enabled"; - FetchSearchResultProto.SearchHit.NestedIdentity proto = FetchSearchResultProto.SearchHit.NestedIdentity.parseFrom(in); - if (proto.hasField()) { - field = new Text(proto.getField()); - } else { - field = null; - } - if (proto.hasOffset()) { - offset = proto.getOffset(); - } else { - offset = -1; - } - if (proto.hasChild()) { - child = new NestedIdentity(proto.getChild().toByteArray()); - } else { - child = null; - } - } - /** * Returns the nested field in the source this hit originates from */ diff --git a/server/src/main/java/org/opensearch/search/SearchHits.java b/server/src/main/java/org/opensearch/search/SearchHits.java index 630a7c90f8b3d..cd5854898fccd 100644 --- a/server/src/main/java/org/opensearch/search/SearchHits.java +++ b/server/src/main/java/org/opensearch/search/SearchHits.java @@ -32,13 +32,9 @@ package org.opensearch.search; -import com.google.protobuf.ByteString; import org.apache.lucene.search.SortField; import org.apache.lucene.search.TotalHits; import org.apache.lucene.search.TotalHits.Relation; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.SuppressForbidden; -import org.opensearch.OpenSearchException; import org.opensearch.common.Nullable; import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.lucene.Lucene; @@ -50,12 +46,9 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.rest.action.search.RestSearchAction; -import org.opensearch.server.proto.FetchSearchResultProto; -import org.opensearch.server.proto.QuerySearchResultProto; +import org.opensearch.search.serializer.SearchHitsProtobufSerializer; import java.io.IOException; -import java.io.OutputStream; -import java.math.BigInteger; import java.util.ArrayList; import java.util.Arrays; import java.util.Iterator; @@ -112,84 +105,10 @@ public SearchHits( this.collapseField = collapseField; this.collapseValues = collapseValues; if (FeatureFlags.isEnabled(FeatureFlags.PROTOBUF_SETTING)) { - this.searchHitsProto = convertHitsToProto(this); + this.searchHitsProto = SearchHitsProtobufSerializer.convertHitsToProto(this); } } - public static FetchSearchResultProto.SearchHits convertHitsToProto(SearchHits hits) { - List searchHitList = new ArrayList<>(); - for (SearchHit hit : hits) { - searchHitList.add(SearchHit.convertHitToProto(hit)); - } - QuerySearchResultProto.TotalHits.Builder totalHitsBuilder = QuerySearchResultProto.TotalHits.newBuilder(); - if (hits.getTotalHits() != null) { - totalHitsBuilder.setValue(hits.getTotalHits().value); - totalHitsBuilder.setRelation( - hits.getTotalHits().relation == Relation.EQUAL_TO - ? QuerySearchResultProto.TotalHits.Relation.EQUAL_TO - : QuerySearchResultProto.TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO - ); - } - FetchSearchResultProto.SearchHits.Builder searchHitsBuilder = FetchSearchResultProto.SearchHits.newBuilder(); - searchHitsBuilder.setMaxScore(hits.getMaxScore()); - searchHitsBuilder.addAllHits(searchHitList); - searchHitsBuilder.setTotalHits(totalHitsBuilder.build()); - if (hits.getSortFields() != null && hits.getSortFields().length > 0) { - for (SortField sortField : hits.getSortFields()) { - FetchSearchResultProto.SortField.Builder sortFieldBuilder = FetchSearchResultProto.SortField.newBuilder(); - if (sortField.getField() != null) { - sortFieldBuilder.setField(sortField.getField()); - } - sortFieldBuilder.setType(FetchSearchResultProto.SortField.Type.valueOf(sortField.getType().name())); - searchHitsBuilder.addSortFields(sortFieldBuilder.build()); - } - } - if (hits.getCollapseField() != null) { - searchHitsBuilder.setCollapseField(hits.getCollapseField()); - for (Object value : hits.getCollapseValues()) { - FetchSearchResultProto.CollapseValue.Builder collapseValueBuilder = FetchSearchResultProto.CollapseValue.newBuilder(); - try { - collapseValueBuilder = readCollapseValueForProtobuf(value, collapseValueBuilder); - } catch (IOException e) { - throw new OpenSearchException(e); - } - searchHitsBuilder.addCollapseValues(collapseValueBuilder.build()); - } - } - return searchHitsBuilder.build(); - } - - private static FetchSearchResultProto.CollapseValue.Builder readCollapseValueForProtobuf( - Object collapseValue, - FetchSearchResultProto.CollapseValue.Builder collapseValueBuilder - ) throws IOException { - Class type = collapseValue.getClass(); - if (type == String.class) { - collapseValueBuilder.setCollapseString((String) collapseValue); - } else if (type == Integer.class || type == Short.class) { - collapseValueBuilder.setCollapseInt((Integer) collapseValue); - } else if (type == Long.class) { - collapseValueBuilder.setCollapseLong((Long) collapseValue); - } else if (type == Float.class) { - collapseValueBuilder.setCollapseFloat((Float) collapseValue); - } else if (type == Double.class) { - collapseValueBuilder.setCollapseDouble((Double) collapseValue); - } else if (type == Byte.class) { - byte b = (Byte) collapseValue; - collapseValueBuilder.setCollapseBytes(ByteString.copyFrom(new byte[] { b })); - } else if (type == Boolean.class) { - collapseValueBuilder.setCollapseBool((Boolean) collapseValue); - } else if (type == BytesRef.class) { - collapseValueBuilder.setCollapseBytes(ByteString.copyFrom(((BytesRef) collapseValue).bytes)); - } else if (type == BigInteger.class) { - BigInteger bigInt = (BigInteger) collapseValue; - collapseValueBuilder.setCollapseBytes(ByteString.copyFrom(bigInt.toByteArray())); - } else { - throw new IOException("Can't handle sort field value of type [" + type + "]"); - } - return collapseValueBuilder; - } - public SearchHits(StreamInput in) throws IOException { if (in.readBoolean()) { totalHits = Lucene.readTotalHits(in); @@ -212,50 +131,6 @@ public SearchHits(StreamInput in) throws IOException { collapseValues = in.readOptionalArray(Lucene::readSortValue, Object[]::new); } - @SuppressForbidden(reason = "serialization of object to protobuf") - public SearchHits(byte[] in) throws IOException { - assert FeatureFlags.isEnabled(FeatureFlags.PROTOBUF) : "protobuf feature flag is not enabled"; - this.searchHitsProto = org.opensearch.server.proto.FetchSearchResultProto.SearchHits.parseFrom(in); - this.hits = new SearchHit[this.searchHitsProto.getHitsCount()]; - for (int i = 0; i < this.searchHitsProto.getHitsCount(); i++) { - this.hits[i] = new SearchHit(this.searchHitsProto.getHits(i).toByteArray()); - } - this.totalHits = new TotalHits( - this.searchHitsProto.getTotalHits().getValue(), - Relation.valueOf(this.searchHitsProto.getTotalHits().getRelation().toString()) - ); - this.maxScore = this.searchHitsProto.getMaxScore(); - this.sortFields = this.searchHitsProto.getSortFieldsList() - .stream() - .map(sortField -> new SortField(sortField.getField(), SortField.Type.valueOf(sortField.getType().toString()))) - .toArray(SortField[]::new); - this.collapseField = this.searchHitsProto.getCollapseField(); - this.collapseValues = new Object[this.searchHitsProto.getCollapseValuesCount()]; - for (int i = 0; i < this.searchHitsProto.getCollapseValuesCount(); i++) { - this.collapseValues[i] = readCollapseValueFromProtobuf(this.searchHitsProto.getCollapseValues(i)); - } - } - - private Object readCollapseValueFromProtobuf(FetchSearchResultProto.CollapseValue collapseValue) throws IOException { - if (collapseValue.hasCollapseString()) { - return collapseValue.getCollapseString(); - } else if (collapseValue.hasCollapseInt()) { - return collapseValue.getCollapseInt(); - } else if (collapseValue.hasCollapseLong()) { - return collapseValue.getCollapseLong(); - } else if (collapseValue.hasCollapseFloat()) { - return collapseValue.getCollapseFloat(); - } else if (collapseValue.hasCollapseDouble()) { - return collapseValue.getCollapseDouble(); - } else if (collapseValue.hasCollapseBytes()) { - return new BytesRef(collapseValue.getCollapseBytes().toByteArray()); - } else if (collapseValue.hasCollapseBool()) { - return collapseValue.getCollapseBool(); - } else { - throw new IOException("Can't handle sort field value of type [" + collapseValue + "]"); - } - } - @Override public void writeTo(StreamOutput out) throws IOException { final boolean hasTotalHits = totalHits != null; @@ -475,7 +350,4 @@ private static Relation parseRelation(String relation) { } } - public void writeTo(OutputStream out) throws IOException { - out.write(searchHitsProto.toByteArray()); - } } diff --git a/server/src/main/java/org/opensearch/search/SearchSortValues.java b/server/src/main/java/org/opensearch/search/SearchSortValues.java index 1eef677ae6863..d03cc80b90de3 100644 --- a/server/src/main/java/org/opensearch/search/SearchSortValues.java +++ b/server/src/main/java/org/opensearch/search/SearchSortValues.java @@ -32,13 +32,9 @@ package org.opensearch.search; -import com.google.protobuf.ByteString; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.SuppressForbidden; -import org.opensearch.OpenSearchException; import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.lucene.Lucene; -import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; @@ -47,12 +43,8 @@ import org.opensearch.core.xcontent.XContentParser; import org.opensearch.core.xcontent.XContentParserUtils; import org.opensearch.search.SearchHit.Fields; -import org.opensearch.server.proto.FetchSearchResultProto; -import java.io.ByteArrayInputStream; import java.io.IOException; -import java.io.InputStream; -import java.io.ObjectInputStream; import java.util.Arrays; import java.util.Objects; @@ -75,6 +67,11 @@ public class SearchSortValues implements ToXContentFragment, Writeable { this.rawSortValues = EMPTY_ARRAY; } + public SearchSortValues(Object[] sortValues, Object[] rawSortValues) { + this.formattedSortValues = Objects.requireNonNull(sortValues, "sort values must not be empty"); + this.rawSortValues = rawSortValues; + } + public SearchSortValues(Object[] rawSortValues, DocValueFormat[] sortValueFormats) { Objects.requireNonNull(rawSortValues); Objects.requireNonNull(sortValueFormats); @@ -102,34 +99,6 @@ public SearchSortValues(Object[] rawSortValues, DocValueFormat[] sortValueFormat this.rawSortValues = in.readArray(Lucene::readSortValue, Object[]::new); } - @SuppressForbidden(reason = "We need to read from a byte array") - SearchSortValues(byte[] in) throws IOException { - assert FeatureFlags.isEnabled(FeatureFlags.PROTOBUF) : "protobuf feature flag is not enabled"; - FetchSearchResultProto.SearchHit.SearchSortValues searchSortValues = FetchSearchResultProto.SearchHit.SearchSortValues.parseFrom( - in - ); - this.formattedSortValues = new Object[searchSortValues.getFormattedSortValuesCount()]; - for (int i = 0; i < searchSortValues.getFormattedSortValuesCount(); i++) { - ByteString formattedSortValue = searchSortValues.getFormattedSortValues(i); - InputStream is = new ByteArrayInputStream(formattedSortValue.toByteArray()); - try (ObjectInputStream ois = new ObjectInputStream(is)) { - this.formattedSortValues[i] = ois.readObject(); - } catch (ClassNotFoundException e) { - throw new OpenSearchException(e); - } - } - this.rawSortValues = new Object[searchSortValues.getRawSortValuesCount()]; - for (int i = 0; i < searchSortValues.getRawSortValuesCount(); i++) { - ByteString rawSortValue = searchSortValues.getRawSortValues(i); - InputStream is = new ByteArrayInputStream(rawSortValue.toByteArray()); - try (ObjectInputStream ois = new ObjectInputStream(is)) { - this.rawSortValues[i] = ois.readObject(); - } catch (ClassNotFoundException e) { - throw new OpenSearchException(e); - } - } - } - @Override public void writeTo(StreamOutput out) throws IOException { out.writeArray(Lucene::writeSortValue, this.formattedSortValues); diff --git a/server/src/main/java/org/opensearch/search/fetch/FetchSearchResult.java b/server/src/main/java/org/opensearch/search/fetch/FetchSearchResult.java index a9c70d336225d..b46b2c5ca12d8 100644 --- a/server/src/main/java/org/opensearch/search/fetch/FetchSearchResult.java +++ b/server/src/main/java/org/opensearch/search/fetch/FetchSearchResult.java @@ -42,9 +42,11 @@ import org.opensearch.search.SearchShardTarget; import org.opensearch.search.internal.ShardSearchContextId; import org.opensearch.search.query.QuerySearchResult; +import org.opensearch.search.serializer.SearchHitsProtobufSerializer; import org.opensearch.server.proto.FetchSearchResultProto; import org.opensearch.server.proto.ShardSearchRequestProto; +import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -79,7 +81,8 @@ public FetchSearchResult(InputStream in) throws IOException { this.fetchSearchResultProto.getContextId().getSessionId(), this.fetchSearchResultProto.getContextId().getId() ); - hits = new SearchHits(this.fetchSearchResultProto.getHits().toByteArray()); + SearchHitsProtobufSerializer protobufSerializer = new SearchHitsProtobufSerializer(); + hits = protobufSerializer.createSearchHits(new ByteArrayInputStream(this.fetchSearchResultProto.getHits().toByteArray())); } public FetchSearchResult(ShardSearchContextId id, SearchShardTarget shardTarget) { @@ -106,7 +109,9 @@ public void hits(SearchHits hits) { assert assertNoSearchTarget(hits); this.hits = hits; if (FeatureFlags.isEnabled(FeatureFlags.PROTOBUF_SETTING) && this.fetchSearchResultProto != null) { - this.fetchSearchResultProto = this.fetchSearchResultProto.toBuilder().setHits(SearchHits.convertHitsToProto(hits)).build(); + this.fetchSearchResultProto = this.fetchSearchResultProto.toBuilder() + .setHits(SearchHitsProtobufSerializer.convertHitsToProto(hits)) + .build(); } } @@ -121,7 +126,8 @@ public SearchHits hits() { if (FeatureFlags.isEnabled(FeatureFlags.PROTOBUF_SETTING) && this.fetchSearchResultProto != null) { SearchHits hits; try { - hits = new SearchHits(this.fetchSearchResultProto.getHits().toByteArray()); + SearchHitsProtobufSerializer protobufSerializer = new SearchHitsProtobufSerializer(); + hits = protobufSerializer.createSearchHits(new ByteArrayInputStream(this.fetchSearchResultProto.getHits().toByteArray())); return hits; } catch (IOException e) { throw new RuntimeException(e); diff --git a/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/HighlightField.java b/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/HighlightField.java index 7ea5f5fdca7d0..30effe2826d76 100644 --- a/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/HighlightField.java +++ b/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/HighlightField.java @@ -33,7 +33,6 @@ package org.opensearch.search.fetch.subphase.highlight; import org.opensearch.common.annotation.PublicApi; -import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.common.ParsingException; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; @@ -42,7 +41,6 @@ import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.server.proto.FetchSearchResultProto; import java.io.IOException; import java.util.ArrayList; @@ -79,21 +77,6 @@ public HighlightField(StreamInput in) throws IOException { } } - public HighlightField(byte[] in) throws IOException { - assert FeatureFlags.isEnabled(FeatureFlags.PROTOBUF) : "protobuf feature flag is not enabled"; - FetchSearchResultProto.SearchHit.HighlightField highlightField = FetchSearchResultProto.SearchHit.HighlightField.parseFrom(in); - name = highlightField.getName(); - if (highlightField.getFragmentsCount() == 0) { - fragments = Text.EMPTY_ARRAY; - } else { - List values = new ArrayList<>(); - for (String fragment : highlightField.getFragmentsList()) { - values.add(new Text(fragment)); - } - fragments = values.toArray(new Text[0]); - } - } - public HighlightField(String name, Text[] fragments) { this.name = Objects.requireNonNull(name, "missing highlight field name"); this.fragments = fragments; diff --git a/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/serializer/HighlightFieldProtobufSerializer.java b/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/serializer/HighlightFieldProtobufSerializer.java new file mode 100644 index 0000000000000..dbf9ac5ac48ee --- /dev/null +++ b/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/serializer/HighlightFieldProtobufSerializer.java @@ -0,0 +1,44 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.search.fetch.subphase.highlight.serializer; + +import org.opensearch.core.common.text.Text; +import org.opensearch.search.fetch.subphase.highlight.HighlightField; +import org.opensearch.server.proto.FetchSearchResultProto; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; + +public class HighlightFieldProtobufSerializer implements HighlightFieldSerializer { + + @Override + public HighlightField createHighLightField(InputStream inputStream) throws IOException { + FetchSearchResultProto.SearchHit.HighlightField highlightField = FetchSearchResultProto.SearchHit.HighlightField.parseFrom( + inputStream + ); + String name = highlightField.getName(); + Text[] fragments = Text.EMPTY_ARRAY; + if (highlightField.getFragmentsCount() > 0) { + List values = new ArrayList<>(); + for (String fragment : highlightField.getFragmentsList()) { + values.add(new Text(fragment)); + } + fragments = values.toArray(new Text[0]); + } + return new HighlightField(name, fragments); + } + +} diff --git a/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/serializer/HighlightFieldSerializer.java b/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/serializer/HighlightFieldSerializer.java new file mode 100644 index 0000000000000..d62016969885d --- /dev/null +++ b/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/serializer/HighlightFieldSerializer.java @@ -0,0 +1,24 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.search.fetch.subphase.highlight.serializer; + +import org.opensearch.search.fetch.subphase.highlight.HighlightField; + +import java.io.IOException; +import java.io.InputStream; + +public interface HighlightFieldSerializer { + + HighlightField createHighLightField(InputStream inputStream) throws IOException; +} diff --git a/server/src/main/java/org/opensearch/search/serializer/NestedIdentityProtobufSerializer.java b/server/src/main/java/org/opensearch/search/serializer/NestedIdentityProtobufSerializer.java new file mode 100644 index 0000000000000..fc6902193af84 --- /dev/null +++ b/server/src/main/java/org/opensearch/search/serializer/NestedIdentityProtobufSerializer.java @@ -0,0 +1,49 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.search.serializer; + +import org.opensearch.search.SearchHit.NestedIdentity; +import org.opensearch.server.proto.FetchSearchResultProto; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; + +public class NestedIdentityProtobufSerializer implements NestedIdentitySerializer { + + @Override + public NestedIdentity createNestedIdentity(InputStream inputStream) throws IOException { + FetchSearchResultProto.SearchHit.NestedIdentity proto = FetchSearchResultProto.SearchHit.NestedIdentity.parseFrom(inputStream); + String field; + int offset; + NestedIdentity child; + if (proto.hasField()) { + field = proto.getField(); + } else { + field = null; + } + if (proto.hasOffset()) { + offset = proto.getOffset(); + } else { + offset = -1; + } + if (proto.hasChild()) { + child = createNestedIdentity(new ByteArrayInputStream(proto.getChild().toByteArray())); + } else { + child = null; + } + return new NestedIdentity(field, offset, child); + } + +} diff --git a/server/src/main/java/org/opensearch/search/serializer/NestedIdentitySerializer.java b/server/src/main/java/org/opensearch/search/serializer/NestedIdentitySerializer.java new file mode 100644 index 0000000000000..7631fc61b28c7 --- /dev/null +++ b/server/src/main/java/org/opensearch/search/serializer/NestedIdentitySerializer.java @@ -0,0 +1,23 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.search.serializer; + +import org.opensearch.search.SearchHit.NestedIdentity; + +import java.io.IOException; + +public interface NestedIdentitySerializer { + + public NestedIdentity createNestedIdentity(T inputStream) throws IOException, Exception; +} diff --git a/server/src/main/java/org/opensearch/search/serializer/SearchHitProtobufSerializer.java b/server/src/main/java/org/opensearch/search/serializer/SearchHitProtobufSerializer.java new file mode 100644 index 0000000000000..78e2f76e6a8d1 --- /dev/null +++ b/server/src/main/java/org/opensearch/search/serializer/SearchHitProtobufSerializer.java @@ -0,0 +1,179 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.search.serializer; + +import com.google.protobuf.ByteString; +import org.apache.lucene.search.Explanation; +import org.opensearch.OpenSearchParseException; +import org.opensearch.action.OriginalIndices; +import org.opensearch.common.document.DocumentField; +import org.opensearch.common.document.serializer.DocumentFieldProtobufSerializer; +import org.opensearch.common.lucene.Lucene; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.search.SearchHit; +import org.opensearch.search.SearchHit.NestedIdentity; +import org.opensearch.search.SearchHits; +import org.opensearch.search.SearchShardTarget; +import org.opensearch.search.SearchSortValues; +import org.opensearch.search.fetch.subphase.highlight.HighlightField; +import org.opensearch.search.fetch.subphase.highlight.serializer.HighlightFieldProtobufSerializer; +import org.opensearch.server.proto.FetchSearchResultProto; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.stream.Collectors; + +public class SearchHitProtobufSerializer implements SearchHitSerializer { + + private FetchSearchResultProto.SearchHit searchHitProto; + + @Override + public SearchHit createSearchHit(InputStream inputStream) throws IOException { + this.searchHitProto = FetchSearchResultProto.SearchHit.parseFrom(inputStream); + int docId = -1; + float score = this.searchHitProto.getScore(); + String id = this.searchHitProto.getId(); + NestedIdentity nestedIdentity; + if (!this.searchHitProto.hasNestedIdentity() && this.searchHitProto.getNestedIdentity().toByteArray().length > 0) { + NestedIdentityProtobufSerializer protobufSerializer = new NestedIdentityProtobufSerializer(); + nestedIdentity = protobufSerializer.createNestedIdentity( + new ByteArrayInputStream(this.searchHitProto.getNestedIdentity().toByteArray()) + ); + } else { + nestedIdentity = null; + } + long version = this.searchHitProto.getVersion(); + long seqNo = this.searchHitProto.getSeqNo(); + long primaryTerm = this.searchHitProto.getPrimaryTerm(); + BytesReference source = BytesReference.fromByteBuffer(ByteBuffer.wrap(this.searchHitProto.getSource().toByteArray())); + if (source.length() == 0) { + source = null; + } + Map documentFields = new HashMap<>(); + DocumentFieldProtobufSerializer protobufSerializer = new DocumentFieldProtobufSerializer(); + this.searchHitProto.getDocumentFieldsMap().forEach((k, v) -> { + try { + documentFields.put(k, protobufSerializer.createDocumentField(new ByteArrayInputStream(v.toByteArray()))); + } catch (IOException e) { + throw new OpenSearchParseException("failed to parse document field", e); + } + }); + Map metaFields = new HashMap<>(); + this.searchHitProto.getMetaFieldsMap().forEach((k, v) -> { + try { + metaFields.put(k, protobufSerializer.createDocumentField(new ByteArrayInputStream(v.toByteArray()))); + } catch (IOException e) { + throw new OpenSearchParseException("failed to parse document field", e); + } + }); + Map highlightFields = new HashMap<>(); + HighlightFieldProtobufSerializer highlightFieldProtobufSerializer = new HighlightFieldProtobufSerializer(); + this.searchHitProto.getHighlightFieldsMap().forEach((k, v) -> { + try { + highlightFields.put(k, highlightFieldProtobufSerializer.createHighLightField(new ByteArrayInputStream(v.toByteArray()))); + } catch (IOException e) { + throw new OpenSearchParseException("failed to parse highlight field", e); + } + }); + SearchSortValuesProtobufSerializer sortValueProtobufSerializer = new SearchSortValuesProtobufSerializer(); + SearchSortValues sortValues = sortValueProtobufSerializer.createSearchSortValues( + new ByteArrayInputStream(this.searchHitProto.getSortValues().toByteArray()) + ); + Map matchedQueries = new HashMap<>(); + if (this.searchHitProto.getMatchedQueriesCount() > 0) { + matchedQueries = new LinkedHashMap<>(this.searchHitProto.getMatchedQueriesCount()); + for (String query : this.searchHitProto.getMatchedQueriesList()) { + matchedQueries.put(query, Float.NaN); + } + } + if (this.searchHitProto.getMatchedQueriesWithScoresCount() > 0) { + Map tempMap = this.searchHitProto.getMatchedQueriesWithScoresMap() + .entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue())); + matchedQueries = tempMap.entrySet() + .stream() + .sorted(Map.Entry.comparingByKey()) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (oldValue, newValue) -> oldValue, LinkedHashMap::new)); + } + Explanation explanation = null; + if (this.searchHitProto.hasExplanation()) { + explanation = Lucene.readExplanation(this.searchHitProto.getExplanation().toByteArray()); + } + SearchShardTarget searchShardTarget = new SearchShardTarget( + this.searchHitProto.getShard().getNodeId(), + new ShardId( + this.searchHitProto.getShard().getShardId().getIndexName(), + this.searchHitProto.getShard().getShardId().getIndexUUID(), + this.searchHitProto.getShard().getShardId().getShardId() + ), + this.searchHitProto.getShard().getClusterAlias(), + OriginalIndices.NONE + ); + Map innerHits; + if (this.searchHitProto.getInnerHitsCount() > 0) { + innerHits = new HashMap<>(); + this.searchHitProto.getInnerHitsMap().forEach((k, v) -> { + try { + SearchHitsProtobufSerializer protobufHitsFactory = new SearchHitsProtobufSerializer(); + innerHits.put(k, protobufHitsFactory.createSearchHits(new ByteArrayInputStream(v.toByteArray()))); + } catch (IOException e) { + throw new OpenSearchParseException("failed to parse inner hits", e); + } + }); + } else { + innerHits = null; + } + SearchHit searchHit = new SearchHit(docId, id, nestedIdentity, documentFields, metaFields); + searchHit.score(score); + searchHit.version(version); + searchHit.setSeqNo(seqNo); + searchHit.setPrimaryTerm(primaryTerm); + searchHit.sourceRef(source); + searchHit.highlightFields(highlightFields); + searchHit.sortValues(sortValues); + searchHit.matchedQueriesWithScores(matchedQueries); + searchHit.explanation(explanation); + searchHit.shard(searchShardTarget); + searchHit.setInnerHits(innerHits); + return searchHit; + } + + public static FetchSearchResultProto.SearchHit convertHitToProto(SearchHit hit) { + FetchSearchResultProto.SearchHit.Builder searchHitBuilder = FetchSearchResultProto.SearchHit.newBuilder(); + if (hit.getIndex() != null) { + searchHitBuilder.setIndex(hit.getIndex()); + } + searchHitBuilder.setId(hit.getId()); + searchHitBuilder.setScore(hit.getScore()); + searchHitBuilder.setSeqNo(hit.getSeqNo()); + searchHitBuilder.setPrimaryTerm(hit.getPrimaryTerm()); + searchHitBuilder.setVersion(hit.getVersion()); + searchHitBuilder.setDocId(hit.docId()); + if (hit.getSourceRef() != null) { + searchHitBuilder.setSource(ByteString.copyFrom(hit.getSourceRef().toBytesRef().bytes)); + } + for (Map.Entry entry : hit.getFields().entrySet()) { + searchHitBuilder.putDocumentFields(entry.getKey(), DocumentField.convertDocumentFieldToProto(entry.getValue())); + } + return searchHitBuilder.build(); + } + +} diff --git a/server/src/main/java/org/opensearch/search/serializer/SearchHitSerializer.java b/server/src/main/java/org/opensearch/search/serializer/SearchHitSerializer.java new file mode 100644 index 0000000000000..0f66eeb9d03da --- /dev/null +++ b/server/src/main/java/org/opensearch/search/serializer/SearchHitSerializer.java @@ -0,0 +1,24 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.search.serializer; + +import org.opensearch.search.SearchHit; + +import java.io.IOException; + +public interface SearchHitSerializer { + + SearchHit createSearchHit(T inputStream) throws IOException; + +} diff --git a/server/src/main/java/org/opensearch/search/serializer/SearchHitsProtobufSerializer.java b/server/src/main/java/org/opensearch/search/serializer/SearchHitsProtobufSerializer.java new file mode 100644 index 0000000000000..b61fc194200b5 --- /dev/null +++ b/server/src/main/java/org/opensearch/search/serializer/SearchHitsProtobufSerializer.java @@ -0,0 +1,157 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.search.serializer; + +import com.google.protobuf.ByteString; +import org.apache.lucene.search.SortField; +import org.apache.lucene.search.TotalHits; +import org.apache.lucene.search.TotalHits.Relation; +import org.apache.lucene.util.BytesRef; +import org.opensearch.OpenSearchException; +import org.opensearch.search.SearchHit; +import org.opensearch.search.SearchHits; +import org.opensearch.server.proto.FetchSearchResultProto; +import org.opensearch.server.proto.QuerySearchResultProto; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.List; + +public class SearchHitsProtobufSerializer implements SearchHitsSerializer { + + private FetchSearchResultProto.SearchHits searchHitsProto; + + @Override + public SearchHits createSearchHits(InputStream inputStream) throws IOException { + this.searchHitsProto = FetchSearchResultProto.SearchHits.parseFrom(inputStream); + SearchHit[] hits = new SearchHit[this.searchHitsProto.getHitsCount()]; + SearchHitProtobufSerializer protobufSerializer = new SearchHitProtobufSerializer(); + for (int i = 0; i < this.searchHitsProto.getHitsCount(); i++) { + hits[i] = protobufSerializer.createSearchHit(new ByteArrayInputStream(this.searchHitsProto.getHits(i).toByteArray())); + } + TotalHits totalHits = new TotalHits( + this.searchHitsProto.getTotalHits().getValue(), + Relation.valueOf(this.searchHitsProto.getTotalHits().getRelation().toString()) + ); + float maxScore = this.searchHitsProto.getMaxScore(); + SortField[] sortFields = this.searchHitsProto.getSortFieldsList() + .stream() + .map(sortField -> new SortField(sortField.getField(), SortField.Type.valueOf(sortField.getType().toString()))) + .toArray(SortField[]::new); + String collapseField = this.searchHitsProto.getCollapseField(); + Object[] collapseValues = new Object[this.searchHitsProto.getCollapseValuesCount()]; + for (int i = 0; i < this.searchHitsProto.getCollapseValuesCount(); i++) { + collapseValues[i] = readSortValueFromProtobuf(this.searchHitsProto.getCollapseValues(i)); + } + return new SearchHits(hits, totalHits, maxScore, sortFields, collapseField, collapseValues); + } + + public static Object readSortValueFromProtobuf(FetchSearchResultProto.SortValue collapseValue) throws IOException { + if (collapseValue.hasCollapseString()) { + return collapseValue.getCollapseString(); + } else if (collapseValue.hasCollapseInt()) { + return collapseValue.getCollapseInt(); + } else if (collapseValue.hasCollapseLong()) { + return collapseValue.getCollapseLong(); + } else if (collapseValue.hasCollapseFloat()) { + return collapseValue.getCollapseFloat(); + } else if (collapseValue.hasCollapseDouble()) { + return collapseValue.getCollapseDouble(); + } else if (collapseValue.hasCollapseBytes()) { + return new BytesRef(collapseValue.getCollapseBytes().toByteArray()); + } else if (collapseValue.hasCollapseBool()) { + return collapseValue.getCollapseBool(); + } else { + throw new IOException("Can't handle sort field value of type [" + collapseValue + "]"); + } + } + + public static FetchSearchResultProto.SearchHits convertHitsToProto(SearchHits hits) { + List searchHitList = new ArrayList<>(); + for (SearchHit hit : hits) { + searchHitList.add(SearchHitProtobufSerializer.convertHitToProto(hit)); + } + QuerySearchResultProto.TotalHits.Builder totalHitsBuilder = QuerySearchResultProto.TotalHits.newBuilder(); + if (hits.getTotalHits() != null) { + totalHitsBuilder.setValue(hits.getTotalHits().value); + totalHitsBuilder.setRelation( + hits.getTotalHits().relation == Relation.EQUAL_TO + ? QuerySearchResultProto.TotalHits.Relation.EQUAL_TO + : QuerySearchResultProto.TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO + ); + } + FetchSearchResultProto.SearchHits.Builder searchHitsBuilder = FetchSearchResultProto.SearchHits.newBuilder(); + searchHitsBuilder.setMaxScore(hits.getMaxScore()); + searchHitsBuilder.addAllHits(searchHitList); + searchHitsBuilder.setTotalHits(totalHitsBuilder.build()); + if (hits.getSortFields() != null && hits.getSortFields().length > 0) { + for (SortField sortField : hits.getSortFields()) { + FetchSearchResultProto.SortField.Builder sortFieldBuilder = FetchSearchResultProto.SortField.newBuilder(); + if (sortField.getField() != null) { + sortFieldBuilder.setField(sortField.getField()); + } + sortFieldBuilder.setType(FetchSearchResultProto.SortField.Type.valueOf(sortField.getType().name())); + searchHitsBuilder.addSortFields(sortFieldBuilder.build()); + } + } + if (hits.getCollapseField() != null) { + searchHitsBuilder.setCollapseField(hits.getCollapseField()); + for (Object value : hits.getCollapseValues()) { + FetchSearchResultProto.SortValue.Builder collapseValueBuilder = FetchSearchResultProto.SortValue.newBuilder(); + try { + collapseValueBuilder = readSortValueForProtobuf(value, collapseValueBuilder); + } catch (IOException e) { + throw new OpenSearchException(e); + } + searchHitsBuilder.addCollapseValues(collapseValueBuilder.build()); + } + } + return searchHitsBuilder.build(); + } + + public static FetchSearchResultProto.SortValue.Builder readSortValueForProtobuf( + Object collapseValue, + FetchSearchResultProto.SortValue.Builder collapseValueBuilder + ) throws IOException { + Class type = collapseValue.getClass(); + if (type == String.class) { + collapseValueBuilder.setCollapseString((String) collapseValue); + } else if (type == Integer.class || type == Short.class) { + collapseValueBuilder.setCollapseInt((Integer) collapseValue); + } else if (type == Long.class) { + collapseValueBuilder.setCollapseLong((Long) collapseValue); + } else if (type == Float.class) { + collapseValueBuilder.setCollapseFloat((Float) collapseValue); + } else if (type == Double.class) { + collapseValueBuilder.setCollapseDouble((Double) collapseValue); + } else if (type == Byte.class) { + byte b = (Byte) collapseValue; + collapseValueBuilder.setCollapseBytes(ByteString.copyFrom(new byte[] { b })); + } else if (type == Boolean.class) { + collapseValueBuilder.setCollapseBool((Boolean) collapseValue); + } else if (type == BytesRef.class) { + collapseValueBuilder.setCollapseBytes(ByteString.copyFrom(((BytesRef) collapseValue).bytes)); + } else if (type == BigInteger.class) { + BigInteger bigInt = (BigInteger) collapseValue; + collapseValueBuilder.setCollapseBytes(ByteString.copyFrom(bigInt.toByteArray())); + } else { + throw new IOException("Can't handle sort field value of type [" + type + "]"); + } + return collapseValueBuilder; + } + +} diff --git a/server/src/main/java/org/opensearch/search/serializer/SearchHitsSerializer.java b/server/src/main/java/org/opensearch/search/serializer/SearchHitsSerializer.java new file mode 100644 index 0000000000000..f61af18682627 --- /dev/null +++ b/server/src/main/java/org/opensearch/search/serializer/SearchHitsSerializer.java @@ -0,0 +1,24 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.search.serializer; + +import org.opensearch.search.SearchHits; + +import java.io.IOException; + +public interface SearchHitsSerializer { + + SearchHits createSearchHits(T inputStream) throws IOException; + +} diff --git a/server/src/main/java/org/opensearch/search/serializer/SearchSortValuesProtobufSerializer.java b/server/src/main/java/org/opensearch/search/serializer/SearchSortValuesProtobufSerializer.java new file mode 100644 index 0000000000000..dec522ba4a7ef --- /dev/null +++ b/server/src/main/java/org/opensearch/search/serializer/SearchSortValuesProtobufSerializer.java @@ -0,0 +1,40 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.search.serializer; + +import org.opensearch.search.SearchSortValues; +import org.opensearch.server.proto.FetchSearchResultProto; + +import java.io.IOException; +import java.io.InputStream; + +public class SearchSortValuesProtobufSerializer implements SearchSortValuesSerializer { + + @Override + public SearchSortValues createSearchSortValues(InputStream inputStream) throws IOException { + FetchSearchResultProto.SearchHit.SearchSortValues searchSortValues = FetchSearchResultProto.SearchHit.SearchSortValues.parseFrom( + inputStream + ); + Object[] formattedSortValues = new Object[searchSortValues.getFormattedSortValuesCount()]; + for (int i = 0; i < searchSortValues.getFormattedSortValuesCount(); i++) { + formattedSortValues[i] = SearchHitsProtobufSerializer.readSortValueFromProtobuf(searchSortValues.getFormattedSortValues(i)); + } + Object[] rawSortValues = new Object[searchSortValues.getRawSortValuesCount()]; + for (int i = 0; i < searchSortValues.getRawSortValuesCount(); i++) { + rawSortValues[i] = SearchHitsProtobufSerializer.readSortValueFromProtobuf(searchSortValues.getRawSortValues(i)); + } + return new SearchSortValues(formattedSortValues, rawSortValues); + } + +} diff --git a/server/src/main/java/org/opensearch/search/serializer/SearchSortValuesSerializer.java b/server/src/main/java/org/opensearch/search/serializer/SearchSortValuesSerializer.java new file mode 100644 index 0000000000000..6a9a23700635d --- /dev/null +++ b/server/src/main/java/org/opensearch/search/serializer/SearchSortValuesSerializer.java @@ -0,0 +1,24 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.search.serializer; + +import org.opensearch.search.SearchSortValues; + +import java.io.IOException; + +public interface SearchSortValuesSerializer { + + SearchSortValues createSearchSortValues(T inputStream) throws IOException; + +} diff --git a/server/src/main/java/org/opensearch/transport/BaseInboundMessage.java b/server/src/main/java/org/opensearch/transport/BaseInboundMessage.java index 112b55fdf72be..db0ddfc3f0cd0 100644 --- a/server/src/main/java/org/opensearch/transport/BaseInboundMessage.java +++ b/server/src/main/java/org/opensearch/transport/BaseInboundMessage.java @@ -8,12 +8,15 @@ package org.opensearch.transport; +import org.opensearch.common.annotation.ExperimentalApi; + /** * Base class for inbound data as a message. * Different implementations are used for different protocols. * * @opensearch.internal */ +@ExperimentalApi public interface BaseInboundMessage { /** diff --git a/server/src/main/proto/server/search/FetchSearchResultProto.proto b/server/src/main/proto/server/search/FetchSearchResultProto.proto index b126ca439e9f2..88f306d31a173 100644 --- a/server/src/main/proto/server/search/FetchSearchResultProto.proto +++ b/server/src/main/proto/server/search/FetchSearchResultProto.proto @@ -30,7 +30,7 @@ message SearchHits { repeated SearchHit hits = 4; repeated SortField sortFields = 5; optional string collapseField = 6; - repeated CollapseValue collapseValues = 7; + repeated SortValue collapseValues = 7; } message SearchHit { @@ -71,8 +71,8 @@ message SearchHit { } message SearchSortValues { - repeated bytes formattedSortValues = 1; - repeated bytes rawSortValues = 2; + repeated SortValue formattedSortValues = 1; + repeated SortValue rawSortValues = 2; } message Explanation { @@ -105,7 +105,7 @@ message SortField { } } -message CollapseValue { +message SortValue { optional string collapseString = 1; optional int32 collapseInt = 2; optional int64 collapseLong = 3; From 117572c7b8d71992af4416b4b33e3647936c6ce3 Mon Sep 17 00:00:00 2001 From: Vacha Shah Date: Tue, 26 Mar 2024 18:25:47 +0000 Subject: [PATCH 33/36] Fixing precommit Signed-off-by: Vacha Shah --- .../core/common/io/stream/BytesWriteable.java | 5 ----- .../common/document/DocumentField.java | 13 ------------- .../DocumentFieldProtobufSerializer.java | 18 +++++++++++++----- .../serializer/DocumentFieldSerializer.java | 8 +++----- .../document/serializer/package-info.java | 10 ++++++++++ .../HighlightFieldProtobufSerializer.java | 10 ++++------ .../serializer/HighlightFieldSerializer.java | 13 +++++-------- .../highlight/serializer/package-info.java | 10 ++++++++++ .../NestedIdentityProtobufSerializer.java | 8 +++----- .../serializer/NestedIdentitySerializer.java | 8 +++----- .../SearchHitProtobufSerializer.java | 13 +++++++------ .../search/serializer/SearchHitSerializer.java | 8 +++----- .../SearchHitsProtobufSerializer.java | 8 +++----- .../serializer/SearchHitsSerializer.java | 8 +++----- .../SearchSortValuesProtobufSerializer.java | 8 +++----- .../serializer/SearchSortValuesSerializer.java | 8 +++----- .../search/serializer/package-info.java | 10 ++++++++++ .../transport/ProtobufMessageHandler.java | 5 ----- .../proto/server/NodeToNodeMessageProto.proto | 3 --- .../server/search/FetchSearchResultProto.proto | 3 --- .../search/QueryFetchSearchResultProto.proto | 3 --- .../server/search/QuerySearchResultProto.proto | 3 --- .../search/ShardSearchRequestProto.proto | 3 --- 23 files changed, 83 insertions(+), 103 deletions(-) create mode 100644 server/src/main/java/org/opensearch/common/document/serializer/package-info.java create mode 100644 server/src/main/java/org/opensearch/search/fetch/subphase/highlight/serializer/package-info.java create mode 100644 server/src/main/java/org/opensearch/search/serializer/package-info.java diff --git a/libs/core/src/main/java/org/opensearch/core/common/io/stream/BytesWriteable.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/BytesWriteable.java index 001af4e672b1d..8b4f9a6aaf6ef 100644 --- a/libs/core/src/main/java/org/opensearch/core/common/io/stream/BytesWriteable.java +++ b/libs/core/src/main/java/org/opensearch/core/common/io/stream/BytesWriteable.java @@ -6,11 +6,6 @@ * compatible open source license. */ -/* - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - package org.opensearch.core.common.io.stream; import org.opensearch.common.annotation.ExperimentalApi; diff --git a/server/src/main/java/org/opensearch/common/document/DocumentField.java b/server/src/main/java/org/opensearch/common/document/DocumentField.java index 383648e175dc0..5cdc2bba8be16 100644 --- a/server/src/main/java/org/opensearch/common/document/DocumentField.java +++ b/server/src/main/java/org/opensearch/common/document/DocumentField.java @@ -33,7 +33,6 @@ package org.opensearch.common.document; import org.opensearch.common.annotation.PublicApi; -import org.opensearch.common.document.serializer.DocumentFieldProtobufSerializer; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; @@ -42,7 +41,6 @@ import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.get.GetResult; import org.opensearch.search.SearchHit; -import org.opensearch.server.proto.FetchSearchResultProto; import java.io.IOException; import java.util.ArrayList; @@ -66,7 +64,6 @@ public class DocumentField implements Writeable, ToXContentFragment, Iterable values; - private FetchSearchResultProto.SearchHit.DocumentField documentField; public DocumentField(StreamInput in) throws IOException { name = in.readString(); @@ -78,16 +75,6 @@ public DocumentField(String name, List values) { this.values = Objects.requireNonNull(values, "values must not be null"); } - public static FetchSearchResultProto.SearchHit.DocumentField convertDocumentFieldToProto(DocumentField documentField) { - FetchSearchResultProto.SearchHit.DocumentField.Builder builder = FetchSearchResultProto.SearchHit.DocumentField.newBuilder(); - builder.setName(documentField.getName()); - for (Object value : documentField.getValues()) { - FetchSearchResultProto.DocumentFieldValue.Builder valueBuilder = FetchSearchResultProto.DocumentFieldValue.newBuilder(); - builder.addValues(DocumentFieldProtobufSerializer.convertDocumentFieldValueToProto(value, valueBuilder)); - } - return builder.build(); - } - /** * The name of the field. */ diff --git a/server/src/main/java/org/opensearch/common/document/serializer/DocumentFieldProtobufSerializer.java b/server/src/main/java/org/opensearch/common/document/serializer/DocumentFieldProtobufSerializer.java index 614837ce7c9d3..e606773d07826 100644 --- a/server/src/main/java/org/opensearch/common/document/serializer/DocumentFieldProtobufSerializer.java +++ b/server/src/main/java/org/opensearch/common/document/serializer/DocumentFieldProtobufSerializer.java @@ -6,11 +6,6 @@ * compatible open source license. */ -/* - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - package org.opensearch.common.document.serializer; import com.google.protobuf.ByteString; @@ -33,6 +28,9 @@ import java.util.List; import java.util.Map; +/** + * Serializer for {@link DocumentField} to/from protobuf. + */ public class DocumentFieldProtobufSerializer implements DocumentFieldSerializer { private FetchSearchResultProto.SearchHit.DocumentField documentField; @@ -132,4 +130,14 @@ public static DocumentFieldValue.Builder convertDocumentFieldValueToProto(Object return valueBuilder; } + public static FetchSearchResultProto.SearchHit.DocumentField convertDocumentFieldToProto(DocumentField documentField) { + FetchSearchResultProto.SearchHit.DocumentField.Builder builder = FetchSearchResultProto.SearchHit.DocumentField.newBuilder(); + builder.setName(documentField.getName()); + for (Object value : documentField.getValues()) { + FetchSearchResultProto.DocumentFieldValue.Builder valueBuilder = FetchSearchResultProto.DocumentFieldValue.newBuilder(); + builder.addValues(DocumentFieldProtobufSerializer.convertDocumentFieldValueToProto(value, valueBuilder)); + } + return builder.build(); + } + } diff --git a/server/src/main/java/org/opensearch/common/document/serializer/DocumentFieldSerializer.java b/server/src/main/java/org/opensearch/common/document/serializer/DocumentFieldSerializer.java index 04d0b5a81bcf6..48916c4b87b51 100644 --- a/server/src/main/java/org/opensearch/common/document/serializer/DocumentFieldSerializer.java +++ b/server/src/main/java/org/opensearch/common/document/serializer/DocumentFieldSerializer.java @@ -6,17 +6,15 @@ * compatible open source license. */ -/* - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - package org.opensearch.common.document.serializer; import org.opensearch.common.document.DocumentField; import java.io.IOException; +/** + * Serializer for {@link DocumentField} which can be implemented for different types of serialization. + */ public interface DocumentFieldSerializer { DocumentField createDocumentField(T inputStream) throws IOException; diff --git a/server/src/main/java/org/opensearch/common/document/serializer/package-info.java b/server/src/main/java/org/opensearch/common/document/serializer/package-info.java new file mode 100644 index 0000000000000..e8419ac59bb03 --- /dev/null +++ b/server/src/main/java/org/opensearch/common/document/serializer/package-info.java @@ -0,0 +1,10 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** Serializer package for documents. */ +package org.opensearch.common.document.serializer; diff --git a/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/serializer/HighlightFieldProtobufSerializer.java b/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/serializer/HighlightFieldProtobufSerializer.java index dbf9ac5ac48ee..74557d5618ce0 100644 --- a/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/serializer/HighlightFieldProtobufSerializer.java +++ b/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/serializer/HighlightFieldProtobufSerializer.java @@ -6,11 +6,6 @@ * compatible open source license. */ -/* - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - package org.opensearch.search.fetch.subphase.highlight.serializer; import org.opensearch.core.common.text.Text; @@ -22,7 +17,10 @@ import java.util.ArrayList; import java.util.List; -public class HighlightFieldProtobufSerializer implements HighlightFieldSerializer { +/** + * Serializer for {@link HighlightField} to/from protobuf. + */ +public class HighlightFieldProtobufSerializer implements HighlightFieldSerializer { @Override public HighlightField createHighLightField(InputStream inputStream) throws IOException { diff --git a/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/serializer/HighlightFieldSerializer.java b/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/serializer/HighlightFieldSerializer.java index d62016969885d..21a6afdac565f 100644 --- a/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/serializer/HighlightFieldSerializer.java +++ b/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/serializer/HighlightFieldSerializer.java @@ -6,19 +6,16 @@ * compatible open source license. */ -/* - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - package org.opensearch.search.fetch.subphase.highlight.serializer; import org.opensearch.search.fetch.subphase.highlight.HighlightField; import java.io.IOException; -import java.io.InputStream; -public interface HighlightFieldSerializer { +/** + * Serializer for {@link HighlightField} which can be implemented for different types of serialization. + */ +public interface HighlightFieldSerializer { - HighlightField createHighLightField(InputStream inputStream) throws IOException; + HighlightField createHighLightField(T inputStream) throws IOException; } diff --git a/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/serializer/package-info.java b/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/serializer/package-info.java new file mode 100644 index 0000000000000..dc08282a8954f --- /dev/null +++ b/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/serializer/package-info.java @@ -0,0 +1,10 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** Serializer package for highlights. */ +package org.opensearch.search.fetch.subphase.highlight.serializer; diff --git a/server/src/main/java/org/opensearch/search/serializer/NestedIdentityProtobufSerializer.java b/server/src/main/java/org/opensearch/search/serializer/NestedIdentityProtobufSerializer.java index fc6902193af84..0e1081c9cc8d8 100644 --- a/server/src/main/java/org/opensearch/search/serializer/NestedIdentityProtobufSerializer.java +++ b/server/src/main/java/org/opensearch/search/serializer/NestedIdentityProtobufSerializer.java @@ -6,11 +6,6 @@ * compatible open source license. */ -/* - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - package org.opensearch.search.serializer; import org.opensearch.search.SearchHit.NestedIdentity; @@ -20,6 +15,9 @@ import java.io.IOException; import java.io.InputStream; +/** + * Serializer for {@link NestedIdentity} to/from protobuf. + */ public class NestedIdentityProtobufSerializer implements NestedIdentitySerializer { @Override diff --git a/server/src/main/java/org/opensearch/search/serializer/NestedIdentitySerializer.java b/server/src/main/java/org/opensearch/search/serializer/NestedIdentitySerializer.java index 7631fc61b28c7..5ee30337bb4b6 100644 --- a/server/src/main/java/org/opensearch/search/serializer/NestedIdentitySerializer.java +++ b/server/src/main/java/org/opensearch/search/serializer/NestedIdentitySerializer.java @@ -6,17 +6,15 @@ * compatible open source license. */ -/* - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - package org.opensearch.search.serializer; import org.opensearch.search.SearchHit.NestedIdentity; import java.io.IOException; +/** + * Serializer for {@link NestedIdentity} which can be implemented for different types of serialization. + */ public interface NestedIdentitySerializer { public NestedIdentity createNestedIdentity(T inputStream) throws IOException, Exception; diff --git a/server/src/main/java/org/opensearch/search/serializer/SearchHitProtobufSerializer.java b/server/src/main/java/org/opensearch/search/serializer/SearchHitProtobufSerializer.java index 78e2f76e6a8d1..e06e31bee249f 100644 --- a/server/src/main/java/org/opensearch/search/serializer/SearchHitProtobufSerializer.java +++ b/server/src/main/java/org/opensearch/search/serializer/SearchHitProtobufSerializer.java @@ -6,11 +6,6 @@ * compatible open source license. */ -/* - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - package org.opensearch.search.serializer; import com.google.protobuf.ByteString; @@ -40,6 +35,9 @@ import java.util.Map; import java.util.stream.Collectors; +/** + * Serializer for {@link SearchHit} to/from protobuf. + */ public class SearchHitProtobufSerializer implements SearchHitSerializer { private FetchSearchResultProto.SearchHit searchHitProto; @@ -171,7 +169,10 @@ public static FetchSearchResultProto.SearchHit convertHitToProto(SearchHit hit) searchHitBuilder.setSource(ByteString.copyFrom(hit.getSourceRef().toBytesRef().bytes)); } for (Map.Entry entry : hit.getFields().entrySet()) { - searchHitBuilder.putDocumentFields(entry.getKey(), DocumentField.convertDocumentFieldToProto(entry.getValue())); + searchHitBuilder.putDocumentFields( + entry.getKey(), + DocumentFieldProtobufSerializer.convertDocumentFieldToProto(entry.getValue()) + ); } return searchHitBuilder.build(); } diff --git a/server/src/main/java/org/opensearch/search/serializer/SearchHitSerializer.java b/server/src/main/java/org/opensearch/search/serializer/SearchHitSerializer.java index 0f66eeb9d03da..217266f720079 100644 --- a/server/src/main/java/org/opensearch/search/serializer/SearchHitSerializer.java +++ b/server/src/main/java/org/opensearch/search/serializer/SearchHitSerializer.java @@ -6,17 +6,15 @@ * compatible open source license. */ -/* - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - package org.opensearch.search.serializer; import org.opensearch.search.SearchHit; import java.io.IOException; +/** + * Serializer for {@link SearchHit} which can be implemented for different types of serialization. + */ public interface SearchHitSerializer { SearchHit createSearchHit(T inputStream) throws IOException; diff --git a/server/src/main/java/org/opensearch/search/serializer/SearchHitsProtobufSerializer.java b/server/src/main/java/org/opensearch/search/serializer/SearchHitsProtobufSerializer.java index b61fc194200b5..d32043d9caa49 100644 --- a/server/src/main/java/org/opensearch/search/serializer/SearchHitsProtobufSerializer.java +++ b/server/src/main/java/org/opensearch/search/serializer/SearchHitsProtobufSerializer.java @@ -6,11 +6,6 @@ * compatible open source license. */ -/* - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - package org.opensearch.search.serializer; import com.google.protobuf.ByteString; @@ -31,6 +26,9 @@ import java.util.ArrayList; import java.util.List; +/** + * Serializer for {@link SearchHits} to/from protobuf. + */ public class SearchHitsProtobufSerializer implements SearchHitsSerializer { private FetchSearchResultProto.SearchHits searchHitsProto; diff --git a/server/src/main/java/org/opensearch/search/serializer/SearchHitsSerializer.java b/server/src/main/java/org/opensearch/search/serializer/SearchHitsSerializer.java index f61af18682627..f09e369aaeee1 100644 --- a/server/src/main/java/org/opensearch/search/serializer/SearchHitsSerializer.java +++ b/server/src/main/java/org/opensearch/search/serializer/SearchHitsSerializer.java @@ -6,17 +6,15 @@ * compatible open source license. */ -/* - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - package org.opensearch.search.serializer; import org.opensearch.search.SearchHits; import java.io.IOException; +/** + * Serializer for {@link SearchHits} which can be implemented for different types of serialization. + */ public interface SearchHitsSerializer { SearchHits createSearchHits(T inputStream) throws IOException; diff --git a/server/src/main/java/org/opensearch/search/serializer/SearchSortValuesProtobufSerializer.java b/server/src/main/java/org/opensearch/search/serializer/SearchSortValuesProtobufSerializer.java index dec522ba4a7ef..007e951cbf10a 100644 --- a/server/src/main/java/org/opensearch/search/serializer/SearchSortValuesProtobufSerializer.java +++ b/server/src/main/java/org/opensearch/search/serializer/SearchSortValuesProtobufSerializer.java @@ -6,11 +6,6 @@ * compatible open source license. */ -/* - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - package org.opensearch.search.serializer; import org.opensearch.search.SearchSortValues; @@ -19,6 +14,9 @@ import java.io.IOException; import java.io.InputStream; +/** + * Serializer for {@link SearchSortValues} to/from protobuf. + */ public class SearchSortValuesProtobufSerializer implements SearchSortValuesSerializer { @Override diff --git a/server/src/main/java/org/opensearch/search/serializer/SearchSortValuesSerializer.java b/server/src/main/java/org/opensearch/search/serializer/SearchSortValuesSerializer.java index 6a9a23700635d..31feb206eb74d 100644 --- a/server/src/main/java/org/opensearch/search/serializer/SearchSortValuesSerializer.java +++ b/server/src/main/java/org/opensearch/search/serializer/SearchSortValuesSerializer.java @@ -6,17 +6,15 @@ * compatible open source license. */ -/* - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - package org.opensearch.search.serializer; import org.opensearch.search.SearchSortValues; import java.io.IOException; +/** + * Serializer for {@link SearchSortValues} which can be implemented for different types of serialization. + */ public interface SearchSortValuesSerializer { SearchSortValues createSearchSortValues(T inputStream) throws IOException; diff --git a/server/src/main/java/org/opensearch/search/serializer/package-info.java b/server/src/main/java/org/opensearch/search/serializer/package-info.java new file mode 100644 index 0000000000000..25a4d1935016e --- /dev/null +++ b/server/src/main/java/org/opensearch/search/serializer/package-info.java @@ -0,0 +1,10 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** Serializer package for search. */ +package org.opensearch.search.serializer; diff --git a/server/src/main/java/org/opensearch/transport/ProtobufMessageHandler.java b/server/src/main/java/org/opensearch/transport/ProtobufMessageHandler.java index d3262f4d3e5c4..3aa8be4d094e2 100644 --- a/server/src/main/java/org/opensearch/transport/ProtobufMessageHandler.java +++ b/server/src/main/java/org/opensearch/transport/ProtobufMessageHandler.java @@ -6,11 +6,6 @@ * compatible open source license. */ -/* - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - package org.opensearch.transport; import org.apache.logging.log4j.LogManager; diff --git a/server/src/main/proto/server/NodeToNodeMessageProto.proto b/server/src/main/proto/server/NodeToNodeMessageProto.proto index e0d46bac94216..e83143202dce7 100644 --- a/server/src/main/proto/server/NodeToNodeMessageProto.proto +++ b/server/src/main/proto/server/NodeToNodeMessageProto.proto @@ -4,9 +4,6 @@ * The OpenSearch Contributors require contributions made to * this file be licensed under the Apache-2.0 license or a * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. */ syntax = "proto3"; diff --git a/server/src/main/proto/server/search/FetchSearchResultProto.proto b/server/src/main/proto/server/search/FetchSearchResultProto.proto index 88f306d31a173..f0f4f495ad179 100644 --- a/server/src/main/proto/server/search/FetchSearchResultProto.proto +++ b/server/src/main/proto/server/search/FetchSearchResultProto.proto @@ -4,9 +4,6 @@ * The OpenSearch Contributors require contributions made to * this file be licensed under the Apache-2.0 license or a * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. */ syntax = "proto3"; diff --git a/server/src/main/proto/server/search/QueryFetchSearchResultProto.proto b/server/src/main/proto/server/search/QueryFetchSearchResultProto.proto index 08d7d9941022c..deac135c0e3d0 100644 --- a/server/src/main/proto/server/search/QueryFetchSearchResultProto.proto +++ b/server/src/main/proto/server/search/QueryFetchSearchResultProto.proto @@ -4,9 +4,6 @@ * The OpenSearch Contributors require contributions made to * this file be licensed under the Apache-2.0 license or a * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. */ syntax = "proto3"; diff --git a/server/src/main/proto/server/search/QuerySearchResultProto.proto b/server/src/main/proto/server/search/QuerySearchResultProto.proto index 32ed57d4d9063..e4fc82207042c 100644 --- a/server/src/main/proto/server/search/QuerySearchResultProto.proto +++ b/server/src/main/proto/server/search/QuerySearchResultProto.proto @@ -4,9 +4,6 @@ * The OpenSearch Contributors require contributions made to * this file be licensed under the Apache-2.0 license or a * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. */ syntax = "proto3"; diff --git a/server/src/main/proto/server/search/ShardSearchRequestProto.proto b/server/src/main/proto/server/search/ShardSearchRequestProto.proto index 0975a96d77167..114e9154ee5d9 100644 --- a/server/src/main/proto/server/search/ShardSearchRequestProto.proto +++ b/server/src/main/proto/server/search/ShardSearchRequestProto.proto @@ -4,9 +4,6 @@ * The OpenSearch Contributors require contributions made to * this file be licensed under the Apache-2.0 license or a * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. */ syntax = "proto3"; From 46eeaf68a34f8f3644e334c566fba8cd25fbafe6 Mon Sep 17 00:00:00 2001 From: Vacha Shah Date: Wed, 27 Mar 2024 20:46:16 +0000 Subject: [PATCH 34/36] Addressing comments Signed-off-by: Vacha Shah --- .../processor/ApiAnnotationProcessor.java | 15 +++++++++------ .../org/opensearch/common/util/FeatureFlags.java | 6 ++++-- .../java/org/opensearch/search/SearchHit.java | 11 ----------- .../java/org/opensearch/search/SearchHits.java | 7 ------- 4 files changed, 13 insertions(+), 26 deletions(-) diff --git a/libs/common/src/main/java/org/opensearch/common/annotation/processor/ApiAnnotationProcessor.java b/libs/common/src/main/java/org/opensearch/common/annotation/processor/ApiAnnotationProcessor.java index 0d4e342bd9dc6..6264d00f01887 100644 --- a/libs/common/src/main/java/org/opensearch/common/annotation/processor/ApiAnnotationProcessor.java +++ b/libs/common/src/main/java/org/opensearch/common/annotation/processor/ApiAnnotationProcessor.java @@ -179,11 +179,6 @@ private void process(ExecutableElement executable, ReferenceType ref) { return; } - // Skip protobuf generated classes used in public apis - if (ref.toString().contains("Proto")) { - return; - } - if (ref instanceof DeclaredType) { final DeclaredType declaredType = (DeclaredType) ref; @@ -243,7 +238,15 @@ private boolean inspectable(ExecutableElement executable) { */ private boolean inspectable(Element element) { final PackageElement pckg = processingEnv.getElementUtils().getPackageOf(element); - return pckg.getQualifiedName().toString().startsWith(OPENSEARCH_PACKAGE); + return pckg.getQualifiedName().toString().startsWith(OPENSEARCH_PACKAGE) + && !element.getEnclosingElement() + .getAnnotationMirrors() + .stream() + .anyMatch( + m -> m.getAnnotationType() + .toString() /* ClassSymbol.toString() returns class name */ + .equalsIgnoreCase("javax.annotation.Generated") + ); } /** diff --git a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java index e43dfd3b3b063..f9da0968699b5 100644 --- a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java +++ b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java @@ -103,6 +103,8 @@ public class FeatureFlags { public static final Setting PLUGGABLE_CACHE_SETTING = Setting.boolSetting(PLUGGABLE_CACHE, false, Property.NodeScope); + public static final Setting PROTOBUF_SETTING = Setting.boolSetting(PROTOBUF, false, Property.NodeScope, Property.Dynamic); + private static final List> ALL_FEATURE_FLAG_SETTINGS = List.of( REMOTE_STORE_MIGRATION_EXPERIMENTAL_SETTING, EXTENSIONS_SETTING, @@ -110,7 +112,8 @@ public class FeatureFlags { TELEMETRY_SETTING, DATETIME_FORMATTER_CACHING_SETTING, WRITEABLE_REMOTE_INDEX_SETTING, - PLUGGABLE_CACHE_SETTING + PLUGGABLE_CACHE_SETTING, + PROTOBUF_SETTING ); /** * Should store the settings from opensearch.yml. @@ -166,5 +169,4 @@ public static boolean isEnabled(Setting featureFlag) { } } - public static final Setting PROTOBUF_SETTING = Setting.boolSetting(PROTOBUF, false, Property.NodeScope, Property.Dynamic); } diff --git a/server/src/main/java/org/opensearch/search/SearchHit.java b/server/src/main/java/org/opensearch/search/SearchHit.java index 3cd85488bd0c6..6391353cfe5b1 100644 --- a/server/src/main/java/org/opensearch/search/SearchHit.java +++ b/server/src/main/java/org/opensearch/search/SearchHit.java @@ -39,7 +39,6 @@ import org.opensearch.common.Nullable; import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.document.DocumentField; -import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.ParseField; import org.opensearch.core.common.ParsingException; @@ -68,8 +67,6 @@ import org.opensearch.rest.action.search.RestSearchAction; import org.opensearch.search.fetch.subphase.highlight.HighlightField; import org.opensearch.search.lookup.SourceLookup; -import org.opensearch.search.serializer.SearchHitProtobufSerializer; -import org.opensearch.server.proto.FetchSearchResultProto; import org.opensearch.transport.RemoteClusterAware; import java.io.IOException; @@ -142,8 +139,6 @@ public final class SearchHit implements Writeable, ToXContentObject, Iterable innerHits; - private FetchSearchResultProto.SearchHit searchHitProto; - // used only in tests public SearchHit(int docId) { this(docId, null, null, null); @@ -169,9 +164,6 @@ public SearchHit( this.nestedIdentity = nestedIdentity; this.documentFields = documentFields == null ? emptyMap() : documentFields; this.metaFields = metaFields == null ? emptyMap() : metaFields; - if (FeatureFlags.isEnabled(FeatureFlags.PROTOBUF)) { - this.searchHitProto = SearchHitProtobufSerializer.convertHitToProto(this); - } } public SearchHit(StreamInput in) throws IOException { @@ -456,9 +448,6 @@ public void setDocumentField(String fieldName, DocumentField field) { if (fieldName == null || field == null) return; if (documentFields.isEmpty()) this.documentFields = new HashMap<>(); this.documentFields.put(fieldName, field); - if (FeatureFlags.isEnabled(FeatureFlags.PROTOBUF)) { - this.searchHitProto = SearchHitProtobufSerializer.convertHitToProto(this); - } } public DocumentField removeDocumentField(String fieldName) { diff --git a/server/src/main/java/org/opensearch/search/SearchHits.java b/server/src/main/java/org/opensearch/search/SearchHits.java index cd5854898fccd..0b4ee3851bd61 100644 --- a/server/src/main/java/org/opensearch/search/SearchHits.java +++ b/server/src/main/java/org/opensearch/search/SearchHits.java @@ -38,7 +38,6 @@ import org.opensearch.common.Nullable; import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.lucene.Lucene; -import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; @@ -46,7 +45,6 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.rest.action.search.RestSearchAction; -import org.opensearch.search.serializer.SearchHitsProtobufSerializer; import java.io.IOException; import java.util.ArrayList; @@ -84,8 +82,6 @@ public static SearchHits empty(boolean withTotalHits) { @Nullable private final Object[] collapseValues; - private org.opensearch.server.proto.FetchSearchResultProto.SearchHits searchHitsProto; - public SearchHits(SearchHit[] hits, @Nullable TotalHits totalHits, float maxScore) { this(hits, totalHits, maxScore, null, null, null); } @@ -104,9 +100,6 @@ public SearchHits( this.sortFields = sortFields; this.collapseField = collapseField; this.collapseValues = collapseValues; - if (FeatureFlags.isEnabled(FeatureFlags.PROTOBUF_SETTING)) { - this.searchHitsProto = SearchHitsProtobufSerializer.convertHitsToProto(this); - } } public SearchHits(StreamInput in) throws IOException { From ed00f89d2ed6bae335f885ed8aa8123f0da1fb39 Mon Sep 17 00:00:00 2001 From: Vacha Shah Date: Tue, 9 Apr 2024 19:58:14 +0000 Subject: [PATCH 35/36] Merging latest changes from main Signed-off-by: Vacha Shah --- .../search/fetch/FetchSearchResult.java | 1 - .../search/fetch/QueryFetchSearchResult.java | 8 +-- .../search/internal/ShardSearchRequest.java | 1 - .../search/query/QuerySearchResult.java | 8 +-- .../transport/BaseInboundMessage.java | 37 ------------- .../opensearch/transport/InboundHandler.java | 9 ++++ .../opensearch/transport/InboundPipeline.java | 7 ++- .../opensearch/transport/OutboundHandler.java | 13 ++--- .../opensearch/transport/TcpTransport.java | 8 --- .../ProtobufInboundBytesHandler.java | 53 +++++++++++++++++++ .../ProtobufInboundMessage.java} | 26 ++++----- .../ProtobufMessageHandler.java | 35 ++++++++---- .../transport/InboundHandlerTests.java | 3 +- .../transport/InboundPipelineTests.java | 24 ++------- .../transport/OutboundHandlerTests.java | 11 ++-- 15 files changed, 133 insertions(+), 111 deletions(-) delete mode 100644 server/src/main/java/org/opensearch/transport/BaseInboundMessage.java create mode 100644 server/src/main/java/org/opensearch/transport/protobufprotocol/ProtobufInboundBytesHandler.java rename server/src/main/java/org/opensearch/transport/{NodeToNodeMessage.java => protobufprotocol/ProtobufInboundMessage.java} (90%) rename server/src/main/java/org/opensearch/transport/{ => protobufprotocol}/ProtobufMessageHandler.java (83%) diff --git a/server/src/main/java/org/opensearch/search/fetch/FetchSearchResult.java b/server/src/main/java/org/opensearch/search/fetch/FetchSearchResult.java index b46b2c5ca12d8..681398d36e07b 100644 --- a/server/src/main/java/org/opensearch/search/fetch/FetchSearchResult.java +++ b/server/src/main/java/org/opensearch/search/fetch/FetchSearchResult.java @@ -75,7 +75,6 @@ public FetchSearchResult(StreamInput in) throws IOException { public FetchSearchResult(InputStream in) throws IOException { super(in); - assert FeatureFlags.isEnabled(FeatureFlags.PROTOBUF) : "protobuf feature flag is not enabled"; this.fetchSearchResultProto = FetchSearchResultProto.FetchSearchResult.parseFrom(in); contextId = new ShardSearchContextId( this.fetchSearchResultProto.getContextId().getSessionId(), diff --git a/server/src/main/java/org/opensearch/search/fetch/QueryFetchSearchResult.java b/server/src/main/java/org/opensearch/search/fetch/QueryFetchSearchResult.java index b5e0a820da9af..8531fe027abd4 100644 --- a/server/src/main/java/org/opensearch/search/fetch/QueryFetchSearchResult.java +++ b/server/src/main/java/org/opensearch/search/fetch/QueryFetchSearchResult.java @@ -40,7 +40,8 @@ import org.opensearch.search.internal.ShardSearchContextId; import org.opensearch.search.query.QuerySearchResult; import org.opensearch.server.proto.QueryFetchSearchResultProto; -import org.opensearch.transport.BaseInboundMessage; +import org.opensearch.transport.nativeprotocol.NativeInboundMessage; +import org.opensearch.transport.protobufprotocol.ProtobufInboundMessage; import java.io.IOException; import java.io.InputStream; @@ -65,7 +66,6 @@ public QueryFetchSearchResult(StreamInput in) throws IOException { public QueryFetchSearchResult(InputStream in) throws IOException { super(in); - assert FeatureFlags.isEnabled(FeatureFlags.PROTOBUF) : "protobuf feature flag is not enabled"; this.queryFetchSearchResultProto = QueryFetchSearchResultProto.QueryFetchSearchResult.parseFrom(in); queryResult = new QuerySearchResult(in); fetchResult = new FetchSearchResult(in); @@ -125,9 +125,9 @@ public void writeTo(StreamOutput out) throws IOException { @Override public String getProtocol() { if (FeatureFlags.isEnabled(FeatureFlags.PROTOBUF_SETTING)) { - return BaseInboundMessage.PROTOBUF_PROTOCOL; + return ProtobufInboundMessage.PROTOBUF_PROTOCOL; } - return BaseInboundMessage.NATIVE_PROTOCOL; + return NativeInboundMessage.NATIVE_PROTOCOL; } public QueryFetchSearchResultProto.QueryFetchSearchResult response() { diff --git a/server/src/main/java/org/opensearch/search/internal/ShardSearchRequest.java b/server/src/main/java/org/opensearch/search/internal/ShardSearchRequest.java index dde9f7130afa5..a42224b5d94de 100644 --- a/server/src/main/java/org/opensearch/search/internal/ShardSearchRequest.java +++ b/server/src/main/java/org/opensearch/search/internal/ShardSearchRequest.java @@ -272,7 +272,6 @@ public ShardSearchRequest(StreamInput in) throws IOException { } public ShardSearchRequest(byte[] in) throws IOException { - assert FeatureFlags.isEnabled(FeatureFlags.PROTOBUF) : "protobuf feature flag is not enabled"; ShardSearchRequestProto.ShardSearchRequest searchRequestProto = ShardSearchRequestProto.ShardSearchRequest.parseFrom(in); this.clusterAlias = searchRequestProto.getClusterAlias(); shardId = new ShardId( diff --git a/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java b/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java index 7ae21e6667caf..0cc151766084c 100644 --- a/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java +++ b/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java @@ -60,7 +60,8 @@ import org.opensearch.server.proto.ShardSearchRequestProto; import org.opensearch.server.proto.ShardSearchRequestProto.AliasFilter; import org.opensearch.server.proto.ShardSearchRequestProto.ShardSearchRequest.SearchType; -import org.opensearch.transport.BaseInboundMessage; +import org.opensearch.transport.nativeprotocol.NativeInboundMessage; +import org.opensearch.transport.protobufprotocol.ProtobufInboundMessage; import java.io.IOException; import java.io.InputStream; @@ -124,7 +125,6 @@ public QuerySearchResult(StreamInput in) throws IOException { public QuerySearchResult(InputStream in) throws IOException { super(in); - assert FeatureFlags.isEnabled(FeatureFlags.PROTOBUF) : "protobuf feature flag is not enabled"; this.querySearchResultProto = QuerySearchResultProto.QuerySearchResult.parseFrom(in); isNull = this.querySearchResultProto.getIsNull(); if (!isNull) { @@ -628,8 +628,8 @@ public QuerySearchResult(QuerySearchResultProto.QuerySearchResult querySearchRes @Override public String getProtocol() { if (FeatureFlags.isEnabled(FeatureFlags.PROTOBUF_SETTING)) { - return BaseInboundMessage.PROTOBUF_PROTOCOL; + return ProtobufInboundMessage.PROTOBUF_PROTOCOL; } - return BaseInboundMessage.NATIVE_PROTOCOL; + return NativeInboundMessage.NATIVE_PROTOCOL; } } diff --git a/server/src/main/java/org/opensearch/transport/BaseInboundMessage.java b/server/src/main/java/org/opensearch/transport/BaseInboundMessage.java deleted file mode 100644 index db0ddfc3f0cd0..0000000000000 --- a/server/src/main/java/org/opensearch/transport/BaseInboundMessage.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.transport; - -import org.opensearch.common.annotation.ExperimentalApi; - -/** - * Base class for inbound data as a message. - * Different implementations are used for different protocols. - * - * @opensearch.internal - */ -@ExperimentalApi -public interface BaseInboundMessage { - - /** - * The protocol used to encode this message - */ - static String NATIVE_PROTOCOL = "native"; - static String PROTOBUF_PROTOCOL = "protobuf"; - - /** - * @return the protocol used to encode this message - */ - public String getProtocol(); - - /** - * Set the protocol used to encode this message - */ - public void setProtocol(); -} diff --git a/server/src/main/java/org/opensearch/transport/InboundHandler.java b/server/src/main/java/org/opensearch/transport/InboundHandler.java index 6492900c49a0e..d5d7f614c37e6 100644 --- a/server/src/main/java/org/opensearch/transport/InboundHandler.java +++ b/server/src/main/java/org/opensearch/transport/InboundHandler.java @@ -33,10 +33,13 @@ package org.opensearch.transport; import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.telemetry.tracing.Tracer; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.nativeprotocol.NativeInboundMessage; +import org.opensearch.transport.protobufprotocol.ProtobufInboundMessage; +import org.opensearch.transport.protobufprotocol.ProtobufMessageHandler; import java.io.IOException; import java.util.Map; @@ -80,6 +83,12 @@ public class InboundHandler { keepAlive ) ); + if (FeatureFlags.isEnabled(FeatureFlags.PROTOBUF_SETTING)) { + this.protocolMessageHandlers.put( + ProtobufInboundMessage.PROTOBUF_PROTOCOL, + new ProtobufMessageHandler(threadPool, responseHandlers) + ); + } } void setMessageListener(TransportMessageListener listener) { diff --git a/server/src/main/java/org/opensearch/transport/InboundPipeline.java b/server/src/main/java/org/opensearch/transport/InboundPipeline.java index f4c410671b08a..7f23c9e3db1a9 100644 --- a/server/src/main/java/org/opensearch/transport/InboundPipeline.java +++ b/server/src/main/java/org/opensearch/transport/InboundPipeline.java @@ -36,12 +36,12 @@ import org.opensearch.common.bytes.ReleasableBytesReference; import org.opensearch.common.lease.Releasable; import org.opensearch.common.lease.Releasables; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.util.PageCacheRecycler; import org.opensearch.core.common.breaker.CircuitBreaker; -import org.opensearch.core.common.bytes.CompositeBytesReference; import org.opensearch.transport.nativeprotocol.NativeInboundBytesHandler; +import org.opensearch.transport.protobufprotocol.ProtobufInboundBytesHandler; -import java.io.ByteArrayInputStream; import java.io.IOException; import java.util.ArrayDeque; import java.util.List; @@ -99,6 +99,9 @@ public InboundPipeline( this.aggregator = aggregator; this.protocolBytesHandlers = List.of(new NativeInboundBytesHandler(pending, decoder, aggregator, statsTracker)); this.messageHandler = messageHandler; + if (FeatureFlags.isEnabled(FeatureFlags.PROTOBUF_SETTING)) { + protocolBytesHandlers.add(new ProtobufInboundBytesHandler()); + } } @Override diff --git a/server/src/main/java/org/opensearch/transport/OutboundHandler.java b/server/src/main/java/org/opensearch/transport/OutboundHandler.java index ec3a0efd3c834..e5bb7764c70dd 100644 --- a/server/src/main/java/org/opensearch/transport/OutboundHandler.java +++ b/server/src/main/java/org/opensearch/transport/OutboundHandler.java @@ -54,6 +54,7 @@ import org.opensearch.search.fetch.QueryFetchSearchResult; import org.opensearch.search.query.QuerySearchResult; import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.protobufprotocol.ProtobufInboundMessage; import java.io.IOException; import java.util.Set; @@ -149,13 +150,13 @@ void sendResponse( ) throws IOException { Version version = Version.min(this.version, nodeVersion); ActionListener listener = ActionListener.wrap(() -> messageListener.onResponseSent(requestId, action, response)); - if ((response.getProtocol()).equals(BaseInboundMessage.PROTOBUF_PROTOCOL) && version.onOrAfter(Version.V_3_0_0)) { + if ((response.getProtocol()).equals(ProtobufInboundMessage.PROTOBUF_PROTOCOL) && version.onOrAfter(Version.V_3_0_0)) { if (response instanceof QueryFetchSearchResult) { QueryFetchSearchResult queryFetchSearchResult = (QueryFetchSearchResult) response; if (queryFetchSearchResult.response() != null) { byte[] bytes = new byte[1]; bytes[0] = 1; - NodeToNodeMessage protobufMessage = new NodeToNodeMessage( + ProtobufInboundMessage protobufMessage = new ProtobufInboundMessage( requestId, bytes, Version.CURRENT, @@ -171,7 +172,7 @@ void sendResponse( if (querySearchResult.response() != null) { byte[] bytes = new byte[1]; bytes[0] = 1; - NodeToNodeMessage protobufMessage = new NodeToNodeMessage( + ProtobufInboundMessage protobufMessage = new ProtobufInboundMessage( requestId, bytes, Version.CURRENT, @@ -231,7 +232,7 @@ private void sendMessage(TcpChannel channel, OutboundMessage networkMessage, Act internalSend(channel, sendContext); } - private void sendProtobufMessage(TcpChannel channel, NodeToNodeMessage message, ActionListener listener) throws IOException { + private void sendProtobufMessage(TcpChannel channel, ProtobufInboundMessage message, ActionListener listener) throws IOException { ProtobufMessageSerializer serializer = new ProtobufMessageSerializer(message, bigArrays); SendContext sendContext = new SendContext(channel, serializer, listener, serializer); internalSend(channel, sendContext); @@ -288,11 +289,11 @@ public void close() { private static class ProtobufMessageSerializer implements CheckedSupplier, Releasable { - private final NodeToNodeMessage message; + private final ProtobufInboundMessage message; private final BigArrays bigArrays; private volatile ReleasableBytesStreamOutput bytesStreamOutput; - private ProtobufMessageSerializer(NodeToNodeMessage message, BigArrays bigArrays) { + private ProtobufMessageSerializer(ProtobufInboundMessage message, BigArrays bigArrays) { this.message = message; this.bigArrays = bigArrays; } diff --git a/server/src/main/java/org/opensearch/transport/TcpTransport.java b/server/src/main/java/org/opensearch/transport/TcpTransport.java index 7fb600a484035..e32bba5e836d3 100644 --- a/server/src/main/java/org/opensearch/transport/TcpTransport.java +++ b/server/src/main/java/org/opensearch/transport/TcpTransport.java @@ -806,14 +806,6 @@ public static int readMessageLength(BytesReference networkBytes) throws IOExcept } } - public static String determineTransportProtocol(BytesReference headerBuffer) { - if (headerBuffer.get(0) == 'O' && headerBuffer.get(1) == 'S' && headerBuffer.get(2) == 'P') { - return BaseInboundMessage.PROTOBUF_PROTOCOL; - } else { - return BaseInboundMessage.NATIVE_PROTOCOL; - } - } - private static int readHeaderBuffer(BytesReference headerBuffer) throws IOException { if (headerBuffer.get(0) != 'E' || headerBuffer.get(1) != 'S') { if (appearsToBeHTTPRequest(headerBuffer)) { diff --git a/server/src/main/java/org/opensearch/transport/protobufprotocol/ProtobufInboundBytesHandler.java b/server/src/main/java/org/opensearch/transport/protobufprotocol/ProtobufInboundBytesHandler.java new file mode 100644 index 0000000000000..a0f54d645a378 --- /dev/null +++ b/server/src/main/java/org/opensearch/transport/protobufprotocol/ProtobufInboundBytesHandler.java @@ -0,0 +1,53 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.transport.protobufprotocol; + +import org.opensearch.common.bytes.ReleasableBytesReference; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.transport.InboundBytesHandler; +import org.opensearch.transport.ProtocolInboundMessage; +import org.opensearch.transport.TcpChannel; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.util.function.BiConsumer; + +/** + * Handler for inbound bytes for the protobuf protocol. + */ +public class ProtobufInboundBytesHandler implements InboundBytesHandler { + + public void ProtobufInboundBytesHandler() {} + + @Override + public void doHandleBytes( + TcpChannel channel, + ReleasableBytesReference reference, + BiConsumer messageHandler + ) throws IOException { + // removing the first byte we added for protobuf message + byte[] incomingBytes = BytesReference.toBytes(reference.slice(3, reference.length() - 3)); + ProtobufInboundMessage protobufMessage = new ProtobufInboundMessage(new ByteArrayInputStream(incomingBytes)); + messageHandler.accept(channel, protobufMessage); + } + + @Override + public boolean canHandleBytes(ReleasableBytesReference reference) { + if (reference.get(0) == 'O' && reference.get(1) == 'S' && reference.get(2) == 'P') { + return true; + } + return false; + } + + @Override + public void close() { + // no-op + } + +} diff --git a/server/src/main/java/org/opensearch/transport/NodeToNodeMessage.java b/server/src/main/java/org/opensearch/transport/protobufprotocol/ProtobufInboundMessage.java similarity index 90% rename from server/src/main/java/org/opensearch/transport/NodeToNodeMessage.java rename to server/src/main/java/org/opensearch/transport/protobufprotocol/ProtobufInboundMessage.java index 943007f7913c6..3f650ff61d3ab 100644 --- a/server/src/main/java/org/opensearch/transport/NodeToNodeMessage.java +++ b/server/src/main/java/org/opensearch/transport/protobufprotocol/ProtobufInboundMessage.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.transport; +package org.opensearch.transport.protobufprotocol; import com.google.protobuf.ByteString; import org.opensearch.Version; @@ -18,6 +18,8 @@ import org.opensearch.server.proto.NodeToNodeMessageProto.NodeToNodeMessage.ResponseHandlersList; import org.opensearch.server.proto.QueryFetchSearchResultProto.QueryFetchSearchResult; import org.opensearch.server.proto.QuerySearchResultProto.QuerySearchResult; +import org.opensearch.transport.ProtocolInboundMessage; +import org.opensearch.transport.TcpHeader; import java.io.IOException; import java.io.InputStream; @@ -34,13 +36,17 @@ * * @opensearch.internal */ -public class NodeToNodeMessage implements BaseInboundMessage { +public class ProtobufInboundMessage implements ProtocolInboundMessage { + + /** + * The protocol used to encode this message + */ + public static String PROTOBUF_PROTOCOL = "protobuf"; private final NodeToNodeMessageProto.NodeToNodeMessage message; private static final byte[] PREFIX = { (byte) 'E', (byte) 'S' }; - private String protocol; - public NodeToNodeMessage( + public ProtobufInboundMessage( long requestId, byte[] status, Version version, @@ -77,7 +83,7 @@ public NodeToNodeMessage( .build(); } - public NodeToNodeMessage( + public ProtobufInboundMessage( long requestId, byte[] status, Version version, @@ -114,7 +120,7 @@ public NodeToNodeMessage( .build(); } - public NodeToNodeMessage(InputStream in) throws IOException { + public ProtobufInboundMessage(InputStream in) throws IOException { this.message = NodeToNodeMessageProto.NodeToNodeMessage.parseFrom(in); } @@ -122,7 +128,7 @@ public void writeTo(OutputStream out) throws IOException { this.message.writeTo(out); } - BytesReference serialize(BytesStreamOutput bytesStream) throws IOException { + public BytesReference serialize(BytesStreamOutput bytesStream) throws IOException { NodeToNodeMessageProto.NodeToNodeMessage message = getMessage(); TcpHeader.writeHeaderForProtobuf(bytesStream); message.writeTo(bytesStream); @@ -135,7 +141,7 @@ public NodeToNodeMessageProto.NodeToNodeMessage getMessage() { @Override public String toString() { - return "NodeToNodeMessage [message=" + message + "]"; + return "ProtobufInboundMessage [message=" + message + "]"; } public org.opensearch.server.proto.NodeToNodeMessageProto.NodeToNodeMessage.Header getHeader() { @@ -163,8 +169,4 @@ public String getProtocol() { return PROTOBUF_PROTOCOL; } - @Override - public void setProtocol() { - this.protocol = PROTOBUF_PROTOCOL; - } } diff --git a/server/src/main/java/org/opensearch/transport/ProtobufMessageHandler.java b/server/src/main/java/org/opensearch/transport/protobufprotocol/ProtobufMessageHandler.java similarity index 83% rename from server/src/main/java/org/opensearch/transport/ProtobufMessageHandler.java rename to server/src/main/java/org/opensearch/transport/protobufprotocol/ProtobufMessageHandler.java index 3aa8be4d094e2..a945bab7a345a 100644 --- a/server/src/main/java/org/opensearch/transport/ProtobufMessageHandler.java +++ b/server/src/main/java/org/opensearch/transport/protobufprotocol/ProtobufMessageHandler.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.transport; +package org.opensearch.transport.protobufprotocol; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -19,6 +19,15 @@ import org.opensearch.search.query.QuerySearchResult; import org.opensearch.server.proto.QueryFetchSearchResultProto.QueryFetchSearchResult; import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.ProtocolInboundMessage; +import org.opensearch.transport.ProtocolMessageHandler; +import org.opensearch.transport.RemoteTransportException; +import org.opensearch.transport.ResponseHandlerFailureTransportException; +import org.opensearch.transport.TcpChannel; +import org.opensearch.transport.Transport; +import org.opensearch.transport.TransportMessageListener; +import org.opensearch.transport.TransportResponseHandler; +import org.opensearch.transport.TransportSerializationException; import java.io.IOException; import java.net.InetSocketAddress; @@ -30,7 +39,7 @@ * * @opensearch.internal */ -public class ProtobufMessageHandler { +public class ProtobufMessageHandler implements ProtocolMessageHandler { private static final Logger logger = LogManager.getLogger(ProtobufMessageHandler.class); @@ -41,7 +50,7 @@ public class ProtobufMessageHandler { private volatile long slowLogThresholdMs = Long.MAX_VALUE; - ProtobufMessageHandler(ThreadPool threadPool, Transport.ResponseHandlers responseHandlers) { + public ProtobufMessageHandler(ThreadPool threadPool, Transport.ResponseHandlers responseHandlers) { this.threadPool = threadPool; this.responseHandlers = responseHandlers; } @@ -58,16 +67,24 @@ void setSlowLogThreshold(TimeValue slowLogThreshold) { this.slowLogThresholdMs = slowLogThreshold.getMillis(); } - public void messageReceivedProtobuf(TcpChannel channel, NodeToNodeMessage message, long startTime) throws IOException { + @Override + public void messageReceived( + TcpChannel channel, + ProtocolInboundMessage message, + long startTime, + long slowLogThresholdMs, + TransportMessageListener messageListener + ) throws IOException { + ProtobufInboundMessage nodeToNodeMessage = (ProtobufInboundMessage) message; final InetSocketAddress remoteAddress = channel.getRemoteAddress(); - final org.opensearch.server.proto.NodeToNodeMessageProto.NodeToNodeMessage.Header header = message.getHeader(); + final org.opensearch.server.proto.NodeToNodeMessageProto.NodeToNodeMessage.Header header = nodeToNodeMessage.getHeader(); ThreadContext threadContext = threadPool.getThreadContext(); try (ThreadContext.StoredContext existing = threadContext.stashContext()) { // Place the context with the headers from the message final Tuple, Map>> headers = new Tuple, Map>>( - message.getRequestHeaders(), - message.getResponseHandlers() + nodeToNodeMessage.getRequestHeaders(), + nodeToNodeMessage.getResponseHandlers() ); threadContext.setHeaders(headers); threadContext.putTransient("_remote_address", remoteAddress); @@ -75,7 +92,7 @@ public void messageReceivedProtobuf(TcpChannel channel, NodeToNodeMessage messag long requestId = header.getRequestId(); TransportResponseHandler handler = responseHandlers.onResponseReceived(requestId, messageListener); if (handler != null) { - handleProtobufResponse(requestId, remoteAddress, message, handler); + handleProtobufResponse(requestId, remoteAddress, nodeToNodeMessage, handler); } } finally { final long took = threadPool.relativeTimeInMillis() - startTime; @@ -94,7 +111,7 @@ public void messageReceivedProtobuf(TcpChannel channel, NodeToNodeMessage messag private void handleProtobufResponse( final long requestId, InetSocketAddress remoteAddress, - final NodeToNodeMessage message, + final ProtobufInboundMessage message, final TransportResponseHandler handler ) throws IOException { try { diff --git a/server/src/test/java/org/opensearch/transport/InboundHandlerTests.java b/server/src/test/java/org/opensearch/transport/InboundHandlerTests.java index 892909b094eeb..d78de1d85648b 100644 --- a/server/src/test/java/org/opensearch/transport/InboundHandlerTests.java +++ b/server/src/test/java/org/opensearch/transport/InboundHandlerTests.java @@ -61,6 +61,7 @@ import org.opensearch.test.VersionUtils; import org.opensearch.threadpool.TestThreadPool; import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.protobufprotocol.ProtobufInboundMessage; import org.junit.After; import org.junit.Before; @@ -328,7 +329,7 @@ public QueryFetchSearchResult read(InputStream in) throws IOException { BytesReference fullResponseBytes = channel.getMessageCaptor().get(); byte[] incomingBytes = BytesReference.toBytes(fullResponseBytes.slice(3, fullResponseBytes.length() - 3)); - NodeToNodeMessage nodeToNodeMessage = new NodeToNodeMessage(new ByteArrayInputStream(incomingBytes)); + ProtobufInboundMessage nodeToNodeMessage = new ProtobufInboundMessage(new ByteArrayInputStream(incomingBytes)); handler.inboundMessage(channel, nodeToNodeMessage); QueryFetchSearchResult result = responseCaptor.get(); assertNotNull(result); diff --git a/server/src/test/java/org/opensearch/transport/InboundPipelineTests.java b/server/src/test/java/org/opensearch/transport/InboundPipelineTests.java index 8e0880ea85326..03f10bb702144 100644 --- a/server/src/test/java/org/opensearch/transport/InboundPipelineTests.java +++ b/server/src/test/java/org/opensearch/transport/InboundPipelineTests.java @@ -104,13 +104,7 @@ public void testPipelineHandling() throws IOException { final TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); circuitBreaker.startBreaking(); final InboundAggregator aggregator = new InboundAggregator(() -> circuitBreaker, canTripBreaker); - final InboundPipeline pipeline = new InboundPipeline( - statsTracker, - millisSupplier, - decoder, - aggregator, - messageHandler - ); + final InboundPipeline pipeline = new InboundPipeline(statsTracker, millisSupplier, decoder, aggregator, messageHandler); final FakeTcpChannel channel = new FakeTcpChannel(); final int iterations = randomIntBetween(5, 10); @@ -226,13 +220,7 @@ public void testDecodeExceptionIsPropagated() throws IOException { final InboundDecoder decoder = new InboundDecoder(Version.CURRENT, PageCacheRecycler.NON_RECYCLING_INSTANCE); final Supplier breaker = () -> new NoopCircuitBreaker("test"); final InboundAggregator aggregator = new InboundAggregator(breaker, (Predicate) action -> true); - final InboundPipeline pipeline = new InboundPipeline( - statsTracker, - millisSupplier, - decoder, - aggregator, - messageHandler - ); + final InboundPipeline pipeline = new InboundPipeline(statsTracker, millisSupplier, decoder, aggregator, messageHandler); try (BytesStreamOutput streamOutput = new BytesStreamOutput()) { String actionName = "actionName"; @@ -286,13 +274,7 @@ public void testEnsureBodyIsNotPrematurelyReleased() throws IOException { final InboundDecoder decoder = new InboundDecoder(Version.CURRENT, PageCacheRecycler.NON_RECYCLING_INSTANCE); final Supplier breaker = () -> new NoopCircuitBreaker("test"); final InboundAggregator aggregator = new InboundAggregator(breaker, (Predicate) action -> true); - final InboundPipeline pipeline = new InboundPipeline( - statsTracker, - millisSupplier, - decoder, - aggregator, - messageHandler - ); + final InboundPipeline pipeline = new InboundPipeline(statsTracker, millisSupplier, decoder, aggregator, messageHandler); try (BytesStreamOutput streamOutput = new BytesStreamOutput()) { String actionName = "actionName"; diff --git a/server/src/test/java/org/opensearch/transport/OutboundHandlerTests.java b/server/src/test/java/org/opensearch/transport/OutboundHandlerTests.java index ec40e95fe45c1..a9d8d3c45b9f9 100644 --- a/server/src/test/java/org/opensearch/transport/OutboundHandlerTests.java +++ b/server/src/test/java/org/opensearch/transport/OutboundHandlerTests.java @@ -68,6 +68,7 @@ import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.TestThreadPool; import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.protobufprotocol.ProtobufInboundMessage; import org.junit.After; import org.junit.Before; @@ -121,7 +122,7 @@ public void setUp() throws Exception { } catch (IOException e) { throw new AssertionError(e); } - }, Version.CURRENT); + }); } @After @@ -294,7 +295,7 @@ public void testSendProtobufResponse() throws IOException { FetchSearchResult fetchResult = createFetchSearchResult(); QueryFetchSearchResult response = new QueryFetchSearchResult(queryResult, fetchResult); System.setProperty(FeatureFlags.PROTOBUF, "true"); - assertTrue((response.getProtocol()).equals(BaseInboundMessage.PROTOBUF_PROTOCOL)); + assertTrue((response.getProtocol()).equals(ProtobufInboundMessage.PROTOBUF_PROTOCOL)); AtomicLong requestIdRef = new AtomicLong(); AtomicReference actionRef = new AtomicReference<>(); @@ -315,9 +316,9 @@ public void onResponseSent(long requestId, String action, TransportResponse resp final Supplier breaker = () -> new NoopCircuitBreaker("test"); final InboundAggregator aggregator = new InboundAggregator(breaker, (Predicate) requestCanTripBreaker -> true); InboundPipeline inboundPipeline = new InboundPipeline(statsTracker, millisSupplier, decoder, aggregator, (c, m) -> { - NodeToNodeMessage m1 = (NodeToNodeMessage) m; + ProtobufInboundMessage m1 = (ProtobufInboundMessage) m; protobufMessage.set(BytesReference.fromByteBuffer(ByteBuffer.wrap(m1.getMessage().toByteArray()))); - }, Version.CURRENT); + }); BytesReference reference = channel.getMessageCaptor().get(); ActionListener sendListener = channel.getListenerCaptor().get(); if (randomBoolean()) { @@ -331,7 +332,7 @@ public void onResponseSent(long requestId, String action, TransportResponse resp inboundPipeline.handleBytes(channel, new ReleasableBytesReference(reference, () -> {})); final BytesReference responseBytes = protobufMessage.get(); - final NodeToNodeMessage message = new NodeToNodeMessage(new ByteArrayInputStream(responseBytes.toBytesRef().bytes)); + final ProtobufInboundMessage message = new ProtobufInboundMessage(new ByteArrayInputStream(responseBytes.toBytesRef().bytes)); assertEquals(version.toString(), message.getMessage().getVersion()); assertEquals(requestId, message.getHeader().getRequestId()); assertNotNull(message.getRequestHeaders()); From 324c58ff9b586a5281f59053e39964f45fa10a03 Mon Sep 17 00:00:00 2001 From: Vacha Shah Date: Thu, 11 Apr 2024 18:07:14 +0000 Subject: [PATCH 36/36] Fixing after merging to main Signed-off-by: Vacha Shah --- .../opensearch/common/util/FeatureFlags.java | 6 --- .../org/opensearch/search/SearchHits.java | 1 - .../search/internal/ShardSearchRequest.java | 1 - .../opensearch/transport/InboundHandler.java | 42 ++++++++++++------- .../opensearch/transport/InboundPipeline.java | 14 +++++-- .../ProtobufInboundBytesHandler.java | 2 +- .../protobufprotocol/package-info.java | 10 +++++ .../transport/InboundHandlerTests.java | 25 ++++++++--- 8 files changed, 68 insertions(+), 33 deletions(-) create mode 100644 server/src/main/java/org/opensearch/transport/protobufprotocol/package-info.java diff --git a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java index f9da0968699b5..9dbc340e54689 100644 --- a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java +++ b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java @@ -61,11 +61,6 @@ public class FeatureFlags { */ public static final String WRITEABLE_REMOTE_INDEX = "opensearch.experimental.feature.writeable_remote_index.enabled"; - /** - * Gates the optimization to enable bloom filters for doc id lookup. - */ - public static final String DOC_ID_FUZZY_SET = "opensearch.experimental.optimize_doc_id_lookup.fuzzy_set.enabled"; - /** * Gates the functionality of integrating protobuf within search API and node-to-node communication. */ @@ -168,5 +163,4 @@ public static boolean isEnabled(Setting featureFlag) { return featureFlag.getDefault(Settings.EMPTY); } } - } diff --git a/server/src/main/java/org/opensearch/search/SearchHits.java b/server/src/main/java/org/opensearch/search/SearchHits.java index 0b4ee3851bd61..8232643b353f5 100644 --- a/server/src/main/java/org/opensearch/search/SearchHits.java +++ b/server/src/main/java/org/opensearch/search/SearchHits.java @@ -342,5 +342,4 @@ private static Relation parseRelation(String relation) { throw new IllegalArgumentException("invalid total hits relation: " + relation); } } - } diff --git a/server/src/main/java/org/opensearch/search/internal/ShardSearchRequest.java b/server/src/main/java/org/opensearch/search/internal/ShardSearchRequest.java index a42224b5d94de..0674101dc888f 100644 --- a/server/src/main/java/org/opensearch/search/internal/ShardSearchRequest.java +++ b/server/src/main/java/org/opensearch/search/internal/ShardSearchRequest.java @@ -46,7 +46,6 @@ import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.unit.TimeValue; -import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.common.Strings; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.StreamInput; diff --git a/server/src/main/java/org/opensearch/transport/InboundHandler.java b/server/src/main/java/org/opensearch/transport/InboundHandler.java index d5d7f614c37e6..d58d80857da0b 100644 --- a/server/src/main/java/org/opensearch/transport/InboundHandler.java +++ b/server/src/main/java/org/opensearch/transport/InboundHandler.java @@ -70,23 +70,35 @@ public class InboundHandler { Tracer tracer ) { this.threadPool = threadPool; - this.protocolMessageHandlers = Map.of( - NativeInboundMessage.NATIVE_PROTOCOL, - new NativeMessageHandler( - threadPool, - outboundHandler, - namedWriteableRegistry, - handshaker, - requestHandlers, - responseHandlers, - tracer, - keepAlive - ) - ); if (FeatureFlags.isEnabled(FeatureFlags.PROTOBUF_SETTING)) { - this.protocolMessageHandlers.put( + this.protocolMessageHandlers = Map.of( ProtobufInboundMessage.PROTOBUF_PROTOCOL, - new ProtobufMessageHandler(threadPool, responseHandlers) + new ProtobufMessageHandler(threadPool, responseHandlers), + NativeInboundMessage.NATIVE_PROTOCOL, + new NativeMessageHandler( + threadPool, + outboundHandler, + namedWriteableRegistry, + handshaker, + requestHandlers, + responseHandlers, + tracer, + keepAlive + ) + ); + } else { + this.protocolMessageHandlers = Map.of( + NativeInboundMessage.NATIVE_PROTOCOL, + new NativeMessageHandler( + threadPool, + outboundHandler, + namedWriteableRegistry, + handshaker, + requestHandlers, + responseHandlers, + tracer, + keepAlive + ) ); } } diff --git a/server/src/main/java/org/opensearch/transport/InboundPipeline.java b/server/src/main/java/org/opensearch/transport/InboundPipeline.java index 7f23c9e3db1a9..0e154d571f832 100644 --- a/server/src/main/java/org/opensearch/transport/InboundPipeline.java +++ b/server/src/main/java/org/opensearch/transport/InboundPipeline.java @@ -97,11 +97,15 @@ public InboundPipeline( this.statsTracker = statsTracker; this.decoder = decoder; this.aggregator = aggregator; - this.protocolBytesHandlers = List.of(new NativeInboundBytesHandler(pending, decoder, aggregator, statsTracker)); - this.messageHandler = messageHandler; if (FeatureFlags.isEnabled(FeatureFlags.PROTOBUF_SETTING)) { - protocolBytesHandlers.add(new ProtobufInboundBytesHandler()); + this.protocolBytesHandlers = List.of( + new ProtobufInboundBytesHandler(), + new NativeInboundBytesHandler(pending, decoder, aggregator, statsTracker) + ); + } else { + this.protocolBytesHandlers = List.of(new NativeInboundBytesHandler(pending, decoder, aggregator, statsTracker)); } + this.messageHandler = messageHandler; } @Override @@ -129,6 +133,10 @@ public void handleBytes(TcpChannel channel, ReleasableBytesReference reference) } public void doHandleBytes(TcpChannel channel, ReleasableBytesReference reference) throws IOException { + channel.getChannelStats().markAccessed(relativeTimeInMillis.getAsLong()); + statsTracker.markBytesRead(reference.length()); + pending.add(reference.retain()); + // If we don't have a current handler, we should try to find one based on the protocol of the incoming bytes. if (currentHandler == null) { for (InboundBytesHandler handler : protocolBytesHandlers) { diff --git a/server/src/main/java/org/opensearch/transport/protobufprotocol/ProtobufInboundBytesHandler.java b/server/src/main/java/org/opensearch/transport/protobufprotocol/ProtobufInboundBytesHandler.java index a0f54d645a378..0b3d268001f69 100644 --- a/server/src/main/java/org/opensearch/transport/protobufprotocol/ProtobufInboundBytesHandler.java +++ b/server/src/main/java/org/opensearch/transport/protobufprotocol/ProtobufInboundBytesHandler.java @@ -23,7 +23,7 @@ */ public class ProtobufInboundBytesHandler implements InboundBytesHandler { - public void ProtobufInboundBytesHandler() {} + public ProtobufInboundBytesHandler() {} @Override public void doHandleBytes( diff --git a/server/src/main/java/org/opensearch/transport/protobufprotocol/package-info.java b/server/src/main/java/org/opensearch/transport/protobufprotocol/package-info.java new file mode 100644 index 0000000000000..8bccdc2be0b79 --- /dev/null +++ b/server/src/main/java/org/opensearch/transport/protobufprotocol/package-info.java @@ -0,0 +1,10 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** Protobuf transport protocol package. */ +package org.opensearch.transport.protobufprotocol; diff --git a/server/src/test/java/org/opensearch/transport/InboundHandlerTests.java b/server/src/test/java/org/opensearch/transport/InboundHandlerTests.java index d78de1d85648b..c6c9a33041a66 100644 --- a/server/src/test/java/org/opensearch/transport/InboundHandlerTests.java +++ b/server/src/test/java/org/opensearch/transport/InboundHandlerTests.java @@ -87,6 +87,9 @@ public class InboundHandlerTests extends OpenSearchTestCase { private final Version version = Version.CURRENT; private TaskManager taskManager; + private NamedWriteableRegistry namedWriteableRegistry; + private TransportHandshaker handshaker; + private TransportKeepAlive keepAlive; private Transport.ResponseHandlers responseHandlers; private Transport.RequestHandlers requestHandlers; private InboundHandler handler; @@ -105,8 +108,8 @@ public void sendMessage(BytesReference reference, ActionListener listener) } } }; - NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(Collections.emptyList()); - TransportHandshaker handshaker = new TransportHandshaker(version, threadPool, (n, c, r, v) -> {}); + namedWriteableRegistry = new NamedWriteableRegistry(Collections.emptyList()); + handshaker = new TransportHandshaker(version, threadPool, (n, c, r, v) -> {}); outboundHandler = new OutboundHandler( "node", version, @@ -115,7 +118,7 @@ public void sendMessage(BytesReference reference, ActionListener listener) threadPool, BigArrays.NON_RECYCLING_INSTANCE ); - TransportKeepAlive keepAlive = new TransportKeepAlive(threadPool, outboundHandler::sendBytes); + keepAlive = new TransportKeepAlive(threadPool, outboundHandler::sendBytes); requestHandlers = new Transport.RequestHandlers(); responseHandlers = new Transport.ResponseHandlers(); handler = new InboundHandler( @@ -251,6 +254,17 @@ public TestResponse read(StreamInput in) throws IOException { @SuppressForbidden(reason = "manipulates system properties for testing") public void testProtobufResponse() throws Exception { + System.setProperty(FeatureFlags.PROTOBUF_SETTING.getKey(), "true"); + InboundHandler inboundHandler = new InboundHandler( + threadPool, + outboundHandler, + namedWriteableRegistry, + handshaker, + keepAlive, + requestHandlers, + responseHandlers, + NoopTracer.INSTANCE + ); String action = "test-request"; int headerSize = TcpHeader.headerSize(version); AtomicReference requestCaptor = new AtomicReference<>(); @@ -314,7 +328,7 @@ public QueryFetchSearchResult read(InputStream in) throws IOException { Header requestHeader = new Header(fullRequestBytes.length() - 6, requestId, TransportStatus.setRequest((byte) 0), version); InboundMessage requestMessage = new InboundMessage(requestHeader, ReleasableBytesReference.wrap(requestContent), () -> {}); requestHeader.finishParsingHeader(requestMessage.openOrGetStreamInput()); - handler.inboundMessage(channel, requestMessage); + inboundHandler.inboundMessage(channel, requestMessage); TransportChannel transportChannel = channelCaptor.get(); assertEquals(Version.CURRENT, transportChannel.getVersion()); @@ -324,13 +338,12 @@ public QueryFetchSearchResult read(InputStream in) throws IOException { QuerySearchResult queryResult = OutboundHandlerTests.createQuerySearchResult(); FetchSearchResult fetchResult = OutboundHandlerTests.createFetchSearchResult(); QueryFetchSearchResult response = new QueryFetchSearchResult(queryResult, fetchResult); - System.setProperty(FeatureFlags.PROTOBUF, "true"); transportChannel.sendResponse(response); BytesReference fullResponseBytes = channel.getMessageCaptor().get(); byte[] incomingBytes = BytesReference.toBytes(fullResponseBytes.slice(3, fullResponseBytes.length() - 3)); ProtobufInboundMessage nodeToNodeMessage = new ProtobufInboundMessage(new ByteArrayInputStream(incomingBytes)); - handler.inboundMessage(channel, nodeToNodeMessage); + inboundHandler.inboundMessage(channel, nodeToNodeMessage); QueryFetchSearchResult result = responseCaptor.get(); assertNotNull(result); assertEquals(queryResult.getMaxScore(), result.queryResult().getMaxScore(), 0.0);