diff --git a/CHANGELOG.md b/CHANGELOG.md index d14b24f54ac03..0ebd7bd3ec4cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -86,6 +86,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - [Search Pipelines] Add `default_search_pipeline` index setting ([#7470](https://github.com/opensearch-project/OpenSearch/pull/7470)) - [Search Pipelines] Add RenameFieldResponseProcessor for Search Pipelines ([#7377](https://github.com/opensearch-project/OpenSearch/pull/7377)) - [Search Pipelines] Split search pipeline processor factories by type ([#7597](https://github.com/opensearch-project/OpenSearch/pull/7597)) +- [Search Pipelines] Add script processor ([#7607](https://github.com/opensearch-project/OpenSearch/pull/7607)) - Add descending order search optimization through reverse segment read. ([#7244](https://github.com/opensearch-project/OpenSearch/pull/7244)) - Add 'unsigned_long' numeric field type ([#6237](https://github.com/opensearch-project/OpenSearch/pull/6237)) - Add back primary shard preference for queries ([#7375](https://github.com/opensearch-project/OpenSearch/pull/7375)) diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/71_context_api.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/71_context_api.yml index 0413661fc586c..478ca9ae8abf4 100644 --- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/71_context_api.yml +++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/71_context_api.yml @@ -2,7 +2,7 @@ - do: scripts_painless_context: {} - match: { contexts.0: aggregation_selector} - - match: { contexts.22: update} + - match: { contexts.23: update} --- "Action to get all API values for score context": diff --git a/modules/search-pipeline-common/build.gradle b/modules/search-pipeline-common/build.gradle index cc655e10ada92..fe3c097ff6886 100644 --- a/modules/search-pipeline-common/build.gradle +++ b/modules/search-pipeline-common/build.gradle @@ -15,9 +15,11 @@ apply plugin: 'opensearch.internal-cluster-test' opensearchplugin { description 'Module for search pipeline processors that do not require additional security permissions or have large dependencies and resources' classname 'org.opensearch.search.pipeline.common.SearchPipelineCommonModulePlugin' + extendedPlugins = ['lang-painless'] } dependencies { + compileOnly project(':modules:lang-painless') } restResources { diff --git a/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/FilterQueryRequestProcessor.java b/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/FilterQueryRequestProcessor.java index 0ca090780bb60..7deb8faa03af6 100644 --- a/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/FilterQueryRequestProcessor.java +++ b/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/FilterQueryRequestProcessor.java @@ -40,6 +40,11 @@ public class FilterQueryRequestProcessor extends AbstractProcessor implements Se final QueryBuilder filterQuery; + /** + * Returns the type of the processor. + * + * @return The processor type. + */ @Override public String getType() { return TYPE; @@ -57,6 +62,14 @@ public FilterQueryRequestProcessor(String tag, String description, QueryBuilder this.filterQuery = filterQuery; } + /** + * Modifies the search request by adding a filtered query to the existing query, if any, and sets it as the new query + * in the search request's SearchSourceBuilder. + * + * @param request The search request to be processed. + * @return The modified search request. + * @throws Exception if an error occurs while processing the request. + */ @Override public SearchRequest processRequest(SearchRequest request) throws Exception { QueryBuilder originalQuery = null; diff --git a/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/ScriptRequestProcessor.java b/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/ScriptRequestProcessor.java new file mode 100644 index 0000000000000..015411e0701a4 --- /dev/null +++ b/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/ScriptRequestProcessor.java @@ -0,0 +1,182 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.pipeline.common; + +import org.opensearch.action.search.SearchRequest; + +import org.opensearch.common.Nullable; +import org.opensearch.common.bytes.BytesReference; +import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.common.xcontent.json.JsonXContent; + +import org.opensearch.script.Script; +import org.opensearch.script.ScriptException; +import org.opensearch.script.ScriptService; +import org.opensearch.script.ScriptType; +import org.opensearch.script.SearchScript; +import org.opensearch.search.pipeline.Processor; +import org.opensearch.search.pipeline.SearchRequestProcessor; +import org.opensearch.search.pipeline.common.helpers.SearchRequestMap; + +import java.io.InputStream; +import java.util.Arrays; +import java.util.Map; + +import static org.opensearch.ingest.ConfigurationUtils.newConfigurationException; + +/** + * Processor that evaluates a script with a search request in its context + * and then returns the modified search request. + */ +public final class ScriptRequestProcessor extends AbstractProcessor implements SearchRequestProcessor { + /** + * Key to reference this processor type from a search pipeline. + */ + public static final String TYPE = "script"; + + private final Script script; + private final ScriptService scriptService; + private final SearchScript precompiledSearchScript; + + /** + * Processor that evaluates a script with a search request in its context + * + * @param tag The processor's tag. + * @param description The processor's description. + * @param script The {@link Script} to execute. + * @param precompiledSearchScript The {@link Script} precompiled + * @param scriptService The {@link ScriptService} used to execute the script. + */ + ScriptRequestProcessor( + String tag, + String description, + Script script, + @Nullable SearchScript precompiledSearchScript, + ScriptService scriptService + ) { + super(tag, description); + this.script = script; + this.precompiledSearchScript = precompiledSearchScript; + this.scriptService = scriptService; + } + + /** + * Executes the script with the search request in context. + * + * @param request The search request passed into the script context. + * @return The modified search request. + * @throws Exception if an error occurs while processing the request. + */ + @Override + public SearchRequest processRequest(SearchRequest request) throws Exception { + // assert request is not null and source is not null + if (request == null || request.source() == null) { + throw new IllegalArgumentException("search request must not be null"); + } + final SearchScript searchScript; + if (precompiledSearchScript == null) { + SearchScript.Factory factory = scriptService.compile(script, SearchScript.CONTEXT); + searchScript = factory.newInstance(script.getParams()); + } else { + searchScript = precompiledSearchScript; + } + // execute the script with the search request in context + searchScript.execute(Map.of("_source", new SearchRequestMap(request))); + return request; + } + + /** + * Returns the type of the processor. + * + * @return The processor type. + */ + @Override + public String getType() { + return TYPE; + } + + /** + * Returns the script used by the processor. + * + * @return The script. + */ + Script getScript() { + return script; + } + + /** + * Returns the precompiled search script used by the processor. + * + * @return The precompiled search script. + */ + SearchScript getPrecompiledSearchScript() { + return precompiledSearchScript; + } + + /** + * Factory class for creating {@link ScriptRequestProcessor}. + */ + public static final class Factory implements Processor.Factory { + private final ScriptService scriptService; + + /** + * Constructs a new Factory instance with the specified {@link ScriptService}. + * + * @param scriptService The {@link ScriptService} used to execute scripts. + */ + public Factory(ScriptService scriptService) { + this.scriptService = scriptService; + } + + /** + * Creates a new instance of {@link ScriptRequestProcessor}. + * + * @param registry The registry of processor factories. + * @param processorTag The processor's tag. + * @param description The processor's description. + * @param config The configuration options for the processor. + * @return The created {@link ScriptRequestProcessor} instance. + * @throws Exception if an error occurs during the creation process. + */ + @Override + public ScriptRequestProcessor create( + Map> registry, + String processorTag, + String description, + Map config + ) throws Exception { + try ( + XContentBuilder builder = XContentBuilder.builder(JsonXContent.jsonXContent).map(config); + InputStream stream = BytesReference.bytes(builder).streamInput(); + XContentParser parser = XContentType.JSON.xContent() + .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, stream) + ) { + Script script = Script.parse(parser); + + Arrays.asList("id", "source", "inline", "lang", "params", "options").forEach(config::remove); + + // verify script is able to be compiled before successfully creating processor. + SearchScript searchScript = null; + try { + final SearchScript.Factory factory = scriptService.compile(script, SearchScript.CONTEXT); + if (ScriptType.INLINE.equals(script.getType())) { + searchScript = factory.newInstance(script.getParams()); + } + } catch (ScriptException e) { + throw newConfigurationException(TYPE, processorTag, null, e); + } + return new ScriptRequestProcessor(processorTag, description, script, searchScript, scriptService); + } + } + } +} diff --git a/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/SearchPipelineCommonModulePlugin.java b/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/SearchPipelineCommonModulePlugin.java index aa56714085b48..dc25de460fdba 100644 --- a/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/SearchPipelineCommonModulePlugin.java +++ b/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/SearchPipelineCommonModulePlugin.java @@ -26,9 +26,20 @@ public class SearchPipelineCommonModulePlugin extends Plugin implements SearchPi */ public SearchPipelineCommonModulePlugin() {} + /** + * Returns a map of processor factories. + * + * @param parameters The parameters required for creating the processor factories. + * @return A map of processor factories, where the keys are the processor types and the values are the corresponding factory instances. + */ @Override public Map> getRequestProcessors(Processor.Parameters parameters) { - return Map.of(FilterQueryRequestProcessor.TYPE, new FilterQueryRequestProcessor.Factory(parameters.namedXContentRegistry)); + return Map.of( + FilterQueryRequestProcessor.TYPE, + new FilterQueryRequestProcessor.Factory(parameters.namedXContentRegistry), + ScriptRequestProcessor.TYPE, + new ScriptRequestProcessor.Factory(parameters.scriptService) + ); } @Override diff --git a/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/helpers/SearchRequestMap.java b/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/helpers/SearchRequestMap.java new file mode 100644 index 0000000000000..7af3ac66be146 --- /dev/null +++ b/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/helpers/SearchRequestMap.java @@ -0,0 +1,395 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.pipeline.common.helpers; + +import org.opensearch.action.search.SearchRequest; +import org.opensearch.search.builder.SearchSourceBuilder; + +import java.util.Collection; +import java.util.Map; +import java.util.Set; +import java.util.function.BiConsumer; +import java.util.function.BiFunction; +import java.util.function.Function; + +/** + * A custom implementation of {@link Map} that provides access to the properties of a {@link SearchRequest}'s + * {@link SearchSourceBuilder}. The class allows retrieving and modifying specific properties of the search request. + */ +public class SearchRequestMap implements Map { + private static final String UNSUPPORTED_OP_ERR = " Method not supported in Search pipeline script"; + + private final SearchSourceBuilder source; + + /** + * Constructs a new instance of the {@link SearchRequestMap} with the provided {@link SearchRequest}. + * + * @param searchRequest The SearchRequest containing the SearchSourceBuilder to be accessed. + */ + public SearchRequestMap(SearchRequest searchRequest) { + source = searchRequest.source(); + } + + /** + * Retrieves the number of properties in the SearchSourceBuilder. + * + * @return The number of properties in the SearchSourceBuilder. + * @throws UnsupportedOperationException always, as the method is not supported. + */ + @Override + public int size() { + throw new UnsupportedOperationException("size" + UNSUPPORTED_OP_ERR); + } + + /** + * Checks if the SearchSourceBuilder is empty. + * + * @return {@code true} if the SearchSourceBuilder is empty, {@code false} otherwise. + */ + @Override + public boolean isEmpty() { + return source == null; + } + + /** + * Checks if the SearchSourceBuilder contains the specified property. + * + * @param key The property to check for. + * @return {@code true} if the SearchSourceBuilder contains the specified property, {@code false} otherwise. + */ + @Override + public boolean containsKey(Object key) { + return get(key) != null; + } + + /** + * Checks if the SearchSourceBuilder contains the specified value. + * + * @param value The value to check for. + * @return {@code true} if the SearchSourceBuilder contains the specified value, {@code false} otherwise. + * @throws UnsupportedOperationException always, as the method is not supported. + */ + @Override + public boolean containsValue(Object value) { + throw new UnsupportedOperationException("containsValue" + UNSUPPORTED_OP_ERR); + } + + /** + * Retrieves the value associated with the specified property from the SearchSourceBuilder. + * + * @param key The SearchSourceBuilder property whose value is to be retrieved. + * @return The value associated with the specified property or null if the property has not been initialized. + * @throws IllegalArgumentException if the property name is not a String. + * @throws SearchRequestMapProcessingException if the property is not supported. + */ + @Override + public Object get(Object key) { + if (!(key instanceof String)) { + throw new IllegalArgumentException("key must be a String"); + } + // This is the explicit implementation of fetch value from source + switch ((String) key) { + case "from": + return source.from(); + case "size": + return source.size(); + case "explain": + return source.explain(); + case "version": + return source.version(); + case "seq_no_primary_term": + return source.seqNoAndPrimaryTerm(); + case "track_scores": + return source.trackScores(); + case "track_total_hits": + return source.trackTotalHitsUpTo(); + case "min_score": + return source.minScore(); + case "terminate_after": + return source.terminateAfter(); + case "profile": + return source.profile(); + default: + throw new SearchRequestMapProcessingException("Unsupported key: " + key); + } + } + + /** + * Sets the value for the specified property in the SearchSourceBuilder. + * + * @param key The property whose value is to be set. + * @param value The value to be set for the specified property. + * @return The original value associated with the property, or null if none existed. + * @throws IllegalArgumentException if the property is not a String. + * @throws SearchRequestMapProcessingException if the property is not supported or an error occurs during the setting. + */ + @Override + public Object put(String key, Object value) { + Object originalValue = get(key); + try { + switch (key) { + case "from": + source.from((Integer) value); + break; + case "size": + source.size((Integer) value); + break; + case "explain": + source.explain((Boolean) value); + break; + case "version": + source.version((Boolean) value); + break; + case "seq_no_primary_term": + source.seqNoAndPrimaryTerm((Boolean) value); + break; + case "track_scores": + source.trackScores((Boolean) value); + break; + case "track_total_hits": + source.trackTotalHitsUpTo((Integer) value); + break; + case "min_score": + source.minScore((Float) value); + break; + case "terminate_after": + source.terminateAfter((Integer) value); + break; + case "profile": + source.profile((Boolean) value); + break; + case "stats": // Not modifying stats, sorts, docvalue_fields, etc. as they require more complex handling + case "sort": + case "timeout": + case "docvalue_fields": + case "indices_boost": + default: + throw new SearchRequestMapProcessingException("Unsupported SearchRequest source property: " + key); + } + } catch (Exception e) { + throw new SearchRequestMapProcessingException("Error while setting value for SearchRequest source property: " + key, e); + } + return originalValue; + } + + /** + * Removes the specified property from the SearchSourceBuilder. + * + * @param key The name of the property that will be removed. + * @return The value associated with the property before it was removed, or null if the property was not found. + * @throws UnsupportedOperationException always, as the method is not supported. + */ + @Override + public Object remove(Object key) { + throw new UnsupportedOperationException("remove" + UNSUPPORTED_OP_ERR); + } + + /** + * Sets all the properties from the specified map to the SearchSourceBuilder. + * + * @param m The map containing the properties to be set. + * @throws UnsupportedOperationException always, as the method is not supported. + */ + @Override + public void putAll(Map m) { + throw new UnsupportedOperationException("putAll" + UNSUPPORTED_OP_ERR); + } + + /** + * Removes all properties from the SearchSourceBuilder. + * + * @throws UnsupportedOperationException always, as the method is not supported. + */ + @Override + public void clear() { + throw new UnsupportedOperationException("clear" + UNSUPPORTED_OP_ERR); + } + + /** + * Returns a set view of the property names in the SearchSourceBuilder. + * + * @return A set view of the property names in the SearchSourceBuilder. + * @throws UnsupportedOperationException always, as the method is not supported. + */ + @Override + public Set keySet() { + throw new UnsupportedOperationException("keySet" + UNSUPPORTED_OP_ERR); + } + + /** + * Returns a collection view of the property values in the SearchSourceBuilder. + * + * @return A collection view of the property values in the SearchSourceBuilder. + * @throws UnsupportedOperationException always, as the method is not supported. + */ + @Override + public Collection values() { + throw new UnsupportedOperationException("values" + UNSUPPORTED_OP_ERR); + } + + /** + * Returns a set view of the properties in the SearchSourceBuilder. + * + * @return A set view of the properties in the SearchSourceBuilder. + * @throws UnsupportedOperationException always, as the method is not supported. + */ + @Override + public Set> entrySet() { + throw new UnsupportedOperationException("entrySet" + UNSUPPORTED_OP_ERR); + } + + /** + * Returns the value to which the specified property has, or the defaultValue if the property is not present in the + * SearchSourceBuilder. + * + * @param key The property whose associated value is to be returned. + * @param defaultValue The default value to be returned if the property is not present. + * @return The value to which the specified property has, or the defaultValue if the property is not present. + * @throws UnsupportedOperationException always, as the method is not supported. + */ + @Override + public Object getOrDefault(Object key, Object defaultValue) { + throw new UnsupportedOperationException("getOrDefault" + UNSUPPORTED_OP_ERR); + } + + /** + * Performs the given action for each property in the SearchSourceBuilder until all properties have been processed or the + * action throws an exception + * + * @param action The action to be performed for each property. + * @throws UnsupportedOperationException always, as the method is not supported. + */ + @Override + public void forEach(BiConsumer action) { + throw new UnsupportedOperationException("forEach" + UNSUPPORTED_OP_ERR); + } + + /** + * Replaces each property's value with the result of invoking the given function on that property until all properties have + * been processed or the function throws an exception. + * + * @param function The function to apply to each property. + * @throws UnsupportedOperationException always, as the method is not supported. + */ + @Override + public void replaceAll(BiFunction function) { + throw new UnsupportedOperationException("replaceAll" + UNSUPPORTED_OP_ERR); + } + + /** + * If the specified property is not already associated with a value, associates it with the given value and returns null, + * else returns the current value. + * + * @param key The property whose value is to be set if absent. + * @param value The value to be associated with the specified property. + * @return The current value associated with the property, or null if the property is not present. + * @throws UnsupportedOperationException always, as the method is not supported. + */ + @Override + public Object putIfAbsent(String key, Object value) { + throw new UnsupportedOperationException("putIfAbsent" + UNSUPPORTED_OP_ERR); + } + + /** + * Removes the property only if it has the given value. + * + * @param key The property to be removed. + * @param value The value expected to be associated with the property. + * @return {@code true} if the entry was removed, {@code false} otherwise. + * @throws UnsupportedOperationException always, as the method is not supported. + */ + @Override + public boolean remove(Object key, Object value) { + throw new UnsupportedOperationException("remove" + UNSUPPORTED_OP_ERR); + } + + /** + * Replaces the specified property only if it has the given value. + * + * @param key The property to be replaced. + * @param oldValue The value expected to be associated with the property. + * @param newValue The value to be associated with the property. + * @return {@code true} if the property was replaced, {@code false} otherwise. + * @throws UnsupportedOperationException always, as the method is not supported. + */ + @Override + public boolean replace(String key, Object oldValue, Object newValue) { + throw new UnsupportedOperationException("replace" + UNSUPPORTED_OP_ERR); + } + + /** + * Replaces the specified property only if it has the given value. + * + * @param key The property to be replaced. + * @param value The value to be associated with the property. + * @return The previous value associated with the property, or null if the property was not found. + * @throws UnsupportedOperationException always, as the method is not supported. + */ + @Override + public Object replace(String key, Object value) { + throw new UnsupportedOperationException("replace" + UNSUPPORTED_OP_ERR); + } + + /** + * The computed value associated with the property, or null if the property is not present. + * + * @param key The property whose value is to be computed if absent. + * @param mappingFunction The function to compute a value based on the property. + * @return The computed value associated with the property, or null if the property is not present. + * @throws UnsupportedOperationException always, as the method is not supported. + */ + @Override + public Object computeIfAbsent(String key, Function mappingFunction) { + throw new UnsupportedOperationException("computeIfAbsent" + UNSUPPORTED_OP_ERR); + } + + /** + * If the value for the specified property is present, attempts to compute a new mapping given the property and its current + * mapped value. + * + * @param key The property for which the mapping is to be computed. + * @param remappingFunction The function to compute a new mapping. + * @return The new value associated with the property, or null if the property is not present. + * @throws UnsupportedOperationException always, as the method is not supported. + */ + @Override + public Object computeIfPresent(String key, BiFunction remappingFunction) { + throw new UnsupportedOperationException("computeIfPresent" + UNSUPPORTED_OP_ERR); + } + + /** + * If the value for the specified property is present, attempts to compute a new mapping given the property and its current + * mapped value, or removes the property if the computed value is null. + * + * @param key The property for which the mapping is to be computed. + * @param remappingFunction The function to compute a new mapping. + * @return The new value associated with the property, or null if the property is not present. + * @throws UnsupportedOperationException always, as the method is not supported. + */ + @Override + public Object compute(String key, BiFunction remappingFunction) { + throw new UnsupportedOperationException("compute" + UNSUPPORTED_OP_ERR); + } + + /** + * If the specified property is not already associated with a value or is associated with null, associates it with the + * given non-null value. Otherwise, replaces the associated value with the results of applying the given + * remapping function to the current and new values. + * + * @param key The property for which the mapping is to be merged. + * @param value The non-null value to be merged with the existing value. + * @param remappingFunction The function to merge the existing and new values. + * @return The new value associated with the property, or null if the property is not present. + * @throws UnsupportedOperationException always, as the method is not supported. + */ + @Override + public Object merge(String key, Object value, BiFunction remappingFunction) { + throw new UnsupportedOperationException("merge" + UNSUPPORTED_OP_ERR); + } +} diff --git a/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/helpers/SearchRequestMapProcessingException.java b/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/helpers/SearchRequestMapProcessingException.java new file mode 100644 index 0000000000000..cb1e45a20b624 --- /dev/null +++ b/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/helpers/SearchRequestMapProcessingException.java @@ -0,0 +1,39 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.pipeline.common.helpers; + +import org.opensearch.OpenSearchException; +import org.opensearch.OpenSearchWrapperException; + +/** + * An exception that indicates an error occurred while processing a {@link SearchRequestMap}. + */ +public class SearchRequestMapProcessingException extends OpenSearchException implements OpenSearchWrapperException { + + /** + * Constructs a new SearchRequestMapProcessingException with the specified message. + * + * @param msg The error message. + * @param args Arguments to substitute in the error message. + */ + public SearchRequestMapProcessingException(String msg, Object... args) { + super(msg, args); + } + + /** + * Constructs a new SearchRequestMapProcessingException with the specified message and cause. + * + * @param msg The error message. + * @param cause The cause of the exception. + * @param args Arguments to substitute in the error message. + */ + public SearchRequestMapProcessingException(String msg, Throwable cause, Object... args) { + super(msg, cause, args); + } +} diff --git a/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/helpers/package-info.java b/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/helpers/package-info.java new file mode 100644 index 0000000000000..b960ff72a9e2b --- /dev/null +++ b/modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/helpers/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Provides helper classes and utilities for working with search pipeline processors. + */ +package org.opensearch.search.pipeline.common.helpers; diff --git a/modules/search-pipeline-common/src/test/java/org/opensearch/search/pipeline/common/ScriptRequestProcessorTests.java b/modules/search-pipeline-common/src/test/java/org/opensearch/search/pipeline/common/ScriptRequestProcessorTests.java new file mode 100644 index 0000000000000..2fb3b2345e7e2 --- /dev/null +++ b/modules/search-pipeline-common/src/test/java/org/opensearch/search/pipeline/common/ScriptRequestProcessorTests.java @@ -0,0 +1,131 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.pipeline.common; + +import org.junit.Before; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.settings.Settings; +import org.opensearch.script.MockScriptEngine; +import org.opensearch.script.Script; +import org.opensearch.script.ScriptModule; +import org.opensearch.script.ScriptService; +import org.opensearch.script.SearchScript; +import org.opensearch.script.ScriptType; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.search.pipeline.common.helpers.SearchRequestMap; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.Collections; +import java.util.Map; +import java.util.HashMap; + +import static org.hamcrest.core.Is.is; +import java.util.concurrent.TimeUnit; + +public class ScriptRequestProcessorTests extends OpenSearchTestCase { + + private ScriptService scriptService; + private Script script; + private SearchScript searchScript; + + @Before + public void setupScripting() { + String scriptName = "search_script"; + scriptService = new ScriptService( + Settings.builder().build(), + Map.of(Script.DEFAULT_SCRIPT_LANG, new MockScriptEngine(Script.DEFAULT_SCRIPT_LANG, Map.of(scriptName, ctx -> { + Object sourceObj = ctx.get("_source"); + if (sourceObj instanceof Map) { + Map source = (SearchRequestMap) sourceObj; + + // Update all modifiable source fields + Integer from = (Integer) source.get("from"); + source.put("from", from + 10); + + Integer size = (Integer) source.get("size"); + source.put("size", size + 10); + + Boolean explain = (Boolean) source.get("explain"); + source.put("explain", !explain); + + Boolean version = (Boolean) source.get("version"); + source.put("version", !version); + + Boolean seqNoAndPrimaryTerm = (Boolean) source.get("seq_no_primary_term"); + source.put("seq_no_primary_term", !seqNoAndPrimaryTerm); + + Boolean trackScores = (Boolean) source.get("track_scores"); + source.put("track_scores", !trackScores); + + Integer trackTotalHitsUpTo = (Integer) source.get("track_total_hits"); + source.put("track_total_hits", trackTotalHitsUpTo + 1); + + Float minScore = (Float) source.get("min_score"); + source.put("min_score", minScore + 1.0f); + + Integer terminateAfter = (Integer) source.get("terminate_after"); + source.put("terminate_after", terminateAfter + 1); + } + return null; + }), Collections.emptyMap())), + new HashMap<>(ScriptModule.CORE_CONTEXTS) + ); + script = new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, scriptName, Collections.emptyMap()); + searchScript = scriptService.compile(script, SearchScript.CONTEXT).newInstance(script.getParams()); + } + + public void testScriptingWithoutPrecompiledScriptFactory() throws Exception { + ScriptRequestProcessor processor = new ScriptRequestProcessor(randomAlphaOfLength(10), null, script, null, scriptService); + SearchRequest searchRequest = new SearchRequest(); + searchRequest.source(createSearchSourceBuilder()); + + assertNotNull(searchRequest); + processor.processRequest(searchRequest); + assertSearchRequest(searchRequest); + } + + public void testScriptingWithPrecompiledIngestScript() throws Exception { + ScriptRequestProcessor processor = new ScriptRequestProcessor(randomAlphaOfLength(10), null, script, searchScript, scriptService); + SearchRequest searchRequest = new SearchRequest(); + searchRequest.source(createSearchSourceBuilder()); + + assertNotNull(searchRequest); + processor.processRequest(searchRequest); + assertSearchRequest(searchRequest); + } + + private SearchSourceBuilder createSearchSourceBuilder() { + SearchSourceBuilder source = new SearchSourceBuilder(); + source.from(10); + source.size(20); + source.explain(true); + source.version(true); + source.seqNoAndPrimaryTerm(true); + source.trackScores(true); + source.trackTotalHitsUpTo(3); + source.minScore(1.0f); + source.timeout(new TimeValue(60, TimeUnit.SECONDS)); + source.terminateAfter(5); + return source; + } + + private void assertSearchRequest(SearchRequest searchRequest) { + assertThat(searchRequest.source().from(), is(20)); + assertThat(searchRequest.source().size(), is(30)); + assertThat(searchRequest.source().explain(), is(false)); + assertThat(searchRequest.source().version(), is(false)); + assertThat(searchRequest.source().seqNoAndPrimaryTerm(), is(false)); + assertThat(searchRequest.source().trackScores(), is(false)); + assertThat(searchRequest.source().trackTotalHitsUpTo(), is(4)); + assertThat(searchRequest.source().minScore(), is(2.0f)); + assertThat(searchRequest.source().timeout(), is(new TimeValue(60, TimeUnit.SECONDS))); + assertThat(searchRequest.source().terminateAfter(), is(6)); + } +} diff --git a/modules/search-pipeline-common/src/test/java/org/opensearch/search/pipeline/common/helpers/SearchRequestMapTests.java b/modules/search-pipeline-common/src/test/java/org/opensearch/search/pipeline/common/helpers/SearchRequestMapTests.java new file mode 100644 index 0000000000000..5572f28335e1c --- /dev/null +++ b/modules/search-pipeline-common/src/test/java/org/opensearch/search/pipeline/common/helpers/SearchRequestMapTests.java @@ -0,0 +1,149 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.search.pipeline.common.helpers; + +import org.opensearch.action.search.SearchRequest; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.test.AbstractBuilderTestCase; + +public class SearchRequestMapTests extends AbstractBuilderTestCase { + + public void testEmptyMap() { + SearchRequest searchRequest = new SearchRequest(); + SearchRequestMap map = new SearchRequestMap(searchRequest); + + assertTrue(map.isEmpty()); + } + + public void testGet() { + SearchRequest searchRequest = new SearchRequest(); + SearchSourceBuilder source = new SearchSourceBuilder(); + source.from(10); + source.size(20); + source.explain(true); + source.version(true); + source.seqNoAndPrimaryTerm(true); + source.trackScores(true); + source.trackTotalHitsUpTo(3); + source.minScore(1.0f); + source.terminateAfter(5); + searchRequest.source(source); + + SearchRequestMap map = new SearchRequestMap(searchRequest); + + assertEquals(10, map.get("from")); + assertEquals(20, map.get("size")); + assertEquals(true, map.get("explain")); + assertEquals(true, map.get("version")); + assertEquals(true, map.get("seq_no_primary_term")); + assertEquals(true, map.get("track_scores")); + assertEquals(3, map.get("track_total_hits")); + assertEquals(1.0f, map.get("min_score")); + assertEquals(5, map.get("terminate_after")); + } + + public void testPut() { + SearchRequest searchRequest = new SearchRequest(); + SearchSourceBuilder source = new SearchSourceBuilder(); + searchRequest.source(source); + + SearchRequestMap map = new SearchRequestMap(searchRequest); + + assertEquals(-1, map.put("from", 10)); + assertEquals(10, map.get("from")); + + assertEquals(-1, map.put("size", 20)); + assertEquals(20, map.get("size")); + + assertNull(map.put("explain", true)); + assertEquals(true, map.get("explain")); + + assertNull(map.put("version", true)); + assertEquals(true, map.get("version")); + + assertNull(map.put("seq_no_primary_term", true)); + assertEquals(true, map.get("seq_no_primary_term")); + + assertEquals(false, map.put("track_scores", true)); + assertEquals(true, map.get("track_scores")); + + assertNull(map.put("track_total_hits", 3)); + assertEquals(3, map.get("track_total_hits")); + + assertNull(map.put("min_score", 1.0f)); + assertEquals(1.0f, map.get("min_score")); + + assertEquals(0, map.put("terminate_after", 5)); + assertEquals(5, map.get("terminate_after")); + } + + public void testUnsupportedOperationException() { + SearchRequest searchRequest = new SearchRequest(); + SearchSourceBuilder source = new SearchSourceBuilder(); + searchRequest.source(source); + + SearchRequestMap map = new SearchRequestMap(searchRequest); + + assertThrows(UnsupportedOperationException.class, () -> map.size()); + assertThrows(UnsupportedOperationException.class, () -> map.containsValue(null)); + assertThrows(UnsupportedOperationException.class, () -> map.remove(null)); + assertThrows(UnsupportedOperationException.class, () -> map.putAll(null)); + assertThrows(UnsupportedOperationException.class, map::clear); + assertThrows(UnsupportedOperationException.class, map::keySet); + assertThrows(UnsupportedOperationException.class, map::values); + assertThrows(UnsupportedOperationException.class, map::entrySet); + assertThrows(UnsupportedOperationException.class, () -> map.getOrDefault(null, null)); + assertThrows(UnsupportedOperationException.class, () -> map.forEach(null)); + assertThrows(UnsupportedOperationException.class, () -> map.replaceAll(null)); + assertThrows(UnsupportedOperationException.class, () -> map.putIfAbsent(null, null)); + assertThrows(UnsupportedOperationException.class, () -> map.remove(null, null)); + assertThrows(UnsupportedOperationException.class, () -> map.replace(null, null, null)); + assertThrows(UnsupportedOperationException.class, () -> map.replace(null, null)); + assertThrows(UnsupportedOperationException.class, () -> map.computeIfAbsent(null, null)); + assertThrows(UnsupportedOperationException.class, () -> map.computeIfPresent(null, null)); + assertThrows(UnsupportedOperationException.class, () -> map.compute(null, null)); + assertThrows(UnsupportedOperationException.class, () -> map.merge(null, null, null)); + } + + public void testIllegalArgumentException() { + SearchRequest searchRequest = new SearchRequest(); + SearchSourceBuilder source = new SearchSourceBuilder(); + searchRequest.source(source); + + SearchRequestMap map = new SearchRequestMap(searchRequest); + + try { + map.get(1); + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { + // expected + } + } + + public void testSearchRequestMapProcessingException() { + SearchRequest searchRequest = new SearchRequest(); + SearchSourceBuilder source = new SearchSourceBuilder(); + searchRequest.source(source); + + SearchRequestMap map = new SearchRequestMap(searchRequest); + + try { + map.get("unsupported_key"); + fail("Expected SearchRequestMapProcessingException"); + } catch (SearchRequestMapProcessingException e) { + // expected + } + + try { + map.put("unsupported_key", 10); + fail("Expected SearchRequestMapProcessingException"); + } catch (SearchRequestMapProcessingException e) { + // expected + } + } +} diff --git a/modules/search-pipeline-common/src/yamlRestTest/resources/rest-api-spec/test/search_pipeline/10_basic.yml b/modules/search-pipeline-common/src/yamlRestTest/resources/rest-api-spec/test/search_pipeline/10_basic.yml index 644181d601ea4..ca53f6cd6a7e8 100644 --- a/modules/search-pipeline-common/src/yamlRestTest/resources/rest-api-spec/test/search_pipeline/10_basic.yml +++ b/modules/search-pipeline-common/src/yamlRestTest/resources/rest-api-spec/test/search_pipeline/10_basic.yml @@ -13,4 +13,5 @@ - contains: { nodes.$cluster_manager.modules: { name: search-pipeline-common } } - contains: { nodes.$cluster_manager.search_pipelines.request_processors: { type: filter_query } } + - contains: { nodes.$cluster_manager.search_pipelines.request_processors: { type: script } } - contains: { nodes.$cluster_manager.search_pipelines.response_processors: { type: rename_field } } diff --git a/modules/search-pipeline-common/src/yamlRestTest/resources/rest-api-spec/test/search_pipeline/50_script_processor.yml b/modules/search-pipeline-common/src/yamlRestTest/resources/rest-api-spec/test/search_pipeline/50_script_processor.yml new file mode 100644 index 0000000000000..bba52285fd58d --- /dev/null +++ b/modules/search-pipeline-common/src/yamlRestTest/resources/rest-api-spec/test/search_pipeline/50_script_processor.yml @@ -0,0 +1,96 @@ +--- +teardown: + - do: + search_pipeline.delete: + id: "my_pipeline" + ignore: 404 + +--- +"Test empty script in script processor": + - do: + catch: bad_request + search_pipeline.put: + id: "my_pipeline" + body: > + { + "description": "_description", + "request_processors": [ + { + "script" : { + "lang": "painless", + "source" : "" + } + } + ] + } + + - match: { status: 400 } + - match: { error.root_cause.0.type: "script_exception" } + +--- +"Test supported search source builder fields": + - do: + search_pipeline.put: + id: "my_pipeline" + body: > + { + "description": "_description", + "request_processors": [ + { + "script" : { + "lang" : "painless", + "source" : "ctx._source['size'] += 10; ctx._source['from'] -= 1; ctx._source['explain'] = !ctx._source['explain']; ctx._source['version'] = !ctx._source['version']; ctx._source['seq_no_primary_term'] = !ctx._source['seq_no_primary_term']; ctx._source['track_scores'] = !ctx._source['track_scores']; ctx._source['track_total_hits'] = 1; ctx._source['min_score'] -= 0.9; ctx._source['terminate_after'] += 2; ctx._source['profile'] = !ctx._source['profile'];" + } + } + ] + } + - match: { acknowledged: true } + + - do: + index: + index: test + id: 1 + body: { + "field": 2 + } + - do: + index: + index: test + id: 2 + body: { + "field": 3 + } + + - do: + indices.refresh: + index: test + + - do: + search: + index: test + search_pipeline: "my_pipeline" + body: { + "from": 1, + "size": 1, + "explain": true, + "version": true, + "seq_no_primary_term": true, + "track_scores": true, + "track_total_hits": true, + "min_score": 1.0, + "timeout": "60s", + "terminate_after": 2, + "profile": true + } + - length: { hits.hits: 2 } + - match: { _shards.total: 1 } + - match: { hits.total.value: 1 } + - match: { hits.hits.0._score: 1.0 } + - match: { hits.hits.1._score: 1.0 } + - is_false: hits.hits.0._explanation + - is_false: hits.hits.1._explanation + - is_false: hits.hits.0._seq_no + - is_false: hits.hits.1._seq_no + - is_false: hits.hits.0._primary_term + - is_false: hits.hits.1._primary_term + - is_false: profile diff --git a/server/src/main/java/org/opensearch/script/ScriptModule.java b/server/src/main/java/org/opensearch/script/ScriptModule.java index b5527f6d8d07d..a192e9553016b 100644 --- a/server/src/main/java/org/opensearch/script/ScriptModule.java +++ b/server/src/main/java/org/opensearch/script/ScriptModule.java @@ -68,6 +68,7 @@ public class ScriptModule { SignificantTermsHeuristicScoreScript.CONTEXT, IngestScript.CONTEXT, IngestConditionalScript.CONTEXT, + SearchScript.CONTEXT, FilterScript.CONTEXT, SimilarityScript.CONTEXT, SimilarityWeightScript.CONTEXT, diff --git a/server/src/main/java/org/opensearch/script/SearchScript.java b/server/src/main/java/org/opensearch/script/SearchScript.java new file mode 100644 index 0000000000000..f40cc8ef080a4 --- /dev/null +++ b/server/src/main/java/org/opensearch/script/SearchScript.java @@ -0,0 +1,55 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.script; + +import org.opensearch.common.unit.TimeValue; + +import java.util.Map; + +/** + * A script used by the Search Script Processor. + * + * @opensearch.internal + */ +public abstract class SearchScript { + + public static final String[] PARAMETERS = { "ctx" }; + + /** The context used to compile {@link SearchScript} factories. */ + public static final ScriptContext CONTEXT = new ScriptContext<>( + "search", + Factory.class, + 200, + TimeValue.timeValueMillis(0), + ScriptCache.UNLIMITED_COMPILATION_RATE.asTuple() + ); + + /** The generic runtime parameters for the script. */ + private final Map params; + + public SearchScript(Map params) { + this.params = params; + } + + /** Return the parameters for this script. */ + public Map getParams() { + return params; + } + + public abstract void execute(Map ctx); + + /** + * Factory for search script + * + * @opensearch.internal + */ + public interface Factory { + SearchScript newInstance(Map params); + } +} diff --git a/test/framework/src/main/java/org/opensearch/script/MockScriptEngine.java b/test/framework/src/main/java/org/opensearch/script/MockScriptEngine.java index 9f8abdf5f2c8a..98912e53c9d6a 100644 --- a/test/framework/src/main/java/org/opensearch/script/MockScriptEngine.java +++ b/test/framework/src/main/java/org/opensearch/script/MockScriptEngine.java @@ -177,6 +177,14 @@ public void execute(Map ctx) { } }; return context.factoryClazz.cast(factory); + } else if (context.instanceClazz.equals(SearchScript.class)) { + SearchScript.Factory factory = parameters -> new SearchScript(parameters) { + @Override + public void execute(Map ctx) { + script.apply(ctx); + } + }; + return context.factoryClazz.cast(factory); } else if (context.instanceClazz.equals(AggregationScript.class)) { return context.factoryClazz.cast(new MockAggregationScript(script)); } else if (context.instanceClazz.equals(IngestConditionalScript.class)) {