-
Notifications
You must be signed in to change notification settings - Fork 143
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Initiate MLInferencelngestProcessor (#2205)
* Initiate MLModelIngestProcessor Signed-off-by: Mingshi Liu <[email protected]> add more tests Signed-off-by: Mingshi Liu <[email protected]> * add more tests Signed-off-by: Mingshi Liu <[email protected]> add yaml tests and nested objects tests Signed-off-by: Mingshi Liu <[email protected]> add IT tests Signed-off-by: Mingshi Liu <[email protected]> * use GroupListener and add DEFAULT_MAX_PREDICTION_TASKS Signed-off-by: Mingshi Liu <[email protected]> * add javadoc Signed-off-by: Mingshi Liu <[email protected]> * avoid calling execute(IngestDocument ingestDocument)-s Signed-off-by: Mingshi Liu <[email protected]> * not rewriting dotpath to json path Signed-off-by: Mingshi Liu <[email protected]> * change mapping order, input_map-model input as key, output_map-document field as key Signed-off-by: Mingshi Liu <[email protected]> * use StringUtils.toJson Signed-off-by: Mingshi Liu <[email protected]> --------- Signed-off-by: Mingshi Liu <[email protected]>
- Loading branch information
Showing
16 changed files
with
2,450 additions
and
27 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
83 changes: 83 additions & 0 deletions
83
plugin/src/main/java/org/opensearch/ml/processor/InferenceProcessorAttributes.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,83 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
| package org.opensearch.ml.processor; | ||
|
|
||
| import java.util.List; | ||
| import java.util.Map; | ||
|
|
||
| import lombok.Getter; | ||
| import lombok.Setter; | ||
|
|
||
| @Setter | ||
| @Getter | ||
| public class InferenceProcessorAttributes { | ||
|
|
||
| protected List<Map<String, String>> inputMaps; | ||
|
|
||
| protected List<Map<String, String>> outputMaps; | ||
|
|
||
| protected String modelId; | ||
| protected int maxPredictionTask; | ||
|
|
||
| protected Map<String, String> modelConfigMaps; | ||
| public static final String MODEL_ID = "model_id"; | ||
| /** | ||
| * The list of maps that support one or more prediction tasks mapping. | ||
| * The mappings also support JSON path for nested objects. | ||
| * | ||
| * input_map is used to construct model inputs, where the keys represent the model input fields, | ||
| * and the values represent the corresponding document fields. | ||
| * | ||
| * Example input_map: | ||
| * | ||
| * "input_map": [ | ||
| * { | ||
| * "input": "book.title" | ||
| * }, | ||
| * { | ||
| * "input": "book.text" | ||
| * } | ||
| * ] | ||
| */ | ||
| public static final String INPUT_MAP = "input_map"; | ||
| /** | ||
| * output_map is used to construct document fields, where the keys represent the document fields, | ||
| * and the values represent the corresponding model output fields. | ||
| * | ||
| * Example output_map: | ||
| * | ||
| * "output_map": [ | ||
| * { | ||
| * "book.title_language": "response.language" | ||
| * }, | ||
| * { | ||
| * "book.text_language": "response.language" | ||
| * } | ||
| * ] | ||
| * | ||
| */ | ||
| public static final String OUTPUT_MAP = "output_map"; | ||
| public static final String MODEL_CONFIG = "model_config"; | ||
| public static final String MAX_PREDICTION_TASKS = "max_prediction_tasks"; | ||
|
|
||
| /** | ||
| * Utility class containing shared parameters for MLModelIngest/SearchProcessor | ||
| * */ | ||
|
|
||
| public InferenceProcessorAttributes( | ||
| String modelId, | ||
| List<Map<String, String>> inputMaps, | ||
| List<Map<String, String>> outputMaps, | ||
| Map<String, String> modelConfigMaps, | ||
| int maxPredictionTask | ||
| ) { | ||
| this.modelId = modelId; | ||
| this.modelConfigMaps = modelConfigMaps; | ||
| this.inputMaps = inputMaps; | ||
| this.outputMaps = outputMaps; | ||
| this.maxPredictionTask = maxPredictionTask; | ||
| } | ||
|
|
||
| } |
Oops, something went wrong.