From c25306888dba0e84754f54989227ce200eb14744 Mon Sep 17 00:00:00 2001 From: Daniel Widdis Date: Thu, 21 Sep 2023 10:25:56 -0700 Subject: [PATCH] Add parse methods for the Template XContent Signed-off-by: Daniel Widdis --- .../flowframework/template/Template.java | 197 +++++++++++++++--- .../flowframework/template/WorkflowEdge.java | 35 ++++ .../flowframework/template/WorkflowNode.java | 63 +++++- .../flowframework/workflow/Workflow.java | 67 +++++- .../flowframework/template/GraphJsonUtil.java | 16 ++ 5 files changed, 347 insertions(+), 31 deletions(-) create mode 100644 src/test/java/org/opensearch/flowframework/template/GraphJsonUtil.java diff --git a/src/main/java/org/opensearch/flowframework/template/Template.java b/src/main/java/org/opensearch/flowframework/template/Template.java index 25e25b691..205fa61b7 100644 --- a/src/main/java/org/opensearch/flowframework/template/Template.java +++ b/src/main/java/org/opensearch/flowframework/template/Template.java @@ -11,53 +11,111 @@ import org.opensearch.Version; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; import org.opensearch.flowframework.workflow.Workflow; import java.io.IOException; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Map.Entry; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; + /** * The Template is the central data structure which configures workflows. This object is used to parse JSON communicated via REST API. */ public class Template implements ToXContentObject { + private static final String WORKFLOWS_FIELD = "workflows"; + private static final String USER_INPUTS_FIELD = "user_inputs"; + private static final String COMPATIBILITY_FIELD = "compatibility"; + private static final String TEMPLATE_FIELD = "template"; + private static final String VERSION_FIELD = "version"; + private static final String OPERATIONS_FIELD = "operations"; + private static final String USE_CASE_FIELD = "use_case"; + private static final String DESCRIPTION_FIELD = "description"; + private static final String NAME_FIELD = "name"; // TODO: Some of thse are placeholders based on the template design - // Current code is only using user inputs and workflows - private String name = null; - private String description = null; - private String useCase = null; - private String[] operations = null; // probably an ENUM actually - private Version templateVersion = null; - private Version[] compatibilityVersion = null; - private Map userInputs = new HashMap<>(); - private Map workflows = new HashMap<>(); + // Change as needed to match defined template + private final String name; + private final String description; + private final String useCase; // probably an ENUM actually + private final String[] operations; // probably an ENUM actually + private final Version templateVersion; + private final Version[] compatibilityVersion; + private final Map userInputs; + private final Map workflows; + + /** + * Instantiate the object representing a use case template + * + * @param name The template's name + * @param description A description of the template's use case + * @param useCase A string defining the internal use case type + * @param operations Expected operations of this template. Should match defined workflows. + * @param templateVersion The version of this template + * @param compatibilityVersion OpenSearch version compatibility of this template + * @param userInputs Optional user inputs to apply globally + * @param workflows Workflow graph definitions corresponding to the defined operations. + */ + public Template( + String name, + String description, + String useCase, + String[] operations, + Version templateVersion, + Version[] compatibilityVersion, + Map userInputs, + Map workflows + ) { + this.name = name; + this.description = description; + this.useCase = useCase; + this.operations = operations; + this.templateVersion = templateVersion; + this.compatibilityVersion = compatibilityVersion; + this.userInputs = userInputs; + this.workflows = workflows; + } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { XContentBuilder xContentBuilder = builder.startObject(); - xContentBuilder.field("name", this.name); - xContentBuilder.field("description", this.description); - xContentBuilder.field("use_case", this.useCase); - xContentBuilder.field("operations", this.operations); - - xContentBuilder.startObject("version"); - xContentBuilder.field("template", this.templateVersion); - xContentBuilder.startArray("compatibility"); - for (Version v : this.compatibilityVersion) { - xContentBuilder.value(v); + xContentBuilder.field(NAME_FIELD, this.name); + xContentBuilder.field(DESCRIPTION_FIELD, this.description); + xContentBuilder.field(USE_CASE_FIELD, this.useCase); + xContentBuilder.startArray(OPERATIONS_FIELD); + for (String op : this.operations) { + xContentBuilder.value(op); } xContentBuilder.endArray(); - xContentBuilder.endObject(); - xContentBuilder.startObject("user_inputs"); - for (Entry e : userInputs.entrySet()) { - xContentBuilder.field(e.getKey(), e.getValue()); + if (this.templateVersion != null || this.compatibilityVersion.length > 0) { + xContentBuilder.startObject(VERSION_FIELD); + if (this.templateVersion != null) { + xContentBuilder.field(TEMPLATE_FIELD, this.templateVersion); + } + if (this.compatibilityVersion.length > 0) { + xContentBuilder.startArray(COMPATIBILITY_FIELD); + for (Version v : this.compatibilityVersion) { + xContentBuilder.value(v); + } + xContentBuilder.endArray(); + } + xContentBuilder.endObject(); + } + + if (!this.userInputs.isEmpty()) { + xContentBuilder.startObject(USER_INPUTS_FIELD); + for (Entry e : userInputs.entrySet()) { + xContentBuilder.field(e.getKey(), e.getValue()); + } + xContentBuilder.endObject(); } - xContentBuilder.endObject(); - xContentBuilder.startObject("workflows"); + xContentBuilder.startObject(WORKFLOWS_FIELD); for (Entry e : workflows.entrySet()) { xContentBuilder.field(e.getKey(), e.getValue(), params); } @@ -66,4 +124,93 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return xContentBuilder.endObject(); } + /** + * Parse raw json content into a workflow node instance. + * + * @param parser json based content parser + * @throws IOException if content can't be parsed correctly + */ + public static Template parse(XContentParser parser) throws IOException { + String name = null; + String description = ""; + String useCase = ""; + String[] operations = new String[0]; + Version templateVersion = null; + Version[] compatibilityVersion = new Version[0]; + Map userInputs = new HashMap<>(); + Map workflows = new HashMap<>(); + + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); + while (parser.nextToken() != XContentParser.Token.END_OBJECT) { + String fieldName = parser.currentName(); + parser.nextToken(); + switch (fieldName) { + case NAME_FIELD: + name = parser.text(); + break; + case DESCRIPTION_FIELD: + description = parser.text(); + break; + case USE_CASE_FIELD: + useCase = parser.text(); + break; + case OPERATIONS_FIELD: + ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser); + List operationsList = new ArrayList<>(); + while (parser.nextToken() != XContentParser.Token.END_ARRAY) { + operationsList.add(parser.text()); + } + operations = operationsList.toArray(new String[0]); + break; + case VERSION_FIELD: + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); + while (parser.nextToken() != XContentParser.Token.END_OBJECT) { + String versionFieldName = parser.currentName(); + parser.nextToken(); + switch (versionFieldName) { + case TEMPLATE_FIELD: + templateVersion = Version.fromString(parser.text()); + break; + case COMPATIBILITY_FIELD: + ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser); + List compatibilityList = new ArrayList<>(); + while (parser.nextToken() != XContentParser.Token.END_ARRAY) { + compatibilityList.add(Version.fromString(parser.text())); + } + compatibilityVersion = compatibilityList.toArray(new Version[0]); + break; + default: + throw new IOException("Unable to parse field [" + fieldName + "] in a version object."); + } + + } + break; + case USER_INPUTS_FIELD: + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); + while (parser.nextToken() != XContentParser.Token.END_OBJECT) { + String inputFieldName = parser.currentName(); + parser.nextToken(); + userInputs.put(inputFieldName, parser.text()); + } + break; + case WORKFLOWS_FIELD: + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); + while (parser.nextToken() != XContentParser.Token.END_OBJECT) { + String workflowFieldName = parser.currentName(); + parser.nextToken(); + workflows.put(workflowFieldName, Workflow.parse(parser)); + } + break; + + default: + throw new IOException("Unable to parse field [" + fieldName + "] in a template object."); + } + } + if (name == null) { + throw new IOException("An template object requires a name."); + } + + return new Template(name, description, useCase, operations, templateVersion, compatibilityVersion, userInputs, workflows); + } + } diff --git a/src/main/java/org/opensearch/flowframework/template/WorkflowEdge.java b/src/main/java/org/opensearch/flowframework/template/WorkflowEdge.java index 1a7b202b7..247000c12 100644 --- a/src/main/java/org/opensearch/flowframework/template/WorkflowEdge.java +++ b/src/main/java/org/opensearch/flowframework/template/WorkflowEdge.java @@ -10,10 +10,13 @@ import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; import java.io.IOException; import java.util.Objects; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; + /** * This represents an edge between process nodes (steps) in a workflow graph in the {@link Template}. */ @@ -43,6 +46,38 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return xContentBuilder.endObject(); } + /** + * Parse raw json content into a workflow edge instance. + * + * @param parser json based content parser + * @throws IOException if content can't be parsed correctly + */ + public static WorkflowEdge parse(XContentParser parser) throws IOException { + String source = null; + String destination = null; + + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); + while (parser.nextToken() != XContentParser.Token.END_OBJECT) { + String fieldName = parser.currentName(); + parser.nextToken(); + switch (fieldName) { + case SOURCE_FIELD: + source = parser.text(); + break; + case DEST_FIELD: + destination = parser.text(); + break; + default: + throw new IOException("Unable to parse field [" + fieldName + "] in an edge object."); + } + } + if (source == null || destination == null) { + throw new IOException("An edge object requires both a source and dest field."); + } + + return new WorkflowEdge(source, destination); + } + /** * Gets the source node id. * diff --git a/src/main/java/org/opensearch/flowframework/template/WorkflowNode.java b/src/main/java/org/opensearch/flowframework/template/WorkflowNode.java index 128cf1164..28d368cfb 100644 --- a/src/main/java/org/opensearch/flowframework/template/WorkflowNode.java +++ b/src/main/java/org/opensearch/flowframework/template/WorkflowNode.java @@ -10,12 +10,15 @@ import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; import java.io.IOException; import java.util.HashMap; import java.util.Map; import java.util.Map.Entry; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; + /** * This represents a process node (step) in a workflow graph in the {@link Template}. * It will have a one-to-one correspondence with a {@link ProcessNode}, @@ -28,9 +31,22 @@ public class WorkflowNode implements ToXContentFragment { private static final String TYPE_FIELD = "type"; private static final String ID_FIELD = "id"; - private String id = null; // unique id - private String type = null; // maps to a WorkflowStep - private Map inputs = new HashMap<>(); // maps to WorkflowData + private final String id; // unique id + private final String type; // maps to a WorkflowStep + private final Map inputs; // maps to WorkflowData + + /** + * Create this node with the id and type, and any user input. + * + * @param id A unique string identifying this node + * @param type The type of {@link WorkflowStep} to create for the corresponding {@link ProcessNode} + * @param inputs Optional input to populate params in {@link WorkflowData} + */ + public WorkflowNode(String id, String type, Map inputs) { + this.id = id; + this.type = type; + this.inputs = inputs; + } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { @@ -46,4 +62,45 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return xContentBuilder.endObject(); } + + /** + * Parse raw json content into a workflow node instance. + * + * @param parser json based content parser + * @throws IOException if content can't be parsed correctly + */ + public static WorkflowNode parse(XContentParser parser) throws IOException { + String id = null; + String type = null; + Map inputs = new HashMap<>(); + + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); + while (parser.nextToken() != XContentParser.Token.END_OBJECT) { + String fieldName = parser.currentName(); + parser.nextToken(); + switch (fieldName) { + case ID_FIELD: + id = parser.text(); + break; + case TYPE_FIELD: + type = parser.text(); + break; + case INPUTS_FIELD: + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); + while (parser.nextToken() != XContentParser.Token.END_OBJECT) { + String inputFieldName = parser.currentName(); + parser.nextToken(); + inputs.put(inputFieldName, parser.text()); + } + break; + default: + throw new IOException("Unable to parse field [" + fieldName + "] in a node object."); + } + } + if (id == null || type == null) { + throw new IOException("An node object requires both an id and type field."); + } + + return new WorkflowNode(id, type, inputs); + } } diff --git a/src/main/java/org/opensearch/flowframework/workflow/Workflow.java b/src/main/java/org/opensearch/flowframework/workflow/Workflow.java index 6666ff10a..b3cfcca40 100644 --- a/src/main/java/org/opensearch/flowframework/workflow/Workflow.java +++ b/src/main/java/org/opensearch/flowframework/workflow/Workflow.java @@ -10,14 +10,19 @@ import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; import org.opensearch.flowframework.template.WorkflowEdge; import org.opensearch.flowframework.template.WorkflowNode; import java.io.IOException; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Map.Entry; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; + /** * This represents an object in the workflows section of a {@link Template}. */ @@ -25,11 +30,18 @@ public class Workflow implements ToXContentFragment { private static final String USER_PARAMS_FIELD = "user_params"; private static final String NODES_FIELD = "nodes"; + // TODO: private static final String STEPS_FIELD = "steps"; private static final String EDGES_FIELD = "edges"; - private Map userParams = new HashMap<>(); - private WorkflowNode[] nodes = null; - private WorkflowEdge[] edges = null; + private final Map userParams; + private final WorkflowNode[] nodes; + private final WorkflowEdge[] edges; + + public Workflow(Map userParams, WorkflowNode[] nodes, WorkflowEdge[] edges) { + this.userParams = userParams; + this.nodes = nodes; + this.edges = edges; + } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { @@ -56,4 +68,53 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return xContentBuilder.endObject(); } + /** + * Parse raw json content into a workflow instance. + * + * @param parser json based content parser + * @throws IOException if content can't be parsed correctly + */ + public static Workflow parse(XContentParser parser) throws IOException { + Map userParams = new HashMap<>(); + WorkflowNode[] nodes = null; + WorkflowEdge[] edges = null; + + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); + while (parser.nextToken() != XContentParser.Token.END_OBJECT) { + String fieldName = parser.currentName(); + parser.nextToken(); + switch (fieldName) { + case USER_PARAMS_FIELD: + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); + while (parser.nextToken() != XContentParser.Token.END_OBJECT) { + String userParamFieldName = parser.currentName(); + parser.nextToken(); + userParams.put(userParamFieldName, parser.text()); + } + break; + case NODES_FIELD: + ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser); + List nodesList = new ArrayList<>(); + while (parser.nextToken() != XContentParser.Token.END_ARRAY) { + nodesList.add(WorkflowNode.parse(parser)); + } + nodes = nodesList.toArray(new WorkflowNode[0]); + break; + case EDGES_FIELD: + ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser); + List edgesList = new ArrayList<>(); + while (parser.nextToken() != XContentParser.Token.END_ARRAY) { + edgesList.add(WorkflowEdge.parse(parser)); + } + edges = edgesList.toArray(new WorkflowEdge[0]); + break; + } + + } + if (nodes == null || nodes.length == 0) { + throw new IOException("A workflow must have at least one node."); + } + // TODO: if edges are empty, create edges by iterating over nodes and adding one between each pair + return new Workflow(userParams, nodes, edges); + } } diff --git a/src/test/java/org/opensearch/flowframework/template/GraphJsonUtil.java b/src/test/java/org/opensearch/flowframework/template/GraphJsonUtil.java new file mode 100644 index 000000000..6945691a0 --- /dev/null +++ b/src/test/java/org/opensearch/flowframework/template/GraphJsonUtil.java @@ -0,0 +1,16 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.flowframework.template; + +/** + * Utility methods to create a JSON string useful for testing nodes and edges + */ +public class GraphJsonUtil { + +}