diff --git a/Docs/api/index.html b/Docs/api/index.html
index f9d4c96e..cc40f606 100644
--- a/Docs/api/index.html
+++ b/Docs/api/index.html
@@ -354,25 +354,33 @@ Example data
"id" : "aeiou"
},
"spec" : {
- "src" : {
- "apiVersion" : "aeiou",
- "outputTask" : "aeiou",
- "tasks" : {
- "key" : {
- "args" : [ { } ],
- "name" : "aeiou",
- "type" : "aeiou",
- "dependencies" : [ "aeiou" ]
- }
+ "apiVersion" : "aeiou",
+ "outputTask" : "aeiou",
+ "tasks" : {
+ "key" : {
+ "inputs" : {
+ "key" : {
+ "type" : "aeiou",
+ "value" : "aeiou"
+ }
+ },
+ "name" : "aeiou",
+ "type" : "aeiou",
+ "dependencies" : {
+ "key" : {
+ "alias" : "aeiou",
+ "type" : { }
+ }
+ },
+ "dependencies_await" : 0
}
- },
- "name" : "aeiou",
- "version" : "aeiou"
+ }
},
"status" : {
"resolvedTasks" : {
"key" : {
"src" : "aeiou",
+ "runtime" : "aeiou",
"resolved" : "aeiou"
}
},
@@ -582,22 +590,32 @@ Example data
"id" : "aeiou"
},
"spec" : {
- "input" : "aeiou",
+ "inputs" : {
+ "key" : {
+ "type" : "aeiou",
+ "value" : "aeiou"
+ }
+ },
"workflowId" : "aeiou"
},
"status" : {
- "output" : "aeiou",
+ "output" : "",
"tasks" : {
"key" : {
"metadata" : "",
"spec" : {
- "input" : "aeiou",
- "functionId" : "aeiou",
- "functionName" : "aeiou",
+ "inputs" : {
+ "key" : ""
+ },
+ "type" : {
+ "src" : "aeiou",
+ "runtime" : "aeiou",
+ "resolved" : "aeiou"
+ },
"taskId" : "aeiou"
},
"status" : {
- "output" : "aeiou",
+ "output" : "",
"status" : { },
"updatedAt" : "2000-01-23T04:56:07.000+00:00"
}
@@ -693,8 +711,6 @@ Query parameters
workflowId (optional)
-
Query Parameter —
input (optional)
-
Query Parameter —
@@ -715,22 +731,32 @@ Example data
"id" : "aeiou"
},
"spec" : {
- "input" : "aeiou",
+ "inputs" : {
+ "key" : {
+ "type" : "aeiou",
+ "value" : "aeiou"
+ }
+ },
"workflowId" : "aeiou"
},
"status" : {
- "output" : "aeiou",
+ "output" : "",
"tasks" : {
"key" : {
"metadata" : "",
"spec" : {
- "input" : "aeiou",
- "functionId" : "aeiou",
- "functionName" : "aeiou",
+ "inputs" : {
+ "key" : ""
+ },
+ "type" : {
+ "src" : "aeiou",
+ "runtime" : "aeiou",
+ "resolved" : "aeiou"
+ },
"taskId" : "aeiou"
},
"status" : {
- "output" : "aeiou",
+ "output" : "",
"status" : { },
"updatedAt" : "2000-01-23T04:56:07.000+00:00"
}
@@ -795,22 +821,32 @@ Example data
"id" : "aeiou"
},
"spec" : {
- "input" : "aeiou",
+ "inputs" : {
+ "key" : {
+ "type" : "aeiou",
+ "value" : "aeiou"
+ }
+ },
"workflowId" : "aeiou"
},
"status" : {
- "output" : "aeiou",
+ "output" : "",
"tasks" : {
"key" : {
"metadata" : "",
"spec" : {
- "input" : "aeiou",
- "functionId" : "aeiou",
- "functionName" : "aeiou",
+ "inputs" : {
+ "key" : ""
+ },
+ "type" : {
+ "src" : "aeiou",
+ "runtime" : "aeiou",
+ "resolved" : "aeiou"
+ },
"taskId" : "aeiou"
},
"status" : {
- "output" : "aeiou",
+ "output" : "",
"status" : { },
"updatedAt" : "2000-01-23T04:56:07.000+00:00"
}
@@ -891,15 +927,16 @@ Table of Contents
FunctionInvocationStatusStatus -
ObjectMetadata - Common
Task -
- TaskParameters -
+ TaskDependencyParameters -
+ TaskDependencyParametersDependencyType -
TaskTypeDef -
+ TypedValue -
Workflow - Workflow Model
- WorkflowDefinition -
WorkflowInvocation -
WorkflowInvocationSpec - Workflow Invocation Model
WorkflowInvocationStatus -
WorkflowInvocationStatusStatus -
- WorkflowSpec -
+ WorkflowSpec - Workflow Definition
WorkflowStatus - Internal
WorkflowStatusStatus -
apiserverHealth -
@@ -914,7 +951,7 @@ Table of Contents
-
It contains no indication which workflow/task it is executed for
+
metadata (optional)
spec (optional)
@@ -925,10 +962,9 @@
name (optional)
-
args (optional)
-
dependencies (optional)
+
inputs (optional)
+
dependencies (optional)
+
dependencies_await (optional)
-
+
+
+
+
type (optional)
+
alias (optional)
+
+
+
+
Copy of protobuf's Any, to avoid protobuf requirement of a protobuf-based type.
+
+
type (optional)
+
value (optional)
+
+
-
-
The workflowDefinition contains the definition of a workflow.
-
Ideally the source code (json, yaml) can be converted directly to this message.
-Naming, triggers and versioning of the workflow itself is out of the scope of this data structure, which is delegated
-to the user/system upon the creation of a workflow.
-
-
-
apiVersion (optional)
String apiVersion describes what version is of the workflow definition.
-
By default the workflow engine will assume the latest version to be used.
-
-
tasks (optional)
-
outputTask (optional)
-
-
-
-
+
+
The workflowDefinition contains the definition of a workflow.
+
Ideally the source code (json, yaml) can be converted directly to this message.
+Naming, triggers and versioning of the workflow itself is out of the scope of this data structure, which is delegated
+to the user/system upon the creation of a workflow.
+
-
name (optional)
-
version (optional)
-
src (optional)
+
apiVersion (optional)
String apiVersion describes what version is of the workflow definition.
+
By default the workflow engine will assume the latest version to be used.
+
+
tasks (optional)
+
outputTask (optional)
diff --git a/Docs/wip/datatransformations.md b/Docs/wip/datatransformations.md
new file mode 100644
index 00000000..42248b6f
--- /dev/null
+++ b/Docs/wip/datatransformations.md
@@ -0,0 +1,40 @@
+# Data Transformers
+
+A data transformer allows a user can perform common data transformations on input data.
+Examples of such data transformations are selecting a sub-selection of a data object or concatenating strings.
+This document tracks the design of the data transformers for Fission Workflow.
+
+## Use Cases
+- Reference data from other tasks. For example: `$.tasks.foo.output`
+- Create subselections of data. For example: `$.tasks.foo.output.user.id`
+- Transform data. For example: `concat($.tasks.foo.output.user.firstName, ' ', $.tasks.foo.output.lastName)`
+
+## Data Types
+- string
+- bool
+- object
+- array
+- int
+
+### Special types
+- `expr`: referencing and manipulating data.
+- `raw`: default type, or fallback in case no other type could be assigned. Does not support selectors or most functions.
+
+## Design
+- There needs to be some symbol to differentiate functions from literals (or reverse token for literals)
+- The language and type determine which functions are available to invoke. For now that is solely JSON.
+- The TypedValue is similar to how Any works in Protobuf 3.
+- Selectors and transformers are implemented using Otto's Javascript engine.
+
+## Implementation
+- TypedValue, Type consists out of `/`. E.g. a JSON string is represented as `json/string`.
+It is allowed to just specify the language, allowing a parser to determine the actual type.
+
+### Functionality
+- [x] Transform Go primitive data types (interface{}) <-> TypedValue.
+- [x] Fission Proxy: Transform Go primitive data types (interface{}) <-> HTTP request.
+- [x] Add data transformation interface
+- [x] Parse and resolve expressions
+- [x] Add selector-based support
+- [ ] Add utility functions to the expression-parser. Status:
+- [ ] Allow data transformers to be run as regular tasks
diff --git a/api/swagger/apiserver.swagger.json b/api/swagger/apiserver.swagger.json
index 513e521b..5c94ad19 100644
--- a/api/swagger/apiserver.swagger.json
+++ b/api/swagger/apiserver.swagger.json
@@ -74,12 +74,6 @@
"in": "query",
"required": false,
"type": "string"
- },
- {
- "name": "input",
- "in": "query",
- "required": false,
- "type": "string"
}
],
"tags": [
@@ -282,24 +276,23 @@
"$ref": "#/definitions/FunctionInvocationStatus"
}
},
- "description": "It contains no indication which workflow/task it is executed for",
"title": "Function Invocation Model"
},
"FunctionInvocationSpec": {
"type": "object",
"properties": {
- "functionId": {
- "type": "string",
+ "type": {
+ "$ref": "#/definitions/TaskTypeDef",
"title": "Id of the function to be invoked (no ambiguatity at this point"
},
- "functionName": {
- "type": "string"
- },
"taskId": {
"type": "string"
},
- "input": {
- "type": "string"
+ "inputs": {
+ "type": "object",
+ "additionalProperties": {
+ "$ref": "#/definitions/TypedValue"
+ }
}
}
},
@@ -314,7 +307,7 @@
"format": "date-time"
},
"output": {
- "type": "string"
+ "$ref": "#/definitions/TypedValue"
}
}
},
@@ -357,24 +350,45 @@
"type": "string",
"title": "Name/identifier of the function"
},
- "args": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/TaskParameters"
+ "inputs": {
+ "type": "object",
+ "additionalProperties": {
+ "$ref": "#/definitions/TypedValue"
}
},
"dependencies": {
- "type": "array",
- "items": {
- "type": "string"
+ "type": "object",
+ "additionalProperties": {
+ "$ref": "#/definitions/TaskDependencyParameters"
},
- "title": "TODO next or after\nDependencies for this task to execute"
+ "title": "Dependencies for this task to execute"
+ },
+ "dependencies_await": {
+ "type": "integer",
+ "format": "int32",
+ "title": "Number of dependencies to wait for"
}
},
"description": "A task is the primitive unit of a workflow, representing an action that needs to be performed in order to continue.\n\nA task as a number of inputs and exactly two outputs\nId is specified outside of task"
},
- "TaskParameters": {
- "type": "object"
+ "TaskDependencyParameters": {
+ "type": "object",
+ "properties": {
+ "type": {
+ "$ref": "#/definitions/TaskDependencyParametersDependencyType"
+ },
+ "alias": {
+ "type": "string"
+ }
+ }
+ },
+ "TaskDependencyParametersDependencyType": {
+ "type": "string",
+ "enum": [
+ "DATA",
+ "CONTROL"
+ ],
+ "default": "DATA"
},
"TaskTypeDef": {
"type": "object",
@@ -382,11 +396,27 @@
"src": {
"type": "string"
},
+ "runtime": {
+ "type": "string"
+ },
"resolved": {
"type": "string"
}
}
},
+ "TypedValue": {
+ "type": "object",
+ "properties": {
+ "type": {
+ "type": "string"
+ },
+ "value": {
+ "type": "string",
+ "format": "byte"
+ }
+ },
+ "description": "Copy of protobuf's Any, to avoid protobuf requirement of a protobuf-based type."
+ },
"Workflow": {
"type": "object",
"properties": {
@@ -402,28 +432,6 @@
},
"title": "Workflow Model"
},
- "WorkflowDefinition": {
- "type": "object",
- "properties": {
- "apiVersion": {
- "type": "string",
- "description": "apiVersion describes what version is of the workflow definition.\n\nBy default the workflow engine will assume the latest version to be used."
- },
- "tasks": {
- "type": "object",
- "additionalProperties": {
- "$ref": "#/definitions/Task"
- },
- "description": "Dependency graph is build into the tasks",
- "title": "TODO Parameters\nActions"
- },
- "outputTask": {
- "type": "string",
- "title": "From which task should the workflow return the output? (todo multiple? implicit?)"
- }
- },
- "description": "The workflowDefinition contains the definition of a workflow.\n\nIdeally the source code (json, yaml) can be converted directly to this message.\nNaming, triggers and versioning of the workflow itself is out of the scope of this data structure, which is delegated\nto the user/system upon the creation of a workflow."
- },
"WorkflowInvocation": {
"type": "object",
"properties": {
@@ -444,8 +452,11 @@
"workflowId": {
"type": "string"
},
- "input": {
- "type": "string"
+ "inputs": {
+ "type": "object",
+ "additionalProperties": {
+ "$ref": "#/definitions/TypedValue"
+ }
}
},
"title": "Workflow Invocation Model"
@@ -467,7 +478,7 @@
}
},
"output": {
- "type": "string"
+ "$ref": "#/definitions/TypedValue"
}
}
},
@@ -486,16 +497,25 @@
"WorkflowSpec": {
"type": "object",
"properties": {
- "name": {
- "type": "string"
+ "apiVersion": {
+ "type": "string",
+ "description": "apiVersion describes what version is of the workflow definition.\n\nBy default the workflow engine will assume the latest version to be used."
},
- "version": {
- "type": "string"
+ "tasks": {
+ "type": "object",
+ "additionalProperties": {
+ "$ref": "#/definitions/Task"
+ },
+ "description": "Dependency graph is build into the tasks",
+ "title": "TODO Parameters\nActions"
},
- "src": {
- "$ref": "#/definitions/WorkflowDefinition"
+ "outputTask": {
+ "type": "string",
+ "title": "From which task should the workflow return the output? Future: multiple? Implicit?"
}
- }
+ },
+ "description": "The workflowDefinition contains the definition of a workflow.\n\nIdeally the source code (json, yaml) can be converted directly to this message.\nNaming, triggers and versioning of the workflow itself is out of the scope of this data structure, which is delegated\nto the user/system upon the creation of a workflow.",
+ "title": "Workflow Definition"
},
"WorkflowStatus": {
"type": "object",