Add design notes + update api

fission · Aug 25, 2017 · fa9d4b7 · fa9d4b7
1 parent 40462d4
commit fa9d4b7
Show file tree

Hide file tree

Showing 3 changed files with 228 additions and 123 deletions.
diff --git a/Docs/api/index.html b/Docs/api/index.html
@@ -354,25 +354,33 @@ <h3 class="field-label">Example data</h3>
     "id" : "aeiou"
   },
   "spec" : {
-    "src" : {
-      "apiVersion" : "aeiou",
-      "outputTask" : "aeiou",
-      "tasks" : {
-        "key" : {
-          "args" : [ { } ],
-          "name" : "aeiou",
-          "type" : "aeiou",
-          "dependencies" : [ "aeiou" ]
-        }
+    "apiVersion" : "aeiou",
+    "outputTask" : "aeiou",
+    "tasks" : {
+      "key" : {
+        "inputs" : {
+          "key" : {
+            "type" : "aeiou",
+            "value" : "aeiou"
+          }
+        },
+        "name" : "aeiou",
+        "type" : "aeiou",
+        "dependencies" : {
+          "key" : {
+            "alias" : "aeiou",
+            "type" : { }
+          }
+        },
+        "dependencies_await" : 0
       }
-    },
-    "name" : "aeiou",
-    "version" : "aeiou"
+    }
   },
   "status" : {
     "resolvedTasks" : {
       "key" : {
         "src" : "aeiou",
+        "runtime" : "aeiou",
         "resolved" : "aeiou"
       }
     },
@@ -582,22 +590,32 @@ <h3 class="field-label">Example data</h3>
     "id" : "aeiou"
   },
   "spec" : {
-    "input" : "aeiou",
+    "inputs" : {
+      "key" : {
+        "type" : "aeiou",
+        "value" : "aeiou"
+      }
+    },
     "workflowId" : "aeiou"
   },
   "status" : {
-    "output" : "aeiou",
+    "output" : "",
     "tasks" : {
       "key" : {
         "metadata" : "",
         "spec" : {
-          "input" : "aeiou",
-          "functionId" : "aeiou",
-          "functionName" : "aeiou",
+          "inputs" : {
+            "key" : ""
+          },
+          "type" : {
+            "src" : "aeiou",
+            "runtime" : "aeiou",
+            "resolved" : "aeiou"
+          },
           "taskId" : "aeiou"
         },
         "status" : {
-          "output" : "aeiou",
+          "output" : "",
           "status" : { },
           "updatedAt" : "2000-01-23T04:56:07.000+00:00"
         }
@@ -693,8 +711,6 @@ <h3 class="field-label">Query parameters</h3>
     <div class="field-items">
       <div class="param">workflowId (optional)</div>
 
-      <div class="param-desc"><span class="param-type">Query Parameter</span> &mdash;  </div><div class="param">input (optional)</div>
-
       <div class="param-desc"><span class="param-type">Query Parameter</span> &mdash;  </div>
     </div>  <!-- field-items -->
 
@@ -715,22 +731,32 @@ <h3 class="field-label">Example data</h3>
     "id" : "aeiou"
   },
   "spec" : {
-    "input" : "aeiou",
+    "inputs" : {
+      "key" : {
+        "type" : "aeiou",
+        "value" : "aeiou"
+      }
+    },
     "workflowId" : "aeiou"
   },
   "status" : {
-    "output" : "aeiou",
+    "output" : "",
     "tasks" : {
       "key" : {
         "metadata" : "",
         "spec" : {
-          "input" : "aeiou",
-          "functionId" : "aeiou",
-          "functionName" : "aeiou",
+          "inputs" : {
+            "key" : ""
+          },
+          "type" : {
+            "src" : "aeiou",
+            "runtime" : "aeiou",
+            "resolved" : "aeiou"
+          },
           "taskId" : "aeiou"
         },
         "status" : {
-          "output" : "aeiou",
+          "output" : "",
           "status" : { },
           "updatedAt" : "2000-01-23T04:56:07.000+00:00"
         }
@@ -795,22 +821,32 @@ <h3 class="field-label">Example data</h3>
     "id" : "aeiou"
   },
   "spec" : {
-    "input" : "aeiou",
+    "inputs" : {
+      "key" : {
+        "type" : "aeiou",
+        "value" : "aeiou"
+      }
+    },
     "workflowId" : "aeiou"
   },
   "status" : {
-    "output" : "aeiou",
+    "output" : "",
     "tasks" : {
       "key" : {
         "metadata" : "",
         "spec" : {
-          "input" : "aeiou",
-          "functionId" : "aeiou",
-          "functionName" : "aeiou",
+          "inputs" : {
+            "key" : ""
+          },
+          "type" : {
+            "src" : "aeiou",
+            "runtime" : "aeiou",
+            "resolved" : "aeiou"
+          },
           "taskId" : "aeiou"
         },
         "status" : {
-          "output" : "aeiou",
+          "output" : "",
           "status" : { },
           "updatedAt" : "2000-01-23T04:56:07.000+00:00"
         }
@@ -891,15 +927,16 @@ <h3>Table of Contents</h3>
     <li><a href="#FunctionInvocationStatusStatus"><code>FunctionInvocationStatusStatus</code> - </a></li>
     <li><a href="#ObjectMetadata"><code>ObjectMetadata</code> - Common</a></li>
     <li><a href="#Task"><code>Task</code> - </a></li>
-    <li><a href="#TaskParameters"><code>TaskParameters</code> - </a></li>
+    <li><a href="#TaskDependencyParameters"><code>TaskDependencyParameters</code> - </a></li>
+    <li><a href="#TaskDependencyParametersDependencyType"><code>TaskDependencyParametersDependencyType</code> - </a></li>
     <li><a href="#TaskTypeDef"><code>TaskTypeDef</code> - </a></li>
+    <li><a href="#TypedValue"><code>TypedValue</code> - </a></li>
     <li><a href="#Workflow"><code>Workflow</code> - Workflow Model</a></li>
-    <li><a href="#WorkflowDefinition"><code>WorkflowDefinition</code> - </a></li>
     <li><a href="#WorkflowInvocation"><code>WorkflowInvocation</code> - </a></li>
     <li><a href="#WorkflowInvocationSpec"><code>WorkflowInvocationSpec</code> - Workflow Invocation Model</a></li>
     <li><a href="#WorkflowInvocationStatus"><code>WorkflowInvocationStatus</code> - </a></li>
     <li><a href="#WorkflowInvocationStatusStatus"><code>WorkflowInvocationStatusStatus</code> - </a></li>
-    <li><a href="#WorkflowSpec"><code>WorkflowSpec</code> - </a></li>
+    <li><a href="#WorkflowSpec"><code>WorkflowSpec</code> - Workflow Definition</a></li>
     <li><a href="#WorkflowStatus"><code>WorkflowStatus</code> - Internal</a></li>
     <li><a href="#WorkflowStatusStatus"><code>WorkflowStatusStatus</code> - </a></li>
     <li><a href="#apiserverHealth"><code>apiserverHealth</code> - </a></li>
@@ -914,7 +951,7 @@ <h3>Table of Contents</h3>
 
   <div class="model">
     <h3><a name="FunctionInvocation"><code>FunctionInvocation</code> - Function Invocation Model</a> <a class="up" href="#__Models">Up</a></h3>
-    <div class='model-description'>It contains no indication which workflow/task it is executed for</div>
+    <div class='model-description'></div>
     <div class="field-items">
       <div class="param">metadata (optional)</div><div class="param-desc"><span class="param-type"><a href="#ObjectMetadata">ObjectMetadata</a></span>  </div>
 <div class="param">spec (optional)</div><div class="param-desc"><span class="param-type"><a href="#FunctionInvocationSpec">FunctionInvocationSpec</a></span>  </div>
@@ -925,10 +962,9 @@ <h3><a name="FunctionInvocation"><code>FunctionInvocation</code> - Function Invo
     <h3><a name="FunctionInvocationSpec"><code>FunctionInvocationSpec</code> - </a> <a class="up" href="#__Models">Up</a></h3>
     <div class='model-description'></div>
     <div class="field-items">
-      <div class="param">functionId (optional)</div><div class="param-desc"><span class="param-type"><a href="#string">String</a></span>  </div>
-<div class="param">functionName (optional)</div><div class="param-desc"><span class="param-type"><a href="#string">String</a></span>  </div>
+      <div class="param">type (optional)</div><div class="param-desc"><span class="param-type"><a href="#TaskTypeDef">TaskTypeDef</a></span>  </div>
 <div class="param">taskId (optional)</div><div class="param-desc"><span class="param-type"><a href="#string">String</a></span>  </div>
-<div class="param">input (optional)</div><div class="param-desc"><span class="param-type"><a href="#string">String</a></span>  </div>
+<div class="param">inputs (optional)</div><div class="param-desc"><span class="param-type"><a href="#TypedValue">map[String, TypedValue]</a></span>  </div>
     </div>  <!-- field-items -->
   </div>
   <div class="model">
@@ -937,7 +973,7 @@ <h3><a name="FunctionInvocationStatus"><code>FunctionInvocationStatus</code> - <
     <div class="field-items">
       <div class="param">status (optional)</div><div class="param-desc"><span class="param-type"><a href="#FunctionInvocationStatusStatus">FunctionInvocationStatusStatus</a></span>  </div>
 <div class="param">updatedAt (optional)</div><div class="param-desc"><span class="param-type"><a href="#DateTime">Date</a></span>  format: date-time</div>
-<div class="param">output (optional)</div><div class="param-desc"><span class="param-type"><a href="#string">String</a></span>  </div>
+<div class="param">output (optional)</div><div class="param-desc"><span class="param-type"><a href="#TypedValue">TypedValue</a></span>  </div>
     </div>  <!-- field-items -->
   </div>
   <div class="model">
@@ -968,12 +1004,21 @@ <h3><a name="Task"><code>Task</code> - </a> <a class="up" href="#__Models">Up</a
 </ul>
  </div>
 <div class="param">name (optional)</div><div class="param-desc"><span class="param-type"><a href="#string">String</a></span>  </div>
-<div class="param">args (optional)</div><div class="param-desc"><span class="param-type"><a href="#TaskParameters">array[TaskParameters]</a></span>  </div>
-<div class="param">dependencies (optional)</div><div class="param-desc"><span class="param-type"><a href="#string">array[String]</a></span>  </div>
+<div class="param">inputs (optional)</div><div class="param-desc"><span class="param-type"><a href="#TypedValue">map[String, TypedValue]</a></span>  </div>
+<div class="param">dependencies (optional)</div><div class="param-desc"><span class="param-type"><a href="#TaskDependencyParameters">map[String, TaskDependencyParameters]</a></span>  </div>
+<div class="param">dependencies_await (optional)</div><div class="param-desc"><span class="param-type"><a href="#integer">Integer</a></span>  format: int32</div>
     </div>  <!-- field-items -->
   </div>
   <div class="model">
-    <h3><a name="TaskParameters"><code>TaskParameters</code> - </a> <a class="up" href="#__Models">Up</a></h3>
+    <h3><a name="TaskDependencyParameters"><code>TaskDependencyParameters</code> - </a> <a class="up" href="#__Models">Up</a></h3>
+    <div class='model-description'></div>
+    <div class="field-items">
+      <div class="param">type (optional)</div><div class="param-desc"><span class="param-type"><a href="#TaskDependencyParametersDependencyType">TaskDependencyParametersDependencyType</a></span>  </div>
+<div class="param">alias (optional)</div><div class="param-desc"><span class="param-type"><a href="#string">String</a></span>  </div>
+    </div>  <!-- field-items -->
+  </div>
+  <div class="model">
+    <h3><a name="TaskDependencyParametersDependencyType"><code>TaskDependencyParametersDependencyType</code> - </a> <a class="up" href="#__Models">Up</a></h3>
     <div class='model-description'></div>
     <div class="field-items">
           </div>  <!-- field-items -->
@@ -983,9 +1028,18 @@ <h3><a name="TaskTypeDef"><code>TaskTypeDef</code> - </a> <a class="up" href="#_
     <div class='model-description'></div>
     <div class="field-items">
       <div class="param">src (optional)</div><div class="param-desc"><span class="param-type"><a href="#string">String</a></span>  </div>
+<div class="param">runtime (optional)</div><div class="param-desc"><span class="param-type"><a href="#string">String</a></span>  </div>
 <div class="param">resolved (optional)</div><div class="param-desc"><span class="param-type"><a href="#string">String</a></span>  </div>
     </div>  <!-- field-items -->
   </div>
+  <div class="model">
+    <h3><a name="TypedValue"><code>TypedValue</code> - </a> <a class="up" href="#__Models">Up</a></h3>
+    <div class='model-description'>Copy of protobuf's Any, to avoid protobuf requirement of a protobuf-based type.</div>
+    <div class="field-items">
+      <div class="param">type (optional)</div><div class="param-desc"><span class="param-type"><a href="#string">String</a></span>  </div>
+<div class="param">value (optional)</div><div class="param-desc"><span class="param-type"><a href="#ByteArray">byte[]</a></span>  format: byte</div>
+    </div>  <!-- field-items -->
+  </div>
   <div class="model">
     <h3><a name="Workflow"><code>Workflow</code> - Workflow Model</a> <a class="up" href="#__Models">Up</a></h3>
     <div class='model-description'></div>
@@ -995,21 +1049,6 @@ <h3><a name="Workflow"><code>Workflow</code> - Workflow Model</a> <a class="up"
 <div class="param">status (optional)</div><div class="param-desc"><span class="param-type"><a href="#WorkflowStatus">WorkflowStatus</a></span>  </div>
     </div>  <!-- field-items -->
   </div>
-  <div class="model">
-    <h3><a name="WorkflowDefinition"><code>WorkflowDefinition</code> - </a> <a class="up" href="#__Models">Up</a></h3>
-    <div class='model-description'><p>The workflowDefinition contains the definition of a workflow.</p>
-<p>Ideally the source code (json, yaml) can be converted directly to this message.
-Naming, triggers and versioning of the workflow itself is out of the scope of this data structure, which is delegated
-to the user/system upon the creation of a workflow.</p>
-</div>
-    <div class="field-items">
-      <div class="param">apiVersion (optional)</div><div class="param-desc"><span class="param-type"><a href="#string">String</a></span> <p>apiVersion describes what version is of the workflow definition.</p>
-<p>By default the workflow engine will assume the latest version to be used.</p>
- </div>
-<div class="param">tasks (optional)</div><div class="param-desc"><span class="param-type"><a href="#Task">map[String, Task]</a></span> Dependency graph is build into the tasks </div>
-<div class="param">outputTask (optional)</div><div class="param-desc"><span class="param-type"><a href="#string">String</a></span>  </div>
-    </div>  <!-- field-items -->
-  </div>
   <div class="model">
     <h3><a name="WorkflowInvocation"><code>WorkflowInvocation</code> - </a> <a class="up" href="#__Models">Up</a></h3>
     <div class='model-description'></div>
@@ -1024,7 +1063,7 @@ <h3><a name="WorkflowInvocationSpec"><code>WorkflowInvocationSpec</code> - Workf
     <div class='model-description'></div>
     <div class="field-items">
       <div class="param">workflowId (optional)</div><div class="param-desc"><span class="param-type"><a href="#string">String</a></span>  </div>
-<div class="param">input (optional)</div><div class="param-desc"><span class="param-type"><a href="#string">String</a></span>  </div>
+<div class="param">inputs (optional)</div><div class="param-desc"><span class="param-type"><a href="#TypedValue">map[String, TypedValue]</a></span>  </div>
     </div>  <!-- field-items -->
   </div>
   <div class="model">
@@ -1034,7 +1073,7 @@ <h3><a name="WorkflowInvocationStatus"><code>WorkflowInvocationStatus</code> - <
       <div class="param">status (optional)</div><div class="param-desc"><span class="param-type"><a href="#WorkflowInvocationStatusStatus">WorkflowInvocationStatusStatus</a></span>  </div>
 <div class="param">updatedAt (optional)</div><div class="param-desc"><span class="param-type"><a href="#DateTime">Date</a></span>  format: date-time</div>
 <div class="param">tasks (optional)</div><div class="param-desc"><span class="param-type"><a href="#FunctionInvocation">map[String, FunctionInvocation]</a></span>  </div>
-<div class="param">output (optional)</div><div class="param-desc"><span class="param-type"><a href="#string">String</a></span>  </div>
+<div class="param">output (optional)</div><div class="param-desc"><span class="param-type"><a href="#TypedValue">TypedValue</a></span>  </div>
     </div>  <!-- field-items -->
   </div>
   <div class="model">
@@ -1044,12 +1083,18 @@ <h3><a name="WorkflowInvocationStatusStatus"><code>WorkflowInvocationStatusStatu
           </div>  <!-- field-items -->
   </div>
   <div class="model">
-    <h3><a name="WorkflowSpec"><code>WorkflowSpec</code> - </a> <a class="up" href="#__Models">Up</a></h3>
-    <div class='model-description'></div>
+    <h3><a name="WorkflowSpec"><code>WorkflowSpec</code> - Workflow Definition</a> <a class="up" href="#__Models">Up</a></h3>
+    <div class='model-description'><p>The workflowDefinition contains the definition of a workflow.</p>
+<p>Ideally the source code (json, yaml) can be converted directly to this message.
+Naming, triggers and versioning of the workflow itself is out of the scope of this data structure, which is delegated
+to the user/system upon the creation of a workflow.</p>
+</div>
     <div class="field-items">
-      <div class="param">name (optional)</div><div class="param-desc"><span class="param-type"><a href="#string">String</a></span>  </div>
-<div class="param">version (optional)</div><div class="param-desc"><span class="param-type"><a href="#string">String</a></span>  </div>
-<div class="param">src (optional)</div><div class="param-desc"><span class="param-type"><a href="#WorkflowDefinition">WorkflowDefinition</a></span>  </div>
+      <div class="param">apiVersion (optional)</div><div class="param-desc"><span class="param-type"><a href="#string">String</a></span> <p>apiVersion describes what version is of the workflow definition.</p>
+<p>By default the workflow engine will assume the latest version to be used.</p>
+ </div>
+<div class="param">tasks (optional)</div><div class="param-desc"><span class="param-type"><a href="#Task">map[String, Task]</a></span> Dependency graph is build into the tasks </div>
+<div class="param">outputTask (optional)</div><div class="param-desc"><span class="param-type"><a href="#string">String</a></span>  </div>
     </div>  <!-- field-items -->
   </div>
   <div class="model">

diff --git a/Docs/wip/datatransformations.md b/Docs/wip/datatransformations.md
@@ -0,0 +1,40 @@
+# Data Transformers
+
+A data transformer allows a user can perform common data transformations on input data.
+Examples of such data transformations are selecting a sub-selection of a data object or concatenating strings.
+This document tracks the design of the data transformers for Fission Workflow.
+
+## Use Cases
+- Reference data from other tasks. For example: `$.tasks.foo.output`
+- Create subselections of data. For example: `$.tasks.foo.output.user.id` 
+- Transform data. For example: `concat($.tasks.foo.output.user.firstName, ' ', $.tasks.foo.output.lastName)`
+
+## Data Types
+- string
+- bool
+- object
+- array
+- int
+
+### Special types
+- `expr`: referencing and manipulating data.
+- `raw`: default type, or fallback in case no other type could be assigned. Does not support selectors or most functions.
+
+## Design
+- There needs to be some symbol to differentiate functions from literals (or reverse token for literals)
+- The language and type determine which functions are available to invoke. For now that is solely JSON.
+- The TypedValue is similar to how Any works in Protobuf 3.
+- Selectors and transformers are implemented using Otto's Javascript engine.
+
+## Implementation
+- TypedValue, Type consists out of `<language/format>/<type>`. E.g. a JSON string is represented as `json/string`. 
+It is allowed to just specify the language, allowing a parser to determine the actual type. 
+
+### Functionality
+- [x] Transform Go primitive data types (interface{}) <-> TypedValue.
+- [x] Fission Proxy: Transform Go primitive data types (interface{}) <-> HTTP request.
+- [x] Add data transformation interface
+- [x] Parse and resolve expressions
+- [x] Add selector-based support
+- [ ] Add utility functions to the expression-parser. Status: 
+- [ ] Allow data transformers to be run as regular tasks