Incorporate Plotly JSON to metamodel. Closes #1745 (#1747)

* WIP- Inital Version of plotly json in metamodel

* WIP- update devProject; increate timeout for devProject

1. Increase Timeout ExecutePipeline.spec.js
2. Update devProject seed to adhere to new MetaModel

* WIP- Fix node id from test seeds

* Fix releases.jsonl

* WIP- Remove Line node creation from ExecuteJob.spec.js

* Fix timeout in ExecutePipeline.spec.js and fix releases.jsonl

1. Remove dependencies in SubplotsOperation in devProject.webgmex
2. Remove src/seeds/project/releases.json
3. Revert timeout in ExecutePipeline.spec.js

* WIP-Add 3D support for DeepforgePlotlyRenderer

* WIP- Fix np masked array imports in plotly_backend.py

* WIP-Refactor fix HelloWorld Job execution ID

* WIP- Increase timeout for sciserver to 10 minutes

* WIP- add PR refactors, pin matplotlib

1. Pin matplotlib to v3.2.2 in environment.worker.yml and devProject
2. Remove this.created in PlotlyGraphWidget
3. Fix releases.jsonl
4. Rename getGraphNode to getGraphNodes in updates.js, use array instead of dictionary
5. Remove extra line in ExecuteJob.spec.js

* WIP- Add beforeLibraryUpdates flag to UpdateGraphContainment

* Add Metadata node as Mixin for correct extraction of older seeds

-- In version 0.21.1 of the pipeline library, Inheritance of the old
SubGraph and below was changed from FCO to Metadata (because of provenance).
So, the current Extractor to extract plotlyJSON expects this inheritance.
For older seeds, a function to add META['pipeline.Metadata'] as a Mixin is added to the
updating capabilities.

-- remove wrongfully pased comment

* Filter provenance nodes from mixin application; Add fix for #1799

* WIP- Fix version check in UpdateGraphContainment

* WIP- Cover cases for layout.title as object

* WIP- displayTitle fix for undefined cases

* WIP- adjust height of a single plot; check falsy values in data attribute

environment.worker.yml

@@ -5,3 +5,4 @@ dependencies:
       - pillow
       - matplotlib==3.2.2
       - simplejson
+      - plotly

src/common/updates/Updates.js

@@ -1,13 +1,51 @@
 /* globals define */
 define([
     'deepforge/storage/index',
+    'deepforge/viz/PlotlyDescExtractor',
+    'deepforge/viz/FigureExtractor',
     './Version',
     'q'
 ], function(
     Storage,
+    PlotlyDescExtractor,
+    FigureExtractor,
     Version,
     Q
 ) {
+    const GRAPH = 'Graph';
+    const getGraphNodes = async function(core, rootNode, graphNodes=[]) {
+        const children = await core.loadChildren(rootNode);
+        for(let i = 0; i < children.length; i++) {
+            if (core.getAttribute(children[i], 'name') === GRAPH && !core.isMetaNode(children[i])) {
+                graphNodes.push(children[i]);
+            }
+            await getGraphNodes(core, children[i], graphNodes);
+        }
+    };
+
+    const addMetadataMixinToNodeSubTree = async function(core, META, node) {
+        const METADATA_NODE_PATH =  core.getPath(META['pipeline.Metadata']);
+        const IMPLICIT_OPERATION_NODE = META['pipeline.ImplicitOperation'];
+        const graphNodeChildren = (await core.loadSubTree(node))
+            .filter(node => {
+                return IMPLICIT_OPERATION_NODE ?
+                    !core.isTypeOf(node, IMPLICIT_OPERATION_NODE) :
+                    true;
+            });
+
+        graphNodeChildren.forEach(node => {
+            core.addMixin(node, METADATA_NODE_PATH);
+        });
+    };
+
+    const getPipelineLibraryVersion = function(core, rootNode) {
+        const pipelineRoot = core.getLibraryRoot(rootNode, 'pipeline');
+        const hasPipelineLibrary = !!pipelineRoot;
+        if (hasPipelineLibrary) {
+            const versionString = core.getAttribute(pipelineRoot, 'version');
+            return new Version(versionString);
+        }
+    };
 
     const allUpdates = [
         {
@@ -48,12 +86,9 @@ define([
         {
             name: 'UpdateDataNodesToUserAssets',
             isNeeded: async function(core, rootNode) {
-                const pipelineRoot = core.getLibraryRoot(rootNode, 'pipeline');
-                const hasPipelineLibrary = !!pipelineRoot;
-                if (hasPipelineLibrary) {
-                    const versionString = core.getAttribute(pipelineRoot, 'version');
-                    const version = new Version(versionString);
-                    return version.lessThan(new Version('0.13.0'));
+                const pipelineLibraryVersion = getPipelineLibraryVersion(core, rootNode);
+                if(pipelineLibraryVersion) {
+                    return pipelineLibraryVersion.lessThan(new Version('0.13.0'));
                 }
             },
             apply: async function(core, rootNode, META) {
@@ -72,6 +107,42 @@ define([
                     }
                 }
             }
+        },
+        {
+            name: 'UpdateGraphContainment',
+            beforeLibraryUpdates: true,
+            isNeeded: async function(core, rootNode) {
+                const pipelineLibraryVersion = getPipelineLibraryVersion(core, rootNode);
+                if (pipelineLibraryVersion) {
+                    return pipelineLibraryVersion.lessThan(new Version('0.22.0')) &&
+                           pipelineLibraryVersion.greaterThan(new Version('0.19.1'));
+                }
+            },
+            apply: async function(core, rootNode, META) {
+                let graphNodes = [];
+                await getGraphNodes(core, rootNode, graphNodes);
+                const coreFigureExtractor = new FigureExtractor.CoreFigureExtractor(core, rootNode);
+                const pipelineVersion = getPipelineLibraryVersion(core, rootNode);
+                const shouldAddMetadataMixin = pipelineVersion ?
+                    pipelineVersion.lessThan(new Version('0.21.1')) :
+                    false;
+
+                for (let i = 0; i < graphNodes.length; i++){
+                    const graphNode = graphNodes[i];
+                    if(shouldAddMetadataMixin){
+                        await addMetadataMixinToNodeSubTree(core, META, graphNode);
+                    }
+                    const desc = await coreFigureExtractor.extract(graphNode);
+                    const plotlyJSON = PlotlyDescExtractor.descToPlotlyJSON(desc);
+                    const parentNode = core.getParent(graphNode);
+                    const updatedGraphNode = core.createNode({
+                        parent: parentNode,
+                        base: META['pipeline.Graph']
+                    });
+                    core.setAttribute(updatedGraphNode, 'data', JSON.stringify(plotlyJSON));
+                    core.deleteNode(graphNode);
+                }
+            }
         }
     ];
 

src/plugins/ExecuteJob/metadata/Figure.js

@@ -1,26 +1,12 @@
 /* globals define */
 define([
-    './Metadata',
+    './Metadata'
 ], function(
-    Metadata,
+    Metadata
 ) {
     class Figure extends Metadata {
         async update(state) {
-            this.core.setAttribute(this.node, 'title', state.title);
-            await this.clearSubGraphs();
-
-            state.axes.forEach(axes => {
-                const axesNode = this.core.createNode({
-                    parent: this.node,
-                    base: axes.is3D ? this.META.Plot3D : this.META.Plot2D
-                });
-                this.setAxesProperties(axesNode, axes);
-                this.addAxesLines(axesNode, this.node, axes);
-                if(!axes.is3D){
-                    this.addAxesImage(axesNode, this.node, axes);
-                }
-                this.addAxesScatterPoints(axesNode, this.node, axes);
-            });
+            this.core.setAttribute(this.node, 'data', JSON.stringify(state));
         }
 
         setAxesProperties(axesNode, axes){

src/plugins/GenerateJob/GenerateJob.js

@@ -103,7 +103,7 @@ define([
         this.result.setSuccess(true);
         this.result.addArtifact(hash);
         callback(null, this.result);
-    }; 
+    };
 
     GenerateJob.prototype.createRunScript = async function (files) {
         let runDebug = Templates.RUN_DEBUG;
@@ -291,7 +291,7 @@ define([
         files.addFile('config.json', JSON.stringify(configs));
         files.addFile('start.js', Templates.START);
         files.addFile('utils.build.js', Templates.UTILS);
-        files.addFile('backend_deepforge.py', Templates.MATPLOTLIB_BACKEND);
+        files.addFile('plotly_backend.py', Templates.MATPLOTLIB_BACKEND);
 
         inputs.forEach(pair => {
             const dataInfo = this.core.getAttribute(pair[2], 'data');

src/plugins/GenerateJob/templates/backend_deepforge.py

@@ -354,9 +354,9 @@ def draw(self):
         """
         Draw the figure using the renderer
         """
-        self.send_deepforge_update()
         renderer = RendererTemplate(self.figure.dpi)
         self.figure.draw(renderer)
+        self.send_deepforge_update()
 
     def send_deepforge_update(self):
         state = self.figure_to_state()

src/plugins/GenerateJob/templates/index.js

@@ -4,7 +4,7 @@ define([
     'text!./run-debug.js',
     'text!./main.ejs',
     'text!./deepforge.ejs',
-    'text!./backend_deepforge.py',
+    'text!./plotly_backend.py',
     'text!./deepforge__init__.py',
     'text!./serialize.ejs',
     'text!./deserialize.ejs',