Merge pull request #620 from mvdbeek/mermaid_diagrams_for_workflows

First pass at generating mermaid diagrams for all workflows

.github/workflows/gh-page-preview.yml

@@ -13,8 +13,6 @@ jobs:
     steps:
       - name: Checkout PR Branch
         uses: actions/checkout@v4
-        with:
-          ref: ${{ github.head_ref }}
 
       - name: Set up Python
         uses: actions/setup-python@v5
@@ -72,4 +70,4 @@ jobs:
         run: |
           git rm -rf __preview/pr-${{ github.event.pull_request.number }}
           git commit -m "Remove preview for PR #${{ github.event.pull_request.number }}"
-          git push origin gh-pages
+          git push origin gh-pages

.gitignore

@@ -2,3 +2,4 @@ tool_test_output.html
 tool_test_output.json
 .DS_Store
 workflow_manifest.json
+.vscode

scripts/create_mermaid.py

@@ -0,0 +1,110 @@
+import argparse
+import json
+import os
+from typing import Literal
+
+from gxformat2.converter import python_to_workflow
+from gxformat2.yaml import ordered_load
+
+STEP_TYPE_TO_SHAPE = {
+    "data_input": "@{ shape: doc }",
+    "data_collection_input": "@{ shape: docs }",
+    "parameter_input": "@{ shape: lean-l }",
+    "tool": "@{ shape: process }",
+    "subworkflow": "@{ shape: subprocess }",
+}
+STEP_TYPE_TO_SYMBOL = {
+    "data_input": "ℹ️ ",
+    "data_collection_input": "ℹ️ ",
+    "parameter_input": "ℹ️ ",
+    "subworkflow": "🛠️ ",
+    "tool": "",
+}
+
+
+def step_to_mermaid_item(
+    step_type: Literal[
+        "parameter_input", "data_input", "data_collection_input", "tool", "subworkflow"
+    ],
+    step_label: str,
+):
+    prefix = STEP_TYPE_TO_SYMBOL[step_type]
+    step_label_anchor = f'["{prefix}{step_label}"]'
+    shape = STEP_TYPE_TO_SHAPE.get(step_type, "")
+    return f"{step_label_anchor}{shape}"
+
+
+def workflow_to_mermaid_diagrams(workflow, workflows = None):
+    """
+    Converts a Galaxy workflow JSON to a Mermaid flowchart diagram.
+
+    Args:
+        workflow_json: The JSON representation of the Galaxy workflow.
+
+    Returns:
+        A string representing the Mermaid flowchart diagram.
+    """
+    if workflows is None:
+        workflows = {}
+
+    mermaid_diagram = ["graph LR"]
+
+    # Create a mapping of step IDs to their labels
+    id_step_labels = {
+        int(step["id"]): step.get("label") or step["name"] or step["content_id"] or step["id"]
+        for step in workflow["steps"].values()
+    }
+
+    # Iterate through each step and its connections
+    for step_id, step in workflow["steps"].items():
+        step_label = id_step_labels.get(int(step_id))
+        mermaid_diagram.append(
+            f'{step_id}{step_to_mermaid_item(step["type"], step_label)}'
+        )
+        for input_connection in step.get("input_connections", {}).values():
+            if not isinstance(input_connection, list):
+                input_connection = [input_connection]
+            for ic in input_connection:
+                mermaid_diagram.append(f"{ic['id']} --> {step_id}")
+
+        if step["type"] == "subworkflow":
+            workflow_to_mermaid_diagrams(step["subworkflow"], workflows=workflows)
+
+    workflows[workflow["name"]] = "\n".join(mermaid_diagram)
+
+    return workflows
+
+
+def walk_directory(directory):
+    """
+    Walk directory and call workflow_to_mermaid on each discovered .ga file.
+    """
+    for root, _, paths in os.walk(directory):
+        for path in paths:
+            if path.endswith((".ga", ".gxwf.yml")):
+                file_path = os.path.join(root, path)
+                with open(file_path, "r") as f:
+                    workflow_data = ordered_load(f)
+                if workflow_data.get("class") == "GalaxyWorkflow":
+                    workflow_data = python_to_workflow(workflow_data, galaxy_interface=None, workflow_directory=os.path.dirname(file_path))
+                mermaid_diagrams = workflow_to_mermaid_diagrams(workflow_data)
+
+                markdown_items = ["# Workflow diagrams\n"]
+                for workflow_name, diagram in reversed(mermaid_diagrams.items()):
+                    markdown_items.append(f"## {workflow_name}\n")
+                    markdown_items.append(f"```mermaid\n{diagram}\n```\n")
+
+                mmd_path = f"{os.path.splitext(file_path)[0]}_diagrams.md"
+                with open(mmd_path, "w") as f:
+                    f.write("\n".join(markdown_items))
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="Process files in a directory")
+    parser.add_argument("directory", type=str, help="Path to the input directory")
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    args = parse_args()
+    walk_directory(args.directory)

scripts/workflow_manifest.py

@@ -1,6 +1,19 @@
 import os
 import json
 import yaml
+from create_mermaid import walk_directory
+
+
+def read_contents(path: str):
+    try:
+        with open(path) as f:
+            return f.read()
+    except FileNotFoundError:
+        print(f"No {os.path.basename(path)} at {path}")
+    except Exception as e:
+        print(
+            f"Error reading file {path}: {e}"
+        )
 
 
 def find_and_load_compliant_workflows(directory):
@@ -57,28 +70,10 @@ def find_and_load_compliant_workflows(directory):
                             f"No workflow file: {os.path.join(root, workflow['primaryDescriptorPath'])}: {e}"
                         )
 
-                    # also try to load a README.md file for each workflow
-                    try:
-                        with open(os.path.join(root, "README.md")) as f:
-                            workflow["readme"] = f.read()
-                    # catch FileNotFound
-                    except FileNotFoundError:
-                        print(f"No README.md at {os.path.join(root, 'README.md')}")
-                    except Exception as e:
-                        print(
-                            f"Error reading file {os.path.join(root, 'README.md')}: {e}"
-                        )
-
-                    # also try to load a CHANGELOG.md file for each workflow
-                    try:
-                        with open(os.path.join(root, "CHANGELOG.md")) as f:
-                            workflow["changelog"] = f.read()
-                    except FileNotFoundError:
-                        print(f"No CHANGELOG.md at {os.path.join(root, 'CHANGELOG.md')}")
-                    except Exception as e:
-                        print(
-                            f"Error reading file {os.path.join(root, 'CHANGELOG.md')}: {e}"
-                        )
+                    # load readme, changelog and diagrams
+                    workflow["readme"] = read_contents(os.path.join(root, "README.md"))
+                    workflow["changelog"] = read_contents(os.path.join(root, "CHANGELOG.md"))
+                    workflow["diagrams"] = read_contents(f"{os.path.splitext(workflow_path)[0]}_diagrams.md")
                     dirname = os.path.dirname(workflow_path).split("/")[-1]
                     workflow["trsID"] = f"#workflow/github.com/iwc-workflows/{dirname}/{workflow['name'] or 'main'}"
 
@@ -108,5 +103,6 @@ def write_to_json(data, filename):
 
 
 if __name__ == "__main__":
+    walk_directory("./workflows")
     workflow_data = find_and_load_compliant_workflows("./workflows")
     write_to_json(workflow_data, "workflow_manifest.json")

website/components/MarkdownRenderer.vue

@@ -0,0 +1,60 @@
+<script setup lang="ts">
+import { ref, nextTick, watch } from "vue";
+import { marked } from "marked";
+import mermaid from "mermaid";
+
+// Props
+const props = defineProps({
+    markdownContent: {
+        type: String,
+        required: true,
+    },
+});
+
+// Refs
+const renderedHtml = ref<string>("");
+
+// Initialize Mermaid
+mermaid.initialize({
+    startOnLoad: false,
+    theme: "neutral",
+});
+
+// Watch Markdown Content for Changes
+watch(
+    () => props.markdownContent,
+    (newContent) => {
+        renderMarkdown(newContent);
+    },
+    { immediate: true },
+);
+
+// Render Markdown Function
+async function renderMarkdown(content: string) {
+    renderedHtml.value = await marked(content);
+    await nextTick();
+    renderMermaidDiagrams();
+}
+
+// Render Mermaid Diagrams
+function renderMermaidDiagrams() {
+    const mermaidElements = document.querySelectorAll(".language-mermaid");
+    mermaidElements.forEach((element) => {
+        const parent = element.parentElement;
+        const code = element.textContent || "";
+        if (parent) {
+            try {
+                const id = `mermaid-${Math.random().toString(36).substr(2, 9)}`;
+                mermaid.render(id, code).then((value) => (parent.innerHTML = value.svg));
+                parent.classList.add("not-prose");
+            } catch (e) {
+                console.error("Mermaid rendering error:", e);
+            }
+        }
+    });
+}
+</script>
+
+<template>
+    <div v-html="renderedHtml"></div>
+</template>

website/models/workflow.ts

@@ -16,6 +16,7 @@ export interface Workflow {
     definition: WorkflowDefinition;
     readme: string;
     changelog: string;
+    diagrams: string;
     trsID: string;
 }
 

website/package.json

@@ -15,6 +15,7 @@
     "@nuxt/ui": "^2.19.2",
     "@pinia/nuxt": "^0.5.5",
     "marked": "^14.1.1",
+    "mermaid": "^11.4.1",
     "nuxt": "^3.11.2",
     "nuxt-icon": "^1.0.0-beta.7",
     "pinia": "^2.2.4",

website/pages/workflow/[id].vue

@@ -1,7 +1,7 @@
 <script setup lang="ts">
 import { ref, computed, onBeforeMount } from "vue";
 import { useRoute } from "vue-router";
-import { marked } from "marked";
+import MarkdownRenderer from "~/components/MarkdownRenderer.vue";
 import Author from "~/components/Author.vue";
 import { useWorkflowStore } from "~/stores/workflows";
 
@@ -17,10 +17,6 @@ const formatDate = (date: string) => {
     });
 };
 
-const parseMarkdown = (content: string) => {
-    return marked(content);
-};
-
 // TODO: Add a component for authors.  For now, just have a computed that grabs names and joins them
 const authors = computed(() => {
     let authorLine = "";
@@ -91,14 +87,14 @@ const tabs = computed(() => [
         label: "Version History",
         content: workflow.value?.changelog || "No CHANGELOG available.",
     },
+    {
+        label: "Diagram",
+        content: workflow.value?.diagrams || "No diagram available",
+    },
     {
         label: "Tools",
         tools: tools || "This tab will show a nice listing of all the tools used in this workflow.",
     },
-    // {
-    //     label: "Preview",
-    //     preview: true,
-    // },
 ]);
 
 /* Instance Selector -- factor out to a component */
@@ -187,13 +183,13 @@ const onInstanceChange = (value: string) => {
                         </template>
                         <template #item="{ item }">
                             <div v-if="item.content" class="mt-6">
-                                <div
-                                    class="prose dark:prose-invert !max-w-none"
-                                    v-html="parseMarkdown(item.content)"></div>
+                                <div class="prose dark:prose-invert !max-w-none">
+                                    <MarkdownRenderer :markdownContent="item.content" />
+                                </div>
                             </div>
                             <div v-else-if="item.tools" class="mt-6">
                                 <div class="prose dark:prose-invert !max-w-none">
-                                    <h3>The following tools are required to run this workflow.</h3>
+                                    <h3>The following tools are used by this workflow.</h3>
                                     <p>
                                         This will eventually be a pretty page with links to each tool in the (new)
                                         toolshed, etc.