Feature/issue 709 custom render plugin (#869)

* POC of Lit SSR for development

* render lit SSR from html

* add renderer API to Greenwood

* mini refactor

* stub out lit renderer plugin

* build and serve support

* update specs

* fix incorrect callback assignment

* refactor SSR worker for Greenwood as a plugin

* support for intercepting and optimizing SSR routes and add dual SSR support for plugins

* template support for SSR routes

* set prerender true in spec

* add minimumal support for using custom renderer to prerender static pages

* lit render SSR specs

* address spec TODOs

* fix specs

* SSR specs for development

* documentation for custom renderer plugin

* remove puppeteer prerendering option from SSR routes and ensure explicit SSR output for lit renderer

* remove demo code

README.md

@@ -16,7 +16,7 @@ Features:
 - 🚫 No JavaScript by default.
 - 📖 Prerendering support for Web Components.
 - ⚒️ Extensible via [plugins](https://www.greenwoodjs.io/plugins/).
-- ⚙️ Supports [SSG, MPA, and SPA](https://www.greenwoodjs.io/docs/configuration/#mode). ([SSR support](https://github.com/ProjectEvergreen/greenwood/discussions/576) coming soon!)
+- ⚙️ Supports [SSG, MPA, SPA, and SSR* (or a hybrid!) project types](https://www.greenwoodjs.io/docs/configuration/#mode).
 
 > Greenwood is currently working towards a [1.0 release](https://github.com/ProjectEvergreen/greenwood/milestone/3) with our recent [`v0.10.0`](https://github.com/ProjectEvergreen/greenwood/releases/tag/v0.10.0) introducing some exciting new changes and concepts  to the project.  If you're interested in learning more about the web and web development (at any skill level!), or interested in checking out our high level roadmap and how Greenwood got where it is today, please see our [Open Beta + RFC Google doc](https://docs.google.com/document/d/1MwDkszKvq81QgIYa8utJgyUgSpLZQx9eKCWjIikvfHU/).  We would love to have your help making Greenwood! ✌️
 
@@ -63,7 +63,7 @@ All of our documentation is on our [website](https://www.greenwoodjs.io/) (which
 We would love your [contribution](.github/CONTRIBUTING.md) to Greenwood!  Please check out our issue tracker for "good first issue" labels or feel to reach out to us on [Slack](https://join.slack.com/t/thegreenhouseio/shared_invite/enQtMzcyMzE2Mjk1MjgwLTU5YmM1MDJiMTg0ODk4MjA4NzUwNWFmZmMxNDY5MTcwM2I0MjYxN2VhOTEwNDU2YWQwOWQzZmY1YzY4MWRlOGI) in the room _"Greenwood"_ or on [Twitter](https://twitter.com/PrjEvergreen).
 
 ## Built With Greenwood
-| Site  | Repo  | Project Details  | 
+| Site  | Repo  | Project Details  |
 |---|---|---|
 | [The Greenhouse I/O](https://www.thegreenhouse.io/)  | [thegreenhouseio/www.thegreenhouse.io](https://github.com/thegreenhouseio/www.thegreenhouse.io)  | Personal portfolio / blog website for @thescientist13 (Greenwood maintainer). |
 | [Contributary](https://www.contributary.community/)  | [ContributaryCommunity/www.contributary.community](https://github.com/ContributaryCommunity/www.contributary.community)  | A website (SPA) for browsing open source projects that are looking for contributions. |

greenwood.config.js

@@ -4,6 +4,7 @@ import { greenwoodPluginImportCss } from '@greenwood/plugin-import-css';
 import { greenwoodPluginImportJson } from '@greenwood/plugin-import-json';
 import { greenwoodPluginPolyfills } from '@greenwood/plugin-polyfills';
 import { greenwoodPluginPostCss } from '@greenwood/plugin-postcss';
+import { greenwoodPluginRendererLit } from '@greenwood/plugin-renderer-lit';
 import rollupPluginAnalyzer from 'rollup-plugin-analyzer';
 import { fileURLToPath, URL } from 'url';
 
@@ -12,7 +13,7 @@ const FAVICON_HREF = '/favicon.ico';
 
 export default {
   workspace: fileURLToPath(new URL('./www', import.meta.url)),
-  mode: 'ssr',
+  mode: 'mpa',
   optimization: 'inline',
   title: 'Greenwood',
   meta: [
@@ -46,7 +47,8 @@ export default {
         ];
       }
     },
-    ...greenwoodPluginIncludeHTML()
+    ...greenwoodPluginIncludeHTML(),
+    greenwoodPluginRendererLit()
   ],
   markdown: {
     plugins: [

package.json

@@ -27,6 +27,9 @@
     "lint:css": "stylelint \"./www/**/*.js\", \"./www/**/*.css\"",
     "lint": "ls-lint && yarn lint:js && yarn lint:ts && yarn lint:css"
   },
+  "resolutions": {
+    "lit": "^2.1.1"
+  },
   "devDependencies": {
     "@ls-lint/ls-lint": "^1.10.0",
     "@typescript-eslint/eslint-plugin": "^4.28.2",

packages/cli/src/lib/resource-interface.js

@@ -79,13 +79,13 @@ class ResourceInterface {
   // ex: add a "banner" to all .js files with a timestamp of the build, or minifying files
   // return true | false
   // eslint-disable-next-line no-unused-vars
-  async shouldOptimize(url, body) {
+  async shouldOptimize(url, body, headers) {
     return Promise.resolve(false);
   }
 
   // return the new body
   // eslint-disable-next-line no-unused-vars
-  async optimize (url, body) {
+  async optimize (url, body, headers) {
     return Promise.resolve(body);
   }
 }

packages/cli/src/lib/ssr-route-worker.js

@@ -1,3 +1,4 @@
+// https://github.com/nodejs/modules/issues/307#issuecomment-858729422
 import { pathToFileURL } from 'url';
 import { workerData, parentPort } from 'worker_threads';
 

packages/cli/src/lifecycles/config.js

@@ -8,6 +8,7 @@ const greenwoodPluginsBasePath = fileURLToPath(new URL('../plugins', import.meta
 
 const greenwoodPlugins = (await Promise.all([
   path.join(greenwoodPluginsBasePath, 'copy'),  
+  path.join(greenwoodPluginsBasePath, 'renderer'),
   path.join(greenwoodPluginsBasePath, 'resource'),
   path.join(greenwoodPluginsBasePath, 'server')
 ].map(async (pluginDirectory) => {
@@ -31,7 +32,7 @@ const greenwoodPlugins = (await Promise.all([
 
 const modes = ['ssg', 'mpa', 'spa', 'ssr'];
 const optimizations = ['default', 'none', 'static', 'inline'];
-const pluginTypes = ['copy', 'context', 'resource', 'rollup', 'server', 'source'];
+const pluginTypes = ['copy', 'context', 'resource', 'rollup', 'server', 'source', 'renderer'];
 const defaultConfig = {
   workspace: path.join(process.cwd(), 'src'),
   devServer: {
@@ -129,6 +130,13 @@ const readAndMergeConfig = async() => {
             }
           });
 
+          // if user provides a custom renderer, replace ours with theirs
+          if (plugins.filter(plugin => plugin.type === 'renderer').length === 1) {
+            customConfig.plugins = customConfig.plugins.filter((plugin) => {
+              return plugin.type !== 'renderer';
+            });
+          }
+
           customConfig.plugins = customConfig.plugins.concat(plugins);
         }
 

packages/cli/src/lifecycles/graph.js

@@ -3,7 +3,7 @@ import fs from 'fs';
 import fm from 'front-matter';
 import path from 'path';
 import toc from 'markdown-toc';
-import { pathToFileURL } from 'url';
+import { Worker } from 'worker_threads';
 
 const generateGraph = async (compilation) => {
 
@@ -29,7 +29,7 @@ const generateGraph = async (compilation) => {
           if (fs.statSync(fullPath).isDirectory()) {
             routes = await walkDirectoryForRoutes(fullPath, routes);
           } else {
-            const { getFrontmatter } = await import(pathToFileURL(fullPath));
+            const routeWorkerUrl = compilation.config.plugins.filter(plugin => plugin.type === 'renderer')[0].provider().workerUrl;
             const relativePagePath = fullPath.substring(routesDir.length - 1, fullPath.length);
             const id = filename.split(path.sep)[filename.split(path.sep).length - 1].replace('.js', '');
             const label = id.split('-')
@@ -44,21 +44,35 @@ const generateGraph = async (compilation) => {
             let title = `${compilation.config.title} - ${label}`;
             let customData = {};
             let imports = [];
+            let ssrFrontmatter;
+
+            await new Promise((resolve, reject) => {
+              const worker = new Worker(routeWorkerUrl, {
+                workerData: {
+                  modulePath: fullPath,
+                  compilation: JSON.stringify(compilation),
+                  route
+                }
+              });
+              worker.on('message', (result) => {
+                if (result.frontmatter) {
+                  ssrFrontmatter = result.frontmatter;
+                }
+                resolve();
+              });
+              worker.on('error', reject);
+              worker.on('exit', (code) => {
+                if (code !== 0) {
+                  reject(new Error(`Worker stopped with exit code ${code}`));
+                }
+              });
+            });
 
-            if (getFrontmatter) {
-              const ssrFmData = await getFrontmatter(compilation, route, label, id);
-
-              template = ssrFmData.template ? ssrFmData.template : template;
-              title = ssrFmData.title ? ssrFmData.title : title;
-              imports = ssrFmData.imports ? ssrFmData.imports : imports;
-              customData = ssrFmData.data ? ssrFmData.data : customData;
-
-              // prune "reserved" attributes that are supported by Greenwood
-              // https://www.greenwoodjs.io/docs/front-matter
-              delete ssrFmData.label;
-              delete ssrFmData.imports;
-              delete ssrFmData.title;
-              delete ssrFmData.template;
+            if (ssrFrontmatter) {
+              template = ssrFrontmatter.template || template;
+              title = ssrFrontmatter.title || title;
+              imports = ssrFrontmatter.imports || imports;
+              customData = ssrFrontmatter.data || customData;
 
               /* Menu Query
                * Custom front matter - Variable Definitions
@@ -68,8 +82,8 @@ const generateGraph = async (compilation) => {
                * linkheadings: flag to tell us where to add page's table of contents as menu items
                * tableOfContents: json object containing page's table of contents(list of headings)
                */
-              customData.menu = ssrFmData.menu || '';
-              customData.index = ssrFmData.index || '';
+              customData.menu = ssrFrontmatter.menu || '';
+              customData.index = ssrFrontmatter.index || '';
             }
 
             /*

packages/cli/src/lifecycles/prerender.js

@@ -1,6 +1,9 @@
 import { BrowserRunner } from '../lib/browser.js';
 import fs from 'fs';
+import htmlparser from 'node-html-parser';
 import path from 'path';
+import { Worker } from 'worker_threads';
+import { pathToFileURL } from 'url';
 
 async function interceptPage(compilation, contents, route) {
   const headers = {
@@ -22,7 +25,7 @@ async function interceptPage(compilation, contents, route) {
     return shouldIntercept
       ? resource.intercept(route, html, headers)
       : htmlPromise;
-  }, Promise.resolve(contents));
+  }, Promise.resolve({ body: contents }));
 
   return htmlIntercepted;
 }
@@ -50,8 +53,6 @@ async function optimizePage(compilation, contents, route, outputPath, outputDir)
       recursive: true
     });
   }
-
-  await fs.promises.writeFile(path.join(outputDir, outputPath), htmlOptimized);
 
   return htmlOptimized;
 }
@@ -70,7 +71,8 @@ async function preRenderCompilation(compilation) {
           .then(async (indexHtml) => {
             console.info(`prerendering complete for page ${route}.`);
 
-            await optimizePage(compilation, indexHtml, route, outputPath, outputDir);
+            const html = await optimizePage(compilation, indexHtml, route, outputPath, outputDir);
+            await fs.promises.writeFile(path.join(outputDir, outputPath), html);
           });
       }));
     } catch (e) {
@@ -106,14 +108,81 @@ async function preRenderCompilation(compilation) {
       const port = compilation.config.devServer.port;
       const outputDir = compilation.context.scratchDir;
       const serverAddress = `http://127.0.0.1:${port}`;
+      const customPrerender = (compilation.config.plugins.filter(plugin => plugin.type === 'renderer' && !plugin.isGreenwoodDefaultPlugin) || []).length === 1
+        ? compilation.config.plugins.filter(plugin => plugin.type === 'renderer')[0].provider(compilation)
+        : {};
 
-      console.info(`Prerendering pages at ${serverAddress}`);
       console.debug('pages to render', `\n ${pages.map(page => page.route).join('\n ')}`);
 
-      await runBrowser(serverAddress, pages, outputDir);
+      if (customPrerender.prerender) {
+        for (const page of pages) {
+          const { outputPath, route } = page;
+          const outputPathDir = path.join(outputDir, route);
+          const htmlResource = compilation.config.plugins.filter((plugin) => {
+            return plugin.name === 'plugin-standard-html';
+          }).map((plugin) => {
+            return plugin.provider(compilation);
+          })[0];
+          let html;
+
+          html = (await htmlResource.serve(page.route)).body;
+          html = (await interceptPage(compilation, html, route)).body;
+
+          const root = htmlparser.parse(html, {
+            script: true,
+            style: true
+          });
+
+          const headScripts = root.querySelectorAll('script')
+            .filter(script => {
+              return script.getAttribute('type') === 'module'
+                && script.getAttribute('src') && script.getAttribute('src').indexOf('http') < 0;
+            }).map(script => {
+              return pathToFileURL(path.join(compilation.context.userWorkspace, script.getAttribute('src').replace(/\.\.\//g, '').replace('./', '')));
+            });
+
+          await new Promise((resolve, reject) => {
+            const worker = new Worker(customPrerender.workerUrl, {
+              workerData: {
+                modulePath: null,
+                compilation: JSON.stringify(compilation),
+                route,
+                prerender: true,
+                htmlContents: html,
+                scripts: JSON.stringify(headScripts)
+              }
+            });
+            worker.on('message', (result) => {
+              if (result.html) {
+                html = result.html;
+              }
+              resolve();
+            });
+            worker.on('error', reject);
+            worker.on('exit', (code) => {
+              if (code !== 0) {
+                reject(new Error(`Worker stopped with exit code ${code}`));
+              }
+            });
+          });
+
+          html = await optimizePage(compilation, html, route, outputPath, outputDir);
+
+          if (!fs.existsSync(outputPathDir)) {
+            fs.mkdirSync(outputPathDir, {
+              recursive: true
+            });
+          }
+
+          await fs.promises.writeFile(path.join(outputDir, outputPath), html);
+        }
+      } else {
+        console.info(`Prerendering pages at ${serverAddress}`);
+        await runBrowser(serverAddress, pages, outputDir);
+        browserRunner.close();
+      }
 
       console.info('done prerendering all pages');
-      browserRunner.close();
 
       resolve();
     } catch (err) {
@@ -135,11 +204,12 @@ async function staticRenderCompilation(compilation) {
 
   await Promise.all(pages.map(async (page) => {
     const { route, outputPath } = page;
-    let response = await htmlResource.serve(route);
+    let html = (await htmlResource.serve(route)).body;
 
-    response = await interceptPage(compilation, response, route);
+    html = (await interceptPage(compilation, html, route)).body;
+    html = await optimizePage(compilation, html, route, outputPath, scratchDir);
 
-    await optimizePage(compilation, response.body, route, outputPath, scratchDir);
+    await fs.promises.writeFile(path.join(scratchDir, outputPath), html);
 
     return Promise.resolve();
   }));