feat(compiler): add support for external templates (#219)

.lgtm.yml

@@ -4,3 +4,4 @@ extraction:
       filters:
       - exclude: "src/compiler/JSCodeTemplate.js"
       - exclude: "src/compiler/JSRuntimeTemplate.js"
+      - exclude: "src/compiler/JSPureTemplate.js"

src/compiler/Compiler.js

@@ -13,7 +13,6 @@
 /* eslint-disable no-await-in-loop */
 
 const path = require('path');
-const crypto = require('crypto');
 const fse = require('fs-extra');
 const { SourceMapGenerator } = require('source-map');
 const TemplateParser = require('../parser/html/TemplateParser');
@@ -25,11 +24,14 @@ const VariableBinding = require('../parser/commands/VariableBinding');
 const RuntimeCall = require('../parser/htl/nodes/RuntimeCall');
 const Identifier = require('../parser/htl/nodes/Identifier');
 const FunctionBlock = require('../parser/commands/FunctionBlock');
+const ExternalCode = require('../parser/commands/ExternalCode');
 const TemplateLoader = require('./TemplateLoader.js');
 const ScriptResolver = require('./ScriptResolver.js');
+const ExternalTemplateLoader = require('./ExternalTemplateLoader.js');
 
 const DEFAULT_TEMPLATE = 'JSCodeTemplate.js';
 const RUNTIME_TEMPLATE = 'JSRuntimeTemplate.js';
+const PURE_TEMPLATE = 'JSPureTemplate.js';
 
 module.exports = class Compiler {
   /**
@@ -63,6 +65,7 @@ module.exports = class Compiler {
     this._moduleImportGenerator = Compiler.defaultModuleGenerator;
     this._templateLoader = TemplateLoader();
     this._scriptResolver = ScriptResolver('.');
+    this._scriptId = 'global';
   }
 
   /**
@@ -179,6 +182,37 @@ module.exports = class Compiler {
     return this;
   }
 
+  /**
+   * Sets the script id. mostly used for template registration.
+   * @param {string} id the script id.
+   * @returns {Compiler}
+   */
+  withScriptId(id) {
+    this._scriptId = id;
+    return this;
+  }
+
+  /**
+   * Creates a compiler suitable to compile external templates.
+   * @returns {Compiler} a compatible compiler
+   */
+  async createTemplateCompiler(templatePath, outputDirectory, scriptId) {
+    const codeTemplate = await fse.readFile(path.join(__dirname, PURE_TEMPLATE), 'utf-8');
+    const compiler = new Compiler()
+      .withDirectory(path.dirname(templatePath))
+      .withRuntimeGlobalName(this._runtimeGlobal)
+      .withRuntimeVar(this._runtimeGlobals)
+      .withScriptResolver(this._scriptResolver)
+      .withScriptId(scriptId)
+      .withCodeTemplate(codeTemplate);
+
+    const extLoader = ExternalTemplateLoader({
+      compiler,
+      outputDirectory,
+    });
+    return compiler.withTemplateLoader(extLoader);
+  }
+
   /**
    * Compiles the specified source file and saves the result, overwriting the
    * file name.
@@ -276,13 +310,14 @@ module.exports = class Compiler {
           const templatePath = await this._scriptResolver(baseDir, c.filename);
           let template = templates[templatePath];
           if (!template) {
-            // hash the template path for a somewhat stable id
-            const id = crypto.createHash('md5').update(templatePath, 'utf-8').digest().toString('hex');
+            // use the template path for a somewhat stable id
+            const id = path.relative(process.cwd(), templatePath);
 
             // load template
             const {
               data: templateSource,
-            } = await this._templateLoader(templatePath);
+              code,
+            } = await this._templateLoader(templatePath, id);
 
             // register template
             template = {
@@ -293,25 +328,29 @@ module.exports = class Compiler {
             // eslint-disable-next-line no-param-reassign
             templates[templatePath] = template;
 
-            // parse the template
-            const res = await this._parse(
-              templateSource, path.dirname(templatePath), mods, templates,
-            );
-
-            // extract the template functions and discard commands outside the functions
-            let inside = false;
-            res.commands.forEach((cmd) => {
-              if (cmd instanceof FunctionBlock.Start) {
-                inside = true;
-                // eslint-disable-next-line no-param-reassign
-                cmd.id = template.id;
-              } else if (cmd instanceof FunctionBlock.End) {
-                inside = false;
-              } else if (!inside) {
-                return;
-              }
-              template.commands.push(cmd);
-            });
+            if (code) {
+              template.commands.push(new ExternalCode(code));
+            } else {
+              // parse the template
+              const res = await this._parse(
+                templateSource, path.dirname(templatePath), mods, templates,
+              );
+
+              // extract the template functions and discard commands outside the functions
+              let inside = false;
+              res.commands.forEach((cmd) => {
+                if (cmd instanceof FunctionBlock.Start) {
+                  inside = true;
+                  // eslint-disable-next-line no-param-reassign
+                  cmd.id = template.id;
+                } else if (cmd instanceof FunctionBlock.End) {
+                  inside = false;
+                } else if (!inside) {
+                  return;
+                }
+                template.commands.push(cmd);
+              });
+            }
           }
           // remember the file id on the use statement
           c.id = template.id;
@@ -360,12 +399,14 @@ module.exports = class Compiler {
     });
 
     const {
-      code, templateCode, mappings, templateMappings, globalTemplateNames,
+      code, templateCode, mappings, templateMappings,
     } = new JSCodeGenVisitor()
       .withIndent('  ')
       .withSourceMap(this._sourceMap)
       .withSourceOffset(this._sourceOffset)
       .withSourceFile(this._sourceFile)
+      .withScriptId(this._scriptId)
+      .withGlobals(this._runtimeGlobals)
       .indent()
       .process(parseResult);
 
@@ -381,14 +422,6 @@ module.exports = class Compiler {
       }
       imports += `  ${exp}\n`;
     });
-    // add global template names
-    globalTemplateNames.forEach((ident) => {
-      if (this._runtimeGlobals.indexOf(ident) >= 0) {
-        imports += `  ${ident} = $.template('global').${ident};\n`;
-      } else {
-        imports += `  let ${ident} = $.template('global').${ident};\n`;
-      }
-    });
 
     let template = this._codeTemplate;
     if (!template) {

src/compiler/DomHandler.js

@@ -18,6 +18,7 @@ module.exports = class DomHandler {
     this._gen = generator;
     this._out = generator.out.bind(generator);
     this._globalTemplates = {};
+    this._currentTemplate = null;
   }
 
   beginDocument() {
@@ -78,23 +79,29 @@ module.exports = class DomHandler {
   }
 
   functionStart(cmd) {
-    const exp = ExpressionFormatter.escapeVariable(ExpressionFormatter.format(cmd.expression));
-    const id = cmd.id || 'global';
-    const functionName = `_template_${id}_${exp.replace(/\./g, '_')}`;
-    this._out(`$.template('${id}', '${exp}', function* ${functionName}(args) { `);
+    const name = ExpressionFormatter.escapeVariable(ExpressionFormatter.format(cmd.expression));
+    const id = cmd.id || this._gen.scriptId;
+    const functionName = `_template_${id.replace(/[^\w]/g, '_')}_${name.replace(/\./g, '_')}`;
+    // this._out(`${exp} = $.template('${id}', '${exp}', function* ${functionName}(args) { `);
+    this._out(`${name} = function* ${functionName}(args) { `);
     this._gen.indent();
     cmd.args.forEach((arg) => {
       this._out(`const ${ExpressionFormatter.escapeVariable(arg)} = args[1]['${arg}'] || '';`);
     });
     this._out('let $t, $n = args[0];');
-    if (!cmd.id) {
-      this._globalTemplates[exp] = true;
-    }
+    this._globalTemplates[name] = id;
+    this._currentTemplate = {
+      id,
+      name,
+      functionName,
+    };
   }
 
   functionEnd() {
     this._gen.outdent();
-    this._out('});');
+    this._out('};');
+    const { id, name } = this._currentTemplate;
+    this._out(`$.template('${id}', '${name}', ${name});`);
   }
 
   functionCall(cmd) {

src/compiler/ExternalTemplateLoader.js

@@ -0,0 +1,43 @@
+/*
+ * Copyright 2020 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+const path = require('path');
+const fse = require('fs-extra');
+
+/**
+ * Creates a template loader that compiles the templates and returns their exec function.
+ */
+module.exports = function createLoader(opts) {
+  const {
+    compiler,
+    outputDirectory,
+  } = opts;
+
+  /**
+   * Load the template.
+   * @param {string} templatePath template path
+   * @param {string} scriptId the script id
+   * @returns {Promise<>} the template source and resolved path
+   */
+  async function load(templatePath, scriptId) {
+    const comp = await compiler.createTemplateCompiler(templatePath, outputDirectory, scriptId);
+    const filename = `${path.basename(templatePath)}.js`;
+    const outfile = path.resolve(outputDirectory, filename);
+    const source = await fse.readFile(templatePath, 'utf-8');
+    const file = await comp.compileToFile(source, outfile, compiler.dir);
+    return {
+      path: file,
+      code: `require(${JSON.stringify(file)})(runtime);`,
+    };
+  }
+
+  return load;
+};

src/compiler/JSCodeGenVisitor.js

@@ -14,6 +14,7 @@ const OutText = require('../parser/commands/OutText');
 const VariableBinding = require('../parser/commands/VariableBinding');
 const FileReference = require('../parser/commands/FileReference');
 const FunctionBlock = require('../parser/commands/FunctionBlock');
+const ExternalCode = require('../parser/commands/ExternalCode');
 const FunctionCall = require('../parser/commands/FunctionCall');
 const Conditional = require('../parser/commands/Conditional');
 const Loop = require('../parser/commands/Loop');
@@ -39,9 +40,12 @@ module.exports = class JSCodeGenVisitor {
     this._templateStack = [];
     this._sourceFile = null;
     this._indentLevel = 0;
+    this._templateIndentLevel = 0;
     this._indents = [];
     this._dom = new DomHandler(this);
     this._enableSourceMaps = false;
+    this._scriptId = 'global';
+    this._runtimeGlobals = [];
   }
 
   withIndent(delim) {
@@ -78,6 +82,20 @@ module.exports = class JSCodeGenVisitor {
     return this;
   }
 
+  withScriptId(id) {
+    this._scriptId = id;
+    return this;
+  }
+
+  withGlobals(value) {
+    this._runtimeGlobals = value;
+    return this;
+  }
+
+  get scriptId() {
+    return this._scriptId;
+  }
+
   indent() {
     this._indent = this._indents[++this._indentLevel] || ''; // eslint-disable-line no-plusplus
     return this;
@@ -105,7 +123,8 @@ module.exports = class JSCodeGenVisitor {
     this._blocks.push(block);
     this._templateStack.push(block);
     this._blk = block;
-    this.setIndent(0);
+    this.setIndent(this._templateIndentLevel);
+    return block;
   }
 
   popBlock() {
@@ -121,18 +140,53 @@ module.exports = class JSCodeGenVisitor {
   process({ commands, templates }) {
     this._dom.beginDocument();
 
+    // create a global block for global templates
+    const globalBlock = this.pushBlock('');
+    this.popBlock();
+
     // first process the main commands
     commands.forEach((c) => {
       c.accept(this);
     });
 
+    // add global vars for templates
+    this.outdent();
+    // eslint-disable-next-line no-underscore-dangle
+    Object.keys(this._dom._globalTemplates).forEach((name) => {
+      if (this._runtimeGlobals.indexOf(name) < 0) {
+        globalBlock.code += `${this._indent}let ${name};\n`;
+        globalBlock.line += 1;
+      }
+    });
+    this.indent();
+    // eslint-disable-next-line no-underscore-dangle
+    this._dom._globalTemplates = [];
+
     // then process the templates
     this._sourceOffset = 0;
+    this._templateIndentLevel = 1;
     Object.values(templates).forEach((t) => {
+      // this is kind of a hack to push code to the template stack
+      const groupBlock = this.pushBlock('');
+      this.outdent();
+      this.out(`(function _template_${t.id.replace(/[^\w]/g, '_')}(){ `);
+      this.popBlock();
       this._sourceFile = t.file;
       t.commands.forEach((c) => {
         c.accept(this);
       });
+      this.pushBlock('');
+      this.outdent();
+      this.out('})();');
+      this.popBlock();
+      // add variable initializers for the template names
+      // eslint-disable-next-line no-underscore-dangle
+      Object.keys(this._dom._globalTemplates).forEach((name) => {
+        groupBlock.code += `${this._indent}let ${name};\n`;
+        groupBlock.line += 1;
+      });
+      // eslint-disable-next-line no-underscore-dangle
+      this._dom._globalTemplates = [];
     });
 
     this._dom.endDocument();
@@ -158,8 +212,6 @@ module.exports = class JSCodeGenVisitor {
       mappings: this._main.map,
       templateCode,
       templateMappings,
-      // eslint-disable-next-line no-underscore-dangle
-      globalTemplateNames: Object.keys(this._dom._globalTemplates),
     };
   }
 
@@ -265,6 +317,10 @@ module.exports = class JSCodeGenVisitor {
       this._dom.doctype(cmd);
     } else if (cmd instanceof Comment) {
       this._dom.comment(cmd);
+    } else if (cmd instanceof ExternalCode) {
+      this.pushBlock('');
+      this.out(cmd.code);
+      this.popBlock();
     } else {
       throw new Error(`unknown command: ${cmd}`);
     }