feat(rosetta): transliterate loose mode (#2892)

When transliterating code samples from a published package tarball, supporting files such as fixtures and literate source files may be missing (typically because they are part of the `.npmignore` file). The new `--loose` option to `jsii transliterate` allows ignoring those problems by: - Ignoring missing fixtures (the behavior is then the same as if no fixture was requested) - Falling back to the `.lit.js` file when the `.lit.ts` file cannot be found, or replacing the source with a placeholder if neither file could be found This addresses the issue outlined in cdklabs/construct-hub#127 --- By submitting this pull request, I confirm that my contribution is made under the terms of the [Apache 2.0 license]. [Apache 2.0 license]: https://www.apache.org/licenses/LICENSE-2.0
aws · Jun 30, 2021 · 43e6dfd · 43e6dfd
1 parent 73cac2c
commit 43e6dfd
Show file tree

Hide file tree

Showing 8 changed files with 394 additions and 21 deletions.
diff --git a/packages/jsii-pacmak/lib/targets/go.ts b/packages/jsii-pacmak/lib/targets/go.ts
@@ -94,10 +94,11 @@ export class Golang extends Target {
     // local build directory for this module. if
     // we do, add a "replace" directive to point to it instead of download from
     // the network.
-    const visit = (pkg: RootPackage) => {
+    const visit = async (pkg: RootPackage) => {
       for (const dep of pkg.packageDependencies) {
         for (const baseDir of dirs) {
-          const moduleDir = tryFindLocalModule(baseDir, dep);
+          // eslint-disable-next-line no-await-in-loop
+          const moduleDir = await tryFindLocalModule(baseDir, dep);
           if (moduleDir) {
             replace[dep.goModuleName] = moduleDir;
 
@@ -107,11 +108,12 @@ export class Golang extends Target {
         }
 
         // recurse to transitive deps ("replace" is only considered at the top level go.mod)
-        visit(dep);
+        // eslint-disable-next-line no-await-in-loop
+        await visit(dep);
       }
     };
 
-    visit(this.goGenerator.rootPackage);
+    await visit(this.goGenerator.rootPackage);
 
     // write `local.go.mod`
 
@@ -198,14 +200,14 @@ class GoGenerator implements IGenerator {
  * @param baseDir the `dist/go` directory
  * @returns `undefined` if not or the module directory otherwise.
  */
-function tryFindLocalModule(baseDir: string, pkg: RootPackage) {
+async function tryFindLocalModule(baseDir: string, pkg: RootPackage) {
   const gomodPath = path.join(baseDir, pkg.packageName, GOMOD_FILENAME);
-  if (!fs.pathExistsSync(gomodPath)) {
+  if (!(await fs.pathExists(gomodPath))) {
     return undefined;
   }
 
   // read `go.mod` and check that it is for the correct module
-  const gomod = fs.readFileSync(gomodPath, 'utf-8').split('\n');
+  const gomod = (await fs.readFile(gomodPath, 'utf-8')).split('\n');
   const isExpectedModule = gomod.find(
     (line) => line.trim() === `module ${pkg.goModuleName}`,
   );

diff --git a/packages/jsii-rosetta/bin/jsii-rosetta.ts b/packages/jsii-rosetta/bin/jsii-rosetta.ts
@@ -189,9 +189,17 @@ function main() {
           })
           .options('strict', {
             alias: 's',
-            type: 'boolean',
+            conflicts: 'loose',
             describe:
               'Fail if an example that needs live transliteration fails to compile (which could cause incorrect transpilation results)',
+            type: 'boolean',
+          })
+          .options('loose', {
+            alias: 'l',
+            conflicts: 'strict',
+            describe:
+              'Ignore missing fixtures and literate markdown files instead of failing',
+            type: 'boolean',
           })
           .option('tablet', {
             alias: 't',

diff --git a/packages/jsii-rosetta/lib/commands/extract.ts b/packages/jsii-rosetta/lib/commands/extract.ts
@@ -28,6 +28,7 @@ export interface ExtractOptions {
 export async function extractSnippets(
   assemblyLocations: string[],
   options: ExtractOptions,
+  loose = false,
 ): Promise<ExtractResult> {
   const only = options.only ?? [];
 
@@ -37,7 +38,7 @@ export async function extractSnippets(
     options.validateAssemblies,
   );
 
-  let snippets = allTypeScriptSnippets(assemblies);
+  let snippets = allTypeScriptSnippets(assemblies, loose);
   if (only.length > 0) {
     snippets = filterSnippets(snippets, only);
   }

diff --git a/packages/jsii-rosetta/lib/commands/transliterate.ts b/packages/jsii-rosetta/lib/commands/transliterate.ts
@@ -11,7 +11,15 @@ import { Translation } from '../tablets/tablets';
 
 export interface TransliterateAssemblyOptions {
   /**
-   * Whather transliteration should fail upon failing to compile an example that
+   * Whether to ignore any missing fixture files or literate markdown documents
+   * referenced by the assembly, instead of failing.
+   *
+   * @default false
+   */
+  readonly loose?: boolean;
+
+  /**
+   * Whether transliteration should fail upon failing to compile an example that
    * required live transliteration.
    *
    * @default false
@@ -46,6 +54,7 @@ export async function transliterateAssembly(
   const rosetta = new Rosetta({
     includeCompilerDiagnostics: true,
     liveConversion: true,
+    loose: options.loose,
     targetLanguages,
   });
   if (options.tablet) {
@@ -71,7 +80,7 @@ export async function transliterateAssembly(
         );
       }
       for (const type of Object.values(result.types ?? {})) {
-        transliterateType(type, rosetta, language, location);
+        transliterateType(type, rosetta, language, location, options.loose);
       }
       // eslint-disable-next-line no-await-in-loop
       await writeJson(
@@ -153,6 +162,7 @@ function transliterateType(
   rosetta: Rosetta,
   language: TargetLanguage,
   workingDirectory: string,
+  loose = false,
 ): void {
   transliterateDocs(type.docs);
   switch (type.kind) {
@@ -193,6 +203,7 @@ function transliterateType(
           true /* strict */,
           { [SnippetParameters.$PROJECT_DIRECTORY]: workingDirectory },
         ),
+        loose,
       );
       const translation = rosetta.translateSnippet(snippet, language);
       if (translation != null) {

diff --git a/packages/jsii-rosetta/lib/fixtures.ts b/packages/jsii-rosetta/lib/fixtures.ts
@@ -12,7 +12,10 @@ import { TypeScriptSnippet, SnippetParameters } from './snippet';
 /**
  * Complete snippets with fixtures, if required
  */
-export function fixturize(snippet: TypeScriptSnippet): TypeScriptSnippet {
+export function fixturize(
+  snippet: TypeScriptSnippet,
+  loose = false,
+): TypeScriptSnippet {
   let source = snippet.visibleSource;
   const parameters = snippet.parameters ?? {};
 
@@ -25,14 +28,14 @@ export function fixturize(snippet: TypeScriptSnippet): TypeScriptSnippet {
   if (literateSource) {
     // Compatibility with the "old school" example inclusion mechanism.
     // Completely load this file and attach a parameter with its directory.
-    source = loadLiterateSource(directory, literateSource);
+    source = loadLiterateSource(directory, literateSource, loose);
     parameters[SnippetParameters.$COMPILATION_DIRECTORY] = path.join(
       directory,
       path.dirname(literateSource),
     );
   } else if (parameters[SnippetParameters.FIXTURE]) {
-    // Explicitly request a fixture
-    source = loadAndSubFixture(directory, parameters.fixture, source, true);
+    // Explicitly requested fixture must exist, unless we are operating in loose mode
+    source = loadAndSubFixture(directory, parameters.fixture, source, !loose);
   } else if (parameters[SnippetParameters.NO_FIXTURE] === undefined) {
     // Don't explicitly request no fixture
     source = loadAndSubFixture(directory, 'default', source, false);
@@ -45,10 +48,22 @@ export function fixturize(snippet: TypeScriptSnippet): TypeScriptSnippet {
   };
 }
 
-function loadLiterateSource(directory: string, literateFileName: string) {
+function loadLiterateSource(
+  directory: string,
+  literateFileName: string,
+  loose = false,
+) {
   const fullPath = path.join(directory, literateFileName);
   const exists = fs.existsSync(fullPath);
   if (!exists) {
+    if (loose) {
+      // In loose mode, we'll fall back to the `.js` file if it exists...
+      const jsFile = fullPath.replace(/\.ts(x?)$/, '.js$1');
+      if (fs.existsSync(jsFile)) {
+        return fs.readFileSync(jsFile, { encoding: 'utf-8' });
+      }
+      return `Missing literate source file ${literateFileName}`;
+    }
     // This couldn't really happen in practice, but do the check anyway
     throw new Error(
       `Sample uses literate source ${literateFileName}, but not found: ${fullPath}`,

diff --git a/packages/jsii-rosetta/lib/jsii/assemblies.ts b/packages/jsii-rosetta/lib/jsii/assemblies.ts
@@ -134,6 +134,7 @@ function removeSlashes(x: string) {
 
 export function* allTypeScriptSnippets(
   assemblies: readonly LoadedAssembly[],
+  loose = false,
 ): IterableIterator<TypeScriptSnippet> {
   for (const { assembly, directory } of assemblies) {
     const strict = enforcesStrictMode(assembly);
@@ -146,7 +147,7 @@ export function* allTypeScriptSnippets(
               [SnippetParameters.$PROJECT_DIRECTORY]: directory,
             },
           );
-          yield fixturize(snippet);
+          yield fixturize(snippet, loose);
           break;
         case 'markdown':
           for (const snippet of extractTypescriptSnippetsFromMarkdown(
@@ -157,7 +158,7 @@ export function* allTypeScriptSnippets(
             const withDirectory = updateParameters(snippet, {
               [SnippetParameters.$PROJECT_DIRECTORY]: directory,
             });
-            yield fixturize(withDirectory);
+            yield fixturize(withDirectory, loose);
           }
       }
     }

diff --git a/packages/jsii-rosetta/lib/rosetta.ts b/packages/jsii-rosetta/lib/rosetta.ts
@@ -41,6 +41,14 @@ export interface RosettaOptions {
    * Whether to include compiler diagnostics in the compilation results.
    */
   readonly includeCompilerDiagnostics?: boolean;
+
+  /**
+   * Whether this Rosetta should operate in "loose" mode, where missing literate
+   * source files and missing fixtures are ignored instead of failing.
+   *
+   * @default false
+   */
+  readonly loose?: boolean;
 }
 
 /**
@@ -60,8 +68,10 @@ export class Rosetta {
   private readonly liveTablet = new LanguageTablet();
   private readonly extractedSnippets = new Map<string, TypeScriptSnippet>();
   private readonly translator: Translator;
+  private readonly loose: boolean;
 
   public constructor(private readonly options: RosettaOptions = {}) {
+    this.loose = !!options.loose;
     this.translator = new Translator(
       options.includeCompilerDiagnostics ?? false,
     );
@@ -114,9 +124,10 @@ export class Rosetta {
     }
 
     if (this.options.liveConversion) {
-      for (const tsnip of allTypeScriptSnippets([
-        { assembly, directory: assemblyDir },
-      ])) {
+      for (const tsnip of allTypeScriptSnippets(
+        [{ assembly, directory: assemblyDir }],
+        this.loose,
+      )) {
         this.extractedSnippets.set(tsnip.visibleSource, tsnip);
       }
     }