feat(jsii-pacmak): option to disallow parallel generation of languages (

#1890)

The build process for each language can be intensive in terms of limited resources,
such as open file handles. In certain environments, 'EMFILE' can happen as a result
of having ot many files concurrently open, failing the build. Limiting parallelism
can reduce (or effectively negate) the risk this happens.

The `EMFILE` problem was encountered by @richardhboyd.

---

By submitting this pull request, I confirm that my contribution is made under the terms of the [Apache 2.0 license].

[Apache 2.0 license]: https://www.apache.org/licenses/LICENSE-2.0

packages/jsii-pacmak/bin/jsii-pacmak.ts

@@ -95,6 +95,12 @@ import { ALL_BUILDERS, TargetName } from '../lib/targets';
         "Translate code samples on-the-fly if they can't be found in the samples tablet",
       default: true,
     })
+    .option('parallel', {
+      type: 'boolean',
+      desc:
+        'Generate all configured targets in parallel (disabling this might help if you encounter EMFILE errors)',
+      default: true,
+    })
     .version(VERSION_DESC)
     .strict().argv;
 
@@ -165,21 +171,25 @@ import { ALL_BUILDERS, TargetName } from '../lib/targets';
 
     // We run all target sets in parallel for minimal wall clock time
     await Promise.all(
-      targetSets.map(async (targetSet) => {
-        logging.info(
-          `Packaging '${targetSet.targetType}' for ${describePackages(
-            targetSet,
-          )}`,
-        );
-        await timers.recordAsync(targetSet.targetType, () =>
-          buildTargetsForLanguage(
-            targetSet.targetType,
-            targetSet.modules,
-            perLanguageDirectory,
-          ),
-        );
-        logging.info(`${targetSet.targetType} finished`);
-      }),
+      mapParallelOrSerial(
+        targetSets,
+        async (targetSet) => {
+          logging.info(
+            `Packaging '${targetSet.targetType}' for ${describePackages(
+              targetSet,
+            )}`,
+          );
+          await timers.recordAsync(targetSet.targetType, () =>
+            buildTargetsForLanguage(
+              targetSet.targetType,
+              targetSet.modules,
+              perLanguageDirectory,
+            ),
+          );
+          logging.info(`${targetSet.targetType} finished`);
+        },
+        { parallel: argv.parallel },
+      ),
     );
   } finally {
     if (argv.clean) {
@@ -266,3 +276,26 @@ function describePackages(target: TargetSet) {
   }
   return `${target.modules.length} modules`;
 }
+
+function mapParallelOrSerial<T, R>(
+  collection: readonly T[],
+  mapper: (item: T) => Promise<R>,
+  { parallel }: { parallel: boolean },
+): Array<Promise<R>> {
+  const result = new Array<Promise<R>>();
+  for (const item of collection) {
+    result.push(
+      result.length === 0 || parallel
+        ? // Running parallel, or first element
+          mapper(item)
+        : // Wait for the previous promise, then make the next one
+          result[result.length - 1].then(
+            () => mapper(item),
+            (error) => {
+              throw error;
+            },
+          ),
+    );
+  }
+  return result;
+}

packages/jsii-pacmak/test/build-test.sh

@@ -49,4 +49,4 @@ done
 # Multiple targets, build all at once into own directory
 clean_dists
 echo "Testing ALL-AT-ONCE build."
-../bin/jsii-pacmak $packagedirs
+../bin/jsii-pacmak --no-parallel $packagedirs