feat: support ES modules (puppeteer#8306)

.versionrc.js

@@ -20,7 +20,8 @@ module.exports = {
     tag: true,
   },
   scripts: {
-    prerelease: 'node utils/remove_version_suffix.js',
+    prerelease:
+      'node utils/remove_version_suffix.js && node utils/generate_version_file.js',
     postbump: 'IS_RELEASE=true npm run doc && git add --update',
   },
 };

compat/README.md

@@ -0,0 +1,16 @@
+# Compatibility layer
+
+This directory provides an additional compatibility layer between ES modules and CommonJS.
+
+## Why?
+
+Both `./cjs/compat.ts` and `./esm/compat.ts` are written as ES modules, but `./cjs/compat.ts` can additionally use NodeJS CommonJS globals such as `__dirname` and `require` while these are disabled in ES module mode. For more information, see [Differences between ES modules and CommonJS](https://nodejs.org/api/esm.html#differences-between-es-modules-and-commonjs).
+
+## Adding exports
+
+In order to add exports, two things need to be done:
+
+- The exports must be declared in `src/compat.ts`.
+- The exports must be realized in `./cjs/compat.ts` and `./esm/compat.ts`.
+
+In the event `compat.ts` becomes too large, you can place declarations in another file. Just make sure `./cjs`, `./esm`, and `src` have the same structure.

compat/cjs/compat.ts

@@ -0,0 +1,16 @@
+import { dirname } from 'path';
+
+let puppeteerDirname: string;
+
+try {
+  // In some environments, like esbuild, this will throw an error.
+  // We suppress the error since the bundled binary is not expected
+  // to be used or installed in this case and, therefore, the
+  // root directory does not have to be known.
+  puppeteerDirname = dirname(require.resolve('./compat'));
+} catch (error) {
+  // Fallback to __dirname.
+  puppeteerDirname = __dirname;
+}
+
+export { puppeteerDirname };

compat/cjs/tsconfig.json

@@ -0,0 +1,9 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "composite": true,
+    "outDir": "../../lib/cjs/puppeteer",
+    "module": "CommonJS"
+  },
+  "references": [{ "path": "../../vendor/tsconfig.cjs.json" }]
+}

compat/esm/compat.ts

@@ -0,0 +1,19 @@
+import { createRequire } from 'module';
+import { dirname } from 'path';
+import { fileURLToPath } from 'url';
+
+const require = createRequire(import.meta.url);
+
+let puppeteerDirname: string;
+
+try {
+  // In some environments, like esbuild, this will throw an error.
+  // We suppress the error since the bundled binary is not expected
+  // to be used or installed in this case and, therefore, the
+  // root directory does not have to be known.
+  puppeteerDirname = dirname(require.resolve('./compat'));
+} catch (error) {
+  puppeteerDirname = dirname(fileURLToPath(import.meta.url));
+}
+
+export { puppeteerDirname };

compat/esm/tsconfig.json

@@ -0,0 +1,9 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "composite": true,
+    "outDir": "../../lib/esm/puppeteer",
+    "module": "esnext"
+  },
+  "references": [{ "path": "../../vendor/tsconfig.esm.json" }]
+}

package.json

@@ -8,7 +8,18 @@
     "headless",
     "automation"
   ],
+  "type": "commonjs",
   "main": "./cjs-entry.js",
+  "exports": {
+    ".": {
+      "import": "./lib/esm/puppeteer/node.js",
+      "require": "./cjs-entry.js"
+    },
+    "./*": {
+      "import": "./*",
+      "require": "./*"
+    }
+  },
   "types": "lib/types.d.ts",
   "repository": "github:puppeteer/puppeteer",
   "engines": {
@@ -23,10 +34,10 @@
     "unit-with-coverage": "cross-env COVERAGE=1 npm run unit",
     "assert-unit-coverage": "cross-env COVERAGE=1 mocha --config mocha-config/coverage-tests.js",
     "funit": "cross-env PUPPETEER_PRODUCT=firefox npm run unit",
-    "test": "npm run tsc && npm run lint --silent && npm run unit-with-coverage && npm run test-browser",
+    "test": "npm run build && npm run lint --silent && npm run unit-with-coverage && npm run test-browser",
     "prepare": "node typescript-if-required.js && ([[ $HUSKY = 0 ]] || husky install)",
     "prepublishOnly": "npm run build",
-    "dev-install": "npm run tsc && node install.js",
+    "dev-install": "npm run build && node install.js",
     "install": "node install.js",
     "eslint": "([ \"$CI\" = true ] && eslint --ext js --ext ts --quiet -f codeframe . || eslint --ext js --ext ts .)",
     "eslint-fix": "eslint --ext js --ext ts --fix .",
@@ -37,15 +48,18 @@
     "doc": "node utils/doclint/cli.js",
     "generate-api-docs-for-testing": "commonmark docs/api.md > docs/api.html",
     "clean-lib": "rimraf lib",
-    "build": "npm run tsc && npm run generate-d-ts",
-    "tsc": "npm run clean-lib && tsc --version && npm run tsc-cjs && npm run tsc-esm",
+    "build": "npm run tsc && npm run generate-d-ts && npm run generate-esm-package-json",
+    "tsc": "npm run clean-lib && tsc --version && (npm run tsc-cjs & npm run tsc-esm) && (npm run tsc-compat-cjs & npm run tsc-compat-esm)",
     "tsc-cjs": "tsc -b src/tsconfig.cjs.json",
     "tsc-esm": "tsc -b src/tsconfig.esm.json",
+    "tsc-compat-cjs": "tsc -b compat/cjs/tsconfig.json",
+    "tsc-compat-esm": "tsc -b compat/esm/tsconfig.json",
     "apply-next-version": "node utils/apply_next_version.js",
     "test-install": "scripts/test-install.sh",
     "clean-docs": "rimraf website/docs && rimraf docs-api-json",
     "generate-d-ts": "npm run clean-docs && api-extractor run --local --verbose",
     "generate-docs": "npm run generate-d-ts && api-documenter markdown -i docs-api-json -o website/docs && node utils/remove-tag.js",
+    "generate-esm-package-json": "echo '{\"type\": \"module\"}' > lib/esm/package.json",
     "ensure-correct-devtools-protocol-revision": "ts-node -s scripts/ensure-correct-devtools-protocol-package",
     "ensure-pinned-deps": "ts-node -s scripts/ensure-pinned-deps",
     "test-types-file": "ts-node -s scripts/test-ts-definition-files.ts",
@@ -58,6 +72,7 @@
     "lib/**/*.d.ts.map",
     "lib/**/*.js",
     "lib/**/*.js.map",
+    "lib/**/package.json",
     "install.js",
     "typescript-if-required.js",
     "cjs-entry.js",

scripts/test-install.sh

@@ -1,41 +1,111 @@
 #!/usr/bin/env sh
 set -e
 
+# All tests are headed by a echo 'Test'.
+# The general schema is:
+# 1. Check we can install from the tarball.
+# 2. The install script works and correctly exits without errors
+# 3. Requiring/importing Puppeteer from Node works.
+
+## Puppeter tests
+
+echo "Setting up Puppeteer tests..."
 ROOTDIR="$(pwd)"
-# Pack the module into a tarball
 npm pack
 tarball="$(realpath puppeteer-*.tgz)"
+
+echo "Testing... Chrome CommonJS"
 TMPDIR="$(mktemp -d)"
 cd $TMPDIR
-# Check we can install from the tarball.
-# This emulates installing from npm and ensures that:
-# 1. we publish the right files in the `files` list from package.json
-# 2. The install script works and correctly exits without errors
-# 3. Requiring Puppeteer from Node works.
 npm install --loglevel silent "${tarball}"
 node --eval="require('puppeteer')"
+node --eval="require('puppeteer/lib/cjs/puppeteer/revisions.js');"
 ls $TMPDIR/node_modules/puppeteer/.local-chromium/
 
-# Again for Firefox
+echo "Testing... Chrome ES Modules"
+TMPDIR="$(mktemp -d)"
+cd $TMPDIR
+echo '{"type":"module"}' >>$TMPDIR/package.json
+npm install --loglevel silent "${tarball}"
+node --input-type="module" --eval="import puppeteer from 'puppeteer'"
+node --input-type="module" --eval="import 'puppeteer/lib/esm/puppeteer/revisions.js';"
+node --input-type="module" --eval="
+import puppeteer from 'puppeteer';
+(async () => {
+  const browser = await puppeteer.launch();
+  const page = await browser.newPage();
+  await page.goto('http://example.com');
+  await page.screenshot({ path: 'example.png' });
+  await browser.close();
+})();
+"
+
+echo "Testing... Chrome Webpack ES Modules"
+TMPDIR="$(mktemp -d)"
+cd $TMPDIR
+echo '{"type": "module"}' >>$TMPDIR/package.json
+npm install --loglevel silent "${tarball}"
+npm install -D --loglevel silent webpack [email protected]
+echo 'export default {
+  mode: "production",
+  entry: "./index.js",
+  target: "node",
+  output: {
+    filename: "bundle.cjs",
+  },
+};' >>$TMPDIR/webpack.config.js
+echo "
+import puppeteer from 'puppeteer';
+(async () => {
+  const browser = await puppeteer.launch();
+  const page = await browser.newPage();
+  await page.goto('http://example.com');
+  await page.screenshot({ path: 'example.png' });
+  await browser.close();
+})();
+" >>$TMPDIR/index.js
+npx webpack
+cp -r node_modules/puppeteer/.local-chromium .
+rm -rf node_modules
+node dist/bundle.cjs
+
+echo "Testing... Firefox CommonJS"
 TMPDIR="$(mktemp -d)"
 cd $TMPDIR
 PUPPETEER_PRODUCT=firefox npm install --loglevel silent "${tarball}"
 node --eval="require('puppeteer')"
-rm "${tarball}"
+node --eval="require('puppeteer/lib/cjs/puppeteer/revisions.js');"
 ls $TMPDIR/node_modules/puppeteer/.local-firefox/linux-*/firefox/firefox
 
-# Again for puppeteer-core
+echo "Testing... Firefox ES Modules"
+TMPDIR="$(mktemp -d)"
+cd $TMPDIR
+echo '{"type":"module"}' >>$TMPDIR/package.json
+PUPPETEER_PRODUCT=firefox npm install --loglevel silent "${tarball}"
+node --input-type="module" --eval="import puppeteer from 'puppeteer'"
+node --input-type="module" --eval="import 'puppeteer/lib/esm/puppeteer/revisions.js';"
+ls $TMPDIR/node_modules/puppeteer/.local-firefox/linux-*/firefox/firefox
+
+## Puppeteer Core tests
+
+echo "Setting up Puppeteer Core tests..."
 cd $ROOTDIR
+rm "${tarball}"
 node ./utils/prepare_puppeteer_core.js
 npm pack
 tarball="$(realpath puppeteer-core-*.tgz)"
+
+echo "Testing... Puppeteer Core CommonJS"
 TMPDIR="$(mktemp -d)"
 cd $TMPDIR
-# Check we can install from the tarball.
-# This emulates installing from npm and ensures that:
-# 1. we publish the right files in the `files` list from package.json
-# 2. The install script works and correctly exits without errors
-# 3. Requiring Puppeteer Core from Node works.
 npm install --loglevel silent "${tarball}"
 node --eval="require('puppeteer-core')"
+node --eval="require('puppeteer-core/lib/cjs/puppeteer/revisions.js');"
 
+echo "Testing... Puppeteer Core ES Modules"
+TMPDIR="$(mktemp -d)"
+cd $TMPDIR
+echo '{"type":"module"}' >>$TMPDIR/package.json
+npm install --loglevel silent "${tarball}"
+node --input-type="module" --eval="import puppeteer from 'puppeteer-core'"
+node --input-type="module" --eval="import 'puppeteer-core/lib/esm/puppeteer/revisions.js';"

src/common/Debug.ts

@@ -54,8 +54,9 @@ import { isNode } from '../environment.js';
  */
 export const debug = (prefix: string): ((...args: unknown[]) => void) => {
   if (isNode) {
-    // eslint-disable-next-line @typescript-eslint/no-var-requires
-    return require('debug')(prefix);
+    return async (...logArgs: unknown[]) => {
+      (await import('debug')).default(prefix)(logArgs);
+    };
   }
 
   return (...logArgs: unknown[]): void => {

src/compat.ts

@@ -0,0 +1,3 @@
+declare const puppeteerDirname: string;
+
+export { puppeteerDirname };

src/constants.ts

@@ -0,0 +1,4 @@
+import { dirname } from 'path';
+import { puppeteerDirname } from './compat.js';
+
+export const rootDirname = dirname(dirname(dirname(puppeteerDirname)));

src/generated/README.md

@@ -0,0 +1,3 @@
+# Generated Artifacts
+
+**Do not edit manually edit any TypeScript files in this folder** All TS files are generated from their respectively named template file (ext. `tmpl`) in the `templates` directory. Edit them there is needed.

src/generated/version.ts

@@ -0,0 +1 @@
+export const packageVersion = '13.7.0';

src/initialize-node.ts

@@ -17,24 +17,11 @@
 import { PuppeteerNode } from './node/Puppeteer.js';
 import { PUPPETEER_REVISIONS } from './revisions.js';
 import { sync } from 'pkg-dir';
-import { dirname } from 'path';
 import { Product } from './common/Product.js';
-
-function resolvePuppeteerRootDirectory(): string | undefined {
-  try {
-    // In some environments, like esbuild, this will throw an error.
-    // We suppress the error since the bundled binary is not expected
-    // to be used or installed in this case and, therefore, the
-    // root directory does not have to be known.
-    return sync(dirname(require.resolve('./initialize-node')));
-  } catch (error) {
-    // Fallback to __dirname.
-    return sync(__dirname);
-  }
-}
+import { rootDirname } from './constants.js';
 
 export const initializePuppeteerNode = (packageName: string): PuppeteerNode => {
-  const puppeteerRootDirectory = resolvePuppeteerRootDirectory();
+  const puppeteerRootDirectory = sync(rootDirname);
   let preferredRevision = PUPPETEER_REVISIONS.chromium;
   const isPuppeteerCore = packageName === 'puppeteer-core';
   // puppeteer-core ignores environment variables

src/node/BrowserFetcher.ts

@@ -35,6 +35,9 @@ import createHttpsProxyAgent, {
 import { getProxyForUrl } from 'proxy-from-env';
 import { assert } from '../common/assert.js';
 
+import tar from 'tar-fs';
+import bzip from 'unbzip2-stream';
+
 const { PUPPETEER_EXPERIMENTAL_CHROMIUM_MAC_ARM } = process.env;
 
 const debugFetcher = debug('puppeteer:fetcher');
@@ -512,10 +515,6 @@ function install(archivePath: string, folderPath: string): Promise<unknown> {
  * @internal
  */
 function extractTar(tarPath: string, folderPath: string): Promise<unknown> {
-  // eslint-disable-next-line @typescript-eslint/no-var-requires
-  const tar = require('tar-fs');
-  // eslint-disable-next-line @typescript-eslint/no-var-requires
-  const bzip = require('unbzip2-stream');
   return new Promise((fulfill, reject) => {
     const tarStream = tar.extract(folderPath);
     tarStream.on('error', reject);

src/node/NodeWebSocketTransport.ts

@@ -13,20 +13,19 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import { ConnectionTransport } from '../common/ConnectionTransport.js';
 import NodeWebSocket from 'ws';
+import { ConnectionTransport } from '../common/ConnectionTransport.js';
+import { packageVersion } from '../generated/version.js';
 
 export class NodeWebSocketTransport implements ConnectionTransport {
   static create(url: string): Promise<NodeWebSocketTransport> {
-    // eslint-disable-next-line @typescript-eslint/no-var-requires
-    const pkg = require('../../../../package.json');
     return new Promise((resolve, reject) => {
       const ws = new NodeWebSocket(url, [], {
         followRedirects: true,
         perMessageDeflate: false,
         maxPayload: 256 * 1024 * 1024, // 256Mb
         headers: {
-          'User-Agent': `Puppeteer ${pkg.version}`,
+          'User-Agent': `Puppeteer ${packageVersion}`,
         },
       });
 

src/templates/version.ts.tmpl

@@ -0,0 +1 @@
+export const packageVersion = 'PACKAGE_VERSION';

utils/apply_next_version.js

@@ -12,7 +12,6 @@ const current_sha = execSync(`git rev-parse HEAD`).toString('utf8');
 if (upstream_sha.trim() !== current_sha.trim()) {
   console.log('REFUSING TO PUBLISH: this is not tip-of-tree!');
   process.exit(1);
-  return;
 }
 
 const package = require('../package.json');

utils/generate_version_file.js

@@ -0,0 +1,9 @@
+const { writeFileSync, readFileSync } = require('fs');
+const { join } = require('path');
+
+writeFileSync(
+  join(__dirname, '../src/generated/version.ts'),
+  readFileSync(join(__dirname, '../src/templates/version.ts.tmpl'), {
+    encoding: 'utf-8',
+  }).replace('PACKAGE_VERSION', require('../package.json').version)
+);