Skip to content

Commit

Permalink
Fix loader cyclical dependency bug (#36)
Browse files Browse the repository at this point in the history 
fixes loader cyclical dependency bug by forgoing module wrapping of file
and embedding file contents within the patch instead.

To ensure the patch still works when embedding file content an AST is
generated to make modifications to the file contents. The general flow
is file content -> AST -> update const named exports to let & add a name
to any non named default export -> regenerate file contents using
updated AST -> embed the file contents in the patch.

Recast (https://github.com/benjamn/recast) is used as the AST
transformer & code generator of choice since it seems to be great at
maintaining original file content structure & thus subsequently
maintaining original file stack trace. Although the use of source maps
might be better.

This PR aims to solve this issue:
DataDog/dd-trace-js#3595
  • Loading branch information
@khanayan123
khanayan123 authored Dec 12, 2023
1 parent 16115e9 commit 4df8a29
Show file tree
Hide file tree
Showing 34 changed files with 649 additions and 64 deletions.
Binary file added .DS_Store
View file
Binary file not shown.
77 changes: 71 additions & 6 deletions hook.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,25 +2,41 @@
//
// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2021 Datadog, Inc.

const fs = require('fs')
const { fileURLToPath } = require('url')
const specifiers = new Map()
const isWin = process.platform === "win32"
const warn = require('./lib/helpers')

// FIXME: Typescript extensions are added temporarily until we find a better
// way of supporting arbitrary extensions
const EXTENSION_RE = /\.(js|mjs|cjs|ts|mts|cts)$/
const EXTENSION_MJS_RE = /\.mjs$/
const EXTENSION_JS_RE = /\.js$/
const NODE_VERSION = process.versions.node.split('.')
const NODE_MAJOR = Number(NODE_VERSION[0])
const NODE_MINOR = Number(NODE_VERSION[1])
const FILE_NAME = 'hook.js'

let entrypoint

let getExports
let getEsmExports
let getPkgJsonTypeModule

if (NODE_MAJOR >= 20 || (NODE_MAJOR == 18 && NODE_MINOR >= 19)) {
getExports = require('./lib/get-exports.js')
} else {
getExports = (url) => import(url).then(Object.keys)
}

if (NODE_MAJOR >= 16) {
getEsmExports = require('./lib/get-esm-exports.js')
getPkgJsonTypeModule = require('./lib/get-pkg-json-type-module.js')
} else {
getEsmExports = undefined
getPkgJsonTypeModule = undefined
}

function hasIitm (url) {
try {
return new URL(url).searchParams.has('iitm')
Expand Down Expand Up @@ -108,12 +124,23 @@ function createHook (meta) {
) {
return url
}


// on Node's 16.0.0-16.12.0, url.format is undefined for the cyclical dependency test files ./test/fixtures/a.mjs & ./test/fixtures/b.mjs
// so explicitly set format to 'module' for files with a .mjs extension or cjs files that have type 'module in their package.json
// so that they can go through the ast parsing patch for Node >= 16
if (NODE_MAJOR === 16 && NODE_MINOR < 13) {
if (
(url.format === undefined && EXTENSION_MJS_RE.test(url.url)) ||
(EXTENSION_JS_RE.test(url.url) && getPkgJsonTypeModule(fileURLToPath(url.url)))
) {
url.format = 'module'
}
}

specifiers.set(url.url, specifier)

return {
url: addIitm(url.url),
url: url.format !== 'module' || NODE_MAJOR < 16 ? addIitm(url.url) : url.url,
shortCircuit: true,
format: url.format
}
Expand All @@ -122,9 +149,12 @@ function createHook (meta) {
const iitmURL = new URL('lib/register.js', meta.url).toString()
async function getSource (url, context, parentGetSource) {
if (hasIitm(url)) {

const realUrl = deleteIitm(url)

const exportNames = await getExports(realUrl, context, parentGetSource)
return {

return {
source: `
import { register } from '${iitmURL}'
import * as namespace from ${JSON.stringify(url)}
Expand All @@ -139,6 +169,41 @@ set.${n} = (v) => {
`).join('\n')}
register(${JSON.stringify(realUrl)}, namespace, set, ${JSON.stringify(specifiers.get(realUrl))})
`
}
} else if (NODE_MAJOR >= 16 && context.format === 'module') {
let fileContents
const realPath = fileURLToPath(url)
try {
fileContents = fs.readFileSync(realPath, 'utf8')
} catch (parseError) {
warn(`Had trouble reading file: ${fileContents}, got error: ${parseError}`, FILE_NAME)
return parentGetSource(url, context, parentGetSource)
}
try {
const outPut = getEsmExports(fileContents, true, url)
fileContents = outPut.code
exportAlias = outPut.exportAlias
} catch (parseError) {
warn(`Tried AST parsing ${realPath}, got error: ${parseError}`, FILE_NAME)
return parentGetSource(url, context, parentGetSource)
}
const src = `${fileContents}
import { register as DATADOG_REGISTER_FUNC } from '${iitmURL}'
{
const set = {}
const namespace = {}
${Object.entries(exportAlias).map(([key, value]) => `
set.${key} = (v) => {
${value} = v
return true
}
namespace.${key} = ${value}
`).join('\n')}
DATADOG_REGISTER_FUNC(${JSON.stringify(url)}, namespace, set, ${JSON.stringify(specifiers.get(url))})
}
`
return {
source: src
}
}

Expand All @@ -147,7 +212,7 @@ register(${JSON.stringify(realUrl)}, namespace, set, ${JSON.stringify(specifiers

// For Node.js 16.12.0 and higher.
async function load (url, context, parentLoad) {
if (hasIitm(url)) {
if (hasIitm(url) || context.format === 'module') {
const { source } = await getSource(url, context, parentLoad)
return {
source,
Expand All @@ -167,7 +232,7 @@ register(${JSON.stringify(realUrl)}, namespace, set, ${JSON.stringify(specifiers
resolve,
getSource,
getFormat (url, context, parentGetFormat) {
if (hasIitm(url)) {
if (hasIitm(url) || context.format === 'module') {
return {
format: 'module'
}
Expand Down
203 changes: 151 additions & 52 deletions lib/get-esm-exports.js
View file
Original file line number Diff line number Diff line change
@@ -1,95 +1,194 @@
'use strict'

const recast = require('recast')
const { Parser } = require('acorn')
const { importAssertions } = require('acorn-import-assertions');
const { importAssertions } = require('acorn-import-assertions')
const warn = require('./helpers')

const TS_EXTENSION_RE = /\.(ts|mts|cts)$/

const acornOpts = {
ecmaVersion: 'latest',
sourceType: 'module'
}

const parser = Parser.extend(importAssertions)
const FILE_NAME = 'get-esm-exports'

function getEsmExports(moduleStr, generate=false, url=undefined) {
const exportSpecifierNames = new Set()
const exportAlias = {}
let ast

// if it's a typescript file, we need to parse it with recasts typescript parser
if (url && TS_EXTENSION_RE.test(url)) {
ast = recast.parse(moduleStr, {parser: require("recast/parsers/typescript")})
} else {
ast = recast.parse(moduleStr, {parser: {
parse(source) {
return parser.parse(source, acornOpts)
}
}})
}

const iitmRenamedExport = 'iitmRenamedExport';

// Loop through the top-level declarations of the AST
for (const statement of ast.program.body) {
if (statement.type === 'ExportNamedDeclaration') {
const node = statement;

if (node.declaration) {
parseDeclaration(node.declaration, exportAlias);
} else {
parseSpecifiers(node.specifiers, exportAlias, exportSpecifierNames);
}
} else if (statement.type === 'ExportDefaultDeclaration') {
const node = statement;

function warn (txt) {
process.emitWarning(txt, 'get-esm-exports')
if (['ObjectExpression', 'ArrayExpression', 'Literal'].includes(node.declaration.type) && generate) {
const variableDeclaration = {
type: 'VariableDeclaration',
kind: 'let',
declarations: [
{
type: 'VariableDeclarator',
id: {
type: 'Identifier',
name: iitmRenamedExport,
},
init: node.declaration,
},
],
};

// Replace the current ExportDefaultDeclaration with the new VariableDeclaration
ast.program.body.splice(ast.program.body.indexOf(statement), 1, variableDeclaration);

const newExportDefaultDeclaration = {
type: 'ExportDefaultDeclaration',
declaration: {
type: 'Identifier',
name: iitmRenamedExport,
},
};

// Insert the new ExportDefaultDeclaration after the VariableDeclaration
ast.program.body.splice(ast.program.body.indexOf(variableDeclaration) + 1, 0, newExportDefaultDeclaration);
} else if (['FunctionDeclaration', 'Identifier', 'ClassDeclaration'].includes(node.declaration.type) && generate) {
node.declaration.id = { type: 'Identifier', name: iitmRenamedExport };
}
exportAlias['default'] = iitmRenamedExport;
} else if (statement.type === 'ExportAllDeclaration') {
const node = statement;
const exportedName = node.exported ? node.exported.name : '*';

exportAlias[exportedName] = exportedName;
} else if (statement.type === 'ExportSpecifier') {
const node = statement;
exportSpecifierNames.add(node.local.name);

if (node.exported.name) {
exportAlias[node.exported.name] = node.local.name;
} else if (node.exported.value) {
exportAlias[node.exported.value] = node.local.name;
} else {
warn('unrecognized specifier export: ' + node.exported, FILE_NAME);
}
}
}

if (exportSpecifierNames.size !== 0 && generate) {
convertExportSpecifierToLet(exportSpecifierNames, ast);
}

if (generate) {
return {
exportAlias: exportAlias,
code: recast.print(ast).code,
};
}

return Object.keys(exportAlias);
}

function getEsmExports (moduleStr) {
const exportedNames = new Set()
const tree = parser.parse(moduleStr, acornOpts)
for (const node of tree.body) {
if (!node.type.startsWith('Export')) continue
switch (node.type) {
case 'ExportNamedDeclaration':
if (node.declaration) {
parseDeclaration(node, exportedNames)
} else {
parseSpecifiers(node, exportedNames)
function convertExportSpecifierToLet(exportSpecifierNames, ast) {
for (const statement of ast.program.body) {
if (statement.type === 'VariableDeclaration') {
const declaration = statement;

if (declaration.kind === 'const') {
for (const declarator of declaration.declarations) {
const variableName = declarator.id.name;
if (exportSpecifierNames.has(variableName)) {
declaration.kind = 'let';
}
}
break
case 'ExportDefaultDeclaration':
exportedNames.add('default')
break
case 'ExportAllDeclaration':
if (node.exported) {
exportedNames.add(node.exported.name)
} else {
exportedNames.add('*')
}
break
default:
warn('unrecognized export type: ' + node.type)
}
}
}
return Array.from(exportedNames)
}

function parseDeclaration (node, exportedNames) {
switch (node.declaration.type) {
function parseDeclaration(declaration, exportAlias) {
switch (declaration.type) {
case 'FunctionDeclaration':
exportedNames.add(node.declaration.id.name)
case 'ClassDeclaration':
exportAlias[declaration.id.name] = declaration.id.name
break
case 'VariableDeclaration':
for (const varDecl of node.declaration.declarations) {
parseVariableDeclaration(varDecl, exportedNames)
for (const varDecl of declaration.declarations) {
if (declaration.kind === 'const') {
declaration.kind = 'let'
}
parseVariableDeclaration(varDecl, exportAlias)
}
break
case 'ClassDeclaration':
exportedNames.add(node.declaration.id.name)
break
default:
warn('unknown declaration type: ' + node.delcaration.type)
warn('unknown declaration type: ' + declaration.type, FILE_NAME)
}
}

function parseVariableDeclaration (node, exportedNames) {
switch (node.id.type) {
function parseVariableDeclaration(varDecl, exportAlias) {
switch (varDecl.id.type) {
case 'Identifier':
exportedNames.add(node.id.name)
exportAlias[varDecl.id.name] = varDecl.id.name
break
case 'ObjectPattern':
for (const prop of node.id.properties) {
exportedNames.add(prop.value.name)
for (const prop of varDecl.id.properties) {
exportAlias[prop.value.name] = prop.value.name
}
break
case 'ArrayPattern':
for (const elem of node.id.elements) {
exportedNames.add(elem.name)
for (const elem of varDecl.id.elements) {
if (elem) {
exportAlias[elem.name] = elem.name
}
}
break
default:
warn('unknown variable declaration type: ' + node.id.type)
warn('unknown variable declaration type: ' + varDecl.id.type, FILE_NAME)
}
}

function parseSpecifiers (node, exportedNames) {
for (const specifier of node.specifiers) {
if (specifier.exported.type === 'Identifier') {
exportedNames.add(specifier.exported.name)
function parseSpecifiers(specifiers, exportAlias, exportSpecifierNames) {
for (const specifier of specifiers) {
if (specifier.type === 'ExportSpecifier') {
exportSpecifierNames.add(specifier.local.name)
if (specifier.exported && specifier.exported.name) {
exportAlias[specifier.exported.name] = specifier.local.name
} else if (specifier.exported && specifier.exported.value) {
exportAlias[specifier.exported.value] = specifier.local.name
} else {
warn('unrecognized specifier export: ' + specifier, FILE_NAME)
}
}
else if (specifier.exported.type === 'Identifier') {
exportAlias[specifier.exported.name] = specifier.exported.name
} else if (specifier.exported.type === 'Literal') {
exportedNames.add(specifier.exported.value)
} else {
warn('unrecognized specifier type: ' + specifier.exported.type)
exportAlias[specifier.exported.value] = specifier.exported.value
}
else {
warn('unrecognized specifier type: ' + specifier.exported.type, FILE_NAME)
}
}
}
Expand Down
Loading

0 comments on commit 4df8a29

Please sign in to comment.