Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(core): improve performance of pnpm lock file parsing #23017

Merged
merged 4 commits into from
Apr 26, 2024
Merged

fix(core): improve performance of pnpm lock file parsing #23017

Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
3d20d33
fix(core): improve performance of pnpm lock file parsing
JamesHenry Apr 25, 2024
242ce3d
chore(core): add failing mixed-key test
Coly010 Apr 26, 2024
fd0e351
fix(core): pnpm parser should handle aliased deps
Coly010 Apr 26, 2024
1c30086
chore(core): optimize loop perf
Coly010 Apr 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
182 changes: 92 additions & 90 deletions packages/nx/src/plugins/js/lock-file/pnpm-parser.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,49 +65,112 @@ export function getPnpmLockfileDependencies(
return getDependencies(data, keyMap, isV6, ctx);
}

function matchPropValue(
record: Record<string, string>,
key: string,
originalPackageName: string
): string | undefined {
if (!record) {
return undefined;
}
const index = Object.values(record).findIndex((version) => version === key);
if (index > -1) {
return Object.keys(record)[index];
}
// check if non-aliased name is found
if (
record[originalPackageName] &&
key.startsWith(`/${originalPackageName}/${record[originalPackageName]}`)
) {
return originalPackageName;
}
}

function matchedDependencyName(
importer: Partial<PackageSnapshot>,
key: string,
originalPackageName: string
): string | undefined {
return (
matchPropValue(importer.dependencies, key, originalPackageName) ||
matchPropValue(importer.optionalDependencies, key, originalPackageName) ||
matchPropValue(importer.peerDependencies, key, originalPackageName)
);
}

function getNodes(
data: Lockfile,
keyMap: Map<string, ProjectGraphExternalNode>,
isV6: boolean
): Record<string, ProjectGraphExternalNode> {
const nodes: Map<string, Map<string, ProjectGraphExternalNode>> = new Map();

Object.entries(data.packages).forEach(([key, snapshot]) => {
findPackageNames(key, snapshot, data).forEach((packageName) => {
for (const [key, snapshot] of Object.entries(data.packages)) {
const originalPackageName = extractNameFromKey(key);
if (!originalPackageName) {
continue;
}
const packageNames = new Set<string>();

// snapshot already has a name
if (snapshot.name) {
packageNames.add(snapshot.name);
}
const rootDependencyName =
matchedDependencyName(data.importers['.'], key, originalPackageName) ||
// only root importers have devDependencies
matchPropValue(
data.importers['.'].devDependencies,
key,
originalPackageName
);
if (rootDependencyName) {
packageNames.add(rootDependencyName);
}

if (packageNames.size === 0) {
packageNames.add(originalPackageName);
}

for (const packageName of packageNames) {
const rawVersion = findVersion(key, packageName);
if (!rawVersion) {
continue;
}
const version = parseBaseVersion(rawVersion, isV6);

// we don't need to keep duplicates, we can just track the keys
const existingNode = nodes.get(packageName)?.get(version);
if (existingNode) {
keyMap.set(key, existingNode);
return;
if (!version) {
continue;
}

const node: ProjectGraphExternalNode = {
type: 'npm',
name: version ? `npm:${packageName}@${version}` : `npm:${packageName}`,
data: {
version,
packageName,
hash:
snapshot.resolution?.['integrity'] ||
hashArray(
snapshot.resolution?.['tarball']
? [snapshot.resolution['tarball']]
: [packageName, version]
),
},
};

keyMap.set(key, node);
if (!nodes.has(packageName)) {
nodes.set(packageName, new Map([[version, node]]));
} else {
nodes.set(packageName, new Map());
}

if (!nodes.get(packageName).has(version)) {
const node: ProjectGraphExternalNode = {
type: 'npm',
name: version
? `npm:${packageName}@${version}`
: `npm:${packageName}`,
data: {
version,
packageName,
hash:
snapshot.resolution?.['integrity'] ||
hashArray(
snapshot.resolution?.['tarball']
? [snapshot.resolution['tarball']]
: [packageName, version]
),
},
};
nodes.get(packageName).set(version, node);
keyMap.set(key, node);
} else {
keyMap.set(key, nodes.get(packageName).get(version));
}
});
});
}
}

const hoistedDeps = loadPnpmHoistedDepsDefinition();
const results: Record<string, ProjectGraphExternalNode> = {};
Expand Down Expand Up @@ -323,67 +386,6 @@ function findVersion(key: string, packageName: string): string {
return key;
}

function findPackageNames(
key: string,
snapshot: PackageSnapshot,
data: Lockfile
): string[] {
const packageNames = new Set<string>();
const originalPackageName = extractNameFromKey(key);

const matchPropValue = (record: Record<string, string>): string => {
if (!record) {
return undefined;
}
const index = Object.values(record).findIndex((version) => version === key);
if (index > -1) {
return Object.keys(record)[index];
}
// check if non aliased name is found
if (
record[originalPackageName] &&
key.startsWith(`/${originalPackageName}/${record[originalPackageName]}`)
) {
return originalPackageName;
}
};

const matchedDependencyName = (
importer: Partial<PackageSnapshot>
): string => {
return (
matchPropValue(importer.dependencies) ||
matchPropValue(importer.optionalDependencies) ||
matchPropValue(importer.peerDependencies)
);
};

// snapshot already has a name
if (snapshot.name) {
packageNames.add(snapshot.name);
}
// it'a a root dependency
const rootDependencyName =
matchedDependencyName(data.importers['.']) ||
// only root importers have devDependencies
matchPropValue(data.importers['.'].devDependencies);
if (rootDependencyName) {
packageNames.add(rootDependencyName);
}
// find a snapshot that has a dependency that points to this snapshot
const snapshots = Object.values(data.packages);
for (let i = 0; i < snapshots.length; i++) {
const dependencyName = matchedDependencyName(snapshots[i]);
if (dependencyName) {
packageNames.add(dependencyName);
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This handled things like string-width-cjs and strip-ansi-cjs.

https://github.com/nrwl/nx/pull/17613/files

It seems like those are missing now.

}
}
if (packageNames.size === 0) {
packageNames.add(originalPackageName);
}
return Array.from(packageNames);
}

function getVersion(key: string, packageName: string): string {
const KEY_NAME_SEPARATOR_LENGTH = 2; // leading and trailing slash

Expand Down