Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Eliminate the need for node_modules rewriting #1374

Closed
wants to merge 9 commits into from
Closed

Eliminate the need for node_modules rewriting #1374

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
cc3185e
introduce standalone stage1 and invoke it from CompatAddons
ef4 Mar 15, 2023
bcd0756
Merge branch 'main' into dont-rewrite-node-modules
ef4 Mar 20, 2023
d7c17fe
flatten the package caches
ef4 Mar 21, 2023
aac1c75
extract RewrittenPackages
ef4 Mar 21, 2023
15940d9
make app.ts go through module-resolver
ef4 Mar 21, 2023
52504ac
This won't be needed
ef4 Mar 21, 2023
ada137d
simplifying staging system
ef4 Mar 21, 2023
665331e
[wip]
ef4 Mar 23, 2023
d8d5a21
Merge branch 'main' into dont-rewrite-node-modules
ef4 May 18, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
238 changes: 22 additions & 216 deletions packages/compat/src/compat-addons.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,15 +1,11 @@
import { Node } from 'broccoli-node-api';
import { join, relative, dirname, isAbsolute, sep } from 'path';
import { emptyDirSync, ensureSymlinkSync, ensureDirSync, realpathSync, copySync, writeJSONSync } from 'fs-extra';
import { Stage, Package, PackageCache, WaitForTrees, mangledEngineRoot } from '@embroider/core';
import V1InstanceCache from './v1-instance-cache';
import { MovedPackageCache } from './moved-package-cache';
import { WaitForTrees, EmberAppInstance } from '@embroider/core';
import { Memoize } from 'typescript-memoize';
import buildCompatAddon from './build-compat-addon';
import Options, { optionsWithDefaults } from './options';
import V1App from './v1-app';
import TreeSync from 'tree-sync';
import { WatchedDir } from 'broccoli-source';
import { convertLegacyAddons } from '.';
import V1InstanceCache from './v1-instance-cache';
import { resolve } from 'path';

// This build stage expects to be run with broccoli memoization enabled in order
// to get good rebuild performance. We turn it on by default here, but you can
Expand All @@ -22,242 +18,52 @@ if (typeof process.env.BROCCOLI_ENABLED_MEMOIZE === 'undefined') {
process.env.BROCCOLI_ENABLED_MEMOIZE = 'true';
}

export default class CompatAddons implements Stage {
export default class CompatAddons {
private didBuild = false;
private destDir: string;
private packageCache: MovedPackageCache;
private treeSyncMap: WeakMap<Package, TreeSync>;
private v1Cache: V1InstanceCache;
private treeSync: TreeSync | undefined;
readonly inputPath: string;

constructor(legacyEmberAppInstance: object, maybeOptions?: Options) {
let options = optionsWithDefaults(maybeOptions);
let v1Cache = V1InstanceCache.forApp(legacyEmberAppInstance, options);
private addons: Node;

// we want this to be stable across builds, because it becomes part of the
// path to all of the files that the stage3 packager sees, and we want to
// benefit from on-disk caching in stage3 packagers.
ensureDirSync(options.workspaceDir!);
this.destDir = realpathSync(options.workspaceDir!);
constructor(legacyEmberAppInstance: EmberAppInstance, maybeOptions?: Options) {
let options = optionsWithDefaults(maybeOptions);

this.packageCache = v1Cache.app.packageCache.moveAddons(this.destDir);
this.inputPath = v1Cache.app.root;
this.treeSyncMap = new WeakMap();
this.v1Cache = v1Cache;
this.addons = convertLegacyAddons(legacyEmberAppInstance, maybeOptions);
this.inputPath = V1InstanceCache.forApp(legacyEmberAppInstance, options).app.root;
}

get tree(): Node {
let movedAddons = [...this.packageCache.moved.keys()].map(oldPkg => buildCompatAddon(oldPkg, this.v1Cache));

// these get watched so that EMBROIDER_REBUILD_ADDONS will still work
// correctly, even for v2 addons that have no v1 addon deps and therefore
// don't need to be moved. We don't consume these trees in our build step,
// we only do this to trigger rebuilds to happen.
let watchedUnmovedAddons = [...this.packageCache.unmovedAddons]
.filter(pkg => pkg.mayRebuild)
.map(pkg => new WatchedDir(pkg.root));

let { synthVendor, synthStyles } = this.getSyntheticPackages(this.v1Cache.app, movedAddons);
return new WaitForTrees(
{ movedAddons, synthVendor, synthStyles, watchedUnmovedAddons },
'@embroider/compat/addons',
this.build.bind(this)
);
}

async ready(): Promise<{ outputPath: string; packageCache: PackageCache }> {
await this.deferReady.promise;
writeJSONSync(join(this.destDir, '.embroider-reuse.json'), {
appDestDir: relative(this.destDir, this.packageCache.appRoot),
});
return {
outputPath: this.packageCache.appRoot,
packageCache: this.packageCache,
};
}

private get appDestDir(): string {
return this.packageCache.appRoot;
}

private get app(): Package {
return this.packageCache.app;
return new WaitForTrees({ addons: this.addons }, '@embroider/compat/addons', this.build.bind(this));
}

private async build(
{
movedAddons,
synthVendor,
synthStyles,
addons,
}: {
movedAddons: string[];
synthVendor: string;
synthStyles: string;
addons: string;
},
changedMap: Map<string, boolean>
) {
// empty the directory only on the first pass
if (!this.didBuild) {
emptyDirSync(this.destDir);
}

[...this.packageCache.moved.entries()].forEach(([oldPkg, newPkg], index) => {
let treeInstance = this.treeSyncMap.get(newPkg);

// we need to pull metadata off the oldPkg, not the newPkg, because the
// newPkg doesn't actually have anything in it yet (including
// package.json)
let isEngine = oldPkg.isEngine();

// Engines get built not into their real package name, but a mangled one.
// Their real one needs to be free for us to merge all their dependencies
// into.
let destination = isEngine ? mangledEngineRoot(newPkg) : newPkg.root;

if (!treeInstance) {
let ignore = ['**/node_modules'];

let rel = relative(destination, this.appDestDir);
if (!rel.startsWith('..') && !isAbsolute(rel)) {
// the app is inside our addon. We must not copy the app as part of
// the addon, because that would overwrite the real app build.
ignore.push(rel);

if (rel === `tests${sep}dummy`) {
// special case: classic dummy apps are weird because they put the
// tests (which are truly part of the app, not the addon) inside the
// addon instead of inside the app.
ignore.push('tests');
}
}

treeInstance = new TreeSync(movedAddons[index], destination, {
ignore,
});

this.treeSyncMap.set(newPkg, treeInstance);
}

if (
!this.didBuild || // always copy on the first build
(newPkg.mayRebuild && changedMap.get(movedAddons[index]))
) {
treeInstance.sync();
if (!this.didBuild && isEngine) {
// The first time we encounter an engine, we also create the empty
// shell for its real module namespace.
copySync(join(destination, 'package.json'), join(newPkg.root, 'package.json'));
}
}
});

// this has to be a separate pass over the packages because
// linkNonCopiedDeps resolves dependencies, so we want all the packages
// already in their new places before they start trying to resolve each
// other.
[...this.packageCache.moved.values()].forEach((newPkg, index) => {
if (
!this.didBuild || // always copy on the first build
(newPkg.mayRebuild && changedMap.get(movedAddons[index]))
) {
// for engines, this isn't the mangled destination (we don't need
// resolvable node_modules there), this is the empty shell of their real
// location
this.linkNonCopiedDeps(newPkg, newPkg.root);
}
});

this.linkNonCopiedDeps(this.app, this.appDestDir);
await this.packageCache.updatePreexistingResolvableSymlinks();

if (changedMap && changedMap.get(synthVendor)) {
copySync(synthVendor, join(this.appDestDir, 'node_modules', '@embroider', 'synthesized-vendor'), {
dereference: true,
overwrite: true,
if (!this.treeSync) {
this.treeSync = new TreeSync(addons, resolve(this.inputPath, 'node_modules/.embroider/addons'), {
ignore: ['**/node_modules'],
});
}

if (changedMap && changedMap.get(synthStyles)) {
copySync(synthStyles, join(this.appDestDir, 'node_modules', '@embroider', 'synthesized-styles'), {
dereference: true,
overwrite: true,
});
}

if (!this.didBuild) {
this.handleNonResolvableDeps();
if (
!this.didBuild || // always copy on the first build
changedMap.get(addons)
) {
this.treeSync.sync();
}
this.didBuild = true;
this.deferReady.resolve();
}

private handleNonResolvableDeps() {
// non-resolvable deps in addons
for (let [oldPkg, newPkg] of this.packageCache.moved.entries()) {
if (!oldPkg.nonResolvableDeps) {
continue;
}
for (let dep of oldPkg.nonResolvableDeps.values()) {
let moved = this.packageCache.moved.get(dep);
if (moved) {
dep = moved;
}
let target = join(newPkg.root, 'node_modules', dep.name);
ensureDirSync(dirname(target));
ensureSymlinkSync(dep.root, target, 'dir');
}
}
// non-resolvable deps in app
if (this.packageCache.app.nonResolvableDeps) {
for (let dep of this.packageCache.app.nonResolvableDeps.values()) {
let moved = this.packageCache.moved.get(dep);
if (moved) {
dep = moved;
}
let target = join(this.appDestDir, 'node_modules', dep.name);
ensureDirSync(dirname(target));
ensureSymlinkSync(dep.root, target, 'dir');
}
}
}

private getSyntheticPackages(v1App: V1App, movedAddons: Node[]): { synthVendor: Node; synthStyles: Node } {
let index = 0;
let upgradedAddonTrees = [];
for (let [oldPkg] of this.packageCache.moved.entries()) {
if (!oldPkg.isV2Ember()) {
upgradedAddonTrees.push(movedAddons[index]);
}
index++;
}
return {
synthVendor: v1App.synthesizeVendorPackage(upgradedAddonTrees),
synthStyles: v1App.synthesizeStylesPackage(upgradedAddonTrees),
};
}

@Memoize()
private get deferReady() {
let resolve: Function;
let promise: Promise<void> = new Promise(r => (resolve = r));
return { resolve: resolve!, promise };
}

@Memoize()
private isMoved(pkg: Package) {
for (let candidate of this.packageCache.moved.values()) {
if (candidate === pkg) {
return true;
}
}
return false;
}

private linkNonCopiedDeps(pkg: Package, destRoot: string) {
for (let dep of pkg.dependencies) {
if (!this.isMoved(dep)) {
ensureSymlinkSync(dep.root, join(destRoot, 'node_modules', dep.packageJSON.name));
}
}
}
}
Loading