From d7c17fee90a5c53704cd77b085b2c2fe2bb698ae Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Tue, 21 Mar 2023 16:47:49 -0400 Subject: [PATCH] flatten the package caches --- packages/compat/src/compat-addons.ts | 2 +- packages/compat/src/compat-app.ts | 3 +- packages/compat/src/moved-package-cache.ts | 332 ------------------ packages/compat/src/standalone-addon-build.ts | 2 +- packages/compat/src/v1-app.ts | 8 +- packages/core/src/build-stage.ts | 11 +- packages/core/src/module-resolver.ts | 16 +- packages/core/src/stage.ts | 5 - packages/core/src/to-broccoli-plugin.ts | 7 +- .../macros/src/babel/macros-babel-plugin.ts | 2 +- packages/macros/src/babel/state.ts | 2 +- packages/macros/src/glimmer/ast-transform.ts | 2 +- packages/macros/src/macros-config.ts | 2 +- packages/shared-internals/src/babel-filter.ts | 2 +- .../shared-internals/src/package-cache.ts | 5 - .../src/template-colocation-plugin.ts | 2 +- 16 files changed, 26 insertions(+), 377 deletions(-) delete mode 100644 packages/compat/src/moved-package-cache.ts diff --git a/packages/compat/src/compat-addons.ts b/packages/compat/src/compat-addons.ts index 42211ec63..e15d51dc8 100644 --- a/packages/compat/src/compat-addons.ts +++ b/packages/compat/src/compat-addons.ts @@ -47,7 +47,7 @@ export default class CompatAddons implements Stage { await this.deferReady.promise; return { outputPath: this.destDir, - packageCache: PackageCache.shared('embroider-stage1', this.inputPath), + packageCache: PackageCache.shared('embroider-unified', this.inputPath), }; } diff --git a/packages/compat/src/compat-app.ts b/packages/compat/src/compat-app.ts index 3fd4206f1..87ef71b2d 100644 --- a/packages/compat/src/compat-app.ts +++ b/packages/compat/src/compat-app.ts @@ -66,7 +66,8 @@ function setup(legacyEmberAppInstance: object, options: Required) { appBootTree, }; - let instantiate = async (root: string, appSrcDir: string, packageCache: PackageCache) => { + let instantiate = async (root: string, appSrcDir: string) => { + let packageCache = PackageCache.shared('embroider-unified', appSrcDir); let appPackage = packageCache.get(appSrcDir); let adapter = new CompatAppAdapter( root, diff --git a/packages/compat/src/moved-package-cache.ts b/packages/compat/src/moved-package-cache.ts deleted file mode 100644 index 997ae870d..000000000 --- a/packages/compat/src/moved-package-cache.ts +++ /dev/null @@ -1,332 +0,0 @@ -import { join, sep, isAbsolute } from 'path'; -import { ensureSymlinkSync, readdirSync, realpathSync, lstatSync } from 'fs-extra'; -import { Memoize } from 'typescript-memoize'; -import { PackageCache, Package, getOrCreate } from '@embroider/core'; -import { MacrosConfig } from '@embroider/macros/src/node'; -import os from 'os'; - -function assertNoTildeExpansion(source: string, target: string) { - if (target.includes('~') && os.platform() !== 'win32') { - throw new Error( - `The symbolic link: ${source}'s target: ${target} contained a bash expansion '~' which is not supported.` - ); - } -} -export class MovablePackageCache extends PackageCache { - constructor(private macrosConfig: MacrosConfig, appRoot: string) { - super(appRoot); - } - - moveAddons(destDir: string): MovedPackageCache { - // start with the plain old app package - let origApp = this.get(this.appRoot); - - // discover the set of all packages that will need to be moved into the - // workspace - let movedSet = new MovedSet(origApp); - - return new MovedPackageCache(this.rootCache, this.resolutionCache, destDir, movedSet, origApp, this.macrosConfig); - } -} - -export class MovedPackageCache extends PackageCache { - readonly app!: Package; - private commonSegmentCount: number; - readonly moved: Map = new Map(); - readonly unmovedAddons: Set; - - constructor( - rootCache: PackageCache['rootCache'], - resolutionCache: PackageCache['resolutionCache'], - private destDir: string, - movedSet: MovedSet, - private origApp: Package, - private macrosConfig: MacrosConfig - ) { - // this is the initial appRoot, which we can't know until just below here - super('not-the-real-root'); - - // that gives us our common segment count, which enables localPath mapping - this.commonSegmentCount = movedSet.commonSegmentCount; - - // so we can now determine where the app will go inside the workspace. THIS - // is where we fix 'not-the-real-root' from above. - this.appRoot = this.localPath(origApp.root); - - this.macrosConfig.packageMoved(origApp.root, this.appRoot); - - for (let originalPkg of movedSet.packages) { - // Update our rootCache so we don't need to rediscover moved packages - let movedPkg; - if (originalPkg === origApp) { - // this wraps the original app package with one that will use moved - // dependencies. The app itself hasn't moved yet, which is why a proxy - // is needed at this level. - movedPkg = packageProxy(origApp, (pkg: Package) => this.moved.get(pkg) || pkg); - this.app = movedPkg; - rootCache.set(movedPkg.root, movedPkg); - } else { - movedPkg = this.movedPackage(originalPkg); - this.moved.set(originalPkg, movedPkg); - this.macrosConfig.packageMoved(originalPkg.root, movedPkg.root); - } - - // Update our resolutionCache so we still know as much about the moved - // packages as we did before we moved them, without redoing package - // resolution. - let resolutions = new Map(); - for (let dep of originalPkg.dependencies) { - if (movedSet.packages.has(dep)) { - resolutions.set(dep.name, this.movedPackage(dep)); - } else { - resolutions.set(dep.name, dep); - } - } - resolutionCache.set(movedPkg, resolutions); - } - this.rootCache = rootCache; - this.resolutionCache = resolutionCache; - this.unmovedAddons = movedSet.unmovedAddons; - } - - private movedPackage(originalPkg: Package): Package { - let newRoot = this.localPath(originalPkg.root); - return getOrCreate(this.rootCache, newRoot, () => new (originalPkg.constructor as any)(newRoot, this, false)); - } - - private localPath(filename: string) { - return join(this.destDir, ...pathSegments(filename).slice(this.commonSegmentCount)); - } - - // hunt for symlinks that may be needed to do node_modules resolution from the - // given path. - async updatePreexistingResolvableSymlinks(): Promise { - let roots = this.originalRoots(); - [...this.candidateDirs()].map(path => { - let links = symlinksInNodeModules(path); - for (let { source, target } of links) { - let pkg = roots.get(target); - if (pkg) { - // we found a symlink that points at a package that was copied. - // Replicate it in the new structure pointing at the new package. - ensureSymlinkSync(pkg.root, this.localPath(source)); - } - } - }); - } - - // places that might have symlinks we need to mimic - private candidateDirs(): Set { - let candidates = new Set() as Set; - let originalPackages = [this.origApp, ...this.moved.keys()]; - for (let pkg of originalPackages) { - let segments = pathSegments(pkg.root); - - let candidate = join(pkg.root, 'node_modules'); - candidates.add(candidate); - - for (let i = segments.length - 1; i >= this.commonSegmentCount; i--) { - if (segments[i - 1] !== 'node_modules') { - let candidate = '/' + join(...segments.slice(0, i), 'node_modules'); - if (candidates.has(candidate)) { - break; - } - candidates.add(candidate); - } - } - } - return candidates; - } - - private originalRoots(): Map { - let originalRoots = new Map(); - for (let [originalPackage, movedPackage] of this.moved.entries()) { - originalRoots.set(originalPackage.root, movedPackage); - } - return originalRoots; - } -} - -function maybeReaddirSync(path: string) { - try { - return readdirSync(path); - } catch (err) { - if (err.code !== 'ENOTDIR' && err.code !== 'ENOENT') { - throw err; - } - return []; - } -} - -function isSymlink(path: string): boolean { - try { - let stat = lstatSync(path); - return stat.isSymbolicLink(); - } catch (err) { - if (err.code !== 'ENOTDIR' && err.code !== 'ENOENT') { - throw err; - } - - return false; - } -} - -function symlinksInNodeModules(path: string): { source: string; target: string }[] { - let results: { source: string; target: string }[] = []; - - // handles the full `node_modules` being symlinked (this is uncommon, but sometimes - // be useful for test harnesses to avoid multiple `npm install` invocations) - let parentIsSymlink = isSymlink(path); - - let names = maybeReaddirSync(path); - - names.map(name => { - let source = join(path, name); - let stats = lstatSync(source); - if (parentIsSymlink || stats.isSymbolicLink()) { - let target = realpathSync(source); - assertNoTildeExpansion(source, target); - - results.push({ source, target }); - } else if (stats.isDirectory() && name.startsWith('@')) { - // handle symlinked scope names (e.g. symlinking `@myorghere` to a shared location) - let isSourceSymlink = isSymlink(source); - let innerNames = maybeReaddirSync(source); - - innerNames.map(innerName => { - let innerSource = join(source, innerName); - let innerStats = lstatSync(innerSource); - if (parentIsSymlink || isSourceSymlink || innerStats.isSymbolicLink()) { - let target = realpathSync(innerSource); - assertNoTildeExpansion(innerSource, target); - - results.push({ source: innerSource, target }); - } - }); - } - }); - - return results; -} - -function pathSegments(filename: string) { - let segments = filename.split(sep); - if (isAbsolute(filename)) { - segments.shift(); - } - return segments; -} - -class MovedSet { - private mustMove: Map = new Map(); - unmovedAddons: Set = new Set(); - - constructor(private app: Package) { - this.check(app); - } - - private check(pkg: Package): boolean { - if (this.mustMove.has(pkg)) { - return this.mustMove.get(pkg)!; - } - - // non-ember packages don't need to move - if (pkg !== this.app && !pkg.isEmberPackage()) { - this.mustMove.set(pkg, false); - return false; - } - - let mustMove = - // The app always moves (because we need a place to mash all the - // addon-provided "app-js" trees), - pkg === this.app || - // For the same reason, engines need to move (we need a place to mash all - // their child addon's provided app-js trees into) - pkg.isEngine() || - // any other ember package that isn't native v2 must move because we've - // got to rewrite them - !pkg.isV2Ember(); - - // this is a partial answer. After we check our children, our own `mustMove` - // may change from false to true. But it's OK that our children see false in - // that case, because they don't need to move on our behalf. - // - // We need to already be in the `this.mustMove` cache at this moment in - // order to avoid infinite recursion if any of our children end up depending - // back on us. - this.mustMove.set(pkg, mustMove); - - for (let dep of pkg.dependencies) { - // or if any of your deps need to move - mustMove = this.check(dep) || mustMove; - } - this.mustMove.set(pkg, mustMove); - - if (!mustMove) { - this.unmovedAddons.add(pkg); - } - - return mustMove; - } - - @Memoize() - get packages(): Set { - let result = new Set() as Set; - for (let [pkg, mustMove] of this.mustMove) { - if (mustMove) { - result.add(pkg); - } - } - return result; - } - - // the npm structure we're shadowing could have a dependency nearly anywhere - // on disk. We want to maintain their relations to each other. So we must find - // the point in the filesystem that contains all of them, which could even be - // "/" (for example, if you npm-linked a dependency that lives in /tmp). - // - // The commonSegmentCount is how many leading path segments are shared by all - // our packages. - @Memoize() - get commonSegmentCount(): number { - return [...this.packages].reduce((longestPrefix, pkg) => { - let candidate = pathSegments(pkg.root); - let shorter, longer; - if (longestPrefix.length > candidate.length) { - shorter = candidate; - longer = longestPrefix; - } else { - shorter = longestPrefix; - longer = candidate; - } - let i = 0; - for (; i < shorter.length; i++) { - if (shorter[i] !== longer[i]) { - break; - } - } - return shorter.slice(0, i); - }, pathSegments(this.app.root)).length; - } -} - -function packageProxy(pkg: Package, getMovedPackage: (pkg: Package) => Package) { - let p: Package = new Proxy(pkg, { - get(pkg: Package, prop: string | number | symbol) { - if (prop === 'dependencies') { - return pkg.dependencies.map(getMovedPackage); - } - if (prop === 'nonResolvableDeps') { - if (!pkg.nonResolvableDeps) { - return pkg.nonResolvableDeps; - } - return new Map([...pkg.nonResolvableDeps.values()].map(dep => [dep.name, getMovedPackage(dep)])); - } - if (prop === 'findDescendants') { - return pkg.findDescendants.bind(p); - } - return (pkg as any)[prop]; - }, - }); - return p; -} diff --git a/packages/compat/src/standalone-addon-build.ts b/packages/compat/src/standalone-addon-build.ts index 51ae9557f..b29e30e20 100644 --- a/packages/compat/src/standalone-addon-build.ts +++ b/packages/compat/src/standalone-addon-build.ts @@ -11,7 +11,7 @@ import type { Node } from 'broccoli-node-api'; export function convertLegacyAddons(emberApp: EmberAppInstance, maybeOptions?: Options) { let options = optionsWithDefaults(maybeOptions); let instanceCache = V1InstanceCache.forApp(emberApp, options); - let packageCache = PackageCache.shared('embroider-stage1', instanceCache.app.root); + let packageCache = PackageCache.shared('embroider-unified', instanceCache.app.root); let v1Addons = findV1Addons(packageCache.get(instanceCache.app.root)); let addonIndex = Object.create(null); diff --git a/packages/compat/src/v1-app.ts b/packages/compat/src/v1-app.ts index 107e52d37..e89af4a08 100644 --- a/packages/compat/src/v1-app.ts +++ b/packages/compat/src/v1-app.ts @@ -15,12 +15,13 @@ import { OutputFileToInputFileMap, PackageInfo, AddonInstance, + PackageCache, } from '@embroider/core'; import { writeJSONSync, ensureDirSync, copySync, readdirSync, pathExistsSync, existsSync } from 'fs-extra'; import AddToTree from './add-to-tree'; import DummyPackage, { OwningAddon } from './dummy-package'; import { TransformOptions } from '@babel/core'; -import { isEmbroiderMacrosPlugin, MacrosConfig } from '@embroider/macros/src/node'; +import { isEmbroiderMacrosPlugin } from '@embroider/macros/src/node'; import resolvePackagePath from 'resolve-package-path'; import Concat from 'broccoli-concat'; import mapKeys from 'lodash/mapKeys'; @@ -30,7 +31,6 @@ import prepHtmlbarsAstPluginsForUnwrap from './prepare-htmlbars-ast-plugins'; import { readFileSync } from 'fs'; import type { Options as HTMLBarsOptions } from 'ember-cli-htmlbars'; import semver from 'semver'; -import { MovablePackageCache } from './moved-package-cache'; import type { Transform } from 'babel-plugin-ember-template-compilation'; @@ -64,10 +64,10 @@ export default class V1App { private _implicitScripts: string[] = []; private _implicitStyles: string[] = []; - packageCache: MovablePackageCache; + packageCache: PackageCache; protected constructor(protected app: EmberAppInstance) { - this.packageCache = new MovablePackageCache(MacrosConfig.for(app, this.root), this.root); + this.packageCache = PackageCache.shared('embroider-unified', this.root); } // always the name from package.json. Not the one that apps may have weirdly diff --git a/packages/core/src/build-stage.ts b/packages/core/src/build-stage.ts index 35ac64f05..7b017695f 100644 --- a/packages/core/src/build-stage.ts +++ b/packages/core/src/build-stage.ts @@ -16,21 +16,16 @@ export default class BuildStage implements Stage { private prevStage: Stage, private inTrees: NamedTrees, private annotation: string, - private instantiate: ( - root: string, - appSrcDir: string, - packageCache: PackageCache - ) => Promise> + private instantiate: (root: string, appSrcDir: string) => Promise> ) {} @Memoize() get tree(): Node { return new WaitForTrees(this.augment(this.inTrees), this.annotation, async treePaths => { if (!this.active) { - let { outputPath, packageCache } = await this.prevStage.ready(); + let { outputPath } = await this.prevStage.ready(); this.outputPath = outputPath; - this.packageCache = packageCache; - this.active = await this.instantiate(outputPath, this.prevStage.inputPath, packageCache); + this.active = await this.instantiate(outputPath, this.prevStage.inputPath); } delete (treePaths as any).__prevStageTree; await this.active.build(this.deAugment(treePaths)); diff --git a/packages/core/src/module-resolver.ts b/packages/core/src/module-resolver.ts index 7cacab99e..949049938 100644 --- a/packages/core/src/module-resolver.ts +++ b/packages/core/src/module-resolver.ts @@ -268,13 +268,13 @@ export class Resolver { } owningPackage(fromFile: string): Package | undefined { - return PackageCache.shared('embroider-stage3', this.options.appRoot).ownerOfFile(fromFile); + return PackageCache.shared('embroider-unified', this.options.appRoot).ownerOfFile(fromFile); } private logicalPackage(owningPackage: V2Package, file: string): V2Package { let logicalLocation = this.reverseSearchAppTree(owningPackage, file); if (logicalLocation) { - let pkg = PackageCache.shared('embroider-stage3', this.options.appRoot).get(logicalLocation.owningEngine.root); + let pkg = PackageCache.shared('embroider-unified', this.options.appRoot).get(logicalLocation.owningEngine.root); if (!pkg.isV2Ember()) { throw new Error(`bug: all engines should be v2 addons by the time we see them here`); } @@ -477,7 +477,7 @@ export class Resolver { // out. @Memoize() private get mergeMap(): MergeMap { - let packageCache = PackageCache.shared('embroider-stage3', this.options.appRoot); + let packageCache = PackageCache.shared('embroider-unified', this.options.appRoot); let result: MergeMap = new Map(); for (let engine of this.options.engines) { let engineModules: Map = new Map(); @@ -597,9 +597,9 @@ export class Resolver { } private handleLegacyAddons(request: R): R { - let packageCache = PackageCache.shared('embroider-stage3', this.options.appRoot); + let packageCache = PackageCache.shared('embroider-unified', this.options.appRoot); - // first we handle output requests from moved packages + // first we handle outbound requests from moved packages let pkg = this.owningPackage(request.fromFile); if (!pkg) { return request; @@ -619,7 +619,7 @@ export class Resolver { if (packageName && packageName !== pkg.name) { // non-relative, non-self request, so check if it aims at a rewritten addon try { - let target = PackageCache.shared('embroider-stage3', this.options.appRoot).resolve(packageName, pkg); + let target = PackageCache.shared('embroider-unified', this.options.appRoot).resolve(packageName, pkg); if (target) { let movedRoot = this.legacyAddonsIndex.v1ToV2.get(target.root); if (movedRoot) { @@ -789,7 +789,7 @@ export class Resolver { if (logicalPackage.meta['auto-upgraded'] && !logicalPackage.hasDependency('ember-auto-import')) { try { - let dep = PackageCache.shared('embroider-stage3', this.options.appRoot).resolve(packageName, logicalPackage); + let dep = PackageCache.shared('embroider-unified', this.options.appRoot).resolve(packageName, logicalPackage); if (!dep.isEmberPackage()) { // classic ember addons can only import non-ember dependencies if they // have ember-auto-import. @@ -876,7 +876,7 @@ export class Resolver { request, this.resolveWithinPackage( request, - PackageCache.shared('embroider-stage3', this.options.appRoot).get(this.options.activeAddons[packageName]) + PackageCache.shared('embroider-unified', this.options.appRoot).get(this.options.activeAddons[packageName]) ) ); } diff --git a/packages/core/src/stage.ts b/packages/core/src/stage.ts index 8a9624ce6..b5c6e8616 100644 --- a/packages/core/src/stage.ts +++ b/packages/core/src/stage.ts @@ -1,5 +1,4 @@ import type { Node } from 'broccoli-node-api'; -import { PackageCache } from '@embroider/shared-internals'; // A build Stage is _kinda_ like a Broccoli transform, and it interoperates with // Broccoli, but it takes a different approach to how stages combine. @@ -27,9 +26,5 @@ export default interface Stage { // This is the actual directory in which the output will be. It's guaranteed // to not change once you get it. readonly outputPath: string; - - // Stages must propagate their PackageCache forward to the next stage so we - // don't repeat a lot of resolving work. - readonly packageCache: PackageCache; }>; } diff --git a/packages/core/src/to-broccoli-plugin.ts b/packages/core/src/to-broccoli-plugin.ts index a08af37ed..64a4cbd59 100644 --- a/packages/core/src/to-broccoli-plugin.ts +++ b/packages/core/src/to-broccoli-plugin.ts @@ -22,12 +22,7 @@ export default function toBroccoliPlugin( async build() { if (!this.packager) { - let { outputPath, packageCache } = await this.stage.ready(); - // We always register a shared stage3 packageCache so it can be used by - // things like babel plugins and template compilers. - if (packageCache) { - packageCache.shareAs('embroider-stage3'); - } + let { outputPath } = await this.stage.ready(); this.packager = new packagerClass( outputPath, this.outputPath, diff --git a/packages/macros/src/babel/macros-babel-plugin.ts b/packages/macros/src/babel/macros-babel-plugin.ts index 385f23807..15007c147 100644 --- a/packages/macros/src/babel/macros-babel-plugin.ts +++ b/packages/macros/src/babel/macros-babel-plugin.ts @@ -18,7 +18,7 @@ export default function main(context: typeof Babel): unknown { enter(path: NodePath, state: State) { initState(t, path, state); - state.packageCache = PackageCache.shared('embroider-stage3', state.opts.appPackageRoot); + state.packageCache = PackageCache.shared('embroider-unified', state.opts.appPackageRoot); }, exit(_: NodePath, state: State) { // @embroider/macros itself has no runtime behaviors and should always be removed diff --git a/packages/macros/src/babel/state.ts b/packages/macros/src/babel/state.ts index c6c91dac4..374c4d157 100644 --- a/packages/macros/src/babel/state.ts +++ b/packages/macros/src/babel/state.ts @@ -55,7 +55,7 @@ export function initState(t: typeof Babel.types, path: NodePath new PackageCache(appRoot)); if (pk.appRoot !== appRoot) { diff --git a/packages/shared-internals/src/template-colocation-plugin.ts b/packages/shared-internals/src/template-colocation-plugin.ts index be2e31522..fba6249bc 100644 --- a/packages/shared-internals/src/template-colocation-plugin.ts +++ b/packages/shared-internals/src/template-colocation-plugin.ts @@ -53,7 +53,7 @@ export default function main(babel: typeof Babel) { let filename = path.hub.file.opts.filename; if (state.opts.packageGuard) { - let owningPackage = PackageCache.shared('embroider-stage3', state.opts.appRoot).ownerOfFile(filename); + let owningPackage = PackageCache.shared('embroider-unified', state.opts.appRoot).ownerOfFile(filename); if (!owningPackage || !owningPackage.isV2Ember() || !owningPackage.meta['auto-upgraded']) { return; }