diff --git a/.yarnrc.yml b/.yarnrc.yml index 4efbf59a3aab..29194e6e00e5 100644 --- a/.yarnrc.yml +++ b/.yarnrc.yml @@ -44,3 +44,8 @@ preferInteractive: true telemetryUserId: yarnpkg/berry yarnPath: scripts/run-yarn.js + +cacheParameters: + matrix: + platform: [darwin, linux, windows, freebsd, openbsd] + arch: [32, 64, arm, arm64, mips64le, ppc64le] diff --git a/packages/acceptance-tests/pkg-tests-fixtures/packages/variants-1.0.0/index.js b/packages/acceptance-tests/pkg-tests-fixtures/packages/variants-1.0.0/index.js new file mode 100644 index 000000000000..a6bf8f586524 --- /dev/null +++ b/packages/acceptance-tests/pkg-tests-fixtures/packages/variants-1.0.0/index.js @@ -0,0 +1,10 @@ +/* @flow */ + +module.exports = require(`./package.json`); + +for (const key of [`dependencies`, `devDependencies`, `peerDependencies`]) { + for (const dep of Object.keys(module.exports[key] || {})) { + // $FlowFixMe The whole point of this file is to be dynamic + module.exports[key][dep] = require(dep); + } +} diff --git a/packages/acceptance-tests/pkg-tests-fixtures/packages/variants-1.0.0/package.json b/packages/acceptance-tests/pkg-tests-fixtures/packages/variants-1.0.0/package.json new file mode 100644 index 000000000000..27d59c425f7b --- /dev/null +++ b/packages/acceptance-tests/pkg-tests-fixtures/packages/variants-1.0.0/package.json @@ -0,0 +1,37 @@ +{ + "name": "variants", + "version": "1.0.0", + "variants": [ + { + "pattern": "variants-%par1-%par2@%version", + "matrix": { + "par1": [ + "a", + "b", + "c" + ], + "par2": [ + 1, + 2 + ] + }, + "exclude": [ + { + "par1": "c", + "par2": 1 + } + ] + }, + { + "pattern": "variants-%par1@%version", + "matrix": { + "par1": [ + "d" + ] + } + }, + { + "pattern": "variants-fallback@%version" + } + ] +} diff --git a/packages/acceptance-tests/pkg-tests-fixtures/packages/variants-a-1-1.0.0/index.js b/packages/acceptance-tests/pkg-tests-fixtures/packages/variants-a-1-1.0.0/index.js new file mode 100644 index 000000000000..a6bf8f586524 --- /dev/null +++ b/packages/acceptance-tests/pkg-tests-fixtures/packages/variants-a-1-1.0.0/index.js @@ -0,0 +1,10 @@ +/* @flow */ + +module.exports = require(`./package.json`); + +for (const key of [`dependencies`, `devDependencies`, `peerDependencies`]) { + for (const dep of Object.keys(module.exports[key] || {})) { + // $FlowFixMe The whole point of this file is to be dynamic + module.exports[key][dep] = require(dep); + } +} diff --git a/packages/acceptance-tests/pkg-tests-fixtures/packages/variants-a-1-1.0.0/package.json b/packages/acceptance-tests/pkg-tests-fixtures/packages/variants-a-1-1.0.0/package.json new file mode 100644 index 000000000000..8f8f25f6b219 --- /dev/null +++ b/packages/acceptance-tests/pkg-tests-fixtures/packages/variants-a-1-1.0.0/package.json @@ -0,0 +1,4 @@ +{ + "name": "variants-a-1", + "version": "1.0.0" +} diff --git a/packages/acceptance-tests/pkg-tests-fixtures/packages/variants-a-2-1.0.0/index.js b/packages/acceptance-tests/pkg-tests-fixtures/packages/variants-a-2-1.0.0/index.js new file mode 100644 index 000000000000..a6bf8f586524 --- /dev/null +++ b/packages/acceptance-tests/pkg-tests-fixtures/packages/variants-a-2-1.0.0/index.js @@ -0,0 +1,10 @@ +/* @flow */ + +module.exports = require(`./package.json`); + +for (const key of [`dependencies`, `devDependencies`, `peerDependencies`]) { + for (const dep of Object.keys(module.exports[key] || {})) { + // $FlowFixMe The whole point of this file is to be dynamic + module.exports[key][dep] = require(dep); + } +} diff --git a/packages/acceptance-tests/pkg-tests-fixtures/packages/variants-a-2-1.0.0/package.json b/packages/acceptance-tests/pkg-tests-fixtures/packages/variants-a-2-1.0.0/package.json new file mode 100644 index 000000000000..6ea60958386d --- /dev/null +++ b/packages/acceptance-tests/pkg-tests-fixtures/packages/variants-a-2-1.0.0/package.json @@ -0,0 +1,4 @@ +{ + "name": "variants-a-2", + "version": "1.0.0" +} diff --git a/packages/acceptance-tests/pkg-tests-fixtures/packages/variants-b-1-1.0.0/index.js b/packages/acceptance-tests/pkg-tests-fixtures/packages/variants-b-1-1.0.0/index.js new file mode 100644 index 000000000000..a6bf8f586524 --- /dev/null +++ b/packages/acceptance-tests/pkg-tests-fixtures/packages/variants-b-1-1.0.0/index.js @@ -0,0 +1,10 @@ +/* @flow */ + +module.exports = require(`./package.json`); + +for (const key of [`dependencies`, `devDependencies`, `peerDependencies`]) { + for (const dep of Object.keys(module.exports[key] || {})) { + // $FlowFixMe The whole point of this file is to be dynamic + module.exports[key][dep] = require(dep); + } +} diff --git a/packages/acceptance-tests/pkg-tests-fixtures/packages/variants-b-1-1.0.0/package.json b/packages/acceptance-tests/pkg-tests-fixtures/packages/variants-b-1-1.0.0/package.json new file mode 100644 index 000000000000..d0020b057783 --- /dev/null +++ b/packages/acceptance-tests/pkg-tests-fixtures/packages/variants-b-1-1.0.0/package.json @@ -0,0 +1,4 @@ +{ + "name": "variants-b-1", + "version": "1.0.0" +} diff --git a/packages/acceptance-tests/pkg-tests-fixtures/packages/variants-b-2-1.0.0/index.js b/packages/acceptance-tests/pkg-tests-fixtures/packages/variants-b-2-1.0.0/index.js new file mode 100644 index 000000000000..a6bf8f586524 --- /dev/null +++ b/packages/acceptance-tests/pkg-tests-fixtures/packages/variants-b-2-1.0.0/index.js @@ -0,0 +1,10 @@ +/* @flow */ + +module.exports = require(`./package.json`); + +for (const key of [`dependencies`, `devDependencies`, `peerDependencies`]) { + for (const dep of Object.keys(module.exports[key] || {})) { + // $FlowFixMe The whole point of this file is to be dynamic + module.exports[key][dep] = require(dep); + } +} diff --git a/packages/acceptance-tests/pkg-tests-fixtures/packages/variants-b-2-1.0.0/package.json b/packages/acceptance-tests/pkg-tests-fixtures/packages/variants-b-2-1.0.0/package.json new file mode 100644 index 000000000000..c7eaeb0da827 --- /dev/null +++ b/packages/acceptance-tests/pkg-tests-fixtures/packages/variants-b-2-1.0.0/package.json @@ -0,0 +1,4 @@ +{ + "name": "variants-b-2", + "version": "1.0.0" +} diff --git a/packages/acceptance-tests/pkg-tests-fixtures/packages/variants-c-2-1.0.0/index.js b/packages/acceptance-tests/pkg-tests-fixtures/packages/variants-c-2-1.0.0/index.js new file mode 100644 index 000000000000..a6bf8f586524 --- /dev/null +++ b/packages/acceptance-tests/pkg-tests-fixtures/packages/variants-c-2-1.0.0/index.js @@ -0,0 +1,10 @@ +/* @flow */ + +module.exports = require(`./package.json`); + +for (const key of [`dependencies`, `devDependencies`, `peerDependencies`]) { + for (const dep of Object.keys(module.exports[key] || {})) { + // $FlowFixMe The whole point of this file is to be dynamic + module.exports[key][dep] = require(dep); + } +} diff --git a/packages/acceptance-tests/pkg-tests-fixtures/packages/variants-c-2-1.0.0/package.json b/packages/acceptance-tests/pkg-tests-fixtures/packages/variants-c-2-1.0.0/package.json new file mode 100644 index 000000000000..2fbd3d857092 --- /dev/null +++ b/packages/acceptance-tests/pkg-tests-fixtures/packages/variants-c-2-1.0.0/package.json @@ -0,0 +1,4 @@ +{ + "name": "variants-c-2", + "version": "1.0.0" +} diff --git a/packages/acceptance-tests/pkg-tests-fixtures/packages/variants-d-1.0.0/index.js b/packages/acceptance-tests/pkg-tests-fixtures/packages/variants-d-1.0.0/index.js new file mode 100644 index 000000000000..a6bf8f586524 --- /dev/null +++ b/packages/acceptance-tests/pkg-tests-fixtures/packages/variants-d-1.0.0/index.js @@ -0,0 +1,10 @@ +/* @flow */ + +module.exports = require(`./package.json`); + +for (const key of [`dependencies`, `devDependencies`, `peerDependencies`]) { + for (const dep of Object.keys(module.exports[key] || {})) { + // $FlowFixMe The whole point of this file is to be dynamic + module.exports[key][dep] = require(dep); + } +} diff --git a/packages/acceptance-tests/pkg-tests-fixtures/packages/variants-d-1.0.0/package.json b/packages/acceptance-tests/pkg-tests-fixtures/packages/variants-d-1.0.0/package.json new file mode 100644 index 000000000000..2827a1bb730f --- /dev/null +++ b/packages/acceptance-tests/pkg-tests-fixtures/packages/variants-d-1.0.0/package.json @@ -0,0 +1,4 @@ +{ + "name": "variants-d", + "version": "1.0.0" +} diff --git a/packages/acceptance-tests/pkg-tests-fixtures/packages/variants-fallback-1.0.0/index.js b/packages/acceptance-tests/pkg-tests-fixtures/packages/variants-fallback-1.0.0/index.js new file mode 100644 index 000000000000..a6bf8f586524 --- /dev/null +++ b/packages/acceptance-tests/pkg-tests-fixtures/packages/variants-fallback-1.0.0/index.js @@ -0,0 +1,10 @@ +/* @flow */ + +module.exports = require(`./package.json`); + +for (const key of [`dependencies`, `devDependencies`, `peerDependencies`]) { + for (const dep of Object.keys(module.exports[key] || {})) { + // $FlowFixMe The whole point of this file is to be dynamic + module.exports[key][dep] = require(dep); + } +} diff --git a/packages/acceptance-tests/pkg-tests-fixtures/packages/variants-fallback-1.0.0/package.json b/packages/acceptance-tests/pkg-tests-fixtures/packages/variants-fallback-1.0.0/package.json new file mode 100644 index 000000000000..289a37cf7fcb --- /dev/null +++ b/packages/acceptance-tests/pkg-tests-fixtures/packages/variants-fallback-1.0.0/package.json @@ -0,0 +1,4 @@ +{ + "name": "variants-fallback", + "version": "1.0.0" +} diff --git a/packages/acceptance-tests/pkg-tests-specs/sources/variants.test.ts b/packages/acceptance-tests/pkg-tests-specs/sources/variants.test.ts new file mode 100644 index 000000000000..b4b456e137f0 --- /dev/null +++ b/packages/acceptance-tests/pkg-tests-specs/sources/variants.test.ts @@ -0,0 +1,88 @@ +import {PortablePath, xfs} from '@yarnpkg/fslib'; +import {yarn} from 'pkg-tests-core'; + +const {writeConfiguration} = yarn; + +describe(`Variant tests`, () => { + test( + `it should install the fallback dependency when no matching pattern can be found`, + makeTemporaryEnv( + { + dependencies: {[`variants`]: `1.0.0`}, + }, + async ({run, source}) => { + await run(`install`); + + await expect(source(`require('variants')`)).resolves.toMatchObject({ + name: `variants-fallback`, + version: `1.0.0`, + }); + }, + ), + ); + + test( + `it should correctly install a dependency matching parameters set in the dependenciesMeta`, + makeTemporaryEnv( + { + dependencies: {[`variants`]: `1.0.0`}, + dependenciesMeta: { + [`variants`]: { + parameters: { + par1: `a`, + par2: 1, + }, + }, + }, + }, + async ({run, source}) => { + await run(`install`); + + await expect(source(`require('variants')`)).resolves.toMatchObject({ + name: `variants-a-1`, + version: `1.0.0`, + }); + }, + ), + ); + + test( + `it should fetch the matrix of cached variants set in the yarnrc`, + makeTemporaryEnv( + { + dependencies: {[`variants`]: `1.0.0`}, + dependenciesMeta: { + [`variants`]: { + parameters: { + par1: `a`, + par2: 1, + }, + }, + }, + }, + async ({path, run}) => { + await writeConfiguration(path, { + cacheParameters: { + matrix: { + par1: [`a`, `b`, `c`], + par2: [1], + }, + }, + }); + + await run(`install`); + + + const cache = await xfs.readdirPromise(`${path}/.yarn/cache` as PortablePath); + + const variantA1 = cache.find(file => file.startsWith(`variants-a-1-npm-1.0.0`)); + const variantB1 = cache.find(file => file.startsWith(`variants-b-1-npm-1.0.0`)); + const variantC1 = cache.find(file => file.startsWith(`variants-c-1-npm-1.0.0`)); + + expect(variantA1).toBeDefined(); + expect(variantB1).toBeDefined(); + expect(variantC1).not.toBeDefined(); // C1 is excluded by the variants package + }, + ), + ); +}); diff --git a/packages/plugin-exec/sources/ExecResolver.ts b/packages/plugin-exec/sources/ExecResolver.ts index ca6d6f8590ea..84f6c7816205 100644 --- a/packages/plugin-exec/sources/ExecResolver.ts +++ b/packages/plugin-exec/sources/ExecResolver.ts @@ -89,6 +89,7 @@ export class ExecResolver implements Resolver { peerDependenciesMeta: manifest.peerDependenciesMeta, bin: manifest.bin, + variants: manifest.variants, }; } } diff --git a/packages/plugin-file/sources/FileResolver.ts b/packages/plugin-file/sources/FileResolver.ts index dd272117338c..770630feb939 100644 --- a/packages/plugin-file/sources/FileResolver.ts +++ b/packages/plugin-file/sources/FileResolver.ts @@ -101,6 +101,7 @@ export class FileResolver implements Resolver { peerDependenciesMeta: manifest.peerDependenciesMeta, bin: manifest.bin, + variants: manifest.variants, }; } } diff --git a/packages/plugin-file/sources/TarballFileResolver.ts b/packages/plugin-file/sources/TarballFileResolver.ts index 214eb0f31742..ddb118a1e157 100644 --- a/packages/plugin-file/sources/TarballFileResolver.ts +++ b/packages/plugin-file/sources/TarballFileResolver.ts @@ -85,6 +85,7 @@ export class TarballFileResolver implements Resolver { peerDependenciesMeta: manifest.peerDependenciesMeta, bin: manifest.bin, + variants: manifest.variants, }; } } diff --git a/packages/plugin-git/sources/GitResolver.ts b/packages/plugin-git/sources/GitResolver.ts index b54422b6219a..16dfe86ccda1 100644 --- a/packages/plugin-git/sources/GitResolver.ts +++ b/packages/plugin-git/sources/GitResolver.ts @@ -62,6 +62,7 @@ export class GitResolver implements Resolver { peerDependenciesMeta: manifest.peerDependenciesMeta, bin: manifest.bin, + variants: manifest.variants, }; } } diff --git a/packages/plugin-http/sources/TarballHttpResolver.ts b/packages/plugin-http/sources/TarballHttpResolver.ts index 78b78db9d13b..14bbce749d01 100644 --- a/packages/plugin-http/sources/TarballHttpResolver.ts +++ b/packages/plugin-http/sources/TarballHttpResolver.ts @@ -71,6 +71,7 @@ export class TarballHttpResolver implements Resolver { peerDependenciesMeta: manifest.peerDependenciesMeta, bin: manifest.bin, + variants: manifest.variants, }; } } diff --git a/packages/plugin-link/sources/LinkResolver.ts b/packages/plugin-link/sources/LinkResolver.ts index 526e64005c37..a4b051019ca2 100644 --- a/packages/plugin-link/sources/LinkResolver.ts +++ b/packages/plugin-link/sources/LinkResolver.ts @@ -70,6 +70,7 @@ export class LinkResolver implements Resolver { peerDependenciesMeta: manifest.peerDependenciesMeta, bin: manifest.bin, + variants: manifest.variants, }; } } diff --git a/packages/plugin-link/sources/RawLinkResolver.ts b/packages/plugin-link/sources/RawLinkResolver.ts index 20b89e0fa40b..b3ae3a7530b2 100644 --- a/packages/plugin-link/sources/RawLinkResolver.ts +++ b/packages/plugin-link/sources/RawLinkResolver.ts @@ -61,6 +61,7 @@ export class RawLinkResolver implements Resolver { peerDependenciesMeta: new Map(), bin: new Map(), + variants: null, }; } } diff --git a/packages/plugin-npm/sources/NpmSemverResolver.ts b/packages/plugin-npm/sources/NpmSemverResolver.ts index 8c4a40649d82..f091ee58d060 100644 --- a/packages/plugin-npm/sources/NpmSemverResolver.ts +++ b/packages/plugin-npm/sources/NpmSemverResolver.ts @@ -150,6 +150,7 @@ export class NpmSemverResolver implements Resolver { if (typeof manifest.raw.deprecated === `string`) opts.report.reportWarningOnce(MessageName.DEPRECATED_PACKAGE, `${structUtils.prettyLocator(opts.project.configuration, locator)} is deprecated: ${manifest.raw.deprecated}`); + return { ...locator, @@ -165,6 +166,8 @@ export class NpmSemverResolver implements Resolver { peerDependenciesMeta: manifest.peerDependenciesMeta, bin: manifest.bin, + + variants: manifest.variants, }; } } diff --git a/packages/variant-fallback/index.js b/packages/variant-fallback/index.js new file mode 100644 index 000000000000..a6bf8f586524 --- /dev/null +++ b/packages/variant-fallback/index.js @@ -0,0 +1,10 @@ +/* @flow */ + +module.exports = require(`./package.json`); + +for (const key of [`dependencies`, `devDependencies`, `peerDependencies`]) { + for (const dep of Object.keys(module.exports[key] || {})) { + // $FlowFixMe The whole point of this file is to be dynamic + module.exports[key][dep] = require(dep); + } +} diff --git a/packages/variant-fallback/package.json b/packages/variant-fallback/package.json new file mode 100644 index 000000000000..b0acae1742b5 --- /dev/null +++ b/packages/variant-fallback/package.json @@ -0,0 +1,4 @@ +{ + "name": "variant-fallback", + "version": "0.12.22" +} diff --git a/packages/variant-test/index.js b/packages/variant-test/index.js new file mode 100644 index 000000000000..a6bf8f586524 --- /dev/null +++ b/packages/variant-test/index.js @@ -0,0 +1,10 @@ +/* @flow */ + +module.exports = require(`./package.json`); + +for (const key of [`dependencies`, `devDependencies`, `peerDependencies`]) { + for (const dep of Object.keys(module.exports[key] || {})) { + // $FlowFixMe The whole point of this file is to be dynamic + module.exports[key][dep] = require(dep); + } +} diff --git a/packages/variant-test/package.json b/packages/variant-test/package.json new file mode 100644 index 000000000000..4d058f8b344c --- /dev/null +++ b/packages/variant-test/package.json @@ -0,0 +1,71 @@ +{ + "name": "variant-test", + "version": "0.12.22", + "variants": [ + { + "pattern": "esbuild-%platform-%arch@%version", + "matrix": { + "platform": [ + "darwin", + "linux", + "windows" + ], + "arch": [ + "32", + "64" + ] + }, + "exclude": [ + { + "platform": "darwin", + "arch": "32" + } + ], + "include": [ + { + "platform": "darwin", + "arch": "arm64" + }, + { + "platform": "android", + "arch": "arm64" + }, + { + "platform": "freebsd", + "arch": "64" + }, + { + "platform": "freebsd", + "arch": "arm64" + }, + { + "platform": "linux", + "arch": "arm" + }, + { + "platform": "linux", + "arch": "arm64" + }, + { + "platform": "linux", + "arch": "mips64le" + }, + { + "platform": "linux", + "arch": "ppc64le" + }, + { + "platform": "openbsd", + "arch": "64" + }, + { + "platform": "windows", + "arch": "arm64" + } + ] + }, + { + "pattern": "variant-fallback@%version" + } + ] +} diff --git a/packages/yarnpkg-cli/package.json b/packages/yarnpkg-cli/package.json index 5628bbc1181b..0eb31a935ea3 100644 --- a/packages/yarnpkg-cli/package.json +++ b/packages/yarnpkg-cli/package.json @@ -30,6 +30,7 @@ "semver": "^7.1.2", "tslib": "^1.13.0", "typanion": "^3.3.0", + "variant-test": "workspace:*", "yup": "^0.32.9" }, "devDependencies": { diff --git a/packages/yarnpkg-core/sources/Configuration.ts b/packages/yarnpkg-core/sources/Configuration.ts index b94cedcd9a25..47fc621665d0 100644 --- a/packages/yarnpkg-core/sources/Configuration.ts +++ b/packages/yarnpkg-core/sources/Configuration.ts @@ -1,5 +1,5 @@ -import {Filename, PortablePath, npath, ppath, xfs} from '@yarnpkg/fslib'; import {DEFAULT_COMPRESSION_LEVEL} from '@yarnpkg/fslib'; +import {Filename, PortablePath, npath, ppath, xfs} from '@yarnpkg/fslib'; import {parseSyml, stringifySyml} from '@yarnpkg/parsers'; import camelcase from 'camelcase'; import {isCI} from 'ci-info'; @@ -8,13 +8,14 @@ import pLimit, {Limit} import {PassThrough, Writable} from 'stream'; import {CorePlugin} from './CorePlugin'; -import {Manifest, PeerDependencyMeta} from './Manifest'; +import {Manifest, PeerDependencyMeta, VariantParameters, Variants} from './Manifest'; import {MultiFetcher} from './MultiFetcher'; import {MultiResolver} from './MultiResolver'; import {Plugin, Hooks} from './Plugin'; import {ProtocolResolver} from './ProtocolResolver'; import {Report} from './Report'; import {TelemetryManager} from './TelemetryManager'; +import {VariantRemapResolver} from './VariantRemapResolver'; import {VirtualFetcher} from './VirtualFetcher'; import {VirtualResolver} from './VirtualResolver'; import {WorkspaceFetcher} from './WorkspaceFetcher'; @@ -26,6 +27,7 @@ import * as nodeUtils import * as semverUtils from './semverUtils'; import * as structUtils from './structUtils'; import {IdentHash, Package, Descriptor, PackageExtension, PackageExtensionType, PackageExtensionStatus} from './types'; +import {compareVariantConfiguration} from './variantUtils'; const IGNORED_ENV_VARIABLES = new Set([ // "binFolder" is the magic location where the parent process stored the @@ -438,6 +440,19 @@ export const coreDefinitions: {[coreSettingName: string]: SettingsDefinition} = default: `throw`, }, + /** + * cacheParameters: + * matrix: + * platform: [win32, osx] + * napi: [6, 4] + * js: [cjs, esm, es6, es2015, es2020] + */ + cacheParameters: { + description: `Cache parameters for variants`, + type: SettingsType.ANY, + default: null, + }, + // Package patching - to fix incorrect definitions packageExtensions: { description: `Map of package corrections to apply on the dependency tree`, @@ -477,6 +492,12 @@ export const coreDefinitions: {[coreSettingName: string]: SettingsDefinition} = }, }, }, + variants: { + description: `Variant information for replacing packages with different ones under certain circumstances`, + type: SettingsType.ANY, + isArray: true, + default: null, + }, }, }, }, @@ -549,11 +570,20 @@ export interface ConfigurationValueMap { enableImmutableCache: boolean; checksumBehavior: string; + cacheParameters: { + matrix: { + [parameter: string]: Array + }, + include: Array + exclude: Array + }; + // Package patching - to fix incorrect definitions packageExtensions: Map, peerDependencies?: Map, peerDependenciesMeta?: Map>, + variants?: Variants }>>; } @@ -1389,6 +1419,7 @@ export class Configuration { new VirtualResolver(), new WorkspaceResolver(), new ProtocolResolver(), + new VariantRemapResolver(), ...pluginResolvers, ]); @@ -1445,6 +1476,8 @@ export class Configuration { extensionsPerRange.push({...baseExtension, type: PackageExtensionType.Dependency, descriptor: dependency}); for (const peerDependency of extension.peerDependencies.values()) extensionsPerRange.push({...baseExtension, type: PackageExtensionType.PeerDependency, descriptor: peerDependency}); + if (extension.variants) + extensionsPerRange.push({...baseExtension, type: PackageExtensionType.Variants, variants: extension.variants}); for (const [selector, meta] of extension.peerDependenciesMeta) { for (const [key, value] of Object.entries(meta)) { @@ -1511,6 +1544,14 @@ export class Configuration { } } break; + case PackageExtensionType.Variants: { + const currentVariants = pkg.variants; + if (currentVariants === null || compareVariantConfiguration(currentVariants, extension.variants)) { + extension.status = PackageExtensionStatus.Active; + pkg.variants = extension.variants; + } + } break; + default: { miscUtils.assertNever(extension); } break; diff --git a/packages/yarnpkg-core/sources/CorePlugin.ts b/packages/yarnpkg-core/sources/CorePlugin.ts index 2a1270eff917..670f5243d222 100644 --- a/packages/yarnpkg-core/sources/CorePlugin.ts +++ b/packages/yarnpkg-core/sources/CorePlugin.ts @@ -1,10 +1,12 @@ -import {MessageName} from './MessageName'; -import {Plugin} from './Plugin'; -import {Project} from './Project'; -import {Resolver, ResolveOptions} from './Resolver'; -import {Workspace} from './Workspace'; -import * as structUtils from './structUtils'; -import {Descriptor, Locator} from './types'; +import {VariantParameters} from './Manifest'; +import {MessageName} from './MessageName'; +import {Plugin} from './Plugin'; +import {Project} from './Project'; +import {Resolver, ResolveOptions} from './Resolver'; +import {Workspace} from './Workspace'; +import * as structUtils from './structUtils'; +import {Descriptor, Locator} from './types'; +import {VariantParameterComparators} from './variantUtils'; export const CorePlugin: Plugin = { hooks: { @@ -32,6 +34,46 @@ export const CorePlugin: Plugin = { return dependency; }, + reduceVariantParameters: (variantParameters: VariantParameters, dependency: Descriptor, project: Project, locator: Locator, initialDependency: Descriptor, {resolver, resolveOptions}: {resolver: Resolver, resolveOptions: ResolveOptions}) => { + if (dependency.scope === null && dependency.name === `electron`) { + return { + ...variantParameters, + electron: dependency.range, + }; + } + + return variantParameters; + }, + + reduceVariantStartingParameters: (variantParameters: VariantParameters, project: Project, workspace: Workspace) => { + // TODO: What else should be there by default? + + const parameters: VariantParameters = { + ...variantParameters, + platform: process.platform, + arch: process.arch, + abi: process.versions.modules, + }; + + if ((process.versions as any).napi) + parameters.napi = (process.versions as any).napi; + + return parameters; + }, + + reduceVariantParameterComparators: (variantParameterComparators: VariantParameterComparators) => { + // TODO: What else should be there by default? + + const parameterComparators: VariantParameterComparators = { + ...variantParameterComparators, + }; + + if ((process.versions as any).napi) + parameterComparators.napi = (parameterValue, possiblityValue) => parseInt(parameterValue) >= parseInt(possiblityValue); + + return parameterComparators; + }, + validateProject: async (project: Project, report: { reportWarning: (name: MessageName, text: string) => void, reportError: (name: MessageName, text: string) => void, diff --git a/packages/yarnpkg-core/sources/Manifest.ts b/packages/yarnpkg-core/sources/Manifest.ts index 6cd65991baaf..a1b2f9d135d4 100644 --- a/packages/yarnpkg-core/sources/Manifest.ts +++ b/packages/yarnpkg-core/sources/Manifest.ts @@ -15,10 +15,62 @@ export interface WorkspaceDefinition { pattern: string; } +export interface Variants { + /** + * Which package to fetch? + * + * A pattern with % delimited strings that will be replaced. + * + * prisma-build-%platform-%napi + */ + pattern: string + /** + * What are the supported values? + * Follows the same idea as GH Actions' matrix + * + * "matrix": { + * "platform": ["darwin", "win32"], + * "napi": ["5", "6"] + * }, + */ + matrix?: VariantMatrix + + /** + * Some combinations that specifically aren't supported + * + * "exclude": [{ + * "platform": "win32", + * "napi": "5" + * }], + */ + exclude?: Array + + /** + * Some combinations that specifically are supported. + * + * "include": [{ + * "platform": "linux", + * "napi": "6" + * }], + * + * If a combination is both included and excluded, it is included. + */ + include?: Array +} + +export interface VariantParameters { + [parameter: string]: string +} + +export interface VariantMatrix { + [parameter: string]: Array +} + export interface DependencyMeta { built?: boolean; optional?: boolean; unplugged?: boolean; + parameters?: VariantParameters } export interface PeerDependencyMeta { @@ -72,6 +124,7 @@ export class Manifest { public peerDependenciesMeta: Map = new Map(); public resolutions: Array<{pattern: Resolution, reference: string}> = []; + public variants: Array | null = null; public files: Set | null = null; public publishConfig: PublishConfig | null = null; @@ -259,6 +312,17 @@ export class Manifest { else this.main = null; + if (typeof data.variants === `object` && data.variants !== null) { + if (Array.isArray(data.variants)) { + this.variants = data.variants; + } else { + this.variants = [data.variants]; // TODO: Actually parse this instead of blindly copying + } + // console.log(`Manfiest found a variants`, this.name); + } else { + this.variants = null; + } + if (typeof data.module === `string`) this.module = normalizeSlashes(data.module); else @@ -809,6 +873,13 @@ export class Manifest { delete data.bin; } + if (this.variants) + data.variants = this.variants; + // console.log(`we have variants for dependency ${this.name}`); + else + delete data.variants; + + if (this.workspaceDefinitions.length > 0) { if (this.raw.workspaces && !Array.isArray(this.raw.workspaces)) { data.workspaces = {...this.raw.workspaces, packages: this.workspaceDefinitions.map(({pattern}) => pattern)}; diff --git a/packages/yarnpkg-core/sources/Plugin.ts b/packages/yarnpkg-core/sources/Plugin.ts index 0ebb5f33f2d6..696a72559764 100644 --- a/packages/yarnpkg-core/sources/Plugin.ts +++ b/packages/yarnpkg-core/sources/Plugin.ts @@ -5,11 +5,13 @@ import {Writable, Readable} import {PluginConfiguration, Configuration, ConfigurationDefinitionMap, PackageExtensionData} from './Configuration'; import {Fetcher} from './Fetcher'; import {Linker} from './Linker'; +import {VariantParameters} from './Manifest'; import {MessageName} from './MessageName'; import {Project, InstallOptions} from './Project'; import {Resolver, ResolveOptions} from './Resolver'; import {Workspace} from './Workspace'; import {Locator, Descriptor} from './types'; +import {VariantParameterComparators} from './variantUtils'; type ProcessEnvironment = {[key: string]: string}; @@ -95,6 +97,36 @@ export type Hooks = { extra: {resolver: Resolver, resolveOptions: ResolveOptions}, ) => Promise, + /** + * Before the resolution runs; should be used to set variant parameters based on dependencies. + */ + reduceVariantParameters?: ( + variantParameters: VariantParameters, + dependency: Descriptor, + project: Project, + locator: Locator, + initialDependency: Descriptor, + extra: {resolver: Resolver, resolveOptions: ResolveOptions}, + ) => Promise, + + /** + * Before the resolution runs; should be used to set starting variant parameters. + */ + reduceVariantStartingParameters?: ( + variantParameters: VariantParameters, + project: Project, + workspace: Workspace, + ) => Promise, + + /** + * Custom comparator functions for variant parameters. + * + * For example, Node is backwards compatible with NAPI versions, they don't need to match exactly. + */ + reduceVariantParameterComparators?: ( + variantParameterComparators: VariantParameterComparators, + ) => Promise, + /** * Called after the `install` method from the `Project` class successfully * completed. diff --git a/packages/yarnpkg-core/sources/Project.ts b/packages/yarnpkg-core/sources/Project.ts index cb3577992132..f1b0a3f6edd3 100644 --- a/packages/yarnpkg-core/sources/Project.ts +++ b/packages/yarnpkg-core/sources/Project.ts @@ -1,41 +1,44 @@ -import {PortablePath, ppath, xfs, normalizeLineEndings, Filename} from '@yarnpkg/fslib'; -import {npath} from '@yarnpkg/fslib'; -import {parseSyml, stringifySyml} from '@yarnpkg/parsers'; -import {UsageError} from 'clipanion'; -import {createHash} from 'crypto'; -import {structuredPatch} from 'diff'; -import pick from 'lodash/pick'; -import pLimit from 'p-limit'; -import semver from 'semver'; -import {promisify} from 'util'; -import v8 from 'v8'; -import zlib from 'zlib'; - -import {Cache} from './Cache'; -import {Configuration} from './Configuration'; -import {Fetcher} from './Fetcher'; -import {Installer, BuildDirective, BuildType} from './Installer'; -import {LegacyMigrationResolver} from './LegacyMigrationResolver'; -import {Linker} from './Linker'; -import {LockfileResolver} from './LockfileResolver'; -import {DependencyMeta, Manifest} from './Manifest'; -import {MessageName} from './MessageName'; -import {MultiResolver} from './MultiResolver'; -import {Report, ReportError} from './Report'; -import {ResolveOptions, Resolver} from './Resolver'; -import {RunInstallPleaseResolver} from './RunInstallPleaseResolver'; -import {ThrowReport} from './ThrowReport'; -import {Workspace} from './Workspace'; -import {isFolderInside} from './folderUtils'; -import * as formatUtils from './formatUtils'; -import * as hashUtils from './hashUtils'; -import * as miscUtils from './miscUtils'; -import * as scriptUtils from './scriptUtils'; -import * as semverUtils from './semverUtils'; -import * as structUtils from './structUtils'; -import {IdentHash, DescriptorHash, LocatorHash, PackageExtensionStatus} from './types'; -import {Descriptor, Ident, Locator, Package} from './types'; -import {LinkType} from './types'; +import {PortablePath, ppath, xfs, normalizeLineEndings, Filename} from '@yarnpkg/fslib'; +import {npath} from '@yarnpkg/fslib'; +import {parseSyml, stringifySyml} from '@yarnpkg/parsers'; +import {generatePrettyJson} from '@yarnpkg/pnp/sources/generatePrettyJson'; +import {UsageError} from 'clipanion'; +import {createHash} from 'crypto'; +import {structuredPatch} from 'diff'; +import pick from 'lodash/pick'; +import pLimit from 'p-limit'; +import semver from 'semver'; +import {promisify} from 'util'; +import v8 from 'v8'; +import zlib from 'zlib'; + +import {Cache} from './Cache'; +import {Configuration} from './Configuration'; +import {Fetcher} from './Fetcher'; +import {Installer, BuildDirective, BuildType} from './Installer'; +import {LegacyMigrationResolver} from './LegacyMigrationResolver'; +import {Linker} from './Linker'; +import {LockfileResolver} from './LockfileResolver'; +import {DependencyMeta, Manifest, VariantParameters} from './Manifest'; +import {MessageName} from './MessageName'; +import {MultiResolver} from './MultiResolver'; +import {Report, ReportError} from './Report'; +import {ResolveOptions, Resolver} from './Resolver'; +import {RunInstallPleaseResolver} from './RunInstallPleaseResolver'; +import {ThrowReport} from './ThrowReport'; +import {WorkspaceResolver} from './WorkspaceResolver'; +import {Workspace} from './Workspace'; +import {isFolderInside} from './folderUtils'; +import * as formatUtils from './formatUtils'; +import * as hashUtils from './hashUtils'; +import * as miscUtils from './miscUtils'; +import * as scriptUtils from './scriptUtils'; +import * as semverUtils from './semverUtils'; +import * as structUtils from './structUtils'; +import {IdentHash, DescriptorHash, LocatorHash, PackageExtensionStatus} from './types'; +import {Descriptor, Ident, Locator, Package} from './types'; +import {LinkType} from './types'; +import {combineVariantMatrix, matchVariantParameters, templateVariantPattern} from './variantUtils'; // When upgraded, the lockfile entries have to be resolved again (but the specific // versions are still pinned, no worry). Bump it when you change the fields within @@ -312,6 +315,7 @@ export class Project { const peerDependenciesMeta = manifest.peerDependenciesMeta; const bin = manifest.bin; + const variants = manifest.variants; if (data.checksum != null) { const checksum = typeof cacheKey !== `undefined` && !data.checksum.includes(`/`) @@ -322,7 +326,7 @@ export class Project { } if (lockfileVersion >= LOCKFILE_VERSION) { - const pkg: Package = {...locator, version, languageName, linkType, dependencies, peerDependencies, dependenciesMeta, peerDependenciesMeta, bin}; + const pkg: Package = {...locator, version, languageName, linkType, dependencies, peerDependencies, dependenciesMeta, peerDependenciesMeta, bin, variants}; this.originalPackages.set(pkg.locatorHash, pkg); } @@ -596,26 +600,7 @@ export class Project { } getDependencyMeta(ident: Ident, version: string | null): DependencyMeta { - const dependencyMeta = {}; - - const dependenciesMeta = this.topLevelWorkspace.manifest.dependenciesMeta; - const dependencyMetaSet = dependenciesMeta.get(structUtils.stringifyIdent(ident)); - - if (!dependencyMetaSet) - return dependencyMeta; - - const defaultMeta = dependencyMetaSet.get(null); - if (defaultMeta) - Object.assign(dependencyMeta, defaultMeta); - - if (version === null || !semver.valid(version)) - return dependencyMeta; - - for (const [range, meta] of dependencyMetaSet) - if (range !== null && range === version) - Object.assign(dependencyMeta, meta); - - return dependencyMeta; + return this.topLevelWorkspace.getDependencyMeta(ident, version); } async findLocatorForLocation(cwd: PortablePath, {strict = false}: {strict?: boolean} = {}) { @@ -691,7 +676,18 @@ export class Project { const resolutionQueue: Array> = []; - const startPackageResolution = async (locator: Locator) => { + const variantParameterComparators = await this.configuration.reduceHook(hooks => { + return hooks.reduceVariantParameterComparators; + }, {}); + + const cacheParameters = this.configuration.get(`cacheParameters`); + const cacheParameterMatrix = combineVariantMatrix(cacheParameters?.matrix ?? {}, cacheParameters?.exclude); + cacheParameterMatrix.push(...(cacheParameters?.include ?? [])); + + if (cacheParameterMatrix.length > 0) + opts.report.reportInfo(MessageName.UNNAMED, `Variant cache may hold up to ${cacheParameterMatrix.length} versions`); + + const startPackageResolution = async (locator: Locator, variantParameters: VariantParameters, workspace: Workspace, pkgParent?: Package) => { const originalPkg = await miscUtils.prettifyAsyncErrors(async () => { return await resolver.resolve(locator, resolveOptions); }, message => { @@ -703,9 +699,166 @@ export class Project { originalPackages.set(originalPkg.locatorHash, originalPkg); - const pkg = this.configuration.normalizePackage(originalPkg); + let pkg = this.configuration.normalizePackage(originalPkg); + + const prettyParent = () => { + if (pkgParent) + return structUtils.prettyLocator(this.configuration, pkgParent); + + const workspaceName = workspace.manifest.name; + + if (workspaceName) + return structUtils.prettyIdent(this.configuration, workspaceName); + + return `workspace without name`; + }; + + if (pkg.variants) { + // If a package has variants, remove the descriptorResolutionPromises so that it gets re-resolved each time it's a dependency, + // since its resolution might be different each time. + descriptorResolutionPromises.delete(structUtils.convertLocatorToDescriptor(pkg).descriptorHash); + packageResolutionPromises.delete(locator.locatorHash); + opts.report.reportInfo(MessageName.UNNAMED, `detected variant dependency, ${prettyParent()}'s dependency ${structUtils.prettyLocator(this.configuration, pkg)}, not caching resolution`); + } + + // See if we need to replace this package + // Only do this for packages that are dependencies, workspaces may be transformed by variants + // but not when they are the workspace being resolved. + if (pkg.variants && pkgParent) { + const pkgWorkspace = workspace.project.tryWorkspaceByLocator(pkg); + + // The package has to have a version + let pkgVersion = pkg.version; + + // If the package is a workspace, access the actual workspace to get the actual version in case it's 0.0.0-use-local + if (pkgWorkspace && pkgWorkspace.manifest.version) + pkgVersion = pkgWorkspace.manifest.version; + + const thisPackageVariantParameters = { + ...variantParameters, + ...(this.getDependencyMeta(pkg, pkgVersion).parameters ?? {}), + ...(pkgVersion ? {version: pkgVersion} : {}), + }; + + // Iterate over the variants, trying to find a match + for (const potentialVariants of pkg.variants) { + // The matrix includes the package version, even if no matrix exists + // This provides our fallback behaviour + const matrix = { + ...potentialVariants.matrix ?? {}, + ...(pkgVersion ? {version: [pkgVersion]} : {}), + }; + + // The parameter combinations possible for this variant matrix + const possibilities = combineVariantMatrix(matrix, potentialVariants.exclude); + possibilities.push(...(potentialVariants.include ?? [])); + const matches = matchVariantParameters(possibilities, thisPackageVariantParameters, variantParameterComparators); + // variantDebug(`pattern: `, potentialVariants.pattern); + // variantDebug(`possibilities: `, possibilities); + // variantDebug(`matches:`, matches); + + // Convert our matches into descriptors + const matchDescriptors = matches.map(match => { + return templateVariantPattern(potentialVariants.pattern, match); + }); + + // Calculate the matches for the cache + const cacheMatches: Array = []; + + for (const cacheParameters of cacheParameterMatrix) { + const mergedCacheParameters = { + ...thisPackageVariantParameters, + ...cacheParameters, + }; + cacheMatches.push(...matchVariantParameters(possibilities, mergedCacheParameters, variantParameterComparators)); + } + + const cacheMatchDescriptors = cacheMatches.map(match => { + const matchParameters = pkgVersion ? {...match, version: pkgVersion} : match; + return templateVariantPattern(potentialVariants.pattern, matchParameters); + }); + + // In parallel, schedule resolution of all our cached descriptors + await Promise.all( + cacheMatchDescriptors.map(async matchDescriptor => { + try { + const alreadyResolved = descriptorResolutionPromises.has(matchDescriptor.descriptorHash); + const cacheEntry = await scheduleDescriptorResolution(matchDescriptor, variantParameters, workspace, pkgParent); + + if (!alreadyResolved) { + opts.report.reportInfo(MessageName.UNNAMED, `Adding variant cache entry:, ${ + structUtils.prettyLocator(this.configuration, pkg) + } -> ${ + structUtils.prettyLocator(this.configuration, cacheEntry) + }`); + } + } catch (resolveFailure) { + // Don't worry about it + opts.report.reportError(MessageName.UNNAMED, `Resolve failure for cache, ${ + resolveFailure + }`); + } + }) + ); + + // For each potential match, try and resolve it, if it succeeds, stop searching + matchLoop: for (const matchDescriptor of matchDescriptors) { + try { + const pkgParentOrWorkspace = pkgParent ?? workspace; + + const boundMatchDescriptor = resolver.bindDescriptor(matchDescriptor, pkgParentOrWorkspace, resolveOptions); + + // Find the old dependency in our parent + parentDependencyLoop: for (const [parentDependencyPkgIdentHash, parentDependencyPkgDescriptor] of pkgParentOrWorkspace.dependencies) { + if (parentDependencyPkgDescriptor.name === pkg.name && parentDependencyPkgDescriptor.scope === pkg.scope) { + // Use the VariantRemapResolver + const remapDescriptor = structUtils.makeDescriptor( + parentDependencyPkgDescriptor, + `variant:${structUtils.stringifyDescriptor(boundMatchDescriptor)}` + ); + + opts.report.reportInfo(MessageName.UNNAMED, `Variant replacement remap descriptor: ${structUtils.prettyDescriptor(this.configuration, remapDescriptor)}`); + + // Resolve it + const resolveAttempt = await scheduleDescriptorResolution(remapDescriptor, variantParameters, workspace, pkgParent); + + // Remap the dependency to the remapDescriptor if it succeeded + pkgParentOrWorkspace.dependencies.set(parentDependencyPkgIdentHash, remapDescriptor); + + opts.report.reportInfo(MessageName.UNNAMED, `Variant replacement: ${prettyParent()}'s dependency ${ + structUtils.prettyLocator(this.configuration, pkg) + } -> ${ + structUtils.prettyLocator(this.configuration, resolveAttempt) + }`); + + // We'll be grabbing _this_ package's dependencies next. + pkg = resolveAttempt; + + opts.report.reportInfo(MessageName.UNNAMED, `Environment used: ${JSON.stringify(thisPackageVariantParameters)}`); + + break parentDependencyLoop; + } + } + + break matchLoop; + } catch (resolveFailure) { + // Don't worry about it + opts.report.reportError(MessageName.UNNAMED, `Variant resolve failure ${resolveFailure}`); + } + } + } + } + + const resolutionQueueNext: Array> = []; for (const [identHash, descriptor] of pkg.dependencies) { + const variantParametersNext = await this.configuration.reduceHook(hooks => { + return hooks.reduceVariantParameters; + }, variantParameters, descriptor, this, pkg, descriptor, { + resolver, + resolveOptions, + }); + const dependency = await this.configuration.reduceHook(hooks => { return hooks.reduceDependency; }, descriptor, this, pkg, descriptor, { @@ -718,29 +871,31 @@ export class Project { const bound = resolver.bindDescriptor(dependency, locator, resolveOptions); pkg.dependencies.set(identHash, bound); + + resolutionQueueNext.push( + scheduleDescriptorResolution(bound, variantParametersNext, workspace, pkg) // This package is the parent for the next resolutions + ); } - resolutionQueue.push(Promise.all([...pkg.dependencies.values()].map(descriptor => { - return scheduleDescriptorResolution(descriptor); - }))); + resolutionQueue.push(Promise.all(resolutionQueueNext)); allPackages.set(pkg.locatorHash, pkg); return pkg; }; - const schedulePackageResolution = async (locator: Locator) => { + const schedulePackageResolution = async (locator: Locator, variantParameters: VariantParameters, workspace: Workspace, pkgParent?: Package) => { const promise = packageResolutionPromises.get(locator.locatorHash); if (typeof promise !== `undefined`) return promise; - const newPromise = Promise.resolve().then(() => startPackageResolution(locator)); + const newPromise = Promise.resolve().then(() => startPackageResolution(locator, variantParameters, workspace, pkgParent)); packageResolutionPromises.set(locator.locatorHash, newPromise); return newPromise; }; - const startDescriptorAliasing = async (descriptor: Descriptor, alias: Descriptor): Promise => { - const resolution = await scheduleDescriptorResolution(alias); + const startDescriptorAliasing = async (descriptor: Descriptor, alias: Descriptor, variantParameters: VariantParameters, workspace: Workspace, pkgParent?: Package): Promise => { + const resolution = await scheduleDescriptorResolution(alias, variantParameters, workspace, pkgParent); allDescriptors.set(descriptor.descriptorHash, descriptor); allResolutions.set(descriptor.descriptorHash, resolution.locatorHash); @@ -748,14 +903,14 @@ export class Project { return resolution; }; - const startDescriptorResolution = async (descriptor: Descriptor): Promise => { + const startDescriptorResolution = async (descriptor: Descriptor, variantParameters: VariantParameters, workspace: Workspace, pkgParent?: Package): Promise => { const alias = this.resolutionAliases.get(descriptor.descriptorHash); if (typeof alias !== `undefined`) - return startDescriptorAliasing(descriptor, this.storedDescriptors.get(alias)!); + return startDescriptorAliasing(descriptor, this.storedDescriptors.get(alias)!, variantParameters, workspace, pkgParent); const resolutionDependencies = resolver.getResolutionDependencies(descriptor, resolveOptions); const resolvedDependencies = new Map(await Promise.all(resolutionDependencies.map(async dependency => { - return [dependency.descriptorHash, await scheduleDescriptorResolution(dependency)] as const; + return [dependency.descriptorHash, await scheduleDescriptorResolution(dependency, variantParameters, workspace, pkgParent)] as const; }))); const candidateResolutions = await miscUtils.prettifyAsyncErrors(async () => { @@ -771,24 +926,28 @@ export class Project { allDescriptors.set(descriptor.descriptorHash, descriptor); allResolutions.set(descriptor.descriptorHash, finalResolution.locatorHash); - return schedulePackageResolution(finalResolution); + return schedulePackageResolution(finalResolution, variantParameters, workspace, pkgParent); }; - const scheduleDescriptorResolution = (descriptor: Descriptor) => { + const scheduleDescriptorResolution = (descriptor: Descriptor, variantParameters: VariantParameters, workspace: Workspace, pkgParent?: Package) => { const promise = descriptorResolutionPromises.get(descriptor.descriptorHash); if (typeof promise !== `undefined`) return promise; allDescriptors.set(descriptor.descriptorHash, descriptor); - const newPromise = Promise.resolve().then(() => startDescriptorResolution(descriptor)); + const newPromise = Promise.resolve().then(() => startDescriptorResolution(descriptor, variantParameters, workspace, pkgParent)); descriptorResolutionPromises.set(descriptor.descriptorHash, newPromise); return newPromise; }; for (const workspace of this.workspaces) { + const startingVariantParameters = await this.configuration.reduceHook(hooks => { + return hooks.reduceVariantStartingParameters; + }, {}, this, workspace); + const workspaceDescriptor = workspace.anchoredDescriptor; - resolutionQueue.push(scheduleDescriptorResolution(workspaceDescriptor)); + resolutionQueue.push(scheduleDescriptorResolution(workspaceDescriptor, startingVariantParameters, workspace)); } while (resolutionQueue.length > 0) { @@ -1042,7 +1201,7 @@ export class Project { for (const descriptor of pkg.dependencies.values()) { const resolution = this.storedResolutions.get(descriptor.descriptorHash); if (typeof resolution === `undefined`) - throw new Error(`Assertion failed: The resolution (${structUtils.prettyDescriptor(this.configuration, descriptor)}, from ${structUtils.prettyLocator(this.configuration, pkg)})should have been registered`); + throw new Error(`Assertion failed: The resolution (${structUtils.prettyDescriptor(this.configuration, descriptor)}, from ${structUtils.prettyLocator(this.configuration, pkg)}) should have been registered`); const dependency = this.storedPackages.get(resolution); if (typeof dependency === `undefined`) @@ -1499,6 +1658,7 @@ export class Project { await this.persistInstallStateFile(); + await this.configuration.triggerHook(hooks => { return hooks.afterAllInstalled; }, this, opts); @@ -1563,6 +1723,8 @@ export class Project { manifest.peerDependenciesMeta = new Map(pkg.peerDependenciesMeta); manifest.bin = new Map(pkg.bin); + manifest.variants = pkg.variants; + let entryChecksum: string | undefined; const checksum = this.storedChecksums.get(pkg.locatorHash); @@ -1872,6 +2034,7 @@ function applyVirtualResolutionMutations({ const thirdPass = []; const fourthPass = []; + // During this first pass we virtualize the descriptors. This allows us // to reference them from their sibling without being order-dependent, // which is required to solve cases where packages with peer dependencies @@ -1904,6 +2067,14 @@ function applyVirtualResolutionMutations({ } } + // if (descriptor.name === `app-builder-bin`) { + // variantDebug(`Found app builder bin at 2018, ${parentPackage.name} is requiring it, its deps are:`); + + // for (const descriptor of Array.from(parentPackage.dependencies.values())) { + // variantDebug(`${structUtils.prettyDescriptor(project.configuration, descriptor)} has has ${descriptor.descriptorHash}`); + // } + // } + const resolution = allResolutions.get(descriptor.descriptorHash); if (!resolution) { // Note that we can't use `getPackageFromDescriptor` (defined below, @@ -2071,7 +2242,7 @@ function applyVirtualResolutionMutations({ : `missing:`; if (typeof resolution === `undefined`) - throw new Error(`Assertion failed: Expected the resolution for ${structUtils.prettyDescriptor(project.configuration, descriptor)} to have been registered`); + throw new Error(`Assertion failed: Expected the resolution for ${structUtils.prettyDescriptor(project.configuration, descriptor)} to have been registered.`); return resolution === top ? `${resolution} (top)` : resolution; }), diff --git a/packages/yarnpkg-core/sources/VariantRemapResolver.ts b/packages/yarnpkg-core/sources/VariantRemapResolver.ts new file mode 100644 index 000000000000..fce4669b2bf3 --- /dev/null +++ b/packages/yarnpkg-core/sources/VariantRemapResolver.ts @@ -0,0 +1,53 @@ +import {Descriptor, Locator, MinimalResolveOptions, ResolveOptions, Resolver, DescriptorHash, Package} from '@yarnpkg/core'; +import {structUtils} from '@yarnpkg/core'; + +export const PROTOCOL = `variant:`; + +export class VariantRemapResolver implements Resolver { + supportsDescriptor(descriptor: Descriptor, opts: MinimalResolveOptions) { + if (!descriptor.range.startsWith(PROTOCOL)) + return false; + + if (!structUtils.tryParseDescriptor(descriptor.range.slice(PROTOCOL.length), true)) + return false; + + return true; + } + + supportsLocator(locator: Locator, opts: MinimalResolveOptions) { + // Once transformed into locators, the descriptors are resolved by the variant's resolver + return false; + } + + shouldPersistResolution(locator: Locator, opts: MinimalResolveOptions): never { + // Once transformed into locators, the descriptors are resolved by the variant's resolver + throw new Error(`Unreachable`); + } + + bindDescriptor(descriptor: Descriptor, fromLocator: Locator, opts: MinimalResolveOptions) { + return descriptor; + } + + getResolutionDependencies(descriptor: Descriptor, opts: MinimalResolveOptions) { + const nextDescriptor = structUtils.parseDescriptor(descriptor.range.slice(PROTOCOL.length), true); + + return opts.resolver.getResolutionDependencies(nextDescriptor, opts); + } + + async getCandidates(descriptor: Descriptor, dependencies: Map, opts: ResolveOptions) { + const nextDescriptor = structUtils.parseDescriptor(descriptor.range.slice(PROTOCOL.length), true); + + return await opts.resolver.getCandidates(nextDescriptor, dependencies, opts); + } + + async getSatisfying(descriptor: Descriptor, references: Array, opts: ResolveOptions) { + const nextDescriptor = structUtils.parseDescriptor(descriptor.range.slice(PROTOCOL.length), true); + + return opts.resolver.getSatisfying(nextDescriptor, references, opts); + } + + resolve(locator: Locator, opts: ResolveOptions): never { + // Once transformed into locators, the descriptors are resolved by the variant's resolver + throw new Error(`Unreachable`); + } +} diff --git a/packages/yarnpkg-core/sources/Workspace.ts b/packages/yarnpkg-core/sources/Workspace.ts index b813e4733d5b..fac3e1cc9ecf 100644 --- a/packages/yarnpkg-core/sources/Workspace.ts +++ b/packages/yarnpkg-core/sources/Workspace.ts @@ -1,14 +1,15 @@ -import {PortablePath, npath, ppath, xfs, Filename} from '@yarnpkg/fslib'; -import globby from 'globby'; - -import {HardDependencies, Manifest} from './Manifest'; -import {Project} from './Project'; -import {WorkspaceResolver} from './WorkspaceResolver'; -import * as hashUtils from './hashUtils'; -import * as semverUtils from './semverUtils'; -import * as structUtils from './structUtils'; -import {IdentHash} from './types'; -import {Descriptor, Locator} from './types'; +import {PortablePath, npath, ppath, xfs, Filename} from '@yarnpkg/fslib'; +import globby from 'globby'; +import semver from 'semver'; + +import {DependencyMeta, HardDependencies, Manifest} from './Manifest'; +import {Project} from './Project'; +import {WorkspaceResolver} from './WorkspaceResolver'; +import * as hashUtils from './hashUtils'; +import * as semverUtils from './semverUtils'; +import * as structUtils from './structUtils'; +import {Ident, IdentHash} from './types'; +import {Descriptor, Locator} from './types'; export class Workspace { public readonly project: Project; @@ -192,4 +193,27 @@ export class Workspace { this.manifest.raw = data; } + + getDependencyMeta(ident: Ident, version: string | null): DependencyMeta { + const dependencyMeta = {}; + + const dependenciesMeta = this.manifest.dependenciesMeta; + const dependencyMetaSet = dependenciesMeta.get(structUtils.stringifyIdent(ident)); + + if (!dependencyMetaSet) + return dependencyMeta; + + const defaultMeta = dependencyMetaSet.get(null); + if (defaultMeta) + Object.assign(dependencyMeta, defaultMeta); + + if (version === null || !semver.valid(version)) + return dependencyMeta; + + for (const [range, meta] of dependencyMetaSet) + if (range !== null && range === version) + Object.assign(dependencyMeta, meta); + + return dependencyMeta; + } } diff --git a/packages/yarnpkg-core/sources/WorkspaceResolver.ts b/packages/yarnpkg-core/sources/WorkspaceResolver.ts index 47a6dfdf34f0..6fc6dca155f3 100644 --- a/packages/yarnpkg-core/sources/WorkspaceResolver.ts +++ b/packages/yarnpkg-core/sources/WorkspaceResolver.ts @@ -65,6 +65,7 @@ export class WorkspaceResolver implements Resolver { peerDependenciesMeta: workspace.manifest.peerDependenciesMeta, bin: workspace.manifest.bin, + variants: workspace.manifest.variants, }; } } diff --git a/packages/yarnpkg-core/sources/formatUtils.ts b/packages/yarnpkg-core/sources/formatUtils.ts index ab3403c9fcc7..5912c521155a 100644 --- a/packages/yarnpkg-core/sources/formatUtils.ts +++ b/packages/yarnpkg-core/sources/formatUtils.ts @@ -170,6 +170,8 @@ const transforms = { return `${structUtils.prettyIdent(configuration, packageExtension.parentDescriptor)} ➤ ${applyColor(configuration, `peerDependencies`, Type.CODE)} ➤ ${structUtils.prettyIdent(configuration, packageExtension.descriptor)}`; case PackageExtensionType.PeerDependencyMeta: return `${structUtils.prettyIdent(configuration, packageExtension.parentDescriptor)} ➤ ${applyColor(configuration, `peerDependenciesMeta`, Type.CODE)} ➤ ${structUtils.prettyIdent(configuration, structUtils.parseIdent(packageExtension.selector))} ➤ ${applyColor(configuration, packageExtension.key, Type.CODE)}`; + case PackageExtensionType.Variants: + return `${structUtils.prettyIdent(configuration, packageExtension.parentDescriptor)} Variant Information`; default: throw new Error(`Assertion failed: Unsupported package extension type: ${(packageExtension as PackageExtension).type}`); } @@ -182,6 +184,8 @@ const transforms = { return `${structUtils.stringifyIdent(packageExtension.parentDescriptor)} >> ${structUtils.stringifyIdent(packageExtension.descriptor)}`; case PackageExtensionType.PeerDependencyMeta: return `${structUtils.stringifyIdent(packageExtension.parentDescriptor)} >> ${packageExtension.selector} / ${packageExtension.key}`; + case PackageExtensionType.Variants: + return `${structUtils.stringifyIdent(packageExtension.parentDescriptor)} > Variant Information`; default: throw new Error(`Assertion failed: Unsupported package extension type: ${(packageExtension as PackageExtension).type}`); } diff --git a/packages/yarnpkg-core/sources/structUtils.ts b/packages/yarnpkg-core/sources/structUtils.ts index fd35b4bd25bd..1c7802159437 100644 --- a/packages/yarnpkg-core/sources/structUtils.ts +++ b/packages/yarnpkg-core/sources/structUtils.ts @@ -127,6 +127,8 @@ export function renamePackage(pkg: Package, locator: Locator): Package { peerDependenciesMeta: new Map(pkg.peerDependenciesMeta), bin: new Map(pkg.bin), + + variants: pkg.variants, }; } diff --git a/packages/yarnpkg-core/sources/types.ts b/packages/yarnpkg-core/sources/types.ts index 41072c9db305..0c03ed44d33c 100644 --- a/packages/yarnpkg-core/sources/types.ts +++ b/packages/yarnpkg-core/sources/types.ts @@ -1,6 +1,6 @@ -import {PortablePath} from '@yarnpkg/fslib'; +import {PortablePath} from '@yarnpkg/fslib'; -import {DependencyMeta, PeerDependencyMeta} from './Manifest'; +import {DependencyMeta, PeerDependencyMeta, Variants} from './Manifest'; /** * Unique hash of a package descriptor. Used as key in various places so that @@ -158,7 +158,7 @@ export interface Package extends Locator { peerDependenciesMeta: Map, /** - * All `bin` entries defined by the package + * All `bin` entries defined by the package * * While we don't need the binaries during the resolution, keeping them * within the lockfile is critical to make `yarn run` fast (otherwise we @@ -167,12 +167,18 @@ export interface Package extends Locator { * called at every keystroke) */ bin: Map, + + /** + * The `variants` defined by the package. Needs to be kept in the lockfile. + */ + variants: Array | null } export enum PackageExtensionType { Dependency = `Dependency`, PeerDependency = `PeerDependency`, PeerDependencyMeta = `PeerDependencyMeta`, + Variants = `Variants`, } export enum PackageExtensionStatus { @@ -185,6 +191,7 @@ export type PackageExtension = ( | {type: PackageExtensionType.Dependency, descriptor: Descriptor} | {type: PackageExtensionType.PeerDependency, descriptor: Descriptor} | {type: PackageExtensionType.PeerDependencyMeta, selector: string, key: keyof PeerDependencyMeta, value: any} + | {type: PackageExtensionType.Variants, variants: Array} ) & { status: PackageExtensionStatus, userProvided: boolean, diff --git a/packages/yarnpkg-core/sources/variantUtils.ts b/packages/yarnpkg-core/sources/variantUtils.ts new file mode 100644 index 000000000000..477a08e773bc --- /dev/null +++ b/packages/yarnpkg-core/sources/variantUtils.ts @@ -0,0 +1,104 @@ +import {structUtils} from "@yarnpkg/core"; + +import {VariantMatrix, VariantParameters, Variants} from "./Manifest"; +import {Descriptor, Ident, Locator} from "./types"; + + +export function combineVariantMatrix(variantMatrix: VariantMatrix, exclusions: Array = [], keyIndex = 0, combinations: Array = [], stack: VariantParameters = {}) { + const keys = Object.keys(variantMatrix); + + // If there's no matrix, there's no combinations + if (keys.length === 0) + return []; + + if (keyIndex === keys.length) { + // Check if this matches an exclusion record + let include = true; + + exclusionLoop: for (const exclusion of exclusions) { + let matchExclusion = true; + exclusionKeyLoop: for (const key of Object.keys(exclusion)) { + // Check if this key of this exclusion criteria matches the parameters we're about to add + if (exclusion[key] !== stack[key]) { + matchExclusion = false; + break exclusionKeyLoop; + } + } + if (matchExclusion) { + include = false; + break exclusionLoop; + } + } + + if (include) { + combinations.push(stack); + } + } else { + const key = keys[keyIndex]; + const candidates = variantMatrix[key]; + + for (const candidate of candidates) { + combineVariantMatrix(variantMatrix, exclusions, keyIndex + 1, combinations, { + ...stack, + [key]: candidate, + }); + } + } + + return combinations; +} + +/** + * Custom functions to determine if parameters are compatible + */ +export interface VariantParameterComparators { + [parameterKey: string]: (parameterValue: string, possibilityValue: string) => boolean +} + +const defaultParameterComparator = (parameterValue: string, possibilityValue: string) => parameterValue === possibilityValue; + +/** + * Given a list of possible parameters, and the current parameters value, match them and return those ones that match + */ +export function matchVariantParameters(possibilities: Array, parameters: VariantParameters, comparators: VariantParameterComparators = {}) { + const matches: Array = []; + + possibilityLoop: for (const possibility of possibilities) { + for (const key of Object.keys(possibility)) { + const comparator = comparators[key] ? comparators[key] : defaultParameterComparator; + + if (!comparator(parameters[key], possibility[key])) { + // If this key doesn't match, skip this possibility + continue possibilityLoop; + } + } + + // If all keys match, return this possiblity + matches.push(possibility); + } + + return matches; +} + +export function templateVariantPattern(pattern: string, parameters: VariantParameters) { + // Parse our pattern, for every parameterKey we have, see if we can replace something + let patternToReplace = pattern; + + for (const parameterKey of Object.keys(parameters)) { + const keyWithPercent = `%${parameterKey}`; + const value = parameters[parameterKey]; + + // Replace every instance of the key with the value + while (patternToReplace.indexOf(keyWithPercent) > 0) { + patternToReplace = patternToReplace.replace(keyWithPercent, value); + } + } + + return structUtils.parseDescriptor(patternToReplace); +} + +// A deep equality +export function compareVariantConfiguration(variantsA: Array | null, variantsB: Array | null) { + // TODO: actually compare + return false; +} diff --git a/packages/yarnpkg-core/tests/variantUtils.test.ts b/packages/yarnpkg-core/tests/variantUtils.test.ts new file mode 100644 index 000000000000..41062ed141c2 --- /dev/null +++ b/packages/yarnpkg-core/tests/variantUtils.test.ts @@ -0,0 +1,86 @@ +import * as variantUtils from '../sources/variantUtils'; + +describe(`variantUtils`, () => { + describe(`combineVariantMatrix`, () => { + it(`should combine a simple matrix`, () => { + expect( + variantUtils.combineVariantMatrix({ + A: [`1`, `2`], + B: [`3`, `4`], + }) + ).toMatchObject([ + {A: `1`, B: `3`}, + {A: `1`, B: `4`}, + {A: `2`, B: `3`}, + {A: `2`, B: `4`}, + ]); + }); + it(`should combine a simple matrix with exclusions`, () => { + expect( + variantUtils.combineVariantMatrix({ + A: [`1`, `2`], + B: [`3`, `4`], + }, [ + { + A: `1`, + B: `3`, + }, + ]) + ).toMatchObject([ + {A: `1`, B: `4`}, + {A: `2`, B: `3`}, + {A: `2`, B: `4`}, + ]); + }); + }); + + describe(`matchVariantParameters`, () => { + it(`should match parameters in the exact case`, () => { + expect( + variantUtils.matchVariantParameters([{ + A: `1`, + }, { + A: `2`, + }, { + A: `3`, + }], { + A: `1`, + }) + ).toMatchObject( + [{A: `1`}], + ); + }); + it(`should match parameters with extraneous ones`, () => { + expect( + variantUtils.matchVariantParameters([{ + A: `1`, + }, { + A: `2`, + }, { + A: `3`, + }], { + A: `1`, + B: `unused`, + C: `unused`, + }) + ).toMatchObject( + [{A: `1`}] + ); + }); + it(`should match parameters with custom comparators`, () => { + expect( + variantUtils.matchVariantParameters([{ + backwardsCompatibleParameter: `3`, + }, { + backwardsCompatibleParameter: `5`, + }], { + backwardsCompatibleParameter: `4`, + }, { + backwardsCompatibleParameter: (parameterValue, possiblityValue) => parseInt(parameterValue) >= parseInt(possiblityValue), + }) + ).toMatchObject( + [{backwardsCompatibleParameter: `3`}] + ); + }); + }); +}); diff --git a/yarn.lock b/yarn.lock index 77166a4e8e85..64d6a8259164 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5283,6 +5283,7 @@ __metadata: tslib: ^1.13.0 typanion: ^3.3.0 typescript: ^4.3.2 + variant-test: "workspace:*" yup: ^0.32.9 peerDependencies: "@yarnpkg/core": ^3.1.0-rc.2 @@ -26516,6 +26517,47 @@ typescript@^4.3.2: languageName: node linkType: hard +"variant-test@workspace:*, variant-test@workspace:packages/variant-test": + version: 0.0.0-use.local + resolution: "variant-test@workspace:packages/variant-test" + languageName: unknown + linkType: soft + variants: + - exclude: + - arch: 32 + platform: darwin + include: + - arch: arm64 + platform: darwin + - arch: arm64 + platform: android + - arch: 64 + platform: freebsd + - arch: arm64 + platform: freebsd + - arch: arm + platform: linux + - arch: arm64 + platform: linux + - arch: mips64le + platform: linux + - arch: ppc64le + platform: linux + - arch: 64 + platform: openbsd + - arch: arm64 + platform: windows + matrix: + arch: + - 32 + - 64 + platform: + - darwin + - linux + - windows + pattern: esbuild-%platform-%arch@%version + - pattern: esbuild-openbsd-64@%version + "vary@npm:^1, vary@npm:~1.1.2": version: 1.1.2 resolution: "vary@npm:1.1.2"