From 0f2579927eb08dc3b87879f7af876f6a286dd119 Mon Sep 17 00:00:00 2001 From: Aramis Sennyey <159921952+aramissennyeydd@users.noreply.github.com> Date: Wed, 18 Sep 2024 17:57:13 -0400 Subject: [PATCH] [rush] move sharded operation config to its own block (#4923) * feat: move sharded operation config to its own block * add changesets * deprecate and support older version * adjust changeset * Update common/changes/@microsoft/rush/sennyeya-shard-schema-updates_2024-09-13-22-46.json Co-authored-by: Ian Clanton-Thuon * adjust upgrading.md * use terminal instance * fix lint error * more lint fixes --------- Co-authored-by: Aramis Sennyey Co-authored-by: Ian Clanton-Thuon --- apps/rush/UPGRADING.md | 42 +++++++++++++++++++ .../projects/a/config/rush-project.json | 10 ++--- .../projects/b/config/rush-project.json | 9 ++-- .../projects/e/config/rush-project.json | 9 +++- ...shard-schema-updates_2024-09-13-22-46.json | 11 +++++ common/reviews/api/rush-lib.api.md | 5 +-- .../src/api/RushProjectConfiguration.ts | 18 ++++---- .../operations/ShardedPhaseOperationPlugin.ts | 10 ++++- .../src/schemas/rush-project.schema.json | 10 +---- 9 files changed, 93 insertions(+), 31 deletions(-) create mode 100644 common/changes/@microsoft/rush/sennyeya-shard-schema-updates_2024-09-13-22-46.json diff --git a/apps/rush/UPGRADING.md b/apps/rush/UPGRADING.md index cb78f05e251..000e3bbb3fc 100644 --- a/apps/rush/UPGRADING.md +++ b/apps/rush/UPGRADING.md @@ -1,5 +1,47 @@ # Upgrade notes for @microsoft/rush +### Rush 5.135.0 + +This release of Rush deprecates the `rush-project.json`'s `operationSettings.sharding.shardOperationSettings` +option in favor of defining a separate operation with a `:shard` suffix. This will only affect projects that +have opted into sharding and have custom sharded operation settings. + +To migrate, +**`rush-project.json`** (OLD) +```json +{ + "operationSettings": [ + { + "operationName": "_phase:build", + "sharding": { + "count": 4, + "shardOperationSettings": { + "weight": 4 + } + }, + } + ] +} +``` + +**`rush-project.json`** (NEW) +```json +{ + "operationSettings": [ + { + "operationName": "_phase:build", + "sharding": { + "count": 4, + }, + }, + { + "operationName": "_phase:build:shard", // note the suffix here + "weight": 4 + } + ] +} +``` + ### Rush 5.60.0 This release of Rush includes a breaking change for the experiment build cache feature. It only affects diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/sharded-repo/projects/a/config/rush-project.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/sharded-repo/projects/a/config/rush-project.json index a4a5351c68b..f6fc90813a8 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/sharded-repo/projects/a/config/rush-project.json +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/sharded-repo/projects/a/config/rush-project.json @@ -6,12 +6,12 @@ "outputFolderNames": ["dist"], "sharding": { "count": 4, - "outputFolderArgumentFormat": "--output-directory=.rush/{phaseName}/shards/{shardIndex}", - - "shardOperationSettings": { - "weight": 4 - } + "outputFolderArgumentFormat": "--output-directory=.rush/{phaseName}/shards/{shardIndex}" } + }, + { + "operationName": "_phase:build:shard", + "weight": 4 } ] } diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/sharded-repo/projects/b/config/rush-project.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/sharded-repo/projects/b/config/rush-project.json index dd223a78a30..105cd2e697c 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/sharded-repo/projects/b/config/rush-project.json +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/sharded-repo/projects/b/config/rush-project.json @@ -5,11 +5,12 @@ "outputFolderNames": ["dist"], "sharding": { "count": 5, - "outputFolderArgumentFormat": "--output-directory=.rush/{phaseName}/shards/{shardIndex}", - "shardOperationSettings": { - "weight": 10 - } + "outputFolderArgumentFormat": "--output-directory=.rush/{phaseName}/shards/{shardIndex}" } + }, + { + "operationName": "_phase:build:shard", + "weight": 10 } ] } diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/sharded-repo/projects/e/config/rush-project.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/sharded-repo/projects/e/config/rush-project.json index 1f1e909883f..aae24577ea5 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/sharded-repo/projects/e/config/rush-project.json +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/sharded-repo/projects/e/config/rush-project.json @@ -4,8 +4,15 @@ "operationName": "_phase:build", "outputFolderNames": ["dist"], "sharding": { - "count": 75 + "count": 75, + "shardOperationSettings": { + "weight": 10 + } } + }, + { + "operationName": "_phase:build:shard", + "weight": 1 } ] } diff --git a/common/changes/@microsoft/rush/sennyeya-shard-schema-updates_2024-09-13-22-46.json b/common/changes/@microsoft/rush/sennyeya-shard-schema-updates_2024-09-13-22-46.json new file mode 100644 index 00000000000..ac11992cbda --- /dev/null +++ b/common/changes/@microsoft/rush/sennyeya-shard-schema-updates_2024-09-13-22-46.json @@ -0,0 +1,11 @@ +{ + "changes": [ + { + "comment": "Deprecate the `sharding.shardOperationSettings` property in the project `config/rush-project.json` in favor of an `operationSettings` entry for an operation with a suffix of `:shard`.", + "type": "none", + "packageName": "@microsoft/rush" + } + ], + "packageName": "@microsoft/rush", + "email": "aramissennyeydd@users.noreply.github.com" +} \ No newline at end of file diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index 9e659294629..b1315c3252d 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -785,9 +785,8 @@ export interface IRushPhaseSharding { count: number; outputFolderArgumentFormat?: string; shardArgumentFormat?: string; - shardOperationSettings?: { - weight?: number; - }; + // @deprecated (undocumented) + shardOperationSettings?: unknown; } // @beta (undocumented) diff --git a/libraries/rush-lib/src/api/RushProjectConfiguration.ts b/libraries/rush-lib/src/api/RushProjectConfiguration.ts index 450af4a683d..279c6f92b64 100644 --- a/libraries/rush-lib/src/api/RushProjectConfiguration.ts +++ b/libraries/rush-lib/src/api/RushProjectConfiguration.ts @@ -65,15 +65,9 @@ export interface IRushPhaseSharding { outputFolderArgumentFormat?: string; /** - * Configuration for the shard operation. All other configuration applies to the collator operation. + * @deprecated Create a separate operation settings object for the shard operation settings with the name `{operationName}:shard`. */ - shardOperationSettings?: { - /** - * How many concurrency units this operation should take up during execution. The maximum concurrent units is - * determined by the -p flag. - */ - weight?: number; - }; + shardOperationSettings?: unknown; } /** @@ -585,6 +579,14 @@ export class RushProjectConfiguration { operationSettingsByOperationName.set(operationName, operationSettings); } } + + for (const [operationName, operationSettings] of operationSettingsByOperationName) { + if (operationSettings.sharding?.shardOperationSettings) { + terminal.writeWarningLine( + `DEPRECATED: The "sharding.shardOperationSettings" field is deprecated. Please create a new operation, '${operationName}:shard' to track shard operation settings.` + ); + } + } } if (hasErrors) { diff --git a/libraries/rush-lib/src/logic/operations/ShardedPhaseOperationPlugin.ts b/libraries/rush-lib/src/logic/operations/ShardedPhaseOperationPlugin.ts index 32e6f0e537f..81ff503b294 100644 --- a/libraries/rush-lib/src/logic/operations/ShardedPhaseOperationPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/ShardedPhaseOperationPlugin.ts @@ -2,6 +2,7 @@ // See LICENSE in the project root for license information. import type { IPhase } from '../../api/CommandLineConfiguration'; +import type { IOperationSettings, RushProjectConfiguration } from '../../api/RushProjectConfiguration'; import type { ICreateOperationsContext, IPhasedCommandPlugin, @@ -45,7 +46,7 @@ export class ShardedPhasedOperationPlugin implements IPhasedCommandPlugin { } function spliceShards(existingOperations: Set, context: ICreateOperationsContext): Set { - const { rushConfiguration } = context; + const { rushConfiguration, projectConfigurations } = context; const getCustomParameterValuesForPhase: (phase: IPhase) => ReadonlyArray = getCustomParameterValuesByPhase(); @@ -58,7 +59,7 @@ function spliceShards(existingOperations: Set, context: ICreateOperat logFilenameIdentifier: baseLogFilenameIdentifier } = operation; if (phase && project && operationSettings?.sharding && !operation.runner) { - const { count: shards, shardOperationSettings } = operationSettings.sharding; + const { count: shards } = operationSettings.sharding; /** * A single operation to reduce the number of edges in the graph when creating shards. @@ -165,12 +166,17 @@ function spliceShards(existingOperations: Set, context: ICreateOperat ); } + const projectConfiguration: RushProjectConfiguration | undefined = projectConfigurations.get(project); for (let shard: number = 1; shard <= shards; shard++) { const outputDirectory: string = outputFolderWithTemplate.replace( TemplateStringRegexes.SHARD_INDEX, shard.toString() ); + const shardOperationSettings: IOperationSettings = + projectConfiguration?.operationSettingsByOperationName.get(shardOperationName) ?? + (operationSettings.sharding.shardOperationSettings as IOperationSettings); + const shardOperation: Operation = new Operation({ project, phase, diff --git a/libraries/rush-lib/src/schemas/rush-project.schema.json b/libraries/rush-lib/src/schemas/rush-project.schema.json index ef22790eaf0..3fe4645d924 100644 --- a/libraries/rush-lib/src/schemas/rush-project.schema.json +++ b/libraries/rush-lib/src/schemas/rush-project.schema.json @@ -93,14 +93,8 @@ }, "shardOperationSettings": { "type": "object", - "additionalProperties": false, - "properties": { - "weight": { - "description": "The number of concurrency units that this operation should take up. The maximum concurrency units is determined by the -p flag.", - "type": "integer", - "minimum": 0 - } - } + "description": "DEPRECATED. Use a separate operationSettings entry with {this operation's name}:shard as the name, ex _phase:build would have a separate operation _phase:build:shard to manage per-shard settings.", + "additionalProperties": true } } },