[rush] move sharded operation config to its own block (#4923)

* feat: move sharded operation config to its own block * add changesets * deprecate and support older version * adjust changeset * Update common/changes/@microsoft/rush/sennyeya-shard-schema-updates_2024-09-13-22-46.json Co-authored-by: Ian Clanton-Thuon <[email protected]> * adjust upgrading.md * use terminal instance * fix lint error * more lint fixes --------- Co-authored-by: Aramis Sennyey <[email protected]> Co-authored-by: Ian Clanton-Thuon <[email protected]>
microsoft · Sep 18, 2024 · 0f25799 · 0f25799
1 parent 07914a7
commit 0f25799
Show file tree

Hide file tree

Showing 9 changed files with 93 additions and 31 deletions.
diff --git a/apps/rush/UPGRADING.md b/apps/rush/UPGRADING.md
@@ -1,5 +1,47 @@
 # Upgrade notes for @microsoft/rush
 
+### Rush 5.135.0
+
+This release of Rush deprecates the `rush-project.json`'s `operationSettings.sharding.shardOperationSettings`
+option in favor of defining a separate operation with a `:shard` suffix. This will only affect projects that
+have opted into sharding and have custom sharded operation settings.
+
+To migrate,
+**`rush-project.json`** (OLD)
+```json
+{
+  "operationSettings": [
+    {
+      "operationName": "_phase:build",
+      "sharding": {
+        "count": 4,
+        "shardOperationSettings": {
+          "weight": 4
+        }
+      },
+    }
+  ]
+}
+```
+
+**`rush-project.json`** (NEW)
+```json
+{
+  "operationSettings": [
+    {
+      "operationName": "_phase:build",
+      "sharding": {
+        "count": 4,
+      },
+    },
+    {
+      "operationName": "_phase:build:shard", // note the suffix here
+      "weight": 4
+    }
+  ]
+}
+```
+
 ### Rush 5.60.0
 
 This release of Rush includes a breaking change for the experiment build cache feature. It only affects

diff --git a/...-cobuild-plugin-integration-test/sandbox/sharded-repo/projects/a/config/rush-project.json b/...-cobuild-plugin-integration-test/sandbox/sharded-repo/projects/a/config/rush-project.json
@@ -6,12 +6,12 @@
       "outputFolderNames": ["dist"],
       "sharding": {
         "count": 4,
-        "outputFolderArgumentFormat": "--output-directory=.rush/{phaseName}/shards/{shardIndex}",
-
-        "shardOperationSettings": {
-          "weight": 4
-        }
+        "outputFolderArgumentFormat": "--output-directory=.rush/{phaseName}/shards/{shardIndex}"
       }
+    },
+    {
+      "operationName": "_phase:build:shard",
+      "weight": 4
     }
   ]
 }
diff --git a/...-cobuild-plugin-integration-test/sandbox/sharded-repo/projects/b/config/rush-project.json b/...-cobuild-plugin-integration-test/sandbox/sharded-repo/projects/b/config/rush-project.json
@@ -5,11 +5,12 @@
       "outputFolderNames": ["dist"],
       "sharding": {
         "count": 5,
-        "outputFolderArgumentFormat": "--output-directory=.rush/{phaseName}/shards/{shardIndex}",
-        "shardOperationSettings": {
-          "weight": 10
-        }
+        "outputFolderArgumentFormat": "--output-directory=.rush/{phaseName}/shards/{shardIndex}"
       }
+    },
+    {
+      "operationName": "_phase:build:shard",
+      "weight": 10
     }
   ]
 }
diff --git a/...-cobuild-plugin-integration-test/sandbox/sharded-repo/projects/e/config/rush-project.json b/...-cobuild-plugin-integration-test/sandbox/sharded-repo/projects/e/config/rush-project.json
@@ -4,8 +4,15 @@
       "operationName": "_phase:build",
       "outputFolderNames": ["dist"],
       "sharding": {
-        "count": 75
+        "count": 75,
+        "shardOperationSettings": {
+          "weight": 10
+        }
       }
+    },
+    {
+      "operationName": "_phase:build:shard",
+      "weight": 1
     }
   ]
 }
diff --git a/common/changes/@microsoft/rush/sennyeya-shard-schema-updates_2024-09-13-22-46.json b/common/changes/@microsoft/rush/sennyeya-shard-schema-updates_2024-09-13-22-46.json
@@ -0,0 +1,11 @@
+{
+  "changes": [
+    {
+      "comment": "Deprecate the `sharding.shardOperationSettings` property in the project `config/rush-project.json` in favor of an `operationSettings` entry for an operation with a suffix of `:shard`.",
+      "type": "none",
+      "packageName": "@microsoft/rush"
+    }
+  ],
+  "packageName": "@microsoft/rush",
+  "email": "[email protected]"
+}
diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md
@@ -785,9 +785,8 @@ export interface IRushPhaseSharding {
     count: number;
     outputFolderArgumentFormat?: string;
     shardArgumentFormat?: string;
-    shardOperationSettings?: {
-        weight?: number;
-    };
+    // @deprecated (undocumented)
+    shardOperationSettings?: unknown;
 }
 
 // @beta (undocumented)

diff --git a/libraries/rush-lib/src/api/RushProjectConfiguration.ts b/libraries/rush-lib/src/api/RushProjectConfiguration.ts
@@ -65,15 +65,9 @@ export interface IRushPhaseSharding {
   outputFolderArgumentFormat?: string;
 
   /**
-   * Configuration for the shard operation. All other configuration applies to the collator operation.
+   * @deprecated Create a separate operation settings object for the shard operation settings with the name `{operationName}:shard`.
    */
-  shardOperationSettings?: {
-    /**
-     * How many concurrency units this operation should take up during execution. The maximum concurrent units is
-     *  determined by the -p flag.
-     */
-    weight?: number;
-  };
+  shardOperationSettings?: unknown;
 }
 
 /**
@@ -585,6 +579,14 @@ export class RushProjectConfiguration {
           operationSettingsByOperationName.set(operationName, operationSettings);
         }
       }
+
+      for (const [operationName, operationSettings] of operationSettingsByOperationName) {
+        if (operationSettings.sharding?.shardOperationSettings) {
+          terminal.writeWarningLine(
+            `DEPRECATED: The "sharding.shardOperationSettings" field is deprecated. Please create a new operation, '${operationName}:shard' to track shard operation settings.`
+          );
+        }
+      }
     }
 
     if (hasErrors) {

diff --git a/libraries/rush-lib/src/logic/operations/ShardedPhaseOperationPlugin.ts b/libraries/rush-lib/src/logic/operations/ShardedPhaseOperationPlugin.ts
@@ -2,6 +2,7 @@
 // See LICENSE in the project root for license information.
 
 import type { IPhase } from '../../api/CommandLineConfiguration';
+import type { IOperationSettings, RushProjectConfiguration } from '../../api/RushProjectConfiguration';
 import type {
   ICreateOperationsContext,
   IPhasedCommandPlugin,
@@ -45,7 +46,7 @@ export class ShardedPhasedOperationPlugin implements IPhasedCommandPlugin {
 }
 
 function spliceShards(existingOperations: Set<Operation>, context: ICreateOperationsContext): Set<Operation> {
-  const { rushConfiguration } = context;
+  const { rushConfiguration, projectConfigurations } = context;
 
   const getCustomParameterValuesForPhase: (phase: IPhase) => ReadonlyArray<string> =
     getCustomParameterValuesByPhase();
@@ -58,7 +59,7 @@ function spliceShards(existingOperations: Set<Operation>, context: ICreateOperat
       logFilenameIdentifier: baseLogFilenameIdentifier
     } = operation;
     if (phase && project && operationSettings?.sharding && !operation.runner) {
-      const { count: shards, shardOperationSettings } = operationSettings.sharding;
+      const { count: shards } = operationSettings.sharding;
 
       /**
        * A single operation to reduce the number of edges in the graph when creating shards.
@@ -165,12 +166,17 @@ function spliceShards(existingOperations: Set<Operation>, context: ICreateOperat
         );
       }
 
+      const projectConfiguration: RushProjectConfiguration | undefined = projectConfigurations.get(project);
       for (let shard: number = 1; shard <= shards; shard++) {
         const outputDirectory: string = outputFolderWithTemplate.replace(
           TemplateStringRegexes.SHARD_INDEX,
           shard.toString()
         );
 
+        const shardOperationSettings: IOperationSettings =
+          projectConfiguration?.operationSettingsByOperationName.get(shardOperationName) ??
+          (operationSettings.sharding.shardOperationSettings as IOperationSettings);
+
         const shardOperation: Operation = new Operation({
           project,
           phase,

diff --git a/libraries/rush-lib/src/schemas/rush-project.schema.json b/libraries/rush-lib/src/schemas/rush-project.schema.json
@@ -93,14 +93,8 @@
               },
               "shardOperationSettings": {
                 "type": "object",
-                "additionalProperties": false,
-                "properties": {
-                  "weight": {
-                    "description": "The number of concurrency units that this operation should take up. The maximum concurrency units is determined by the -p flag.",
-                    "type": "integer",
-                    "minimum": 0
-                  }
-                }
+                "description": "DEPRECATED. Use a separate operationSettings entry with {this operation's name}:shard as the name, ex _phase:build would have a separate operation _phase:build:shard to manage per-shard settings.",
+                "additionalProperties": true
               }
             }
           },