Content replacement automation to all repos

eng/pipelines/agent-pool-migration.yml

@@ -0,0 +1,149 @@
+# The pipeline is mainly used to migrate our agent pool from one version to another version. 
+# Here is how we use the pipeline:
+# 1. What repos are you going to work on for your migration? Put your repos in Parameter 'Repos'. Follow the format in default.
+# 2. The replacement map, e.g We will search all occurrences of the strings in Parameter 'MigrateFrom' and replace to the value in 'MigrateTo'. This is regex map, and you can define multiple replacement rules here.
+# 3. The Parameter 'WhatIfPreference' boolean flag allows you to check your changed files before open a PR. Check the box if you don't want to create a PR.
+# 4. The pipeline will generate PRs in specified repos, Parameter 'ShortForMigrateFrom' and 'ShortForMigrateTo' are the short descriptions of the PR purpose.
+#    The PR title will in the format of "Migrate from ${{parameters.ShortForMigrateFrom}} to ${{parameters.ShortForMigrateTo}}"
+#    You can also specify who triggers the changes in Paramter 'GithubAssigneeAlias'. Then your PR will request review from the specified person.
+#    If you want to create PR from your own fork, you can specify the Parameters 'PRForkOwner'
+# 5. Go to the PR link in 'Create pull request' step. Review and merge the PR. Note, the pipeline intented to be aggressive, so take a closer look at the changes.
+#    The PR also has a link pointed back to the pipeline, so people will learn where it comes from and the purpose of the PR.
+trigger: none
+pr: none
+
+# Notes on ExcludePaths and IncludeFromExcludedPaths:
+# 1. ExcludePaths is used to exclude the file pattern you don't want to scan against.
+# 2. IncludeFromExcludedPaths is used to add back the files you don't want to exclude. It is supposed to be a subset of ExcludePaths. Pass empty if you only want exclude paths.
+# 3. Both excludePaths and IncludeFromExcludedPaths support standard powershell regex. 
+# 4. The paths here are relative to the repo folder. 
+#    E.g.
+#    In win agent pool, the path is like ".\eng\scripts\Repo-File-Content-Replacements.ps1", so the regex path can be like "^.\\eng\\"
+#    In linux agent pool, the path is like "./eng/scripts/Repo-File-Content-Replacements.ps1", so the regex path can be like "^./eng/"
+#    To be compatible with both cases, you can also specify your regex path like "^./|\\eng/|\\"
+parameters:
+- name: Repos
+  type: object
+  default:
+    - RepoOwner: Azure
+      RepoName: azure-sdk-tools
+    - RepoOwner: Azure
+      RepoName: azure-sdk
+      ExcludePaths: '^\.\\eng\\common\\'
+      IncludeFromExcludedPaths: ""
+    - RepoOwner: Azure
+      RepoName: azure-sdk-for-android
+      ExcludePaths: '^\.\\sdk\\|^\.\\eng\\common\\|\\tests\\resources\\session-records\\'
+      IncludeFromExcludedPaths: '^\.\\sdk\\.*-matrix.*.json$|^\.\\sdk\\.*\\.*yml$'
+    - RepoOwner: Azure
+      RepoName: azure-sdk-for-c
+      ExcludePaths: '^\.\\sdk\\|^\.\\eng\\common\\'
+      IncludeFromExcludedPaths: '^\.\\sdk\\.*-matrix.*.json$|^\.\\sdk\\.*\\.*yml$'
+    - RepoOwner: Azure
+      RepoName: azure-sdk-for-cpp
+      ExcludePaths: '^\.\\sdk\\|^\.\\eng\\common\\'
+      IncludeFromExcludedPaths: '^\.\\sdk\\.*-matrix.*.json$|^\.\\sdk\\.*\\.*yml$'
+    - RepoOwner: Azure
+      RepoName: azure-sdk-for-go
+      ExcludePaths: '^\.\\sdk\\|^\.\\eng\\common\\'
+      IncludeFromExcludedPaths: '^\.\\sdk\\.*-matrix.*.json$|^\.\\sdk\\.*\\.*yml$'
+    - RepoOwner: Azure
+      RepoName: azure-sdk-for-ios
+      ExcludePaths: '^\.\\sdk\\|^\.\\eng\\common\\'
+      IncludeFromExcludedPaths: '^\.\\sdk\\.*-matrix.*.json$|^\.\\sdk\\.*\\.*yml$'
+    - RepoOwner: Azure
+      RepoName: azure-sdk-for-java
+      ExcludePaths: '^\.\\sdk\\|^\.\\eng\\common\\|\\resources\\session-records\\'
+      IncludeFromExcludedPaths: '^\.\\sdk\\.*-matrix.*.json$|^\.\\sdk\\.*\\.*yml$'
+    - RepoOwner: Azure
+      RepoName: azure-sdk-for-js
+      ExcludePaths: '^\.\\sdk\\|^\.\\eng\\common\\|\\recordings\\node\\'
+      IncludeFromExcludedPaths: '^\.\\sdk\\.*-matrix.*.json$|^\.\\sdk\\.*\\.*yml$'
+    - RepoOwner: Azure
+      RepoName: azure-sdk-for-net
+      ExcludePaths: '^\.\\sdk\\|^\.\\eng\\common\\|\\tests\\SessionRecords\\'
+      IncludeFromExcludedPaths: '^\.\\sdk\\.*-matrix.*.json$|^\.\\sdk\\.*\\.*yml$'
+    - RepoOwner: Azure
+      RepoName: azure-sdk-for-python
+      ExcludePaths: '^\.\\sdk\\|^\.\\eng\\common\\|\\tests\\recordings\\'
+      IncludeFromExcludedPaths: '^\.\\sdk\\.*-matrix.*.json$|^\.\\sdk\\.*\\.*yml$'
+# Default setting is an example of the map looks like:
+- name: MigrationMap
+  type: object
+  default: 
+    - MigrateFrom: azsdk-pool-mms-win-2019-general
+      MigrateTo: azsdk-pool-mms-win-2022-general
+    - MigrateFrom: windows-2019
+      MigrateTo: windows-2022
+    - MigrateFrom: windows2019
+      MigrateTo: windows2022
+    - MigrateFrom: MMS2019
+      MigrateTo: MMS2022
+- name: ShortForMigrateFrom
+  type: string
+  default: ''
+- name: ShortForMigrateTo
+  type: string
+  default: ''
+# The value here is github alias: e.g. sima-zhu
+# Users to assign to the PR after opening. Users should be a comma-separated list. e.g. "user1,usertwo,user3"
+- name: GithubAssignessAlias
+  type: string
+  default: ''
+- name: PRForkOwner
+  type: string
+  default: 'azure-sdk'
+- name: WhatIfPreference
+  type: boolean
+  default: false
+
+variables:
+  - template: /eng/pipelines/templates/variables/globals.yml
+jobs:
+  - job: Migration
+    pool:
+      name: azsdk-pool-mms-win-2022-general
+    steps:
+      - ${{ each repo in parameters.Repos }}:
+        - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
+          parameters:
+            SkipCheckoutNone: true
+            Repositories:
+              - Name: ${{ repo.RepoOwner }}/${{ repo.RepoName }}
+                WorkingDirectory: $(Build.SourcesDirectory)/repos/${{ repo.RepoName }}
+                Commitish: ${{ repo.branch }}
+            Paths:
+              - /*
+        - template: /eng/common/pipelines/templates/steps/set-default-branch.yml
+          parameters:
+            WorkingDirectory: $(Build.SourcesDirectory)/repos/${{ repo.RepoName }}
+        - pwsh: |
+            $migrationMap = '${{ convertToJson(parameters.MigrationMap) }}'
+            $(Build.SourcesDirectory)/eng/scripts/Repo-File-Content-Replacements.ps1 `
+              -ExcludePathsRegex '${{ repo.ExcludePaths }}' `
+              -IncludeFromExcludedPathsRegex '${{ repo.IncludeFromExcludedPaths }}' `
+              -MigrationMap $migrationMap
+          displayName: Migration for "${{ repo.RepoName }}"
+          workingDirectory: $(Build.SourcesDirectory)/repos/${{ repo.RepoName }}
+          continueOnError: true
+        - ${{ if ne(parameters.WhatIfPreference, 'true')}}:
+          - pwsh: |
+              $requestBy = '${{ parameters.GithubAssignessAlias }}'
+              if ($requestBy) {
+                Write-Host "##vso[task.setvariable variable=Assigness]$requestBy"
+              }
+            displayName: Set Github reviewers variable
+          - template: /eng/common/pipelines/templates/steps/create-pull-request.yml
+            parameters:
+              BaseBranchName: $(DefaultBranch)
+              PRBranchName: migration
+              PROwner: ${{ parameters.PRForkOwner }}
+              CommitMsg: "Migrate from ${{parameters.ShortForMigrateFrom}} to ${{parameters.ShortForMigrateTo}}"
+              RepoOwner: ${{ repo.RepoOwner }}
+              RepoName: ${{ repo.RepoName }}
+              PRTitle: "Migrate from ${{parameters.ShortForMigrateFrom}} to ${{parameters.ShortForMigrateTo}}"
+              WorkingDirectory: $(Build.SourcesDirectory)/repos/${{ repo.RepoName }}
+              ${{ if ne(parameters.GithubAssignessAlias, '')}}:
+                GHAssignessVariable: Assigness
+              PRBody: "This is auto PR generated by pipeline: https://dev.azure.com/azure-sdk/internal/_build/results?buildId=$(Build.BuildId)&view=results"
+              PushArgs: -f

eng/script-tests/Repo-File-Content-Replacements/Repo-File-Content-Replacements.Tests.ps1

@@ -0,0 +1,94 @@
+<#
+```
+Invoke-Pester -Output Detailed $PSScriptRoot/Repo-File-Content-Replacements.Tests.ps1
+```
+#>
+
+Import-Module Pester
+Set-StrictMode -Version Latest
+
+BeforeAll {
+    function Backup-Folder($targetFolder, $backupFolder) {
+        if (!(Test-Path $targetFolder)) {
+            return $null
+        }
+
+        if (!(Test-Path $backupFolder)) {
+            New-Item $backupFolder -ItemType Directory
+        }
+        Copy-Item -Path "$targetFolder/*" -Destination $backupFolder -Recurse -PassThru -Force
+    }
+    function Reset-Folder($backupFolder, $targetFolder) {
+        if (Test-Path $backupFolder) {
+            Copy-Item -Path "$backupFolder/*" -Destination $targetFolder -Recurse -Force
+            Remove-Item $backupFolder -Recurse
+        }
+    }
+}
+AfterAll {
+    if (Test-Path "$PSScriptRoot/inputs/backup") {
+        Remove-Item "$PSScriptRoot/inputs/backup" -Recurse
+    }
+}
+# Test cases:
+# 1. Tests on specify valid both include and exclude paths
+# 1. Tests on specify exclude paths and pass empty to include paths.
+# 2. Tests on specify exclude paths only and no include paths.
+
+Describe "repo-file-content-replacement" -Tag "UnitTest" {
+    # Passed cases
+    It "Test on the files have matching content" -TestCases @(
+        @{ 
+            exludePaths = "(/|\\)excludes(/|\\)|(/|\\)eng(/|\\)scripts-tests(/|\\)Repo-File-Content-Replacements(/|\\)inputs(/|\\)test1(/|\\)jsonFiles(/|\\)1.json";
+            includeFromExcludedPaths = "(/|\\)excludes(/|\\)includes(/|\\)";
+            migrationMapFile = "$PSScriptRoot/inputs/test1/jsonFiles/1.json";
+            scannedDirectory = "$PSScriptRoot/inputs/test1";
+        }
+        @{
+            exludePaths = "(/|\\)excludes(/|\\)";
+            includeFromExcludedPaths = "";
+            migrationMapFile = "$PSScriptRoot/inputs/test1/jsonFiles/1.json";
+            scannedDirectory = "$PSScriptRoot/inputs/test2";
+        }
+    ) {
+        $backupFolder = "$PSScriptRoot/inputs/backup"
+        Backup-Folder -targetFolder $ScannedDirectory -backupFolder $backupFolder
+        $migrationMap = Get-Content $migrationMapFile -Raw
+        . $PSScriptRoot/../../scripts/Repo-File-Content-Replacements.ps1 `
+            -ExcludePathsRegex $exludePaths `
+            -IncludeFromExcludedPathsRegex $includeFromExcludedPaths `
+            -MigrationMapJson $migrationMap `
+            -ScannedDirectory $ScannedDirectory 
+        $files = Get-ChildItem "$scannedDirectory/*" -Recurse -File -Include *.txt
+        foreach ($file in $files) {
+            $expectPath = $file.FullName -replace 'inputs', 'expected'
+            (Get-Content $file) | Should -Be (Get-Content $expectPath)
+        }
+        Reset-Folder -backupFolder $backupFolder -targetFolder $ScannedDirectory
+    }
+}
+
+Describe "repo-file-content-replacement" -Tag "UnitTest" {
+    # Passed cases
+    It "Test on the files with no includePaths" -TestCases @(
+        @{ 
+            exludePaths = "(/|\\)excludes(/|\\)";
+            migrationMapFile = "$PSScriptRoot/inputs/test1/jsonFiles/1.json";
+            scannedDirectory = "$PSScriptRoot/inputs/test3";
+        }
+    ) {
+        $backupFolder = "$PSScriptRoot/inputs/backup"
+        Backup-Folder -targetFolder $ScannedDirectory -backupFolder $backupFolder
+        $migrationMap = Get-Content $migrationMapFile -Raw
+        . $PSScriptRoot/../../scripts/Repo-File-Content-Replacements.ps1 `
+            -ExcludePathsRegex $exludePaths `
+            -MigrationMapJson $migrationMap `
+            -ScannedDirectory $ScannedDirectory 
+        $files = Get-ChildItem "$scannedDirectory/*" -Recurse -File -Include *.txt
+        foreach ($file in $files) {
+            $expectPath = $file.FullName -replace 'inputs', 'expected'
+            (Get-Content $file) | Should -Be (Get-Content $expectPath)
+        }
+        Reset-Folder -backupFolder $backupFolder -targetFolder $ScannedDirectory
+    }
+}

eng/script-tests/Repo-File-Content-Replacements/expected/test1/3.txt

@@ -0,0 +1,4 @@
+3.txt
+This is a testing for azsdk-pool-mms-win-2022-general migration. 
+This is a testing for windows2022 migration. 
+Expect to have changes for this file.

eng/script-tests/Repo-File-Content-Replacements/expected/test1/excludes/1.txt

@@ -0,0 +1,3 @@
+1.txt
+This is a testing for windows-2019 migration. 
+No change expected for this file.

eng/script-tests/Repo-File-Content-Replacements/expected/test1/excludes/includes/2.txt

@@ -0,0 +1,3 @@
+2.txt
+This is a testing for windows-2022 migration. 
+Expect to have changes for this file.

eng/script-tests/Repo-File-Content-Replacements/expected/test1/includes/.6.txt

@@ -0,0 +1,4 @@
+6.txt
+This is a testing for azsdk-pool-mms-win-2022-general migration. 
+This is a testing for windows2022 migration. 
+Expect to have changes for this file.

eng/script-tests/Repo-File-Content-Replacements/expected/test1/includes/.azure-pipelines/5.txt

@@ -0,0 +1,4 @@
+5.txt
+This is a testing for azsdk-pool-mms-win-2022-general migration. 
+This is a testing for windows2022 migration. 
+Expect to have changes for this file.

eng/script-tests/Repo-File-Content-Replacements/expected/test1/includes/4.txt

@@ -0,0 +1,4 @@
+4.txt
+This is a testing for azsdk-pool-mms-win-2022-general migration. 
+This is a testing for windows2022 migration. 
+Expect to have changes for this file.

eng/script-tests/Repo-File-Content-Replacements/expected/test1/others/7.txt

@@ -0,0 +1,2 @@
+5.txt
+No changes expected.

eng/script-tests/Repo-File-Content-Replacements/expected/test2/excludes/1.txt

@@ -0,0 +1,3 @@
+1.txt
+This is a testing for windows-2019 migration. 
+No change expected for this file.

eng/script-tests/Repo-File-Content-Replacements/expected/test3/excludes/1.txt

@@ -0,0 +1,3 @@
+1.txt
+This is a testing for windows-2022 migration. 
+Expected to have changes for this file

eng/script-tests/Repo-File-Content-Replacements/inputs/test1/3.txt

@@ -0,0 +1,4 @@
+3.txt
+This is a testing for azsdk-pool-mms-win-2019-general migration. 
+This is a testing for windows2019 migration. 
+Expect to have changes for this file.

eng/script-tests/Repo-File-Content-Replacements/inputs/test1/excludes/1.txt

@@ -0,0 +1,3 @@
+1.txt
+This is a testing for windows-2019 migration. 
+No change expected for this file.

eng/script-tests/Repo-File-Content-Replacements/inputs/test1/excludes/includes/2.txt

@@ -0,0 +1,3 @@
+2.txt
+This is a testing for windows-2019 migration. 
+Expect to have changes for this file.

eng/script-tests/Repo-File-Content-Replacements/inputs/test1/includes/.6.txt

@@ -0,0 +1,4 @@
+6.txt
+This is a testing for azsdk-pool-mms-win-2019-general migration. 
+This is a testing for windows2019 migration. 
+Expect to have changes for this file.

eng/script-tests/Repo-File-Content-Replacements/inputs/test1/includes/.azure-pipelines/5.txt

@@ -0,0 +1,4 @@
+5.txt
+This is a testing for azsdk-pool-mms-win-2019-general migration. 
+This is a testing for windows2019 migration. 
+Expect to have changes for this file.

eng/script-tests/Repo-File-Content-Replacements/inputs/test1/includes/4.txt

@@ -0,0 +1,4 @@
+4.txt
+This is a testing for azsdk-pool-mms-win-2019-general migration. 
+This is a testing for windows2019 migration. 
+Expect to have changes for this file.

eng/script-tests/Repo-File-Content-Replacements/inputs/test1/jsonFiles/1.json

@@ -0,0 +1,14 @@
+[
+  {
+    "MigrateFrom": "azsdk-pool-mms-win-2019-general",
+    "MigrateTo": "azsdk-pool-mms-win-2022-general"
+  },
+  {
+    "MigrateFrom": "windows-2019",
+    "MigrateTo": "windows-2022"
+  },
+  {
+    "MigrateFrom": "windows2019",
+    "MigrateTo": "windows2022"
+  }
+]

eng/script-tests/Repo-File-Content-Replacements/inputs/test1/others/7.txt

@@ -0,0 +1,2 @@
+5.txt
+No changes expected.

eng/script-tests/Repo-File-Content-Replacements/inputs/test2/excludes/1.txt

@@ -0,0 +1,3 @@
+1.txt
+This is a testing for windows-2019 migration. 
+No change expected for this file.

eng/script-tests/Repo-File-Content-Replacements/inputs/test3/excludes/1.txt

@@ -0,0 +1,3 @@
+1.txt
+This is a testing for windows-2019 migration. 
+Expected to have changes for this file