Content replacement automation to all repos

eng/pipelines/agent-pool-migration.yml

@@ -0,0 +1,149 @@
+# The pipeline is mainly used to migrate our agent pool from one version to another version. 
+# Here is how we use the pipeline:
+# 1. What repos are you going to work on for your migration? Put your repos in Parameter 'Repos'. Follow the format in default.
+# 2. The replacement map, e.g We will search all occurrences of the strings in Parameter 'MigrateFrom' and replace to the value in 'MigrateTo'. This is regex map, and you can define multiple replacement rules here.
+# 3. The Parameter 'WhatIfPreference' boolean flag allows you to check your changed files before open a PR. Check the box if you don't want to create a PR.
+# 4. The pipeline will generate PRs in specified repos, Parameter 'ShortForMigrateFrom' and 'ShortForMigrateTo' are the short descriptions of the PR purpose.
+#    The PR title will in the format of "Migrate from ${{parameters.ShortForMigrateFrom}} to ${{parameters.ShortForMigrateTo}}"
+#    You can also specify who triggers the changes in Paramter 'GithubAssigneeAlias'. Then your PR will request review from the specified person.
+#    If you want to create PR from your own fork, you can specify the Parameters 'PRForkOwner'
+# 5. Go to the PR link in 'Create pull request' step. Review and merge the PR. Note, the pipeline intented to be aggressive, so take a closer look at the changes.
+#    The PR also has a link pointed back to the pipeline, so people will learn where it comes from and the purpose of the PR.
+trigger: none
+pr: none
+
+# Notes on ExcludePaths and IncludeFromExcludedPaths:
+# 1. ExcludePaths is used to exclude the file pattern you don't want to scan against.
+# 2. IncludeFromExcludedPaths is used to add back the files you don't want to exclude. It is supposed to be a subset of ExcludePaths. Pass empty if you only want exclude paths.
+# 3. Both excludePaths and IncludeFromExcludedPaths support standard powershell regex. 
+# 4. The paths here are relative to the repo folder. 
+#    E.g.
+#    In win agent pool, the path is like ".\eng\scripts\Repo-File-Content-Replacements.ps1", so the regex path can be like "^.\\eng\\"
+#    In linux agent pool, the path is like "./eng/scripts/Repo-File-Content-Replacements.ps1", so the regex path can be like "^./eng/"
+#    To be compatible with both cases, you can also specify your regex path like "^./|\\eng/|\\"
+parameters:
+- name: Repos
+  type: object
+  default:
+    - RepoOwner: Azure
+      RepoName: azure-sdk-tools
+    - RepoOwner: Azure
+      RepoName: azure-sdk
+      ExcludePaths: '^\.\\eng\\common\\'
+      IncludeFromExcludedPaths: ""
+    - RepoOwner: Azure
+      RepoName: azure-sdk-for-android
+      ExcludePaths: '^\.\\sdk\\|^\.\\eng\\common\\|\\tests\\resources\\session-records\\'
+      IncludeFromExcludedPaths: '^\.\\sdk\\.*-matrix.*.json$|^\.\\sdk\\.*\\.*yml$'
+    - RepoOwner: Azure
+      RepoName: azure-sdk-for-c
+      ExcludePaths: '^\.\\sdk\\|^\.\\eng\\common\\'
+      IncludeFromExcludedPaths: '^\.\\sdk\\.*-matrix.*.json$|^\.\\sdk\\.*\\.*yml$'
+    - RepoOwner: Azure
+      RepoName: azure-sdk-for-cpp
+      ExcludePaths: '^\.\\sdk\\|^\.\\eng\\common\\'
+      IncludeFromExcludedPaths: '^\.\\sdk\\.*-matrix.*.json$|^\.\\sdk\\.*\\.*yml$'
+    - RepoOwner: Azure
+      RepoName: azure-sdk-for-go
+      ExcludePaths: '^\.\\sdk\\|^\.\\eng\\common\\'
+      IncludeFromExcludedPaths: '^\.\\sdk\\.*-matrix.*.json$|^\.\\sdk\\.*\\.*yml$'
+    - RepoOwner: Azure
+      RepoName: azure-sdk-for-ios
+      ExcludePaths: '^\.\\sdk\\|^\.\\eng\\common\\'
+      IncludeFromExcludedPaths: '^\.\\sdk\\.*-matrix.*.json$|^\.\\sdk\\.*\\.*yml$'
+    - RepoOwner: Azure
+      RepoName: azure-sdk-for-java
+      ExcludePaths: '^\.\\sdk\\|^\.\\eng\\common\\|\\resources\\session-records\\'
+      IncludeFromExcludedPaths: '^\.\\sdk\\.*-matrix.*.json$|^\.\\sdk\\.*\\.*yml$'
+    - RepoOwner: Azure
+      RepoName: azure-sdk-for-js
+      ExcludePaths: '^\.\\sdk\\|^\.\\eng\\common\\|\\recordings\\node\\'
+      IncludeFromExcludedPaths: '^\.\\sdk\\.*-matrix.*.json$|^\.\\sdk\\.*\\.*yml$'
+    - RepoOwner: Azure
+      RepoName: azure-sdk-for-net
+      ExcludePaths: '^\.\\sdk\\|^\.\\eng\\common\\|\\tests\\SessionRecords\\'
+      IncludeFromExcludedPaths: '^\.\\sdk\\.*-matrix.*.json$|^\.\\sdk\\.*\\.*yml$'
+    - RepoOwner: Azure
+      RepoName: azure-sdk-for-python
+      ExcludePaths: '^\.\\sdk\\|^\.\\eng\\common\\|\\tests\\recordings\\'
+      IncludeFromExcludedPaths: '^\.\\sdk\\.*-matrix.*.json$|^\.\\sdk\\.*\\.*yml$'
+# Default setting is an example of the map looks like:
+- name: MigrationMap
+  type: object
+  default: 
+    - MigrateFrom: azsdk-pool-mms-win-2019-general
+      MigrateTo: azsdk-pool-mms-win-2022-general
+    - MigrateFrom: windows-2019
+      MigrateTo: windows-2022
+    - MigrateFrom: windows2019
+      MigrateTo: windows2022
+    - MigrateFrom: MMS2019
+      MigrateTo: MMS2022
+- name: ShortForMigrateFrom
+  type: string
+  default: ''
+- name: ShortForMigrateTo
+  type: string
+  default: ''
+# The value here is github alias: e.g. sima-zhu
+# Users to assign to the PR after opening. Users should be a comma-separated list. e.g. "user1,usertwo,user3"
+- name: GithubAssignessAlias
+  type: string
+  default: ''
+- name: PRForkOwner
+  type: string
+  default: 'azure-sdk'
+- name: WhatIfPreference
+  type: boolean
+  default: false
+
+variables:
+  - template: /eng/pipelines/templates/variables/globals.yml
+jobs:
+  - job: Migration
+    pool:
+      name: azsdk-pool-mms-win-2022-general
+    steps:
+      - ${{ each repo in parameters.Repos }}:
+        - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
+          parameters:
+            SkipCheckoutNone: true
+            Repositories:
+              - Name: ${{ repo.RepoOwner }}/${{ repo.RepoName }}
+                WorkingDirectory: $(Build.SourcesDirectory)/repos/${{ repo.RepoName }}
+                Commitish: ${{ repo.branch }}
+            Paths:
+              - /*
+        - template: /eng/common/pipelines/templates/steps/set-default-branch.yml
+          parameters:
+            WorkingDirectory: $(Build.SourcesDirectory)/repos/${{ repo.RepoName }}
+        - pwsh: |
+            $migrationMap = '${{ convertToJson(parameters.MigrationMap) }}'
+            $(Build.SourcesDirectory)/eng/scripts/Repo-File-Content-Replacements.ps1 `
+              -ExcludePathsRegex '${{ repo.ExcludePaths }}' `
+              -IncludeFromExcludedPathsRegex '${{ repo.IncludeFromExcludedPaths }}' `
+              -MigrationMap $migrationMap
+          displayName: Migration for "${{ repo.RepoName }}"
+          workingDirectory: $(Build.SourcesDirectory)/repos/${{ repo.RepoName }}
+          continueOnError: true
+        - ${{ if ne(parameters.WhatIfPreference, 'true')}}:
+          - pwsh: |
+              $requestBy = '${{ parameters.GithubAssignessAlias }}'
+              if ($requestBy) {
+                Write-Host "##vso[task.setvariable variable=Assigness]$requestBy"
+              }
+            displayName: Set Github reviewers variable
+          - template: /eng/common/pipelines/templates/steps/create-pull-request.yml
+            parameters:
+              BaseBranchName: $(DefaultBranch)
+              PRBranchName: migration
+              PROwner: ${{ parameters.PRForkOwner }}
+              CommitMsg: "Migrate from ${{parameters.ShortForMigrateFrom}} to ${{parameters.ShortForMigrateTo}}"
+              RepoOwner: ${{ repo.RepoOwner }}
+              RepoName: ${{ repo.RepoName }}
+              PRTitle: "Migrate from ${{parameters.ShortForMigrateFrom}} to ${{parameters.ShortForMigrateTo}}"
+              WorkingDirectory: $(Build.SourcesDirectory)/repos/${{ repo.RepoName }}
+              ${{ if ne(parameters.GithubAssignessAlias, '')}}:
+                GHAssignessVariable: Assigness
+              PRBody: "This is auto PR generated by pipeline: https://dev.azure.com/azure-sdk/internal/_build/results?buildId=$(Build.BuildId)&view=results"
+              PushArgs: -f

eng/scripts/ci.yml

@@ -0,0 +1,28 @@
+# NOTE: Please refer to https://aka.ms/azsdk/engsys/ci-yaml before editing this file.
+trigger:
+  branches:
+    include:
+      - main
+      - feature/*
+      - release/*
+      - hotfix/*
+  paths:
+    include:
+      - eng/scripts-tests/*
+
+pr:
+  branches:
+    include:
+      - main
+      - feature/*
+      - release/*
+      - hotfix/*
+  paths:
+    include:
+      - eng/scripts-tests/*
+
+extends:
+  template: /eng/common/pipelines/templates/stages/archetype-sdk-tool-pwsh.yml
+  parameters:
+    TargetDirectory: eng/scripts/
+    TargetTags: 'UnitTest'

eng/scripts/contentreplacements/Repo-File-Content-Replacements.ps1

@@ -0,0 +1,97 @@
+<#
+.SYNOPSIS
+The script is to do file content replacements under current folder.
+
+.DESCRIPTION
+We are having the script to help search all occurrences of specified strings within scope and replace to the target strings.
+The script will scan all files by default. 
+
+.PARAMETER ExcludePathsRegex
+The regex of excluded file paths. It is standard regex applied to the file path of the agent. The file path is relative to repo folder. 
+In win agent, the path is like ".\eng\scripts\Repo-File-Content-Replacements.ps1", so the regex path can be like "^.\\eng\\"
+In linux agent, the path is like "./eng/scripts/Repo-File-Content-Replacements.ps1", so the regex path can be like "^./eng/"
+To compitable with both cases, you can also specify your regex path like "^./|\\eng/|\\"
+e.g we can specified multiple paths using '(^./sdk/)|(^./eng/common/)'
+The script will exclude the specified paths from scanned file list first. 
+
+.PARAMETER IncludeFromExcludedPathsRegex
+The regex of file pattern which is used to add back the files you don't want to exclude. The IncludeFromExcludedPathsRegex is supposed to match a subset of ExcludePathsRegex
+It is standard regex applied to the file path of the agent. The file path is relative to repo folder. 
+In win agent, the path is like ".\eng\scripts\Repo-File-Content-Replacements.ps1", so the regex path can be like "^.\\eng\\"
+In linux agent, the path is like "./eng/scripts/Repo-File-Content-Replacements.ps1", so the regex path can be like "^./eng/"
+To compitable with both cases, you can also specify your regex path like "^./|\\eng/|\\"
+e.g we can specified multiple paths using "(^./sdk/.*/platform-matrix.*.json$)|(^./sdk/.*/.*yml$)"
+
+.PARAMETER MigrationMapJson
+The map includes information of from-to pair. E.g:
+[
+  {
+    "MigrateFrom": "azsdk-pool-mms-win-2022-general",
+    "MigrateTo": "azsdk-pool-mms-win-2022-general"
+  },
+  {
+    "MigrateFrom": "windows-2022",
+    "MigrateTo": "windows-2022"
+  },
+  {
+    "MigrateFrom": "windows2022",
+    "MigrateTo": "windows2022"
+  }
+]
+
+.PARAMETER ScannedDirectory
+The directory to scan against. Usually, it is repo root.
+
+.EXAMPLE
+./eng/scripts/check-spelling-in-changed-files.ps1 
+
+This will run spell check with changes in the current branch with respect to 
+`target_branch_name`
+
+#>
+
+[CmdletBinding()]
+param (
+  [Parameter(Mandatory = $false)]
+  [string]$ExcludePathsRegex = '',
+  [Parameter(Mandatory = $false)]
+  [string]$IncludeFromExcludedPathsRegex = '.*',
+  [Parameter(Mandatory = $true)]
+  [string]$MigrationMapJson,
+  [Parameter(Mandatory = $false)]
+  [string]$ScannedDirectory = '.'
+)
+
+$files = Get-ChildItem -LiteralPath $ScannedDirectory -Recurse -File 
+$newFileCollection = @()
+foreach ($file in $files) {
+  $relativePath = Resolve-Path -LiteralPath $file.FullName -Relative
+  $checkIncludes = $true
+  if (!$ExcludePathsRegex -or $relativePath -notmatch $ExcludePathsRegex) {
+    $newFileCollection += $file
+    $checkIncludes = $false
+  }
+  if ($checkIncludes -and $IncludeFromExcludedPathsRegex -and ($relativePath -match $IncludeFromExcludedPathsRegex)) {
+    $newFileCollection += $file
+  }
+}
+
+# Without -NoEnumerate, a single element array[T] gets unwrapped as a single item T.
+$migrationMap = ConvertFrom-Json $MigrationMapJson -NoEnumerate
+
+Write-Host "The number of matching files: $($newFileCollection.Count)"
+# Scan all Files and check the match
+foreach ($file in $newFileCollection) {
+  $newcontent = Get-Content -LiteralPath $file.FullName -Raw
+  $fileChanged = $False
+  foreach ($migrate in $MigrationMap) {
+    if ($newcontent -match $migrate.MigrateFrom) {
+      $fileChanged = $True
+      $newcontent = $newcontent -replace "$($migrate.MigrateFrom)", "$($migrate.MigrateTo)"
+    }
+  }
+  if ($fileChanged) {
+    Write-Host "File to update: $($file.FullName)"
+    Set-Content -Path $file.FullName -Value $newcontent -NoNewline
+  }
+}

eng/scripts/contentreplacements/tests/Repo-File-Content-Replacements/Repo-File-Content-Replacements.Tests.ps1

@@ -0,0 +1,94 @@
+<#
+```
+Invoke-Pester -Output Detailed $PSScriptRoot/Repo-File-Content-Replacements.Tests.ps1
+```
+#>
+
+Import-Module Pester
+Set-StrictMode -Version Latest
+
+BeforeAll {
+    function Backup-Folder($targetFolder, $backupFolder) {
+        if (!(Test-Path $targetFolder)) {
+            return $null
+        }
+
+        if (!(Test-Path $backupFolder)) {
+            New-Item $backupFolder -ItemType Directory
+        }
+        Copy-Item -Path "$targetFolder/*" -Destination $backupFolder -Recurse -PassThru -Force
+    }
+    function Reset-Folder($backupFolder, $targetFolder) {
+        if (Test-Path $backupFolder) {
+            Copy-Item -Path "$backupFolder/*" -Destination $targetFolder -Recurse -Force
+            Remove-Item $backupFolder -Recurse
+        }
+    }
+}
+AfterAll {
+    if (Test-Path "$PSScriptRoot/inputs/backup") {
+        Remove-Item "$PSScriptRoot/inputs/backup" -Recurse
+    }
+}
+# Test cases:
+# 1. Tests on specify valid both include and exclude paths
+# 1. Tests on specify exclude paths and pass empty to include paths.
+# 2. Tests on specify exclude paths only and no include paths.
+
+Describe "repo-file-content-replacement" -Tag "UnitTest" {
+    # Passed cases
+    It "Test on the files have matching content" -TestCases @(
+        @{ 
+            exludePaths = "(/|\\)excludes(/|\\)|(/|\\)eng(/|\\)scripts-tests(/|\\)Repo-File-Content-Replacements(/|\\)inputs(/|\\)test1(/|\\)jsonFiles(/|\\)1.json";
+            includeFromExcludedPaths = "(/|\\)excludes(/|\\)includes(/|\\)";
+            migrationMapFile = "$PSScriptRoot/inputs/test1/jsonFiles/1.json";
+            scannedDirectory = "$PSScriptRoot/inputs/test1";
+        }
+        @{
+            exludePaths = "(/|\\)excludes(/|\\)";
+            includeFromExcludedPaths = "";
+            migrationMapFile = "$PSScriptRoot/inputs/test1/jsonFiles/1.json";
+            scannedDirectory = "$PSScriptRoot/inputs/test2";
+        }
+    ) {
+        $backupFolder = "$PSScriptRoot/inputs/backup"
+        Backup-Folder -targetFolder $ScannedDirectory -backupFolder $backupFolder
+        $migrationMap = Get-Content $migrationMapFile -Raw
+        . $PSScriptRoot/../../Repo-File-Content-Replacements.ps1 `
+            -ExcludePathsRegex $exludePaths `
+            -IncludeFromExcludedPathsRegex $includeFromExcludedPaths `
+            -MigrationMapJson $migrationMap `
+            -ScannedDirectory $ScannedDirectory 
+        $files = Get-ChildItem "$scannedDirectory/*" -Recurse -File -Include *.txt
+        foreach ($file in $files) {
+            $expectPath = $file.FullName -replace 'inputs', 'expected'
+            (Get-Content $file) | Should -Be (Get-Content $expectPath)
+        }
+        Reset-Folder -backupFolder $backupFolder -targetFolder $ScannedDirectory
+    }
+}
+
+Describe "repo-file-content-replacement" -Tag "UnitTest" {
+    # Passed cases
+    It "Test on the files with no includePaths" -TestCases @(
+        @{ 
+            exludePaths = "(/|\\)excludes(/|\\)";
+            migrationMapFile = "$PSScriptRoot/inputs/test1/jsonFiles/1.json";
+            scannedDirectory = "$PSScriptRoot/inputs/test3";
+        }
+    ) {
+        $backupFolder = "$PSScriptRoot/inputs/backup"
+        Backup-Folder -targetFolder $ScannedDirectory -backupFolder $backupFolder
+        $migrationMap = Get-Content $migrationMapFile -Raw
+        . $PSScriptRoot/../../Repo-File-Content-Replacements.ps1 `
+            -ExcludePathsRegex $exludePaths `
+            -MigrationMapJson $migrationMap `
+            -ScannedDirectory $ScannedDirectory 
+        $files = Get-ChildItem "$scannedDirectory/*" -Recurse -File -Include *.txt
+        foreach ($file in $files) {
+            $expectPath = $file.FullName -replace 'inputs', 'expected'
+            (Get-Content $file) | Should -Be (Get-Content $expectPath)
+        }
+        Reset-Folder -backupFolder $backupFolder -targetFolder $ScannedDirectory
+    }
+}

eng/scripts/contentreplacements/tests/Repo-File-Content-Replacements/expected/test1/3.txt

@@ -0,0 +1,4 @@
+3.txt
+This is a testing for azsdk-pool-mms-win-2022-general migration. 
+This is a testing for windows2022 migration. 
+Expect to have changes for this file.

eng/scripts/contentreplacements/tests/Repo-File-Content-Replacements/expected/test1/excludes/1.txt

@@ -0,0 +1,3 @@
+1.txt
+This is a testing for windows-2019 migration. 
+No change expected for this file.

eng/scripts/contentreplacements/tests/Repo-File-Content-Replacements/expected/test1/excludes/includes/2.txt

@@ -0,0 +1,3 @@
+2.txt
+This is a testing for windows-2022 migration. 
+Expect to have changes for this file.

eng/scripts/contentreplacements/tests/Repo-File-Content-Replacements/expected/test1/includes/.6.txt

@@ -0,0 +1,4 @@
+6.txt
+This is a testing for azsdk-pool-mms-win-2022-general migration. 
+This is a testing for windows2022 migration. 
+Expect to have changes for this file.

eng/scripts/contentreplacements/tests/Repo-File-Content-Replacements/expected/test1/includes/.azure-pipelines/5.txt

@@ -0,0 +1,4 @@
+5.txt
+This is a testing for azsdk-pool-mms-win-2022-general migration. 
+This is a testing for windows2022 migration. 
+Expect to have changes for this file.

eng/scripts/contentreplacements/tests/Repo-File-Content-Replacements/expected/test1/includes/4.txt

@@ -0,0 +1,4 @@
+4.txt
+This is a testing for azsdk-pool-mms-win-2022-general migration. 
+This is a testing for windows2022 migration. 
+Expect to have changes for this file.