Run apiview checks in pr to detect public api changes

cspell.yaml

@@ -116,6 +116,9 @@ words:
   - TCGC
   - VITE
   - uncollapsed
+  - nupkg
+  - Vsts
+  - tsdoc
 ignorePaths:
   - "**/node_modules/**"
   - "**/dist/**"

eng/common/scripts/git-helpers.ps1

@@ -0,0 +1,38 @@
+# cSpell:ignore Committish
+# cSpell:ignore PULLREQUEST
+# cSpell:ignore TARGETBRANCH
+# cSpell:ignore SOURCECOMMITID
+function Get-ChangedFiles {
+    param (
+      [string]$SourceCommittish= "${env:SYSTEM_PULLREQUEST_SOURCECOMMITID}",
+      [string]$TargetCommittish = ("origin/${env:SYSTEM_PULLREQUEST_TARGETBRANCH}" -replace "refs/heads/"),
+      [string]$DiffPath,
+      [string]$DiffFilterType = "d"
+    )
+    # If ${env:SYSTEM_PULLREQUEST_TARGETBRANCH} is empty, then return empty.
+    if (!$TargetCommittish -or ($TargetCommittish -eq "origin/")) {
+      Write-Host "There is no target branch passed in. "
+      return ""
+    }
+
+    # Add config to disable the quote and encoding on file name. 
+    # Ref: https://github.com/msysgit/msysgit/wiki/Git-for-Windows-Unicode-Support#disable-quoted-file-names
+    # Ref: https://github.com/msysgit/msysgit/wiki/Git-for-Windows-Unicode-Support#disable-commit-message-transcoding
+    # Git PR diff: https://docs.github.com/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/about-comparing-branches-in-pull-requests#three-dot-and-two-dot-git-diff-comparisons
+    $command = "git -c core.quotepath=off -c i18n.logoutputencoding=utf-8 diff `"$TargetCommittish...$SourceCommittish`" --name-only --diff-filter=$DiffFilterType"
+    if ($DiffPath) {
+    $command = $command + " -- `'$DiffPath`'"
+    }
+    Write-Host $command
+    $changedFiles = Invoke-Expression -Command $command
+    if(!$changedFiles) {
+      Write-Host "No changed files in git diff between $TargetCommittish and $SourceCommittish"
+    }
+    else {
+    Write-Host "Here are the diff files:"
+    foreach ($file in $changedFiles) {
+        Write-Host "    $file"
+    }
+    }
+    return $changedFiles
+  }

eng/emitters/pipelines/templates/jobs/build-job.yml

@@ -42,6 +42,14 @@ parameters:
     type: boolean
     default: false
 
+  # Language short name
+  - name: LanguageShortName
+    type: string
+
+  - name: BuildArtifactName
+    type: string
+    default: ""
+
 jobs:
   - job: Build_${{ parameters.Os }}_${{ split(parameters.NodeVersion, '.')[0] }}
     ${{ if eq(parameters.Os, 'linux') }}:
@@ -62,11 +70,12 @@ jobs:
           Publish: ${{ parameters.Publish }}
           AdditionalInitializeSteps: ${{ parameters.AdditionalInitializeSteps }}
           Packages: ${{ parameters.Packages }}
+          LanguageShortName: ${{ parameters.LanguageShortName }}
           PackagePath: ${{ parameters.PackagePath }}
           NodeVersion: ${{ parameters.NodeVersion }}
     ${{ if parameters.EmitArtifacts }}:
       templateContext:
         outputs:
           - output: pipelineArtifact
             path: $(Build.ArtifactStagingDirectory)
-            artifact: build_artifacts
+            artifact: ${{ parameters.BuildArtifactName }}

eng/emitters/pipelines/templates/jobs/detect-api-changes.yml

@@ -0,0 +1,40 @@
+parameters:
+  Artifacts: []
+  BuildArtifactName: ""
+  LanguageShortName: "Unknown"
+  DependsOn: ""
+
+jobs:
+  - job: DetectApiChanges
+    dependsOn: ${{ parameters.DependsOn }}
+    pool:
+      name: $(LINUXPOOL)
+      image: $(LINUXVMIMAGE)
+      os: linux
+    steps:
+      - download: current
+        artifact: ${{ parameters.BuildArtifactName }}
+        displayName: Download build artifacts
+      - pwsh: |
+          $apiChangeDetectRequestUrl = "https://apiview.dev/PullRequest/DetectApiChanges"
+          echo "##vso[task.setvariable variable=ApiChangeDetectRequestUrl]$apiChangeDetectRequestUrl"
+        displayName: "Set API change detect request URL"
+        condition: eq(variables['ApiChangeDetectRequestUrl'], '')
+
+      - task: Powershell@2
+        inputs:
+          filePath: $(Build.SourcesDirectory)/eng/emitters/scripts/Detect-Api-Changes.ps1
+          arguments: >
+            -ArtifactList ('${{ convertToJson(parameters.Artifacts) }}' | ConvertFrom-Json | Select-Object Name)
+            -ArtifactPath $(Agent.BuildDirectory)/${{ parameters.BuildArtifactName }}
+            -CommitSha '$(Build.SourceVersion)'
+            -BuildId $(Build.BuildId)
+            -PullRequestNumber $(System.PullRequest.PullRequestNumber)
+            -RepoFullName $(Build.Repository.Name)
+            -APIViewUri $(ApiChangeDetectRequestUrl)
+            -BuildArtifactName ${{ parameters.BuildArtifactName }}
+            -DevopsProject $(System.TeamProject)
+            -LanguageShortName ${{ parameters.LanguageShortName }}
+          pwsh: true
+        displayName: Detect API changes
+        condition: and(succeededOrFailed(), eq(variables['Build.Reason'],'PullRequest'))

eng/emitters/pipelines/templates/jobs/test-job.yml

@@ -32,6 +32,14 @@ parameters:
     type: boolean
     default: false
 
+  # Language short name
+  - name: LanguageShortName
+    type: string
+
+  - name: BuildArtifactName
+    type: string
+    default: ""
+
 jobs:
   - job: Test_${{ parameters.Os }}_${{ split(parameters.NodeVersion, '.')[0] }}
     ${{ if eq(parameters.Os, 'linux') }}:
@@ -52,7 +60,9 @@ jobs:
         parameters:
           AdditionalInitializeSteps: ${{ parameters.AdditionalInitializeSteps }}
           PackagePath: ${{ parameters.PackagePath }}
+          LanguageShortName: ${{ parameters.LanguageShortName }}
           NodeVersion: $(nodeVersion)
+          BuildArtifactName: ${{ parameters.BuildArtifactName }}
           ${{ if ne(length(parameters.TestMatrix), 0) }}:
             TestArgs: $(TestArguments)
           ${{ else }}:

eng/emitters/pipelines/templates/stages/emitter-stages.yml

@@ -61,6 +61,13 @@ parameters:
     type: object
     default: []
 
+  - name: LanguageShortName
+    type: string
+
+  - name: HasNugetPackages
+    type: boolean
+    default: false
+
 stages:
   # Build stage
   # Responsible for building the autorest generator and typespec emitter packages
@@ -84,6 +91,8 @@ stages:
           AdditionalInitializeSteps: ${{ parameters.AdditionalInitializeSteps }}
           Packages: ${{ parameters.Packages }}
           PackagePath: ${{ parameters.PackagePath }}
+          LanguageShortName: ${{ parameters.LanguageShortName }}
+          BuildArtifactName: build_artifacts_${{ parameters.LanguageShortName }}
           NodeVersion: 20.x
           Os: linux
           EmitArtifacts: true # Emit artifacts only for the first job
@@ -95,6 +104,7 @@ stages:
           AdditionalInitializeSteps: ${{ parameters.AdditionalInitializeSteps }}
           Packages: ${{ parameters.Packages }}
           PackagePath: ${{ parameters.PackagePath }}
+          LanguageShortName: ${{ parameters.LanguageShortName }}
           NodeVersion: 18.x
           Os: linux
       - template: /eng/emitters/pipelines/templates/jobs/build-job.yml
@@ -105,6 +115,7 @@ stages:
           AdditionalInitializeSteps: ${{ parameters.AdditionalInitializeSteps }}
           Packages: ${{ parameters.Packages }}
           PackagePath: ${{ parameters.PackagePath }}
+          LanguageShortName: ${{ parameters.LanguageShortName }}
           NodeVersion: 20.x
           Os: windows
       - template: /eng/emitters/pipelines/templates/jobs/build-job.yml
@@ -115,66 +126,18 @@ stages:
           AdditionalInitializeSteps: ${{ parameters.AdditionalInitializeSteps }}
           Packages: ${{ parameters.Packages }}
           PackagePath: ${{ parameters.PackagePath }}
+          LanguageShortName: ${{ parameters.LanguageShortName }}
           NodeVersion: 18.x
           Os: windows
-
-  # Publish stage
-  # Responsible for publishing the packages in `build_artifacts/packages` and producing `emitter-package-lock.json`
-  # Produces the artifact `publish_artifacts` which contains the following:
-  #   emitter-package-lock.json: Created by calling `npm install` using `build_artifacts/emitter-package.json` and will
-  #                              be placed in the `/eng` folder of the sdk repository.
-  - ${{ if ne(parameters.Publish, 'none') }}:
-      - stage: ${{ parameters.StagePrefix }}_Publish
-        displayName: ${{ parameters.StagePrefix }} - Publish
-        condition: and(succeeded(), ${{ parameters.Condition }})
-        dependsOn:
-          - ${{ parameters.DependsOn }}
-          - ${{ parameters.StagePrefix }}_Build
-          - ${{ if and(parameters.PublishDependsOnTest, ne(length(parameters.TestMatrix), 0)) }}:
-              - ${{ parameters.StagePrefix }}_Test
-        variables:
-          toolsRepositoryPath: $(Build.SourcesDirectory)
-          buildArtifactsPath: $(Pipeline.Workspace)/build_artifacts
-        pool:
-          name: $(LINUXPOOL)
-          image: $(LINUXVMIMAGE)
-          os: linux
-        jobs:
-          - job: Publish
-            steps:
-              - checkout: self
-
-              - download: current
-                artifact: build_artifacts
-                displayName: Download build artifacts
-
-              # Create authenticated .npmrc file for publishing
-              - ${{ if eq(parameters.Publish, 'internal') }}:
-                  - template: /eng/emitters/pipelines/templates/steps/create-authenticated-npmrc.yml
-                    parameters:
-                      npmrcPath: $(buildArtifactsPath)/packages/.npmrc
-                      registryUrl: https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-js-test-autorest/npm/registry/
-              - ${{ else }}:
-                  - pwsh: |
-                      "//registry.npmjs.org/:_authToken=$(azure-sdk-npm-token)" | Out-File '.npmrc'
-                    displayName: Authenticate .npmrc for npmjs.org
-                    workingDirectory: $(buildArtifactsPath)/packages
-
-              # per package, publishing using appropriate tool
-              - ${{ each package in parameters.Packages }}:
-                  - ${{ if eq(package.type, 'npm') }}:
-                      - pwsh: |
-                          $file = Resolve-Path "${{ package.file }}"
-                          Write-Host "npm publish $file --verbose --access public --prefix $(buildArtifactsPath)/packages"
-                          npm publish $file --verbose --access public --prefix $(buildArtifactsPath)/packages
-                        displayName: Publish ${{ package.name }}
-                        workingDirectory: $(buildArtifactsPath)/packages
-
-            templateContext:
-              outputs:
-                - output: pipelineArtifact
-                  path: $(Build.ArtifactStagingDirectory)
-                  artifact: publish_artifacts
+      - ${{ if eq(parameters.Publish, 'none') }}:
+          - template: /eng/emitters/pipelines/templates/jobs/detect-api-changes.yml
+            parameters:
+              LanguageShortName: ${{ parameters.LanguageShortName }}
+              DependsOn: Build_linux_20
+              BuildArtifactName: build_artifacts_${{ parameters.LanguageShortName }}
+              Artifacts:
+                - ${{ each package in parameters.Packages }}:
+                    - name: ${{ package.name }}
 
   # Test stage
   # Responsible for running unit and functional tests using a matrix passed in as the parameter `TestMatrix`.
@@ -189,14 +152,16 @@ stages:
           - ${{ parameters.StagePrefix }}_Build
         variables:
           selfRepositoryPath: $(Build.SourcesDirectory)
-          buildArtifactsPath: $(Pipeline.Workspace)/build_artifacts
+          buildArtifactsPath: $(Pipeline.Workspace)/build_artifacts_${{ parameters.LanguageShortName }}
         jobs:
           - ${{ if parameters.UnitTestArgs }}:
               - template: /eng/emitters/pipelines/templates/jobs/test-job.yml
                 parameters:
                   AdditionalInitializeSteps: ${{ parameters.AdditionalInitializeSteps }}
                   PackagePath: ${{ parameters.PackagePath }}
                   TestArgs: ${{ parameters.UnitTestArgs }}
+                  LanguageShortName: ${{ parameters.LanguageShortName }}
+                  BuildArtifactName: build_artifacts_${{ parameters.LanguageShortName }}
                   NodeVersion: 20.x
                   Os: linux
                   EmitArtifacts: true # Emit artifacts only for the first job
@@ -205,20 +170,26 @@ stages:
                   AdditionalInitializeSteps: ${{ parameters.AdditionalInitializeSteps }}
                   PackagePath: ${{ parameters.PackagePath }}
                   TestArgs: ${{ parameters.UnitTestArgs }}
+                  LanguageShortName: ${{ parameters.LanguageShortName }}
+                  BuildArtifactName: build_artifacts_${{ parameters.LanguageShortName }}
                   NodeVersion: 18.x
                   Os: linux
               - template: /eng/emitters/pipelines/templates/jobs/test-job.yml
                 parameters:
                   AdditionalInitializeSteps: ${{ parameters.AdditionalInitializeSteps }}
                   PackagePath: ${{ parameters.PackagePath }}
                   TestArgs: ${{ parameters.UnitTestArgs }}
+                  LanguageShortName: ${{ parameters.LanguageShortName }}
+                  BuildArtifactName: build_artifacts_${{ parameters.LanguageShortName }}
                   NodeVersion: 20.x
                   Os: windows
               - template: /eng/emitters/pipelines/templates/jobs/test-job.yml
                 parameters:
                   AdditionalInitializeSteps: ${{ parameters.AdditionalInitializeSteps }}
                   PackagePath: ${{ parameters.PackagePath }}
                   TestArgs: ${{ parameters.UnitTestArgs }}
+                  LanguageShortName: ${{ parameters.LanguageShortName }}
+                  BuildArtifactName: build_artifacts_${{ parameters.LanguageShortName }}
                   NodeVersion: 18.x
                   Os: windows
 
@@ -234,11 +205,90 @@ stages:
           - ${{ parameters.StagePrefix }}_Build
         variables:
           selfRepositoryPath: $(Build.SourcesDirectory)
-          buildArtifactsPath: $(Pipeline.Workspace)/build_artifacts
+          buildArtifactsPath: $(Pipeline.Workspace)/build_artifacts_${{ parameters.LanguageShortName }}
         jobs:
           - template: /eng/emitters/pipelines/templates/jobs/test-job.yml
             parameters:
               AdditionalInitializeSteps: ${{ parameters.AdditionalInitializeSteps }}
               PackagePath: ${{ parameters.PackagePath }}
+              LanguageShortName: ${{ parameters.LanguageShortName }}
+              BuildArtifactName: build_artifacts_${{ parameters.LanguageShortName }}
               Os: windows
               TestMatrix: ${{ parameters.TestMatrix }}
+
+  # Publish stage
+  # Responsible for publishing the packages in `build_artifacts/packages` and producing `emitter-package-lock.json`
+  # Produces the artifact `publish_artifacts` which contains the following:
+  #   emitter-package-lock.json: Created by calling `npm install` using `build_artifacts/emitter-package.json` and will
+  #                              be placed in the `/eng` folder of the sdk repository.
+  - ${{ if ne(parameters.Publish, 'none') }}:
+      - stage: ${{ parameters.StagePrefix }}_Publish
+        displayName: ${{ parameters.StagePrefix }} - Publish
+        condition: and(succeeded(), ${{ parameters.Condition }})
+        dependsOn:
+          - ${{ parameters.DependsOn }}
+          - ${{ parameters.StagePrefix }}_Build
+          - ${{ if and(parameters.PublishDependsOnTest, ne(length(parameters.TestMatrix), 0)) }}:
+              - ${{ parameters.StagePrefix }}_Test
+              - ${{ parameters.StagePrefix }}_Regen_Test
+        variables:
+          toolsRepositoryPath: $(Build.SourcesDirectory)
+          buildArtifactsPath: $(Pipeline.Workspace)/build_artifacts_${{ parameters.LanguageShortName }}
+        pool:
+          name: $(LINUXPOOL)
+          image: $(LINUXVMIMAGE)
+          os: linux
+        jobs:
+          - job: Publish
+            steps:
+              - checkout: self
+
+              - download: current
+                artifact: build_artifacts_${{ parameters.LanguageShortName }}
+                displayName: Download build artifacts
+
+              # Create authenticated .npmrc file for publishing
+              - ${{ if eq(parameters.Publish, 'internal') }}:
+                  - template: /eng/emitters/pipelines/templates/steps/create-authenticated-npmrc.yml
+                    parameters:
+                      npmrcPath: $(buildArtifactsPath)/packages/.npmrc
+                      registryUrl: https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-js-test-autorest/npm/registry/
+              - ${{ else }}:
+                  - pwsh: |
+                      "//registry.npmjs.org/:_authToken=$(azure-sdk-npm-token)" | Out-File '.npmrc'
+                    displayName: Authenticate .npmrc for npmjs.org
+                    workingDirectory: $(buildArtifactsPath)/packages
+
+              # per package, publishing using appropriate tool
+              - ${{ each package in parameters.Packages }}:
+                  - ${{ if eq(package.type, 'npm') }}:
+                      - pwsh: |
+                          $file = Resolve-Path "${{ package.file }}"
+                          Write-Host "npm publish $file --verbose --access public --prefix $(buildArtifactsPath)/packages"
+                          npm publish $file --verbose --access public --prefix $(buildArtifactsPath)/packages
+                        displayName: Publish ${{ package.name }}
+                        workingDirectory: $(buildArtifactsPath)/packages
+              - ${{ if parameters.HasNugetPackages }}:
+                  - task: 1ES.PublishNuget@1
+                    displayName: Publish Nuget packages
+                    inputs:
+                      packagesToPush: $(buildArtifactsPath)/packages/*.nupkg
+                      packageParentPath: $(buildArtifactsPath)/packages
+                      # todo update so we can publish publicly to nuget
+                      publishVstsFeed: "29ec6040-b234-4e31-b139-33dc4287b756/fa8c16a3-dbe0-4de2-a297-03065ec1ba3f"
+                      nuGetFeedType: internal
+
+              - template: /eng/emitters/pipelines/templates/steps/create-apireview.yml
+                parameters:
+                  Artifacts:
+                    - ${{ each package in parameters.Packages }}:
+                        - name: ${{ package.name }}
+                  ArtifactName: build_artifacts_${{ parameters.LanguageShortName }}
+                  ArtifactPath: $(buildArtifactsPath)
+                  LanguageShortName: ${{ parameters.LanguageShortName }}
+
+            templateContext:
+              outputs:
+                - output: pipelineArtifact
+                  path: $(Build.ArtifactStagingDirectory)
+                  artifact: publish_artifacts