Skip to content

Commit

Permalink
Run apiview checks in pr to detect public api changes (#3283)
Browse files Browse the repository at this point in the history
Fixes #3203
Fixes #3207
  • Loading branch information
m-nash authored May 10, 2024
1 parent 8c44c3a commit 52377dd
Show file tree
Hide file tree
Showing 25 changed files with 1,994 additions and 73 deletions.
3 changes: 3 additions & 0 deletions cspell.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,9 @@ words:
- TCGC
- VITE
- uncollapsed
- nupkg
- Vsts
- tsdoc
ignorePaths:
- "**/node_modules/**"
- "**/dist/**"
Expand Down
38 changes: 38 additions & 0 deletions eng/common/scripts/git-helpers.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
# cSpell:ignore Committish
# cSpell:ignore PULLREQUEST
# cSpell:ignore TARGETBRANCH
# cSpell:ignore SOURCECOMMITID
function Get-ChangedFiles {
param (
[string]$SourceCommittish= "${env:SYSTEM_PULLREQUEST_SOURCECOMMITID}",
[string]$TargetCommittish = ("origin/${env:SYSTEM_PULLREQUEST_TARGETBRANCH}" -replace "refs/heads/"),
[string]$DiffPath,
[string]$DiffFilterType = "d"
)
# If ${env:SYSTEM_PULLREQUEST_TARGETBRANCH} is empty, then return empty.
if (!$TargetCommittish -or ($TargetCommittish -eq "origin/")) {
Write-Host "There is no target branch passed in. "
return ""
}

# Add config to disable the quote and encoding on file name.
# Ref: https://github.com/msysgit/msysgit/wiki/Git-for-Windows-Unicode-Support#disable-quoted-file-names
# Ref: https://github.com/msysgit/msysgit/wiki/Git-for-Windows-Unicode-Support#disable-commit-message-transcoding
# Git PR diff: https://docs.github.com/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/about-comparing-branches-in-pull-requests#three-dot-and-two-dot-git-diff-comparisons
$command = "git -c core.quotepath=off -c i18n.logoutputencoding=utf-8 diff `"$TargetCommittish...$SourceCommittish`" --name-only --diff-filter=$DiffFilterType"
if ($DiffPath) {
$command = $command + " -- `'$DiffPath`'"
}
Write-Host $command
$changedFiles = Invoke-Expression -Command $command
if(!$changedFiles) {
Write-Host "No changed files in git diff between $TargetCommittish and $SourceCommittish"
}
else {
Write-Host "Here are the diff files:"
foreach ($file in $changedFiles) {
Write-Host " $file"
}
}
return $changedFiles
}
11 changes: 10 additions & 1 deletion eng/emitters/pipelines/templates/jobs/build-job.yml
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,14 @@ parameters:
type: boolean
default: false

# Language short name
- name: LanguageShortName
type: string

- name: BuildArtifactName
type: string
default: ""

jobs:
- job: Build_${{ parameters.Os }}_${{ split(parameters.NodeVersion, '.')[0] }}
${{ if eq(parameters.Os, 'linux') }}:
Expand All @@ -62,11 +70,12 @@ jobs:
Publish: ${{ parameters.Publish }}
AdditionalInitializeSteps: ${{ parameters.AdditionalInitializeSteps }}
Packages: ${{ parameters.Packages }}
LanguageShortName: ${{ parameters.LanguageShortName }}
PackagePath: ${{ parameters.PackagePath }}
NodeVersion: ${{ parameters.NodeVersion }}
${{ if parameters.EmitArtifacts }}:
templateContext:
outputs:
- output: pipelineArtifact
path: $(Build.ArtifactStagingDirectory)
artifact: build_artifacts
artifact: ${{ parameters.BuildArtifactName }}
40 changes: 40 additions & 0 deletions eng/emitters/pipelines/templates/jobs/detect-api-changes.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
parameters:
Artifacts: []
BuildArtifactName: ""
LanguageShortName: "Unknown"
DependsOn: ""

jobs:
- job: DetectApiChanges
dependsOn: ${{ parameters.DependsOn }}
pool:
name: $(LINUXPOOL)
image: $(LINUXVMIMAGE)
os: linux
steps:
- download: current
artifact: ${{ parameters.BuildArtifactName }}
displayName: Download build artifacts
- pwsh: |
$apiChangeDetectRequestUrl = "https://apiview.dev/PullRequest/DetectApiChanges"
echo "##vso[task.setvariable variable=ApiChangeDetectRequestUrl]$apiChangeDetectRequestUrl"
displayName: "Set API change detect request URL"
condition: eq(variables['ApiChangeDetectRequestUrl'], '')
- task: Powershell@2
inputs:
filePath: $(Build.SourcesDirectory)/eng/emitters/scripts/Detect-Api-Changes.ps1
arguments: >
-ArtifactList ('${{ convertToJson(parameters.Artifacts) }}' | ConvertFrom-Json | Select-Object Name)
-ArtifactPath $(Agent.BuildDirectory)/${{ parameters.BuildArtifactName }}
-CommitSha '$(Build.SourceVersion)'
-BuildId $(Build.BuildId)
-PullRequestNumber $(System.PullRequest.PullRequestNumber)
-RepoFullName $(Build.Repository.Name)
-APIViewUri $(ApiChangeDetectRequestUrl)
-BuildArtifactName ${{ parameters.BuildArtifactName }}
-DevopsProject $(System.TeamProject)
-LanguageShortName ${{ parameters.LanguageShortName }}
pwsh: true
displayName: Detect API changes
condition: and(succeededOrFailed(), eq(variables['Build.Reason'],'PullRequest'))
10 changes: 10 additions & 0 deletions eng/emitters/pipelines/templates/jobs/test-job.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,14 @@ parameters:
type: boolean
default: false

# Language short name
- name: LanguageShortName
type: string

- name: BuildArtifactName
type: string
default: ""

jobs:
- job: Test_${{ parameters.Os }}_${{ split(parameters.NodeVersion, '.')[0] }}
${{ if eq(parameters.Os, 'linux') }}:
Expand All @@ -52,7 +60,9 @@ jobs:
parameters:
AdditionalInitializeSteps: ${{ parameters.AdditionalInitializeSteps }}
PackagePath: ${{ parameters.PackagePath }}
LanguageShortName: ${{ parameters.LanguageShortName }}
NodeVersion: $(nodeVersion)
BuildArtifactName: ${{ parameters.BuildArtifactName }}
${{ if ne(length(parameters.TestMatrix), 0) }}:
TestArgs: $(TestArguments)
${{ else }}:
Expand Down
170 changes: 110 additions & 60 deletions eng/emitters/pipelines/templates/stages/emitter-stages.yml
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,13 @@ parameters:
type: object
default: []

- name: LanguageShortName
type: string

- name: HasNugetPackages
type: boolean
default: false

stages:
# Build stage
# Responsible for building the autorest generator and typespec emitter packages
Expand All @@ -84,6 +91,8 @@ stages:
AdditionalInitializeSteps: ${{ parameters.AdditionalInitializeSteps }}
Packages: ${{ parameters.Packages }}
PackagePath: ${{ parameters.PackagePath }}
LanguageShortName: ${{ parameters.LanguageShortName }}
BuildArtifactName: build_artifacts_${{ parameters.LanguageShortName }}
NodeVersion: 20.x
Os: linux
EmitArtifacts: true # Emit artifacts only for the first job
Expand All @@ -95,6 +104,7 @@ stages:
AdditionalInitializeSteps: ${{ parameters.AdditionalInitializeSteps }}
Packages: ${{ parameters.Packages }}
PackagePath: ${{ parameters.PackagePath }}
LanguageShortName: ${{ parameters.LanguageShortName }}
NodeVersion: 18.x
Os: linux
- template: /eng/emitters/pipelines/templates/jobs/build-job.yml
Expand All @@ -105,6 +115,7 @@ stages:
AdditionalInitializeSteps: ${{ parameters.AdditionalInitializeSteps }}
Packages: ${{ parameters.Packages }}
PackagePath: ${{ parameters.PackagePath }}
LanguageShortName: ${{ parameters.LanguageShortName }}
NodeVersion: 20.x
Os: windows
- template: /eng/emitters/pipelines/templates/jobs/build-job.yml
Expand All @@ -115,66 +126,18 @@ stages:
AdditionalInitializeSteps: ${{ parameters.AdditionalInitializeSteps }}
Packages: ${{ parameters.Packages }}
PackagePath: ${{ parameters.PackagePath }}
LanguageShortName: ${{ parameters.LanguageShortName }}
NodeVersion: 18.x
Os: windows

# Publish stage
# Responsible for publishing the packages in `build_artifacts/packages` and producing `emitter-package-lock.json`
# Produces the artifact `publish_artifacts` which contains the following:
# emitter-package-lock.json: Created by calling `npm install` using `build_artifacts/emitter-package.json` and will
# be placed in the `/eng` folder of the sdk repository.
- ${{ if ne(parameters.Publish, 'none') }}:
- stage: ${{ parameters.StagePrefix }}_Publish
displayName: ${{ parameters.StagePrefix }} - Publish
condition: and(succeeded(), ${{ parameters.Condition }})
dependsOn:
- ${{ parameters.DependsOn }}
- ${{ parameters.StagePrefix }}_Build
- ${{ if and(parameters.PublishDependsOnTest, ne(length(parameters.TestMatrix), 0)) }}:
- ${{ parameters.StagePrefix }}_Test
variables:
toolsRepositoryPath: $(Build.SourcesDirectory)
buildArtifactsPath: $(Pipeline.Workspace)/build_artifacts
pool:
name: $(LINUXPOOL)
image: $(LINUXVMIMAGE)
os: linux
jobs:
- job: Publish
steps:
- checkout: self

- download: current
artifact: build_artifacts
displayName: Download build artifacts

# Create authenticated .npmrc file for publishing
- ${{ if eq(parameters.Publish, 'internal') }}:
- template: /eng/emitters/pipelines/templates/steps/create-authenticated-npmrc.yml
parameters:
npmrcPath: $(buildArtifactsPath)/packages/.npmrc
registryUrl: https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-js-test-autorest/npm/registry/
- ${{ else }}:
- pwsh: |
"//registry.npmjs.org/:_authToken=$(azure-sdk-npm-token)" | Out-File '.npmrc'
displayName: Authenticate .npmrc for npmjs.org
workingDirectory: $(buildArtifactsPath)/packages
# per package, publishing using appropriate tool
- ${{ each package in parameters.Packages }}:
- ${{ if eq(package.type, 'npm') }}:
- pwsh: |
$file = Resolve-Path "${{ package.file }}"
Write-Host "npm publish $file --verbose --access public --prefix $(buildArtifactsPath)/packages"
npm publish $file --verbose --access public --prefix $(buildArtifactsPath)/packages
displayName: Publish ${{ package.name }}
workingDirectory: $(buildArtifactsPath)/packages
templateContext:
outputs:
- output: pipelineArtifact
path: $(Build.ArtifactStagingDirectory)
artifact: publish_artifacts
- ${{ if eq(parameters.Publish, 'none') }}:
- template: /eng/emitters/pipelines/templates/jobs/detect-api-changes.yml
parameters:
LanguageShortName: ${{ parameters.LanguageShortName }}
DependsOn: Build_linux_20
BuildArtifactName: build_artifacts_${{ parameters.LanguageShortName }}
Artifacts:
- ${{ each package in parameters.Packages }}:
- name: ${{ package.name }}

# Test stage
# Responsible for running unit and functional tests using a matrix passed in as the parameter `TestMatrix`.
Expand All @@ -189,14 +152,16 @@ stages:
- ${{ parameters.StagePrefix }}_Build
variables:
selfRepositoryPath: $(Build.SourcesDirectory)
buildArtifactsPath: $(Pipeline.Workspace)/build_artifacts
buildArtifactsPath: $(Pipeline.Workspace)/build_artifacts_${{ parameters.LanguageShortName }}
jobs:
- ${{ if parameters.UnitTestArgs }}:
- template: /eng/emitters/pipelines/templates/jobs/test-job.yml
parameters:
AdditionalInitializeSteps: ${{ parameters.AdditionalInitializeSteps }}
PackagePath: ${{ parameters.PackagePath }}
TestArgs: ${{ parameters.UnitTestArgs }}
LanguageShortName: ${{ parameters.LanguageShortName }}
BuildArtifactName: build_artifacts_${{ parameters.LanguageShortName }}
NodeVersion: 20.x
Os: linux
EmitArtifacts: true # Emit artifacts only for the first job
Expand All @@ -205,20 +170,26 @@ stages:
AdditionalInitializeSteps: ${{ parameters.AdditionalInitializeSteps }}
PackagePath: ${{ parameters.PackagePath }}
TestArgs: ${{ parameters.UnitTestArgs }}
LanguageShortName: ${{ parameters.LanguageShortName }}
BuildArtifactName: build_artifacts_${{ parameters.LanguageShortName }}
NodeVersion: 18.x
Os: linux
- template: /eng/emitters/pipelines/templates/jobs/test-job.yml
parameters:
AdditionalInitializeSteps: ${{ parameters.AdditionalInitializeSteps }}
PackagePath: ${{ parameters.PackagePath }}
TestArgs: ${{ parameters.UnitTestArgs }}
LanguageShortName: ${{ parameters.LanguageShortName }}
BuildArtifactName: build_artifacts_${{ parameters.LanguageShortName }}
NodeVersion: 20.x
Os: windows
- template: /eng/emitters/pipelines/templates/jobs/test-job.yml
parameters:
AdditionalInitializeSteps: ${{ parameters.AdditionalInitializeSteps }}
PackagePath: ${{ parameters.PackagePath }}
TestArgs: ${{ parameters.UnitTestArgs }}
LanguageShortName: ${{ parameters.LanguageShortName }}
BuildArtifactName: build_artifacts_${{ parameters.LanguageShortName }}
NodeVersion: 18.x
Os: windows

Expand All @@ -234,11 +205,90 @@ stages:
- ${{ parameters.StagePrefix }}_Build
variables:
selfRepositoryPath: $(Build.SourcesDirectory)
buildArtifactsPath: $(Pipeline.Workspace)/build_artifacts
buildArtifactsPath: $(Pipeline.Workspace)/build_artifacts_${{ parameters.LanguageShortName }}
jobs:
- template: /eng/emitters/pipelines/templates/jobs/test-job.yml
parameters:
AdditionalInitializeSteps: ${{ parameters.AdditionalInitializeSteps }}
PackagePath: ${{ parameters.PackagePath }}
LanguageShortName: ${{ parameters.LanguageShortName }}
BuildArtifactName: build_artifacts_${{ parameters.LanguageShortName }}
Os: windows
TestMatrix: ${{ parameters.TestMatrix }}

# Publish stage
# Responsible for publishing the packages in `build_artifacts/packages` and producing `emitter-package-lock.json`
# Produces the artifact `publish_artifacts` which contains the following:
# emitter-package-lock.json: Created by calling `npm install` using `build_artifacts/emitter-package.json` and will
# be placed in the `/eng` folder of the sdk repository.
- ${{ if ne(parameters.Publish, 'none') }}:
- stage: ${{ parameters.StagePrefix }}_Publish
displayName: ${{ parameters.StagePrefix }} - Publish
condition: and(succeeded(), ${{ parameters.Condition }})
dependsOn:
- ${{ parameters.DependsOn }}
- ${{ parameters.StagePrefix }}_Build
- ${{ if and(parameters.PublishDependsOnTest, ne(length(parameters.TestMatrix), 0)) }}:
- ${{ parameters.StagePrefix }}_Test
- ${{ parameters.StagePrefix }}_Regen_Test
variables:
toolsRepositoryPath: $(Build.SourcesDirectory)
buildArtifactsPath: $(Pipeline.Workspace)/build_artifacts_${{ parameters.LanguageShortName }}
pool:
name: $(LINUXPOOL)
image: $(LINUXVMIMAGE)
os: linux
jobs:
- job: Publish
steps:
- checkout: self

- download: current
artifact: build_artifacts_${{ parameters.LanguageShortName }}
displayName: Download build artifacts

# Create authenticated .npmrc file for publishing
- ${{ if eq(parameters.Publish, 'internal') }}:
- template: /eng/emitters/pipelines/templates/steps/create-authenticated-npmrc.yml
parameters:
npmrcPath: $(buildArtifactsPath)/packages/.npmrc
registryUrl: https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-js-test-autorest/npm/registry/
- ${{ else }}:
- pwsh: |
"//registry.npmjs.org/:_authToken=$(azure-sdk-npm-token)" | Out-File '.npmrc'
displayName: Authenticate .npmrc for npmjs.org
workingDirectory: $(buildArtifactsPath)/packages
# per package, publishing using appropriate tool
- ${{ each package in parameters.Packages }}:
- ${{ if eq(package.type, 'npm') }}:
- pwsh: |
$file = Resolve-Path "${{ package.file }}"
Write-Host "npm publish $file --verbose --access public --prefix $(buildArtifactsPath)/packages"
npm publish $file --verbose --access public --prefix $(buildArtifactsPath)/packages
displayName: Publish ${{ package.name }}
workingDirectory: $(buildArtifactsPath)/packages
- ${{ if parameters.HasNugetPackages }}:
- task: 1ES.PublishNuget@1
displayName: Publish Nuget packages
inputs:
packagesToPush: $(buildArtifactsPath)/packages/*.nupkg
packageParentPath: $(buildArtifactsPath)/packages
# todo update so we can publish publicly to nuget
publishVstsFeed: "29ec6040-b234-4e31-b139-33dc4287b756/fa8c16a3-dbe0-4de2-a297-03065ec1ba3f"
nuGetFeedType: internal

- template: /eng/emitters/pipelines/templates/steps/create-apireview.yml
parameters:
Artifacts:
- ${{ each package in parameters.Packages }}:
- name: ${{ package.name }}
ArtifactName: build_artifacts_${{ parameters.LanguageShortName }}
ArtifactPath: $(buildArtifactsPath)
LanguageShortName: ${{ parameters.LanguageShortName }}

templateContext:
outputs:
- output: pipelineArtifact
path: $(Build.ArtifactStagingDirectory)
artifact: publish_artifacts
Loading

0 comments on commit 52377dd

Please sign in to comment.