diff --git a/eng/pipelines/templates/stages/archetype-autorest-preview.yml b/eng/pipelines/templates/stages/archetype-autorest-preview.yml index 6a1388cd268..b0b4d713958 100644 --- a/eng/pipelines/templates/stages/archetype-autorest-preview.yml +++ b/eng/pipelines/templates/stages/archetype-autorest-preview.yml @@ -3,26 +3,21 @@ parameters: - name: BuildPrereleaseVersion type: boolean -# Whether to use the `next` version of TypeSpec. This is passed as a flag to the init script. +# Whether to use the `next` version of TypeSpec. This is passed as a flag to the initialize script. - name: UseTypeSpecNext type: boolean -# The target to publish packages to. Currently supported values are 'internal' and 'public'. -- name: PublishTarget - type: string +# Whether to publish to the internal feed. +- name: PublishInternal + type: boolean # Path to the emitter package's package.json file. If specified, it will be used to generate emitter-package.json in the artifact `build_artifacts`. - name: EmitterPackageJsonPath type: string default: 'not-specified' -# Custom steps to run after the autorest repository is cloned. If custom build steps are specified, the default init and build scripts will not be run. -# The build steps should produce the directory `/artifacts` with contents: -# packages/ -# autorest-csharp-2.0.0-alpha.4.tgz -# Microsoft.Azure.AutoRest.CSharp.2.0.0-alpha.4.nupkg -# typespec-csharp-1.2.3-alpha.4.tgz -- name: BuildSteps +# Custom steps to run in the Build and Test stages after the repositories are cloned. +- name: AutorestInitializationSteps type: stepList default: [] @@ -49,7 +44,30 @@ parameters: type: number default: 10 +# Indicates if regenration matrix should only contain folders with typespec files +- name: OnlyGenerateTypespec + type: boolean + default: false + +# Indicates the build matrix to use for post-build autorest validation +- name: TestMatrix + type: object + default: {} + +# Indicates the build matrix to use for post-build autorest validation +- name: PublishDependsOnTest + type: boolean + default: false + stages: + +# Build stage +# Responsible for building the autorest generator and typespec emitter packages +# Produces the artifact `build_artifacts` which contains the following: +# emitter-package.json: Will be used to generate the emitter-package-lock.json file and placed in the `/eng` folder +# of the sdk repository. +# overrides.json: Contains npm package version overrides for the emitter and generator +# packages/: Contains the packages to publish - stage: Build variables: autorestRepositoryPath: $(Build.SourcesDirectory)/autorest @@ -64,13 +82,6 @@ stages: displayName: 'Get repository details' # Validate parameters and fail early if invalid - - ${{ if notIn(parameters.PublishTarget, 'internal', 'public') }}: - - script: | - echo "Publish target ${{ parameters.PublishTarget }} is not supported" - exit 1 - displayName: 'Unsupported PublishTarget' - condition: always() - - ${{ each package in parameters.Packages }}: - ${{ if notIn(package.type, 'npm', 'nuget') }}: - script: | @@ -81,25 +92,30 @@ stages: - checkout: self path: s/autorest - + - checkout: azure-sdk-tools - - ${{ parameters.BuildSteps }} + - ${{ parameters.AutorestInitializationSteps }} - - ${{ if eq(length(parameters.BuildSteps), 0) }}: - - script: > - npm run ci-init -- --useTypeSpecNext ${{ parameters.UseTypeSpecNext }} - displayName: 'Run init script' + - task: PowerShell@2 + displayName: 'Run initialize script' + inputs: + pwsh: true + filePath: $(autorestRepositoryPath)/eng/scripts/initialize.ps1 + arguments: -UseTypeSpecNext:$${{ parameters.UseTypeSpecNext }} workingDirectory: $(autorestRepositoryPath) - - - script: > - npm run ci-build -- - --publishTarget ${{ parameters.PublishTarget }} - --buildPrereleaseVersion ${{ parameters.BuildPrereleaseVersion }} - --buildNumber $(Build.BuildNumber) - --output $(Build.ArtifactStagingDirectory) - displayName: 'Run build script' - name: build_script + + - task: PowerShell@2 + displayName: 'Run build script' + name: ci_build + inputs: + pwsh: true + filePath: $(autorestRepositoryPath)/eng/scripts/build.ps1 + arguments: > + -BuildNumber "$(Build.BuildNumber)" + -Output "$(Build.ArtifactStagingDirectory)" + -PublishInternal:$${{ parameters.PublishInternal }} + -Prerelease:$${{ parameters.BuildPrereleaseVersion }} workingDirectory: $(autorestRepositoryPath) - ${{ if ne(parameters.EmitterPackageJsonPath, 'not-specified') }}: @@ -110,13 +126,14 @@ stages: filePath: $(toolsRepositoryPath)/eng/scripts/autorest/New-EmitterPackageJson.ps1 arguments: > -PackageJsonPath '${{ parameters.EmitterPackageJsonPath }}' - -OverridesPath $(Build.ArtifactStagingDirectory)/overrides.json + -OverridesPath '$(Build.ArtifactStagingDirectory)/overrides.json' -OutputDirectory '$(Build.ArtifactStagingDirectory)' workingDirectory: $(autorestRepositoryPath) - - publish: $(Build.ArtifactStagingDirectory) - artifact: build_artifacts - displayName: Publish artifacts directory + - template: ../../../common/pipelines/templates/steps/publish-artifact.yml + parameters: + artifactName: build_artifacts + artifactPath: $(Build.ArtifactStagingDirectory) - pwsh: | $branchName = 'autorest-failed-build-$(Build.BuildId)' @@ -146,8 +163,16 @@ stages: condition: failed() workingDirectory: $(autorestRepositoryPath) +# Publish stage +# Responsible for publishing the packages in `build_artifacts/packages` and producing `emitter-package-lock.json` +# Produces the artifact `publish_artifacts` which contains the following: +# emitter-package-lock.json: Created by calling `npm install` using `build_artifacts/emitter-package.json` and will +# be placed in the `/eng` folder of the sdk repository. - stage: Publish - dependsOn: Build + dependsOn: + - Build + - ${{ if and(parameters.PublishDependsOnTest, ne(length(parameters.TestMatrix), 0)) }}: + - Test variables: toolsRepositoryPath: $(Build.SourcesDirectory) buildArtifactsPath: $(Pipeline.Workspace)/build_artifacts @@ -161,12 +186,12 @@ stages: displayName: Download build artifacts # Create authenticated .npmrc file for publishing - - ${{ if eq(parameters.PublishTarget, 'internal') }}: + - ${{ if eq(parameters.PublishInternal, 'true') }}: - template: ../steps/create-authenticated-npmrc.yml parameters: npmrcPath: $(buildArtifactsPath)/packages/.npmrc registryUrl: https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-js-test-autorest/npm/registry/ - - ${{ elseif eq(parameters.PublishTarget, 'public') }}: + - ${{ else }}: - pwsh: | "//registry.npmjs.org/:_authToken=$(azure-sdk-npm-token)" | Out-File '.npmrc' displayName: Authenticate .npmrc for npmjs.org @@ -177,8 +202,8 @@ stages: - ${{ if eq(package.type, 'npm') }}: - pwsh: | $file = Resolve-Path "${{ package.file }}" - Write-Host "npm publish $file --verbose --access public" - npm publish $file --verbose --access public + Write-Host "npm publish $file --verbose --access public --prefix $(buildArtifactsPath)/packages" + npm publish $file --verbose --access public --prefix $(buildArtifactsPath)/packages displayName: Publish ${{ package.name }} workingDirectory: $(buildArtifactsPath)/packages - ${{ elseif eq(package.type, 'nuget') }}: @@ -201,9 +226,60 @@ stages: -EmitterPackageJsonPath "$(buildArtifactsPath)/emitter-package.json" -OutputDirectory "$(Build.ArtifactStagingDirectory)" - - publish: $(Build.ArtifactStagingDirectory) - artifact: publish_artifacts - displayName: Publish artifacts directory + - template: ../../../common/pipelines/templates/steps/publish-artifact.yml + parameters: + artifactName: publish_artifacts + artifactPath: $(Build.ArtifactStagingDirectory) + +# Test stage +# Responsible for running unit and functional tests using a matrix passed in as the parameter `TestMatrix`. +# Will only run if the parameter `TestMatrix` is not empty. +# The contents of the artficact `build_artifacts` are available under the path `$(buildArtifactsPath)`. +- ${{ if ne(length(parameters.TestMatrix), 0) }}: + - stage: Test + dependsOn: Build + variables: + autorestRepositoryPath: $(Build.SourcesDirectory)/autorest + toolsRepositoryPath: $(Build.SourcesDirectory)/azure-sdk-tools + jobs: + - job: Test + strategy: + matrix: ${{ parameters.TestMatrix }} + variables: + buildArtifactsPath: $(Pipeline.Workspace)/build_artifacts + steps: + - checkout: self + path: s/autorest # Use a predictable path for the `self` repo + + - checkout: azure-sdk-tools + + - download: current + artifact: build_artifacts + displayName: Download build artifacts + + - ${{ parameters.AutorestInitializationSteps }} + + - task: PowerShell@2 + displayName: 'Run initialize script' + inputs: + pwsh: true + filePath: $(autorestRepositoryPath)/eng/scripts/initialize.ps1 + arguments: -BuildArtifactsPath '$(buildArtifactsPath)' + workingDirectory: $(autorestRepositoryPath) + + - task: PowerShell@2 + displayName: 'Run test script' + inputs: + pwsh: true + filePath: $(autorestRepositoryPath)/eng/scripts/test.ps1 + arguments: $(TestArguments) + workingDirectory: $(autorestRepositoryPath) + + - template: ../../../common/pipelines/templates/steps/publish-artifact.yml + parameters: + artifactName: test_artifacts_$(System.JobName) + artifactPath: $(Build.ArtifactStagingDirectory) + - stage: Regenerate dependsOn: @@ -276,11 +352,12 @@ stages: -OutputVariableName matrix -JobCount ${{ parameters.RegenerationJobCount }} -MinimumPerJob ${{ parameters.MinimumPerJob }} - -OnlyTypespec ${{ parameters.UseTypeSpecNext }} + -OnlyTypespec ${{ parameters.OnlyGenerateTypespec }} - - publish: $(Build.ArtifactStagingDirectory) - artifact: matrix_artifacts - displayName: Publish matrix artifacts + - template: ../../../common/pipelines/templates/steps/publish-artifact.yml + parameters: + artifactName: matrix_artifacts + artifactPath: $(Build.ArtifactStagingDirectory) - job: Generate dependsOn: Initialize @@ -303,14 +380,9 @@ stages: SkipCheckoutNone: true - download: current - artifact: build_artifacts - displayName: Download build artifacts - - download: current - artifact: publish_artifacts - displayName: Download pubish artifacts - - download: current - artifact: matrix_artifacts - displayName: Download matrix artifacts + displayName: Download pipeline artifacts + + - ${{ parameters.SdkInitializationSteps }} - task: PowerShell@2 displayName: Call regeneration script @@ -335,17 +407,47 @@ stages: displayName: Create PR dependsOn: - Generate + variables: + generateJobResult: $[dependencies.Generate.result] + generatorVersion: $[stageDependencies.Build.Build.outputs['ci_build.generatorVersion']] steps: - checkout: self - checkout: azure-sdk-tools - - task: PowerShell@2 - displayName: Get source description - inputs: - pwsh: true - filePath: $(toolsRepositoryPath)/eng/common/scripts/Get-BuildSourceDescription.ps1 - arguments: > - -Variable 'BuildSourceDescription' + - pwsh: | + $generateJobResult = '$(generateJobResult)' + $generatorVersion = '$(generatorVersion)' + $repoUrl = '$(Build.Repository.Uri)' + $sourceBranch = '$(Build.SourceBranch)' + $buildNumber = '$(Build.SourceBranch)' + $queuedBy = '$(Build.SourceBranch)' + $preRelease = '${{ parameters.BuildPrereleaseVersion }}' -eq 'true' + + $source = "[$sourceBranch]($repoUrl/tree/$sourceBranch)" + if ($sourceBranch -match "^refs/heads/(.+)$") { + $source = "branch: [$($Matches[1])]($repoUrl/tree/$sourceBranch)" + } elseif ($sourceBranch -match "^refs/tags/(.+)$") { + $source = "tag: [$($Matches[1])]($repoUrl/tree/$sourceBranch)" + } elseif ($sourceBranch -match "^refs/pull/(\d+)/(head|merge)$") { + $source = "pull request: $repoUrl/pull/$($Matches[1])" + } + + $prBody = "Triggered from $source" + + $prTitle = $preRelease ` + ? "Update autorest version to prerelease $generatorVersion" ` + : "Update autorest version to $generatorVersion" + + if ($generateJobResult -ne 'Succeeded') { + $prTitle = "Failed: $prTitle" + } + + Write-Host "Setting variable 'PullRequestTitle' to '$prTitle'" + Write-Host "##vso[task.setvariable variable=PullRequestTitle]$prTitle" + + Write-Host "Setting variable 'PullRequestBody' to '$prBody'" + Write-Host "##vso[task.setvariable variable=PullRequestBody]$prBody" + displayName: Get PR title and body - task: PowerShell@2 displayName: Create pull request @@ -359,7 +461,7 @@ stages: -PROwner 'azure-sdk' -PRBranch '$(branchName)' -AuthToken '$(azuresdk-github-pat)' - -PRTitle 'Autorest Regen Preview alpha-$(Build.BuildNumber) by $(Build.QueuedBy)' - -PRBody 'Triggered from $(BuildSourceDescription)' + -PRTitle '$(PullRequestTitle)' + -PRBody '$(PullRequestBody)' -OpenAsDraft $true -PRLabels 'Do Not Merge'