From ae77bf3c583eff1df41b8c6cc64f54588dbfaabc Mon Sep 17 00:00:00 2001 From: Chidozie Ononiwu Date: Tue, 31 Dec 2024 16:25:26 -0800 Subject: [PATCH 1/7] Add common SDK generation pipeline --- .../templates/jobs/sdk-generation.yml | 160 ++++++++++++++++++ 1 file changed, 160 insertions(+) create mode 100644 eng/common/pipelines/templates/jobs/sdk-generation.yml diff --git a/eng/common/pipelines/templates/jobs/sdk-generation.yml b/eng/common/pipelines/templates/jobs/sdk-generation.yml new file mode 100644 index 000000000..35bb3c7ec --- /dev/null +++ b/eng/common/pipelines/templates/jobs/sdk-generation.yml @@ -0,0 +1,160 @@ +parameters: + - name: SpecRepoBranch + type: string + default: 'main' + - name: SpecRepoCommit + type: string + default: 'HEAD' + - name: SdkRepoOwner + type: string + - name: SdkRepoName + type: string + + - name: SdkRepoNameForSpecGen + type: string + - name: SdkRepoBranch + type: string + default: 'main' + - name: ConfigType + type: string + values: + - 'TypeSpec' + - 'AutoRest' + - name: TspConfigPath + type: string + - name: ReadmePath + type: string + - name: ApiVersion + type: string + default: 'not-specified' + - name: CodePushRequired + type: boolean + - name: SdkPrRequired + type: boolean + - name: ApiViewRequired + type: boolean + +jobs: +- job: + displayName: 'SDK Generation' + variables: + - template: /eng/pipelines/templates/variables/image.yml + - name: SpecRepoUrl + value: 'https://github.com/Azure/azure-rest-api-specs' + - name: SpecRepoFullName + value: 'Azure/azure-rest-api-specs' + - name: SpecRepoName + value: 'azure-rest-api-specs' + - name: SpecRepoDirectory + value: $(System.DefaultWorkingDirectory)/$(SpecRepoName) + - name: SdkRepoDirectory + value: $(System.DefaultWorkingDirectory)/${{ parameters.SdkRepoName }} + pool: + name: $(LINUXPOOL) + vmImage: $(LINUXVMIMAGE) + + steps: + - checkout: none + + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + parameters: + Paths: + - '/*' + - '!sdk/**/test-recordings/*' + - '!sdk/**/recordings/*' + - '!sdk/**/SessionRecords/*' + - '!sdk/**/session-records/*' + Repositories: + - Name: $(SpecRepoFullName) + Commitish: ${{ parameters.SpecRepoBranch }} + WorkingDirectory: $(SpecRepoDirectory) + - Name: ${{ parameters.SdkRepoOwner }}/${{ parameters.SdkRepoName }} + Commitish: ${{ parameters.SdkRepoBranch }} + WorkingDirectory: $(SdkRepoDirectory) + + - script: | + if [ "${{ parameters.SpecRepoCommit }}" = "HEAD" ]; then + cd $(SpecRepoDirectory) + default_commit=$(git rev-parse HEAD) + echo "##vso[task.setvariable variable=SpecRepoCommit]$default_commit" + echo "SpecRepoCommit variable set to default commit: $default_commit" + else + echo "##vso[task.setvariable variable=SpecRepoCommit]${{ parameters.SpecRepoCommit }}" + echo "SpecRepoCommit variable set to: ${{ parameters.SpecRepoCommit }}" + fi + displayName: 'Set SpecRepoCommit variable' + + - task: NodeTool@0 + inputs: + versionSpec: '22.x' + displayName: 'Install Node.js' + + - script: | + npm install -g @azure-tools/spec-gen-sdk + displayName: 'Install spec-gen-sdk' + + - script: | + optional_params="" + sdk_gen_info="sdk generation from Config : " + + if [ "${{ parameters.ConfigType }}" = "TypeSpec" ]; then + optional_params="$optional_params --tsp-config-relative-path ${{ parameters.TspConfigPath }}" + sdk_gen_info="$sdk_gen_info '${{ parameters.TspConfigPath }}'," + else + optional_params="$optional_params --readme-relative-path ${{ parameters.ReadmePath }}" + sdk_gen_info="$sdk_gen_info '${{ parameters.ReadmePath }}'," + fi + if [ "${{ parameters.ApiVersion }}" != "not-specified" ]; then + optional_params="$optional_params --api-version ${{ parameters.ApiVersion }}" + sdk_gen_info="$sdk_gen_info API Version: '${{ parameters.ApiVersion }}'," + fi + + sdk_gen_info="$sdk_gen_info and CommitSHA: '$(SpecRepoCommit)'" + echo "##vso[task.setvariable variable=GeneratedSDKInformation]$sdk_gen_info" + echo "Generated SDK Information : $sdk_gen_info" + + spec-gen-sdk \ + --scp "$(SpecRepoDirectory)" \ + --sdp "$(SdkRepoDirectory)" \ + --wf "$(System.DefaultWorkingDirectory)" \ + -l "${{ parameters.SdkRepoNameForSpecGen }}" \ + -c "$(SpecRepoCommit)" \ + $optional_params + displayName: 'Generate SDK' + + - ${{ if eq(parameters.CodePushRequired, true) }}: + - template: /eng/common/pipelines/templates/steps/git-push-changes.yml + parameters: + BaseRepoBranch: sdk-generation-from-$(Build.DefinitionName)_$(Build.BuildId) + BaseRepoOwner: azure-sdk + CommitMsg: $(GeneratedSDKInformation) + TargetRepoOwner: ${{ parameters.SdkRepoOwner }} + TargetRepoName: ${{ parameters.SdkRepoName }} + PushArgs: "--force" + WorkingDirectory: $(SdkRepoDirectory) + ScriptDirectory: $(SdkRepoDirectory)/eng/common/scripts + + - task: PowerShell@2 + displayName: Create pull request + condition: and(succeeded(), eq('${{ parameters.SdkPrRequired }}', true), eq(variables['HasChanges'], 'true')) + inputs: + pwsh: true + workingDirectory: $(SdkRepoDirectory) + filePath: $(SdkRepoDirectory)/eng/common/scripts/Submit-PullRequest.ps1 + arguments: > + -RepoOwner "${{ parameters.SdkRepoOwner }}" + -RepoName "${{ parameters.SdkRepoName }}" + -BaseBranch "main" + -PROwner "azure-sdk" + -PRBranch "sdk-generation-from-$(Build.DefinitionName)_$(Build.BuildId)" + -AuthToken "$(azuresdk-github-pat)" + -PRTitle "sdk-generation-from-$(Build.DefinitionName)_$(Build.BuildId)" + -PRBody "$(GeneratedSDKInformation)" + -OpenAsDraft $true + + - script: | + echo "This step runs only if ApiViewRequired is true" + displayName: 'Conditional Step' + condition: and(succeeded(), eq('${{ parameters.ApiViewRequired }}', true)) + + \ No newline at end of file From d826bd4813877608dd21507f81ecf8d76c427d4c Mon Sep 17 00:00:00 2001 From: Chidozie Ononiwu Date: Thu, 9 Jan 2025 17:28:19 -0800 Subject: [PATCH 2/7] Update Yaml Configuration --- .../templates/jobs/sdk-generation.yml | 138 +++++++++++------- 1 file changed, 89 insertions(+), 49 deletions(-) diff --git a/eng/common/pipelines/templates/jobs/sdk-generation.yml b/eng/common/pipelines/templates/jobs/sdk-generation.yml index 35bb3c7ec..e21f5ee8c 100644 --- a/eng/common/pipelines/templates/jobs/sdk-generation.yml +++ b/eng/common/pipelines/templates/jobs/sdk-generation.yml @@ -1,16 +1,13 @@ parameters: + - name: SpecRepourl + type: string - name: SpecRepoBranch type: string default: 'main' - name: SpecRepoCommit type: string default: 'HEAD' - - name: SdkRepoOwner - type: string - - name: SdkRepoName - type: string - - - name: SdkRepoNameForSpecGen + - name: SdkRepoUrl type: string - name: SdkRepoBranch type: string @@ -31,24 +28,18 @@ parameters: type: boolean - name: SdkPrRequired type: boolean - - name: ApiViewRequired - type: boolean jobs: - job: displayName: 'SDK Generation' variables: - template: /eng/pipelines/templates/variables/image.yml - - name: SpecRepoUrl - value: 'https://github.com/Azure/azure-rest-api-specs' - - name: SpecRepoFullName - value: 'Azure/azure-rest-api-specs' - - name: SpecRepoName - value: 'azure-rest-api-specs' - - name: SpecRepoDirectory - value: $(System.DefaultWorkingDirectory)/$(SpecRepoName) - - name: SdkRepoDirectory - value: $(System.DefaultWorkingDirectory)/${{ parameters.SdkRepoName }} + - name: NodeVersion + value: '22.13.x' + - name: PythonVersion + value: '3.13' + - name: SdkArtifactName + value: SDK_Artifact pool: name: $(LINUXPOOL) vmImage: $(LINUXVMIMAGE) @@ -56,6 +47,49 @@ jobs: steps: - checkout: none + - pwsh: | + $urlPattern = '^https://github\.com/(?[^/]+)/(?[^/]+)' + if ('${{ parameters.SpecRepoUrl }}' -match $urlPattern) { + $specRepoOwner = $Matches['organization'] + Write-Host "##vso[task.setvariable variable=SpecRepoOwner]$specRepoOwner" + Write-Host "SpecRepoOwner variable set to: $specRepoOwner" + + $specRepoName = $Matches['repository'] + Write-Host "##vso[task.setvariable variable=SpecRepoName]$specRepoName" + Write-Host "SpecRepoName variable set to: $specRepoName" + + $specRepoDirectory = "$(System.DefaultWorkingDirectory)/$specRepoName" + Write-Host "##vso[task.setvariable variable=SpecRepoDirectory]$specRepoDirectory" + Write-Host "SpecRepoDirectory variable set to: $specRepoDirectory" + } + + if ('${{ parameters.SdkRepoUrl }}' -match $urlPattern) { + $sdkRepoOwner = $Matches['organization'] + Write-Host "##vso[task.setvariable variable=SdkRepoOwner]$sdkRepoOwner" + Write-Host "SdkRepoOwner variable set to: $sdkRepoOwner" + + $sdkRepoName = $Matches['repository'] + Write-Host "##vso[task.setvariable variable=SdkRepoName]$sdkRepoName" + Write-Host "SdkRepoName variable set to: $sdkRepoName" + + $sdkRepoDirectory = "$(System.DefaultWorkingDirectory)/$sdkRepoName" + Write-Host "##vso[task.setvariable variable=SdkRepoDirectory]$sdkRepoDirectory" + Write-Host "SdkRepoDirectory variable set to: $sdkRepoDirectory" + } + + if ([string]::IsNullOrEmpty($SpecRepoOwner) -or [string]::IsNullOrEmpty($SpecRepoName) -or [string]::IsNullOrEmpty($SdkRepoOwner) -or [string]::IsNullOrEmpty($SdkRepoName)) { + Write-Host "One or more required variables are null or empty. Ensure that SpecRepourl and SdkRepoUrl are set to valid GitHub repository URLs." + Exit 1 + } + + $sdkRepoNameForSpecGen = $sdkRepoName + if ($sdkRepoName.EndsWith('-for-net')) { + $sdkRepoNameForSpecGen = $sdkRepoName + "-track2" + } + Write-Host "##vso[task.setvariable variable=SdkRepoNameForSpecGen]$sdkRepoNameForSpecGen" + Write-Host "SdkRepoNameForSpecGen variable set to: $sdkRepoNameForSpecGen" + displayName: "Create Run Time Variables" + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml parameters: Paths: @@ -65,12 +99,13 @@ jobs: - '!sdk/**/SessionRecords/*' - '!sdk/**/session-records/*' Repositories: - - Name: $(SpecRepoFullName) + - Name: $(SpecRepoOwner)/$(SpecRepoName) Commitish: ${{ parameters.SpecRepoBranch }} WorkingDirectory: $(SpecRepoDirectory) - - Name: ${{ parameters.SdkRepoOwner }}/${{ parameters.SdkRepoName }} + - Name: $(SdkRepoOwner)/$(SdkRepoName) Commitish: ${{ parameters.SdkRepoBranch }} WorkingDirectory: $(SdkRepoDirectory) + SkipCheckoutNone: true - script: | if [ "${{ parameters.SpecRepoCommit }}" = "HEAD" ]; then @@ -86,9 +121,13 @@ jobs: - task: NodeTool@0 inputs: - versionSpec: '22.x' + versionSpec: $(NodeVersion) displayName: 'Install Node.js' + - task: UsePythonVersion@0 + inputs: + versionSpec: $(PythonVersion) + - script: | npm install -g @azure-tools/spec-gen-sdk displayName: 'Install spec-gen-sdk' @@ -117,44 +156,45 @@ jobs: --scp "$(SpecRepoDirectory)" \ --sdp "$(SdkRepoDirectory)" \ --wf "$(System.DefaultWorkingDirectory)" \ - -l "${{ parameters.SdkRepoNameForSpecGen }}" \ + -l "$(SdkRepoNameForSpecGen)" \ -c "$(SpecRepoCommit)" \ + -t $true \ $optional_params displayName: 'Generate SDK' + - task: PublishPipelineArtifact@1 + displayName: Publish SDK ${sdkLanguage} Artifact to Pipeline Artifacts + inputs: + artifactName: $(sdkArtifactName) + targetPath: "$(System.DefaultWorkingDirectory)/generatedSdkArtifacts" + + - ${{ if eq(parameters.CodePushRequired, true) }}: - template: /eng/common/pipelines/templates/steps/git-push-changes.yml parameters: - BaseRepoBranch: sdk-generation-from-$(Build.DefinitionName)_$(Build.BuildId) + BaseRepoBranch: $(PrBranch)-$(Build.BuildId) BaseRepoOwner: azure-sdk CommitMsg: $(GeneratedSDKInformation) - TargetRepoOwner: ${{ parameters.SdkRepoOwner }} - TargetRepoName: ${{ parameters.SdkRepoName }} + TargetRepoOwner: $(SdkRepoOwner) + TargetRepoName: $(SdkRepoName) PushArgs: "--force" WorkingDirectory: $(SdkRepoDirectory) ScriptDirectory: $(SdkRepoDirectory)/eng/common/scripts - - task: PowerShell@2 - displayName: Create pull request - condition: and(succeeded(), eq('${{ parameters.SdkPrRequired }}', true), eq(variables['HasChanges'], 'true')) - inputs: - pwsh: true - workingDirectory: $(SdkRepoDirectory) - filePath: $(SdkRepoDirectory)/eng/common/scripts/Submit-PullRequest.ps1 - arguments: > - -RepoOwner "${{ parameters.SdkRepoOwner }}" - -RepoName "${{ parameters.SdkRepoName }}" - -BaseBranch "main" - -PROwner "azure-sdk" - -PRBranch "sdk-generation-from-$(Build.DefinitionName)_$(Build.BuildId)" - -AuthToken "$(azuresdk-github-pat)" - -PRTitle "sdk-generation-from-$(Build.DefinitionName)_$(Build.BuildId)" - -PRBody "$(GeneratedSDKInformation)" - -OpenAsDraft $true - - - script: | - echo "This step runs only if ApiViewRequired is true" - displayName: 'Conditional Step' - condition: and(succeeded(), eq('${{ parameters.ApiViewRequired }}', true)) - - \ No newline at end of file + - task: PowerShell@2 + displayName: Create pull request + condition: and(succeeded(), eq('${{ parameters.SdkPrRequired }}', true), eq(variables['HasChanges'], 'true')) + inputs: + pwsh: true + workingDirectory: $(SdkRepoDirectory) + filePath: $(SdkRepoDirectory)/eng/common/scripts/Submit-PullRequest.ps1 + arguments: > + -RepoOwner "$(SdkRepoOwner)" + -RepoName "$(SdkRepoName)" + -BaseBranch "main" + -PROwner "azure-sdk" + -PRBranch "$(PrBranch)-$(Build.BuildId)" + -AuthToken "$(azuresdk-github-pat)" + -PRTitle "$(PrTitle)-generated-from-$(Build.DefinitionName)_$(Build.BuildId)" + -PRBody "$(GeneratedSDKInformation)" + -OpenAsDraft $true \ No newline at end of file From afa30b3ea198b0ed8b1310dd7f50fe5be6e84f54 Mon Sep 17 00:00:00 2001 From: Chidozie Ononiwu Date: Tue, 14 Jan 2025 20:10:39 -0800 Subject: [PATCH 3/7] Update SDK generation Yaml config --- .../templates/jobs/sdk-generation.yml | 42 ++++++++----------- 1 file changed, 17 insertions(+), 25 deletions(-) diff --git a/eng/common/pipelines/templates/jobs/sdk-generation.yml b/eng/common/pipelines/templates/jobs/sdk-generation.yml index e21f5ee8c..d9aaa2ca2 100644 --- a/eng/common/pipelines/templates/jobs/sdk-generation.yml +++ b/eng/common/pipelines/templates/jobs/sdk-generation.yml @@ -1,32 +1,21 @@ parameters: - - name: SpecRepourl + - name: SpecRepoUrl type: string - name: SpecRepoBranch type: string - default: 'main' - name: SpecRepoCommit type: string - default: 'HEAD' - - name: SdkRepoUrl - type: string - name: SdkRepoBranch type: string - default: 'main' - name: ConfigType type: string - values: - - 'TypeSpec' - - 'AutoRest' - name: TspConfigPath type: string - name: ReadmePath type: string - name: ApiVersion type: string - default: 'not-specified' - - name: CodePushRequired - type: boolean - - name: SdkPrRequired + - name: SkipPullRequestCreation type: boolean jobs: @@ -38,6 +27,8 @@ jobs: value: '22.13.x' - name: PythonVersion value: '3.13' + - name: SpecGenSdkVersion + value: 'latest' - name: SdkArtifactName value: SDK_Artifact pool: @@ -63,10 +54,12 @@ jobs: Write-Host "SpecRepoDirectory variable set to: $specRepoDirectory" } - if ('${{ parameters.SdkRepoUrl }}' -match $urlPattern) { - $sdkRepoOwner = $Matches['organization'] - Write-Host "##vso[task.setvariable variable=SdkRepoOwner]$sdkRepoOwner" - Write-Host "SdkRepoOwner variable set to: $sdkRepoOwner" + if ('$(SdkRepoUrl)' -match $urlPattern) { + if ('$(SdkRepoOwner)' -eq '') { + $sdkRepoOwner = $Matches['organization'] + Write-Host "##vso[task.setvariable variable=SdkRepoOwner]$sdkRepoOwner" + Write-Host "SdkRepoOwner variable set to: $sdkRepoOwner" + } $sdkRepoName = $Matches['repository'] Write-Host "##vso[task.setvariable variable=SdkRepoName]$sdkRepoName" @@ -100,7 +93,7 @@ jobs: - '!sdk/**/session-records/*' Repositories: - Name: $(SpecRepoOwner)/$(SpecRepoName) - Commitish: ${{ parameters.SpecRepoBranch }} + Commitish: ${{ coalesce(parameters.SpecRepoCommit, parameters.SpecRepoBranch) }} WorkingDirectory: $(SpecRepoDirectory) - Name: $(SdkRepoOwner)/$(SdkRepoName) Commitish: ${{ parameters.SdkRepoBranch }} @@ -129,7 +122,7 @@ jobs: versionSpec: $(PythonVersion) - script: | - npm install -g @azure-tools/spec-gen-sdk + npm install -g @azure-tools/spec-gen-sdk@$(SpecGenSdkVersion) displayName: 'Install spec-gen-sdk' - script: | @@ -148,7 +141,7 @@ jobs: sdk_gen_info="$sdk_gen_info API Version: '${{ parameters.ApiVersion }}'," fi - sdk_gen_info="$sdk_gen_info and CommitSHA: '$(SpecRepoCommit)'" + sdk_gen_info="$sdk_gen_info and CommitSHA: '$(SpecRepoCommit)', in SpecRepo: '${{ parameters.SpecRepoUrl }}'" echo "##vso[task.setvariable variable=GeneratedSDKInformation]$sdk_gen_info" echo "Generated SDK Information : $sdk_gen_info" @@ -163,13 +156,12 @@ jobs: displayName: 'Generate SDK' - task: PublishPipelineArtifact@1 - displayName: Publish SDK ${sdkLanguage} Artifact to Pipeline Artifacts + displayName: Publish SDK Artifact to Pipeline Artifacts inputs: artifactName: $(sdkArtifactName) targetPath: "$(System.DefaultWorkingDirectory)/generatedSdkArtifacts" - - - ${{ if eq(parameters.CodePushRequired, true) }}: + - ${{ if eq(parameters.SkipPullRequestCreation, true) }}: - template: /eng/common/pipelines/templates/steps/git-push-changes.yml parameters: BaseRepoBranch: $(PrBranch)-$(Build.BuildId) @@ -183,7 +175,7 @@ jobs: - task: PowerShell@2 displayName: Create pull request - condition: and(succeeded(), eq('${{ parameters.SdkPrRequired }}', true), eq(variables['HasChanges'], 'true')) + condition: and(succeeded(), eq('${{ parameters.SkipPullRequestCreation }}', true), eq(variables['HasChanges'], 'true')) inputs: pwsh: true workingDirectory: $(SdkRepoDirectory) @@ -195,6 +187,6 @@ jobs: -PROwner "azure-sdk" -PRBranch "$(PrBranch)-$(Build.BuildId)" -AuthToken "$(azuresdk-github-pat)" - -PRTitle "$(PrTitle)-generated-from-$(Build.DefinitionName)_$(Build.BuildId)" + -PRTitle "$(PrTitle)-generated-from-$(Build.DefinitionName)-$(Build.BuildId)" -PRBody "$(GeneratedSDKInformation)" -OpenAsDraft $true \ No newline at end of file From 2364c9a3b475379f1014b40a5a23062fa4755993 Mon Sep 17 00:00:00 2001 From: Chidozie Ononiwu Date: Wed, 15 Jan 2025 19:38:31 -0800 Subject: [PATCH 4/7] Add spec-gen-sdk pipeline configuration --- .../{sdk-generation.yml => spec-gen-sdk.yml} | 54 ++++++++++++------- 1 file changed, 36 insertions(+), 18 deletions(-) rename eng/common/pipelines/templates/jobs/{sdk-generation.yml => spec-gen-sdk.yml} (82%) diff --git a/eng/common/pipelines/templates/jobs/sdk-generation.yml b/eng/common/pipelines/templates/jobs/spec-gen-sdk.yml similarity index 82% rename from eng/common/pipelines/templates/jobs/sdk-generation.yml rename to eng/common/pipelines/templates/jobs/spec-gen-sdk.yml index d9aaa2ca2..fbbbbf572 100644 --- a/eng/common/pipelines/templates/jobs/sdk-generation.yml +++ b/eng/common/pipelines/templates/jobs/spec-gen-sdk.yml @@ -1,28 +1,40 @@ parameters: - - name: SpecRepoUrl - type: string - - name: SpecRepoBranch - type: string - name: SpecRepoCommit type: string - - name: SdkRepoBranch + default: 'HEAD' + - name: SdkRepoCommit type: string + default: 'HEAD' - name: ConfigType type: string - - name: TspConfigPath - type: string - - name: ReadmePath + values: + - 'TypeSpec' + - 'OpenAPI' + default: 'TypeSpec' + displayName: 'Configuration Type' + - name: ConfigPath type: string + default: 'specification/contosowidgetmanager/Contoso.Management/tspconfig.yaml' + displayName: 'Path to TypeSpec or OpenAPI Configuration file' - name: ApiVersion type: string + default: 'not-specified' + displayName: 'API Version of Spec to be used for SDK Generation' - name: SkipPullRequestCreation type: boolean + default: false + +variables: + SdkRepoUrl: $(SdkRepoUrlForLanguage) # This variable is set in the pipeline UI for each language e.g. https://github.com/Azure/azure-sdk-for-net for dotnet jobs: - job: displayName: 'SDK Generation' + variables: - template: /eng/pipelines/templates/variables/image.yml + - name: SpecRepoUrl + value: 'https://github.com/$(Build.Repository.Name)' - name: NodeVersion value: '22.13.x' - name: PythonVersion @@ -31,6 +43,7 @@ jobs: value: 'latest' - name: SdkArtifactName value: SDK_Artifact + pool: name: $(LINUXPOOL) vmImage: $(LINUXVMIMAGE) @@ -40,7 +53,7 @@ jobs: - pwsh: | $urlPattern = '^https://github\.com/(?[^/]+)/(?[^/]+)' - if ('${{ parameters.SpecRepoUrl }}' -match $urlPattern) { + if ('$(SpecRepoUrl)' -match $urlPattern) { $specRepoOwner = $Matches['organization'] Write-Host "##vso[task.setvariable variable=SpecRepoOwner]$specRepoOwner" Write-Host "SpecRepoOwner variable set to: $specRepoOwner" @@ -55,6 +68,11 @@ jobs: } if ('$(SdkRepoUrl)' -match $urlPattern) { + if ('$(SpecRepoUrl)'.EndsWith('-pr') -and (-not '$(SdkRepoUrl)'.EndsWith('-pr'))) { + Write-Host "SdkRepoUrl must be a private repository if SpecRepoUrl is a private repository." + Exit 1 + } + if ('$(SdkRepoOwner)' -eq '') { $sdkRepoOwner = $Matches['organization'] Write-Host "##vso[task.setvariable variable=SdkRepoOwner]$sdkRepoOwner" @@ -93,10 +111,10 @@ jobs: - '!sdk/**/session-records/*' Repositories: - Name: $(SpecRepoOwner)/$(SpecRepoName) - Commitish: ${{ coalesce(parameters.SpecRepoCommit, parameters.SpecRepoBranch) }} + Commitish: ${{ parameters.SpecRepoCommit }} WorkingDirectory: $(SpecRepoDirectory) - Name: $(SdkRepoOwner)/$(SdkRepoName) - Commitish: ${{ parameters.SdkRepoBranch }} + Commitish: ${{ parameters.SdkRepoCommit }} WorkingDirectory: $(SdkRepoDirectory) SkipCheckoutNone: true @@ -130,18 +148,18 @@ jobs: sdk_gen_info="sdk generation from Config : " if [ "${{ parameters.ConfigType }}" = "TypeSpec" ]; then - optional_params="$optional_params --tsp-config-relative-path ${{ parameters.TspConfigPath }}" - sdk_gen_info="$sdk_gen_info '${{ parameters.TspConfigPath }}'," + optional_params="$optional_params --tsp-config-relative-path ${{ parameters.ConfigPath }}" + sdk_gen_info="$sdk_gen_info '${{ parameters.ConfigPath }}'," else - optional_params="$optional_params --readme-relative-path ${{ parameters.ReadmePath }}" - sdk_gen_info="$sdk_gen_info '${{ parameters.ReadmePath }}'," + optional_params="$optional_params --readme-relative-path ${{ parameters.ConfigPath }}" + sdk_gen_info="$sdk_gen_info '${{ parameters.ConfigPath }}'," fi if [ "${{ parameters.ApiVersion }}" != "not-specified" ]; then optional_params="$optional_params --api-version ${{ parameters.ApiVersion }}" sdk_gen_info="$sdk_gen_info API Version: '${{ parameters.ApiVersion }}'," fi - sdk_gen_info="$sdk_gen_info and CommitSHA: '$(SpecRepoCommit)', in SpecRepo: '${{ parameters.SpecRepoUrl }}'" + sdk_gen_info="$sdk_gen_info and CommitSHA: '$(SpecRepoCommit)', in SpecRepo: '$(SpecRepoUrl)'" echo "##vso[task.setvariable variable=GeneratedSDKInformation]$sdk_gen_info" echo "Generated SDK Information : $sdk_gen_info" @@ -161,7 +179,7 @@ jobs: artifactName: $(sdkArtifactName) targetPath: "$(System.DefaultWorkingDirectory)/generatedSdkArtifacts" - - ${{ if eq(parameters.SkipPullRequestCreation, true) }}: + - ${{ if eq(parameters.SkipPullRequestCreation, false) }}: - template: /eng/common/pipelines/templates/steps/git-push-changes.yml parameters: BaseRepoBranch: $(PrBranch)-$(Build.BuildId) @@ -175,7 +193,7 @@ jobs: - task: PowerShell@2 displayName: Create pull request - condition: and(succeeded(), eq('${{ parameters.SkipPullRequestCreation }}', true), eq(variables['HasChanges'], 'true')) + condition: and(succeeded(), eq('${{ parameters.SkipPullRequestCreation }}', false), eq(variables['HasChanges'], 'true')) inputs: pwsh: true workingDirectory: $(SdkRepoDirectory) From e7cae1db056a30ca4657cfe275bfa2029eb3c70a Mon Sep 17 00:00:00 2001 From: Chidozie Ononiwu Date: Thu, 16 Jan 2025 13:46:45 -0800 Subject: [PATCH 5/7] Remove spec-gen-sdk.yml and add archetype-spec-gen-sdk.yml --- ...gen-sdk.yml => archetype-spec-gen-sdk.yml} | 29 +++++++------------ 1 file changed, 10 insertions(+), 19 deletions(-) rename eng/common/pipelines/templates/jobs/{spec-gen-sdk.yml => archetype-spec-gen-sdk.yml} (88%) diff --git a/eng/common/pipelines/templates/jobs/spec-gen-sdk.yml b/eng/common/pipelines/templates/jobs/archetype-spec-gen-sdk.yml similarity index 88% rename from eng/common/pipelines/templates/jobs/spec-gen-sdk.yml rename to eng/common/pipelines/templates/jobs/archetype-spec-gen-sdk.yml index fbbbbf572..79e4f27cb 100644 --- a/eng/common/pipelines/templates/jobs/spec-gen-sdk.yml +++ b/eng/common/pipelines/templates/jobs/archetype-spec-gen-sdk.yml @@ -1,4 +1,8 @@ parameters: + - name: SpecRepoUrl + type: string + - name: SdkRepoUrl + type: string - name: SpecRepoCommit type: string default: 'HEAD' @@ -11,30 +15,21 @@ parameters: - 'TypeSpec' - 'OpenAPI' default: 'TypeSpec' - displayName: 'Configuration Type' + displayName: 'API specification type' - name: ConfigPath type: string default: 'specification/contosowidgetmanager/Contoso.Management/tspconfig.yaml' - displayName: 'Path to TypeSpec or OpenAPI Configuration file' - - name: ApiVersion - type: string - default: 'not-specified' - displayName: 'API Version of Spec to be used for SDK Generation' + displayName: 'Path to API specification file (TypeSpec/OpenAPI)' - name: SkipPullRequestCreation type: boolean default: false -variables: - SdkRepoUrl: $(SdkRepoUrlForLanguage) # This variable is set in the pipeline UI for each language e.g. https://github.com/Azure/azure-sdk-for-net for dotnet - jobs: - job: displayName: 'SDK Generation' variables: - template: /eng/pipelines/templates/variables/image.yml - - name: SpecRepoUrl - value: 'https://github.com/$(Build.Repository.Name)' - name: NodeVersion value: '22.13.x' - name: PythonVersion @@ -53,7 +48,7 @@ jobs: - pwsh: | $urlPattern = '^https://github\.com/(?[^/]+)/(?[^/]+)' - if ('$(SpecRepoUrl)' -match $urlPattern) { + if ('${{ parameters.SpecRepoUrl }}' -match $urlPattern) { $specRepoOwner = $Matches['organization'] Write-Host "##vso[task.setvariable variable=SpecRepoOwner]$specRepoOwner" Write-Host "SpecRepoOwner variable set to: $specRepoOwner" @@ -67,8 +62,8 @@ jobs: Write-Host "SpecRepoDirectory variable set to: $specRepoDirectory" } - if ('$(SdkRepoUrl)' -match $urlPattern) { - if ('$(SpecRepoUrl)'.EndsWith('-pr') -and (-not '$(SdkRepoUrl)'.EndsWith('-pr'))) { + if ('${{ parameters.SdkRepoUrl }}' -match $urlPattern) { + if ('${{ parameters.SpecRepoUrl }}'.EndsWith('-pr') -and (-not '${{ parameters.SdkRepoUrl }}'.EndsWith('-pr'))) { Write-Host "SdkRepoUrl must be a private repository if SpecRepoUrl is a private repository." Exit 1 } @@ -154,12 +149,8 @@ jobs: optional_params="$optional_params --readme-relative-path ${{ parameters.ConfigPath }}" sdk_gen_info="$sdk_gen_info '${{ parameters.ConfigPath }}'," fi - if [ "${{ parameters.ApiVersion }}" != "not-specified" ]; then - optional_params="$optional_params --api-version ${{ parameters.ApiVersion }}" - sdk_gen_info="$sdk_gen_info API Version: '${{ parameters.ApiVersion }}'," - fi - sdk_gen_info="$sdk_gen_info and CommitSHA: '$(SpecRepoCommit)', in SpecRepo: '$(SpecRepoUrl)'" + sdk_gen_info="$sdk_gen_info and CommitSHA: '$(SpecRepoCommit)', in SpecRepo: '${{ parameters.SpecRepoUrl }}'" echo "##vso[task.setvariable variable=GeneratedSDKInformation]$sdk_gen_info" echo "Generated SDK Information : $sdk_gen_info" From 0c40470f184580835e20eef1bae626aa71f93b26 Mon Sep 17 00:00:00 2001 From: Chidozie Ononiwu Date: Thu, 16 Jan 2025 14:51:46 -0800 Subject: [PATCH 6/7] Allow pipelline override of repo organisation --- .../templates/jobs/archetype-spec-gen-sdk.yml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/eng/common/pipelines/templates/jobs/archetype-spec-gen-sdk.yml b/eng/common/pipelines/templates/jobs/archetype-spec-gen-sdk.yml index 79e4f27cb..d76fcca80 100644 --- a/eng/common/pipelines/templates/jobs/archetype-spec-gen-sdk.yml +++ b/eng/common/pipelines/templates/jobs/archetype-spec-gen-sdk.yml @@ -68,11 +68,9 @@ jobs: Exit 1 } - if ('$(SdkRepoOwner)' -eq '') { - $sdkRepoOwner = $Matches['organization'] - Write-Host "##vso[task.setvariable variable=SdkRepoOwner]$sdkRepoOwner" - Write-Host "SdkRepoOwner variable set to: $sdkRepoOwner" - } + $sdkRepoOwner = if ('$(SdkRepoOwner)' -eq '') { $Matches['organization'] } else { '$(SdkRepoOwner)' } + Write-Host "##vso[task.setvariable variable=SdkRepoOwner]$sdkRepoOwner" + Write-Host "SdkRepoOwner variable set to: $sdkRepoOwner" $sdkRepoName = $Matches['repository'] Write-Host "##vso[task.setvariable variable=SdkRepoName]$sdkRepoName" @@ -84,7 +82,7 @@ jobs: } if ([string]::IsNullOrEmpty($SpecRepoOwner) -or [string]::IsNullOrEmpty($SpecRepoName) -or [string]::IsNullOrEmpty($SdkRepoOwner) -or [string]::IsNullOrEmpty($SdkRepoName)) { - Write-Host "One or more required variables are null or empty. Ensure that SpecRepourl and SdkRepoUrl are set to valid GitHub repository URLs." + Write-Host "One or more required variables is empty or invalid. Ensure that SpecRepourl and SdkRepoUrl are set to valid GitHub repository URLs." Exit 1 } @@ -198,4 +196,4 @@ jobs: -AuthToken "$(azuresdk-github-pat)" -PRTitle "$(PrTitle)-generated-from-$(Build.DefinitionName)-$(Build.BuildId)" -PRBody "$(GeneratedSDKInformation)" - -OpenAsDraft $true \ No newline at end of file + -OpenAsDraft $true From 45be2c5750d509dadf4df20b0eb2d115d7a56750 Mon Sep 17 00:00:00 2001 From: Chidozie Ononiwu Date: Thu, 16 Jan 2025 14:55:51 -0800 Subject: [PATCH 7/7] Remove SkipPullRequestCreation from pipeline condition --- eng/common/pipelines/templates/jobs/archetype-spec-gen-sdk.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/pipelines/templates/jobs/archetype-spec-gen-sdk.yml b/eng/common/pipelines/templates/jobs/archetype-spec-gen-sdk.yml index d76fcca80..2eb1ca659 100644 --- a/eng/common/pipelines/templates/jobs/archetype-spec-gen-sdk.yml +++ b/eng/common/pipelines/templates/jobs/archetype-spec-gen-sdk.yml @@ -182,7 +182,7 @@ jobs: - task: PowerShell@2 displayName: Create pull request - condition: and(succeeded(), eq('${{ parameters.SkipPullRequestCreation }}', false), eq(variables['HasChanges'], 'true')) + condition: and(succeeded(), eq(variables['HasChanges'], 'true')) inputs: pwsh: true workingDirectory: $(SdkRepoDirectory)