From ec0555c32417765487759393fc0e4b2bb9e0ebef Mon Sep 17 00:00:00 2001 From: Ben Broderick Phillips Date: Mon, 18 Mar 2024 17:55:42 -0400 Subject: [PATCH] More files --- .../publish-docker-image-isolated.yml | 129 ++++++++ .../stages/archetype-sdk-tool-dotnet.yml | 307 +++++++++--------- tools/http-fault-injector/ci.yml | 1 + 3 files changed, 287 insertions(+), 150 deletions(-) create mode 100644 eng/pipelines/publish-docker-image-isolated.yml diff --git a/eng/pipelines/publish-docker-image-isolated.yml b/eng/pipelines/publish-docker-image-isolated.yml new file mode 100644 index 00000000000..9675d15d117 --- /dev/null +++ b/eng/pipelines/publish-docker-image-isolated.yml @@ -0,0 +1,129 @@ +# Additional sample inputs can be found in `eng/containers/ci.yml`, but here is an example. +# - name: test_proxy_linux +# dockerRepo: 'engsys/testproxy-lin' +# prepareScript: tools/test-proxy/docker/prepare.ps1 +# excludeFromManifest: true/false +# configPath: 'tools/test-proxy/docker' +# stableTags: +# - 'latest' + +# A "ManifestDeployment" is a secondary deployment that can be used to map multiple docker tags to a single platform-sensitive one. It waits until +# all previous tag deployments are created, then triggers a manifest upload that bundles all tags configured in DockerDeployments. To exclude an item from the +# manifest upload, set deployment config property excludeFromManifest to true. +# - name: test_proxy_manifest +# dockerRepo: 'engsys/testproxy' +# stableTags: +# - 'latest' +parameters: + - name: DockerDeployments + type: object + default: [] + - name: ManifestDeployment + type: object + default: [] + - name: ImageTag + type: string + - name: ContainerRegistry + type: string + default: 'azsdkengsys' + - name: Publish + type: boolean + default: true + +stages: + - stage: + displayName: Build/Publish Containers + variables: + - template: /eng/pipelines/templates/variables/globals.yml + - template: /eng/pipelines/templates/variables/image.yml + jobs: + - ${{ each config in parameters.DockerDeployments }}: + - job: container_build_${{ config.name }} + displayName: Build ${{ config.name }} Image + pool: + name: $(LINUXPOOL) + image: $(LINUXVMIMAGE) + os: linux + + ${{ if parameters.Publish }}: + templateContext: + outputs: + - output: containerImage + displayName: Push ${{ config.name }}:${{ parameters.ImageTag }} + condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest')) + image: ${{ parameters.ContainerRegistry }}.azurecr.io/${{ config.dockerRepo }}:${{ parameters.ImageTag }} + - ${{ if config.stableTags }}: + - ${{ each stableTag in config.stableTags }}: + - output: containerImage + displayName: Push ${{ config.name }}:${{ stableTag }} + condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest')) + image: ${{ parameters.ContainerRegistry }}.azurecr.io/${{ config.dockerRepo }}:${{ stableTag }} + + steps: + - ${{ if config.prepareScript }}: + - pwsh: | + ./${{ config.prepareScript }} + displayName: "Run prep script" + - task: 1ES.BuildContainerImage@1 + displayName: Build ${{ config.name }}:${{ parameters.ImageTag }} + inputs: + image: ${{ parameters.ContainerRegistry }}.azurecr.io/${{ config.dockerRepo }}:${{ parameters.ImageTag }} + path: ${{ config.configPath }} + + - ${{ if and(parameters.ManifestDeployment, parameters.Publish) }}: + - ${{ each deployment in parameters.ManifestDeployment }}: + - job: container_build_${{ deployment.name }} + displayName: Build ${{ deployment.name }} Manifest + ${{ if gt(length(parameters.DockerDeployments), 0) }}: + dependsOn: + - ${{ each config in parameters.DockerDeployments }}: + - container_build_${{ config.name }} + pool: + name: $(LINUXPOOL) + image: $(LINUXVMIMAGE) + os: linux + + steps: + - task: Docker@2 + displayName: Login to ACR + inputs: + command: login + containerRegistry: ${{ parameters.ContainerRegistry }} + + - pwsh: | + $configurations = '${{ convertToJson(parameters.DockerDeployments) }}' -replace '\\', '/' + $assembledDependentTags = $(Build.SourcesDirectory)/eng/pipelines/templates/scripts/get-docker-manifest-input.ps1 ` + -DockerDeploymentJson $configurations ` + -ContainerRegistry "${{ parameters.ContainerRegistry }}" ` + -ImageTag "${{ parameters.ImageTag }}" ` + + Write-Host "##vso[task.setvariable variable=ManifestVariable]$assembledDependentTags" + displayName: Generate Manifest Variable + + - pwsh: | + docker manifest create ${{ parameters.ContainerRegistry }}.azurecr.io/${{ deployment.dockerRepo }}:${{ parameters.ImageTag }} $(ManifestVariable) + displayName: Generate Manifest + + - pwsh: | + docker manifest push ${{ parameters.ContainerRegistry }}.azurecr.io/${{ deployment.dockerRepo }}:${{ parameters.ImageTag }} + displayName: Upload Manifest + + - ${{ if deployment.stableTags }}: + - ${{ each stableTag in deployment.stableTags }}: + - pwsh: | + $configurations = '${{ convertToJson(parameters.DockerDeployments) }}' -replace '\\', '/' + $assembledDependentTags = $(Build.SourcesDirectory)/eng/pipelines/templates/scripts/get-docker-manifest-input.ps1 ` + -DockerDeploymentJson $configurations ` + -ContainerRegistry "${{ parameters.ContainerRegistry }}" ` + -ImageTag "${{ stableTag }}" ` + + Write-Host "##vso[task.setvariable variable=ManifestVariable]$assembledDependentTags" + displayName: Generate Manifest Variable + + - pwsh: | + docker manifest create ${{ parameters.ContainerRegistry }}.azurecr.io/${{ deployment.dockerRepo }}:${{ stableTag }} $(ManifestVariable) + displayName: Generate Manifest + + - pwsh: | + docker manifest push ${{ parameters.ContainerRegistry }}.azurecr.io/${{ deployment.dockerRepo }}:${{ stableTag }} + displayName: Upload Manifest diff --git a/eng/pipelines/templates/stages/archetype-sdk-tool-dotnet.yml b/eng/pipelines/templates/stages/archetype-sdk-tool-dotnet.yml index f9607797bc6..c45a806e641 100644 --- a/eng/pipelines/templates/stages/archetype-sdk-tool-dotnet.yml +++ b/eng/pipelines/templates/stages/archetype-sdk-tool-dotnet.yml @@ -49,6 +49,9 @@ parameters: - name: RequireStrongNames type: boolean default: true + - name: Use1ESOfficial + type: boolean + default: false - name: TestMatrix type: object default: @@ -65,156 +68,160 @@ parameters: Image: $(MACVMIMAGE) Os: macOS -stages: - - stage: BuildTestAndPackage - - variables: - - template: /eng/pipelines/templates/variables/globals.yml - - template: /eng/pipelines/templates/variables/image.yml - - name: Warn - ${{ if parameters.NoWarn }}: - value: '' - ${{ if not(parameters.NoWarn) }}: - value: -warnaserror - # A path to directory to contain the output of "dotnet pack" call, - # to be consumed as input by "publish" task. - - name: packagesToPublishDir - value: $(Build.ArtifactStagingDirectory)/packages - - pool: - name: $(LINUXPOOL) - image: $(LINUXVMIMAGE) - os: linux - - jobs: - - job: BuildAndPackage - displayName: Build and Package - - templateContext: - outputs: - - output: pipelineArtifact - displayName: Publish to packages artifact - condition: succeededOrFailed() - artifactName: packages - targetPath: $(packagesToPublishDir) - - steps: - - template: /eng/pipelines/templates/steps/install-dotnet.yml - - - script: 'dotnet pack /p:ArtifactsPackagesDir=$(packagesToPublishDir) $(Warn) -c Release' - displayName: 'Build and Package' - workingDirectory: '${{ coalesce(parameters.PackageDirectory, parameters.ToolDirectory) }}' - env: - DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1 - DOTNET_CLI_TELEMETRY_OPTOUT: 1 - DOTNET_MULTILEVEL_LOOKUP: 0 - - # This step creates "$(packagesToPublishDir)" directory if it doesn't exist. - # This step is necessary since migration to net6.0. This is because since net6.0, - # in case the "Build and Package" above would not output any packages to this directory, - # the "Publish to packages artifact" step below would fail on missing directory. - - pwsh: | - if (!(Test-Path -PathType container "$(packagesToPublishDir)")) { - New-Item -ItemType Directory -Path "$(packagesToPublishDir)" - Write-Host "Created directory ""$(packagesToPublishDir)""" - } else { - Write-Host "Directory ""$(packagesToPublishDir)"" already exists. Nothing to do." - Write-Host "Directory ""$(packagesToPublishDir)"" contents:" - Get-ChildItem $(packagesToPublishDir) | ForEach-Object { Write-Host $_ } - } - displayName: Create dir for packages to publish or list its contents - - - ${{ if ne(length(parameters.StandaloneExeMatrix), 0) }}: - - job: Produce_Executables_linux - - pool: - name: $(LINUXPOOL) - image: $(LINUXVMIMAGE) - os: linux - - steps: - - template: /eng/pipelines/templates/steps/install-dotnet.yml - - - template: /eng/pipelines/templates/steps/produce-net-standalone-packs.yml +extends: + template: /eng/pipelines/templates/stages/1es-redirect.yml + parameters: + Use1ESOfficial: ${{ parameters.Use1ESOfficial }} + stages: + - stage: BuildTestAndPackage + + variables: + - template: /eng/pipelines/templates/variables/globals.yml + - template: /eng/pipelines/templates/variables/image.yml + - name: Warn + ${{ if parameters.NoWarn }}: + value: '' + ${{ if not(parameters.NoWarn) }}: + value: -warnaserror + # A path to directory to contain the output of "dotnet pack" call, + # to be consumed as input by "publish" task. + - name: packagesToPublishDir + value: $(Build.ArtifactStagingDirectory)/packages + + pool: + name: $(LINUXPOOL) + image: $(LINUXVMIMAGE) + os: linux + + jobs: + - job: BuildAndPackage + displayName: Build and Package + + templateContext: + outputs: + - output: pipelineArtifact + displayName: Publish to packages artifact + condition: succeededOrFailed() + artifactName: packages + targetPath: $(packagesToPublishDir) + + steps: + - template: /eng/pipelines/templates/steps/install-dotnet.yml + + - script: 'dotnet pack /p:ArtifactsPackagesDir=$(packagesToPublishDir) $(Warn) -c Release' + displayName: 'Build and Package' + workingDirectory: '${{ coalesce(parameters.PackageDirectory, parameters.ToolDirectory) }}' + env: + DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1 + DOTNET_CLI_TELEMETRY_OPTOUT: 1 + DOTNET_MULTILEVEL_LOOKUP: 0 + + # This step creates "$(packagesToPublishDir)" directory if it doesn't exist. + # This step is necessary since migration to net6.0. This is because since net6.0, + # in case the "Build and Package" above would not output any packages to this directory, + # the "Publish to packages artifact" step below would fail on missing directory. + - pwsh: | + if (!(Test-Path -PathType container "$(packagesToPublishDir)")) { + New-Item -ItemType Directory -Path "$(packagesToPublishDir)" + Write-Host "Created directory ""$(packagesToPublishDir)""" + } else { + Write-Host "Directory ""$(packagesToPublishDir)"" already exists. Nothing to do." + Write-Host "Directory ""$(packagesToPublishDir)"" contents:" + Get-ChildItem $(packagesToPublishDir) | ForEach-Object { Write-Host $_ } + } + displayName: Create dir for packages to publish or list its contents + + - ${{ if ne(length(parameters.StandaloneExeMatrix), 0) }}: + - job: Produce_Executables_linux + + pool: + name: $(LINUXPOOL) + image: $(LINUXVMIMAGE) + os: linux + + steps: + - template: /eng/pipelines/templates/steps/install-dotnet.yml + + - template: /eng/pipelines/templates/steps/produce-net-standalone-packs.yml + parameters: + StagingDirectory: $(Build.ArtifactStagingDirectory) + BuildMatrix: ${{ parameters.StandaloneExeMatrix }} + TargetDirectory: '${{ coalesce(parameters.PackageDirectory, parameters.ToolDirectory) }}' + + - job: Produce_Executables_mac + + pool: + name: $(MACPOOL) + vmImage: $(MACVMIMAGE) + os: macOS + + steps: + - template: /eng/pipelines/templates/steps/install-dotnet.yml + + - template: /eng/pipelines/templates/steps/produce-net-standalone-packs.yml + parameters: + StagingDirectory: $(Build.ArtifactStagingDirectory) + BuildMatrix: ${{ parameters.StandaloneExeMatrix }} + TargetDirectory: '${{ coalesce(parameters.PackageDirectory, parameters.ToolDirectory) }}' + + - ${{ each test in parameters.TestMatrix }}: + - job: Test_${{ test }} + + pool: + ${{ if eq(test.os, 'macOS') }}: + vmImage: ${{ test.Image }} + ${{ else }}: + image: ${{ test.Image }} + name: ${{ test.Pool }} + os: ${{ test.Os }} + + steps: + - template: /eng/pipelines/templates/steps/install-dotnet.yml + + - ${{ parameters.TestPreSteps }} + + - script: 'dotnet test /p:ArtifactsPackagesDir=$(Build.ArtifactStagingDirectory) $(Warn) --logger trx' + displayName: 'Test' + workingDirectory: '${{ coalesce(parameters.TestDirectory, parameters.ToolDirectory) }}' + env: + DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1 + DOTNET_CLI_TELEMETRY_OPTOUT: 1 + DOTNET_MULTILEVEL_LOOKUP: 0 + + - ${{ parameters.TestPostSteps }} + + - task: PublishTestResults@2 + condition: succeededOrFailed() + inputs: + testResultsFiles: '**/*.trx' + testRunTitle: $(System.JobDisplayName) + testResultsFormat: 'VSTest' + mergeTestResults: true + + - ${{ if not(eq(length(parameters.DockerDeployments), 0)) }}: + - template: /eng/pipelines/publish-docker-image-isolated.yml parameters: - StagingDirectory: $(Build.ArtifactStagingDirectory) - BuildMatrix: ${{ parameters.StandaloneExeMatrix }} - TargetDirectory: '${{ coalesce(parameters.PackageDirectory, parameters.ToolDirectory) }}' - - - job: Produce_Executables_mac - - pool: - name: $(MACPOOL) - vmImage: $(MACVMIMAGE) - os: macOS + DockerDeployments: ${{ parameters.DockerDeployments }} + Publish: false + ImageTag: "${{ parameters.DockerTagPrefix }}$(Build.BuildNumber)" - steps: - - template: /eng/pipelines/templates/steps/install-dotnet.yml - - - template: /eng/pipelines/templates/steps/produce-net-standalone-packs.yml - parameters: - StagingDirectory: $(Build.ArtifactStagingDirectory) - BuildMatrix: ${{ parameters.StandaloneExeMatrix }} - TargetDirectory: '${{ coalesce(parameters.PackageDirectory, parameters.ToolDirectory) }}' - - - ${{ each test in parameters.TestMatrix }}: - - job: Test_${{ test }} - - pool: - ${{ if eq(test.os, 'macOS') }}: - vmImage: ${{ test.Image }} - ${{ else }}: - image: ${{ test.Image }} - name: ${{ test.Pool }} - os: ${{ test.Os }} - - steps: - - template: /eng/pipelines/templates/steps/install-dotnet.yml - - - ${{ parameters.TestPreSteps }} - - - script: 'dotnet test /p:ArtifactsPackagesDir=$(Build.ArtifactStagingDirectory) $(Warn) --logger trx' - displayName: 'Test' - workingDirectory: '${{ coalesce(parameters.TestDirectory, parameters.ToolDirectory) }}' - env: - DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1 - DOTNET_CLI_TELEMETRY_OPTOUT: 1 - DOTNET_MULTILEVEL_LOOKUP: 0 - - - ${{ parameters.TestPostSteps }} - - - task: PublishTestResults@2 - condition: succeededOrFailed() - inputs: - testResultsFiles: '**/*.trx' - testRunTitle: $(System.JobDisplayName) - testResultsFormat: 'VSTest' - mergeTestResults: true - - - ${{ if not(eq(length(parameters.DockerDeployments), 0)) }}: - - template: /eng/pipelines/publish-docker-image.yml + - ${{if and(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['System.TeamProject'], 'internal'))}}: + - template: pipelines/stages/net-release-to-feed.yml@azure-sdk-build-tools parameters: - DockerDeployments: ${{ parameters.DockerDeployments }} - Publish: false - ImageTag: "${{ parameters.DockerTagPrefix }}$(Build.BuildNumber)" - - - ${{if and(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['System.TeamProject'], 'internal'))}}: - - template: pipelines/stages/net-release-to-feed.yml@azure-sdk-build-tools - parameters: - # Publish to https://dev.azure.com/azure-sdk/public/_packaging?_a=feed&feed=azure-sdk-for-net - DevOpsFeedId: '29ec6040-b234-4e31-b139-33dc4287b756/fa8c16a3-dbe0-4de2-a297-03065ec1ba3f' - ExeMatrix: ${{ parameters.StandaloneExeMatrix }} - ShouldPublishExecutables: ${{ parameters.ReleaseBinaries }} - ShouldPublishSymbols: ${{ parameters.ShouldPublishSymbols }} - RequireStrongNames: ${{ parameters.RequireStrongNames }} - - - ${{if and(not(eq(length(parameters.DockerDeployments), 0)), ne(variables['Build.Reason'], 'PullRequest'), eq(variables['System.TeamProject'], 'internal'))}}: - - stage: PublishDockerImages - displayName: Publish Docker Images - dependsOn: BuildTestAndPackage - jobs: - - template: /eng/pipelines/publish-docker-image.yml - parameters: - DockerDeployments: ${{ parameters.DockerDeployments }} - ManifestDeployment: ${{ parameters.ManifestDeployment }} - ImageTag: "${{ parameters.DockerTagPrefix }}$(Build.BuildNumber)" + # Publish to https://dev.azure.com/azure-sdk/public/_packaging?_a=feed&feed=azure-sdk-for-net + DevOpsFeedId: '29ec6040-b234-4e31-b139-33dc4287b756/fa8c16a3-dbe0-4de2-a297-03065ec1ba3f' + ExeMatrix: ${{ parameters.StandaloneExeMatrix }} + ShouldPublishExecutables: ${{ parameters.ReleaseBinaries }} + ShouldPublishSymbols: ${{ parameters.ShouldPublishSymbols }} + RequireStrongNames: ${{ parameters.RequireStrongNames }} + + - ${{if and(not(eq(length(parameters.DockerDeployments), 0)), ne(variables['Build.Reason'], 'PullRequest'), eq(variables['System.TeamProject'], 'internal'))}}: + - stage: PublishDockerImages + displayName: Publish Docker Images + dependsOn: BuildTestAndPackage + jobs: + - template: /eng/pipelines/publish-docker-image-isolated.yml + parameters: + DockerDeployments: ${{ parameters.DockerDeployments }} + ManifestDeployment: ${{ parameters.ManifestDeployment }} + ImageTag: "${{ parameters.DockerTagPrefix }}$(Build.BuildNumber)" diff --git a/tools/http-fault-injector/ci.yml b/tools/http-fault-injector/ci.yml index 5e9ca989e80..f170e75ff3a 100644 --- a/tools/http-fault-injector/ci.yml +++ b/tools/http-fault-injector/ci.yml @@ -25,3 +25,4 @@ extends: template: /eng/pipelines/templates/stages/archetype-sdk-tool-dotnet.yml parameters: ToolDirectory: tools/http-fault-injector + Use1ESOfficial: true