Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Migrate image and dotnet tool publish to 1es pipelines #7920

Merged
merged 2 commits into from
Mar 20, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 5 additions & 16 deletions eng/containers/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,39 +3,29 @@ parameters:
type: object
default:
- name: mock_attestation
pool: azsdk-pool-mms-ubuntu-2204-general
vmImage: ubuntu-22.04
dockerRepo: 'keyvault-mock-attestation'
dockerFile: 'tools/keyvault-mock-attestation/Dockerfile'
configPath: 'tools/keyvault-mock-attestation'
stableTags:
- 'latest'
- name: stress_watcher
pool: azsdk-pool-mms-ubuntu-2204-general
vmImage: ubuntu-22.04
dockerRepo: 'stress/watcher'
dockerFile: 'tools/stress-cluster/services/Stress.Watcher/Dockerfile'
configPath: 'tools/stress-cluster/services/Stress.Watcher'
stableTags:
- 'latest'
- name: stress_deploy_test_resources
pool: azsdk-pool-mms-ubuntu-2204-general
vmImage: ubuntu-22.04
dockerRepo: 'stress/deploy-test-resources'
prepareScript: tools/stress-cluster/cluster/kubernetes/stress-test-addons/images/test-resource-deployer/prepare.ps1
dockerFile: 'tools/stress-cluster/cluster/kubernetes/stress-test-addons/images/test-resource-deployer/Dockerfile'
configPath: 'tools/stress-cluster/cluster/kubernetes/stress-test-addons/images/test-resource-deployer'
stableTags:
- 'latest'
- name: http_fault_injector
pool: azsdk-pool-mms-ubuntu-2204-general
vmImage: ubuntu-22.04
dockerRepo: 'stress/httpfaultinjector'
dockerFile: 'tools/http-fault-injector/Dockerfile'
configPath: 'tools/http-fault-injector'
stableTags:
- 'latest'
- name: otel_collector
pool: azsdk-pool-mms-ubuntu-2204-general
vmImage: ubuntu-22.04
dockerRepo: 'stress/otelcollector'
dockerFile: 'tools/stress-cluster/services/otelcollector/Dockerfile'
configPath: 'tools/stress-cluster/services/otelcollector'
stableTags:
- '0.94.0'

Expand Down Expand Up @@ -70,7 +60,6 @@ variables:
value: 'azsdkengsys'
- name: imageTag
value: $(build.buildid)
- template: ../pipelines/templates/variables/globals.yml

extends:
template: ../pipelines/publish-docker-image.yml
Expand Down
63 changes: 36 additions & 27 deletions eng/pipelines/mirror-container-images.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,32 +17,41 @@ parameters:
- source: ubuntu/squid
mirror: azsdkengsys.azurecr.io/mirror/ubuntu/squid

jobs:
- job: MirrorImages
displayName: Mirror Container Images
extends:
template: /eng/pipelines/templates/stages/1es-redirect.yml
parameters:
stages:
- stage:
displayName: Mirror Images
variables:
- template: /eng/pipelines/templates/variables/image.yml
jobs:
- job: MirrorImages
displayName: Mirror Container Images

pool:
name: azsdk-pool-mms-ubuntu-2004-general
vmImage: MMSUbuntu20.04
pool:
name: $(LINUXPOOL)
image: $(LINUXVMIMAGE)
os: linux

steps:
- ${{ each image in parameters.Images }}:
- task: Docker@2
displayName: Login to ${{ split(image.mirror, '.')[0] }}
inputs:
command: login
containerRegistry: ${{ split(image.mirror, '.')[0] }}
- task: Powershell@2
displayName: Mirror ${{ image.source }} to ${{ image.mirror }}
inputs:
pwsh: true
filePath: $(Build.SourcesDirectory)/eng/scripts/mirror-container-image.ps1
${{ if image.changes }}:
arguments: >
-Image ${{ image.source }}
-Mirror ${{ image.mirror }}
-changes '${{ image.changes }}'
${{ else }}:
arguments: >
-Image ${{ image.source }}
-Mirror ${{ image.mirror }}
steps:
- ${{ each image in parameters.Images }}:
- task: Docker@2
displayName: Login to ${{ split(image.mirror, '.')[0] }}
inputs:
command: login
containerRegistry: ${{ split(image.mirror, '.')[0] }}
- task: Powershell@2
displayName: Mirror ${{ image.source }} to ${{ image.mirror }}
inputs:
pwsh: true
filePath: $(Build.SourcesDirectory)/eng/scripts/mirror-container-image.ps1
${{ if image.changes }}:
arguments: >
-Image ${{ image.source }}
-Mirror ${{ image.mirror }}
-changes '${{ image.changes }}'
${{ else }}:
arguments: >
-Image ${{ image.source }}
-Mirror ${{ image.mirror }}
135 changes: 135 additions & 0 deletions eng/pipelines/publish-docker-image-isolated.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,135 @@
# Additional sample inputs can be found in `eng/containers/ci.yml`, but here is an example.
# - name: test_proxy_linux
# dockerRepo: 'engsys/testproxy-lin'
# prepareScript: tools/test-proxy/docker/prepare.ps1
# excludeFromManifest: true/false
# configPath: 'tools/test-proxy/docker'
# stableTags:
# - 'latest'

# A "ManifestDeployment" is a secondary deployment that can be used to map multiple docker tags to a single platform-sensitive one. It waits until
# all previous tag deployments are created, then triggers a manifest upload that bundles all tags configured in DockerDeployments. To exclude an item from the
# manifest upload, set deployment config property excludeFromManifest to true.
# - name: test_proxy_manifest
# dockerRepo: 'engsys/testproxy'
# stableTags:
# - 'latest'
parameters:
- name: DockerDeployments
type: object
default: []
- name: ManifestDeployment
type: object
default: []
- name: ImageTag
type: string
- name: ContainerRegistry
type: string
default: 'azsdkengsys'
- name: Publish
type: boolean
default: true

stages:
- stage:
displayName: Build/Publish Containers
variables:
- template: /eng/pipelines/templates/variables/globals.yml
- template: /eng/pipelines/templates/variables/image.yml
jobs:
- ${{ each config in parameters.DockerDeployments }}:
- job: container_build_${{ config.name }}
displayName: Build ${{ config.name }} Image
pool:
name: $(LINUXPOOL)
image: $(LINUXVMIMAGE)
os: linux

${{ if parameters.Publish }}:
templateContext:
outputs:
- output: containerImage
displayName: Push ${{ config.name }}:${{ parameters.ImageTag }}
condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
image: ${{ parameters.ContainerRegistry }}.azurecr.io/${{ config.dockerRepo }}:${{ parameters.ImageTag }}
- ${{ if config.stableTags }}:
- ${{ each stableTag in config.stableTags }}:
- output: containerImage
displayName: Push ${{ config.name }}:${{ stableTag }}
condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
image: ${{ parameters.ContainerRegistry }}.azurecr.io/${{ config.dockerRepo }}:${{ stableTag }}

steps:
- ${{ if config.prepareScript }}:
- pwsh: |
./${{ config.prepareScript }}
displayName: "Run prep script"
- task: 1ES.BuildContainerImage@1
displayName: Build ${{ config.name }}:${{ parameters.ImageTag }}
inputs:
image: ${{ parameters.ContainerRegistry }}.azurecr.io/${{ config.dockerRepo }}:${{ parameters.ImageTag }}
path: ${{ config.configPath }}
enableNetwork: true
useBuildKit: true
- ${{ if config.stableTags }}:
- ${{ each stableTag in config.stableTags }}:
- task: 1ES.BuildContainerImage@1
displayName: Build ${{ config.name }}:${{ stableTag }}
condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
inputs:
image: ${{ parameters.ContainerRegistry }}.azurecr.io/${{ config.dockerRepo }}:${{ stableTag }}
path: ${{ config.configPath }}
enableNetwork: true
useBuildKit: true

- ${{ if and(parameters.ManifestDeployment, parameters.Publish) }}:
- ${{ each deployment in parameters.ManifestDeployment }}:
- job: container_build_${{ deployment.name }}
displayName: Build ${{ deployment.name }} Manifest
${{ if gt(length(parameters.DockerDeployments), 0) }}:
dependsOn:
- ${{ each config in parameters.DockerDeployments }}:
- container_build_${{ config.name }}
pool:
name: $(LINUXPOOL)
image: $(LINUXVMIMAGE)
os: linux

steps:
- pwsh: |
$configurations = '${{ convertToJson(parameters.DockerDeployments) }}' -replace '\\', '/'
$assembledDependentTags = $(Build.SourcesDirectory)/eng/pipelines/templates/scripts/get-docker-manifest-input.ps1 `
-DockerDeploymentJson $configurations `
-ContainerRegistry "${{ parameters.ContainerRegistry }}" `
-ImageTag "${{ parameters.ImageTag }}" `

Write-Host "##vso[task.setvariable variable=ManifestVariable]$assembledDependentTags"
displayName: Generate Manifest Variable

- pwsh: |
docker manifest create ${{ parameters.ContainerRegistry }}.azurecr.io/${{ deployment.dockerRepo }}:${{ parameters.ImageTag }} $(ManifestVariable)
displayName: Generate Manifest

- pwsh: |
docker manifest push ${{ parameters.ContainerRegistry }}.azurecr.io/${{ deployment.dockerRepo }}:${{ parameters.ImageTag }}
displayName: Upload Manifest

- ${{ if deployment.stableTags }}:
- ${{ each stableTag in deployment.stableTags }}:
- pwsh: |
$configurations = '${{ convertToJson(parameters.DockerDeployments) }}' -replace '\\', '/'
$assembledDependentTags = $(Build.SourcesDirectory)/eng/pipelines/templates/scripts/get-docker-manifest-input.ps1 `
-DockerDeploymentJson $configurations `
-ContainerRegistry "${{ parameters.ContainerRegistry }}" `
-ImageTag "${{ stableTag }}" `

Write-Host "##vso[task.setvariable variable=ManifestVariable]$assembledDependentTags"
displayName: Generate Manifest Variable

- pwsh: |
docker manifest create ${{ parameters.ContainerRegistry }}.azurecr.io/${{ deployment.dockerRepo }}:${{ stableTag }} $(ManifestVariable)
displayName: Generate Manifest

- pwsh: |
docker manifest push ${{ parameters.ContainerRegistry }}.azurecr.io/${{ deployment.dockerRepo }}:${{ stableTag }}
displayName: Upload Manifest
117 changes: 7 additions & 110 deletions eng/pipelines/publish-docker-image.yml
Original file line number Diff line number Diff line change
@@ -1,12 +1,9 @@
# Additional sample inputs can be found in `eng/containers/ci.yml`, but here is an example.
# - name: test_proxy_linux
# pool: azsdk-pool-mms-ubuntu-2204-general
# vmImage: ubuntu-22.04
# dockerRepo: 'engsys/testproxy-lin'
# additionalDockerArgs: ''
# prepareScript: tools/test-proxy/docker/prepare.ps1
# excludeFromManifest: true/false
# dockerFile: 'tools/test-proxy/docker/dockerfile'
# configPath: 'tools/test-proxy/docker'
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just FYI though test-proxy/docker is gone now 👍

# stableTags:
# - 'latest'

Expand All @@ -33,109 +30,9 @@ parameters:
type: boolean
default: true

jobs:
- ${{ each config in parameters.DockerDeployments }}:
- job: container_build_${{ config.name }}
displayName: Build ${{ config.name }} Image
pool:
name: ${{ config.pool }}
vmImage: ${{ config.vmImage }}
steps:
- ${{ if config.prepareScript }}:
- pwsh: |
./${{ config.prepareScript }}
displayName: "Run prep script"
- task: Docker@2
displayName: Build ${{ config.name }}:${{ parameters.ImageTag }}
inputs:
command: build
Dockerfile: ${{ config.dockerFile }}
tags: ${{ parameters.ImageTag }}
arguments: '-t ${{ parameters.ContainerRegistry }}.azurecr.io/${{ config.dockerRepo }}:${{ parameters.ImageTag }} ${{ config.AdditionalDockerArgs }}'

- ${{ if parameters.Publish }}:
- task: Docker@2
displayName: Push ${{ config.name }}:${{ parameters.ImageTag }}
condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
inputs:
containerRegistry: ${{ parameters.ContainerRegistry }}
repository: ${{ config.dockerRepo }}
command: push
tags: ${{ parameters.ImageTag }}

- ${{ if config.stableTags }}:
- ${{ each stableTag in config.stableTags }}:
- task: Docker@2
displayName: Build ${{ config.name }}:${{ stableTag }}
condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
inputs:
command: build
Dockerfile: ${{ config.dockerFile }}
tags: ${{ stableTag }}
arguments: '-t ${{ parameters.ContainerRegistry }}.azurecr.io/${{ config.dockerRepo }}:${{ stableTag }} ${{ config.exclude }}'

- task: Docker@2
displayName: Push ${{ config.name }}:${{ stableTag }}
condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
inputs:
containerRegistry: ${{ parameters.ContainerRegistry }}
repository: ${{ config.dockerRepo }}
command: push
tags: ${{ stableTag }}

- ${{ if and(parameters.ManifestDeployment, parameters.Publish) }}:
- ${{ each deployment in parameters.ManifestDeployment }}:
- job: container_build_${{ deployment.name }}
displayName: Build ${{ deployment.name }} Manifest
${{ if gt(length(parameters.DockerDeployments), 0) }}:
dependsOn:
- ${{ each config in parameters.DockerDeployments }}:
- container_build_${{ config.name }}
pool:
name: azsdk-pool-mms-ubuntu-2204-general
vmImage: ubuntu-22.04

steps:
- task: Docker@2
displayName: Login to ACR
inputs:
command: login
containerRegistry: ${{ parameters.ContainerRegistry }}

- pwsh: |
$configurations = '${{ convertToJson(parameters.DockerDeployments) }}' -replace '\\', '/'
$assembledDependentTags = $(Build.SourcesDirectory)/eng/pipelines/templates/scripts/get-docker-manifest-input.ps1 `
-DockerDeploymentJson $configurations `
-ContainerRegistry "${{ parameters.ContainerRegistry }}" `
-ImageTag "${{ parameters.ImageTag }}" `

Write-Host "##vso[task.setvariable variable=ManifestVariable]$assembledDependentTags"
displayName: Generate Manifest Variable

- pwsh: |
docker manifest create ${{ parameters.ContainerRegistry }}.azurecr.io/${{ deployment.dockerRepo }}:${{ parameters.ImageTag }} $(ManifestVariable)
displayName: Generate Manifest

- pwsh: |
docker manifest push ${{ parameters.ContainerRegistry }}.azurecr.io/${{ deployment.dockerRepo }}:${{ parameters.ImageTag }}
displayName: Upload Manifest

- ${{ if deployment.stableTags }}:
- ${{ each stableTag in deployment.stableTags }}:
- pwsh: |
$configurations = '${{ convertToJson(parameters.DockerDeployments) }}' -replace '\\', '/'
$assembledDependentTags = $(Build.SourcesDirectory)/eng/pipelines/templates/scripts/get-docker-manifest-input.ps1 `
-DockerDeploymentJson $configurations `
-ContainerRegistry "${{ parameters.ContainerRegistry }}" `
-ImageTag "${{ stableTag }}" `

Write-Host "##vso[task.setvariable variable=ManifestVariable]$assembledDependentTags"
displayName: Generate Manifest Variable

- pwsh: |
docker manifest create ${{ parameters.ContainerRegistry }}.azurecr.io/${{ deployment.dockerRepo }}:${{ stableTag }} $(ManifestVariable)
displayName: Generate Manifest

- pwsh: |
docker manifest push ${{ parameters.ContainerRegistry }}.azurecr.io/${{ deployment.dockerRepo }}:${{ stableTag }}
displayName: Upload Manifest
extends:
template: /eng/pipelines/templates/stages/1es-redirect.yml
parameters:
stages:
- template: /eng/pipelines/publish-docker-image-isolated.yml
parameters: ${{ parameters }}
Loading