Skip to content

Commit

Permalink
Migrate to 1es pipelines
Browse files Browse the repository at this point in the history
  • Loading branch information
benbp committed Mar 20, 2024
1 parent aeca529 commit 7184890
Show file tree
Hide file tree
Showing 8 changed files with 433 additions and 307 deletions.
21 changes: 5 additions & 16 deletions eng/containers/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,39 +3,29 @@ parameters:
type: object
default:
- name: mock_attestation
pool: azsdk-pool-mms-ubuntu-2204-general
vmImage: ubuntu-22.04
dockerRepo: 'keyvault-mock-attestation'
dockerFile: 'tools/keyvault-mock-attestation/Dockerfile'
configPath: 'tools/keyvault-mock-attestation'
stableTags:
- 'latest'
- name: stress_watcher
pool: azsdk-pool-mms-ubuntu-2204-general
vmImage: ubuntu-22.04
dockerRepo: 'stress/watcher'
dockerFile: 'tools/stress-cluster/services/Stress.Watcher/Dockerfile'
configPath: 'tools/stress-cluster/services/Stress.Watcher'
stableTags:
- 'latest'
- name: stress_deploy_test_resources
pool: azsdk-pool-mms-ubuntu-2204-general
vmImage: ubuntu-22.04
dockerRepo: 'stress/deploy-test-resources'
prepareScript: tools/stress-cluster/cluster/kubernetes/stress-test-addons/images/test-resource-deployer/prepare.ps1
dockerFile: 'tools/stress-cluster/cluster/kubernetes/stress-test-addons/images/test-resource-deployer/Dockerfile'
configPath: 'tools/stress-cluster/cluster/kubernetes/stress-test-addons/images/test-resource-deployer'
stableTags:
- 'latest'
- name: http_fault_injector
pool: azsdk-pool-mms-ubuntu-2204-general
vmImage: ubuntu-22.04
dockerRepo: 'stress/httpfaultinjector'
dockerFile: 'tools/http-fault-injector/Dockerfile'
configPath: 'tools/http-fault-injector'
stableTags:
- 'latest'
- name: otel_collector
pool: azsdk-pool-mms-ubuntu-2204-general
vmImage: ubuntu-22.04
dockerRepo: 'stress/otelcollector'
dockerFile: 'tools/stress-cluster/services/otelcollector/Dockerfile'
configPath: 'tools/stress-cluster/services/otelcollector'
stableTags:
- '0.94.0'

Expand Down Expand Up @@ -70,7 +60,6 @@ variables:
value: 'azsdkengsys'
- name: imageTag
value: $(build.buildid)
- template: ../pipelines/templates/variables/globals.yml

extends:
template: ../pipelines/publish-docker-image.yml
Expand Down
63 changes: 36 additions & 27 deletions eng/pipelines/mirror-container-images.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,32 +17,41 @@ parameters:
- source: ubuntu/squid
mirror: azsdkengsys.azurecr.io/mirror/ubuntu/squid

jobs:
- job: MirrorImages
displayName: Mirror Container Images
extends:
template: /eng/pipelines/templates/stages/1es-redirect.yml
parameters:
stages:
- stage:
displayName: Mirror Images
variables:
- template: /eng/pipelines/templates/variables/image.yml
jobs:
- job: MirrorImages
displayName: Mirror Container Images

pool:
name: azsdk-pool-mms-ubuntu-2004-general
vmImage: MMSUbuntu20.04
pool:
name: $(LINUXPOOL)
image: $(LINUXVMIMAGE)
os: linux

steps:
- ${{ each image in parameters.Images }}:
- task: Docker@2
displayName: Login to ${{ split(image.mirror, '.')[0] }}
inputs:
command: login
containerRegistry: ${{ split(image.mirror, '.')[0] }}
- task: Powershell@2
displayName: Mirror ${{ image.source }} to ${{ image.mirror }}
inputs:
pwsh: true
filePath: $(Build.SourcesDirectory)/eng/scripts/mirror-container-image.ps1
${{ if image.changes }}:
arguments: >
-Image ${{ image.source }}
-Mirror ${{ image.mirror }}
-changes '${{ image.changes }}'
${{ else }}:
arguments: >
-Image ${{ image.source }}
-Mirror ${{ image.mirror }}
steps:
- ${{ each image in parameters.Images }}:
- task: Docker@2
displayName: Login to ${{ split(image.mirror, '.')[0] }}
inputs:
command: login
containerRegistry: ${{ split(image.mirror, '.')[0] }}
- task: Powershell@2
displayName: Mirror ${{ image.source }} to ${{ image.mirror }}
inputs:
pwsh: true
filePath: $(Build.SourcesDirectory)/eng/scripts/mirror-container-image.ps1
${{ if image.changes }}:
arguments: >
-Image ${{ image.source }}
-Mirror ${{ image.mirror }}
-changes '${{ image.changes }}'
${{ else }}:
arguments: >
-Image ${{ image.source }}
-Mirror ${{ image.mirror }}
135 changes: 135 additions & 0 deletions eng/pipelines/publish-docker-image-isolated.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,135 @@
# Additional sample inputs can be found in `eng/containers/ci.yml`, but here is an example.
# - name: test_proxy_linux
# dockerRepo: 'engsys/testproxy-lin'
# prepareScript: tools/test-proxy/docker/prepare.ps1
# excludeFromManifest: true/false
# configPath: 'tools/test-proxy/docker'
# stableTags:
# - 'latest'

# A "ManifestDeployment" is a secondary deployment that can be used to map multiple docker tags to a single platform-sensitive one. It waits until
# all previous tag deployments are created, then triggers a manifest upload that bundles all tags configured in DockerDeployments. To exclude an item from the
# manifest upload, set deployment config property excludeFromManifest to true.
# - name: test_proxy_manifest
# dockerRepo: 'engsys/testproxy'
# stableTags:
# - 'latest'
parameters:
- name: DockerDeployments
type: object
default: []
- name: ManifestDeployment
type: object
default: []
- name: ImageTag
type: string
- name: ContainerRegistry
type: string
default: 'azsdkengsys'
- name: Publish
type: boolean
default: true

stages:
- stage:
displayName: Build/Publish Containers
variables:
- template: /eng/pipelines/templates/variables/globals.yml
- template: /eng/pipelines/templates/variables/image.yml
jobs:
- ${{ each config in parameters.DockerDeployments }}:
- job: container_build_${{ config.name }}
displayName: Build ${{ config.name }} Image
pool:
name: $(LINUXPOOL)
image: $(LINUXVMIMAGE)
os: linux

${{ if parameters.Publish }}:
templateContext:
outputs:
- output: containerImage
displayName: Push ${{ config.name }}:${{ parameters.ImageTag }}
condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
image: ${{ parameters.ContainerRegistry }}.azurecr.io/${{ config.dockerRepo }}:${{ parameters.ImageTag }}
- ${{ if config.stableTags }}:
- ${{ each stableTag in config.stableTags }}:
- output: containerImage
displayName: Push ${{ config.name }}:${{ stableTag }}
condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
image: ${{ parameters.ContainerRegistry }}.azurecr.io/${{ config.dockerRepo }}:${{ stableTag }}

steps:
- ${{ if config.prepareScript }}:
- pwsh: |
./${{ config.prepareScript }}
displayName: "Run prep script"
- task: 1ES.BuildContainerImage@1
displayName: Build ${{ config.name }}:${{ parameters.ImageTag }}
inputs:
image: ${{ parameters.ContainerRegistry }}.azurecr.io/${{ config.dockerRepo }}:${{ parameters.ImageTag }}
path: ${{ config.configPath }}
enableNetwork: true
useBuildKit: true
- ${{ if config.stableTags }}:
- ${{ each stableTag in config.stableTags }}:
- task: 1ES.BuildContainerImage@1
displayName: Build ${{ config.name }}:${{ stableTag }}
condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
inputs:
image: ${{ parameters.ContainerRegistry }}.azurecr.io/${{ config.dockerRepo }}:${{ stableTag }}
path: ${{ config.configPath }}
enableNetwork: true
useBuildKit: true

- ${{ if and(parameters.ManifestDeployment, parameters.Publish) }}:
- ${{ each deployment in parameters.ManifestDeployment }}:
- job: container_build_${{ deployment.name }}
displayName: Build ${{ deployment.name }} Manifest
${{ if gt(length(parameters.DockerDeployments), 0) }}:
dependsOn:
- ${{ each config in parameters.DockerDeployments }}:
- container_build_${{ config.name }}
pool:
name: $(LINUXPOOL)
image: $(LINUXVMIMAGE)
os: linux

steps:
- pwsh: |
$configurations = '${{ convertToJson(parameters.DockerDeployments) }}' -replace '\\', '/'
$assembledDependentTags = $(Build.SourcesDirectory)/eng/pipelines/templates/scripts/get-docker-manifest-input.ps1 `
-DockerDeploymentJson $configurations `
-ContainerRegistry "${{ parameters.ContainerRegistry }}" `
-ImageTag "${{ parameters.ImageTag }}" `
Write-Host "##vso[task.setvariable variable=ManifestVariable]$assembledDependentTags"
displayName: Generate Manifest Variable
- pwsh: |
docker manifest create ${{ parameters.ContainerRegistry }}.azurecr.io/${{ deployment.dockerRepo }}:${{ parameters.ImageTag }} $(ManifestVariable)
displayName: Generate Manifest
- pwsh: |
docker manifest push ${{ parameters.ContainerRegistry }}.azurecr.io/${{ deployment.dockerRepo }}:${{ parameters.ImageTag }}
displayName: Upload Manifest
- ${{ if deployment.stableTags }}:
- ${{ each stableTag in deployment.stableTags }}:
- pwsh: |
$configurations = '${{ convertToJson(parameters.DockerDeployments) }}' -replace '\\', '/'
$assembledDependentTags = $(Build.SourcesDirectory)/eng/pipelines/templates/scripts/get-docker-manifest-input.ps1 `
-DockerDeploymentJson $configurations `
-ContainerRegistry "${{ parameters.ContainerRegistry }}" `
-ImageTag "${{ stableTag }}" `
Write-Host "##vso[task.setvariable variable=ManifestVariable]$assembledDependentTags"
displayName: Generate Manifest Variable
- pwsh: |
docker manifest create ${{ parameters.ContainerRegistry }}.azurecr.io/${{ deployment.dockerRepo }}:${{ stableTag }} $(ManifestVariable)
displayName: Generate Manifest
- pwsh: |
docker manifest push ${{ parameters.ContainerRegistry }}.azurecr.io/${{ deployment.dockerRepo }}:${{ stableTag }}
displayName: Upload Manifest
117 changes: 7 additions & 110 deletions eng/pipelines/publish-docker-image.yml
Original file line number Diff line number Diff line change
@@ -1,12 +1,9 @@
# Additional sample inputs can be found in `eng/containers/ci.yml`, but here is an example.
# - name: test_proxy_linux
# pool: azsdk-pool-mms-ubuntu-2204-general
# vmImage: ubuntu-22.04
# dockerRepo: 'engsys/testproxy-lin'
# additionalDockerArgs: ''
# prepareScript: tools/test-proxy/docker/prepare.ps1
# excludeFromManifest: true/false
# dockerFile: 'tools/test-proxy/docker/dockerfile'
# configPath: 'tools/test-proxy/docker'
# stableTags:
# - 'latest'

Expand All @@ -33,109 +30,9 @@ parameters:
type: boolean
default: true

jobs:
- ${{ each config in parameters.DockerDeployments }}:
- job: container_build_${{ config.name }}
displayName: Build ${{ config.name }} Image
pool:
name: ${{ config.pool }}
vmImage: ${{ config.vmImage }}
steps:
- ${{ if config.prepareScript }}:
- pwsh: |
./${{ config.prepareScript }}
displayName: "Run prep script"
- task: Docker@2
displayName: Build ${{ config.name }}:${{ parameters.ImageTag }}
inputs:
command: build
Dockerfile: ${{ config.dockerFile }}
tags: ${{ parameters.ImageTag }}
arguments: '-t ${{ parameters.ContainerRegistry }}.azurecr.io/${{ config.dockerRepo }}:${{ parameters.ImageTag }} ${{ config.AdditionalDockerArgs }}'

- ${{ if parameters.Publish }}:
- task: Docker@2
displayName: Push ${{ config.name }}:${{ parameters.ImageTag }}
condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
inputs:
containerRegistry: ${{ parameters.ContainerRegistry }}
repository: ${{ config.dockerRepo }}
command: push
tags: ${{ parameters.ImageTag }}

- ${{ if config.stableTags }}:
- ${{ each stableTag in config.stableTags }}:
- task: Docker@2
displayName: Build ${{ config.name }}:${{ stableTag }}
condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
inputs:
command: build
Dockerfile: ${{ config.dockerFile }}
tags: ${{ stableTag }}
arguments: '-t ${{ parameters.ContainerRegistry }}.azurecr.io/${{ config.dockerRepo }}:${{ stableTag }} ${{ config.exclude }}'

- task: Docker@2
displayName: Push ${{ config.name }}:${{ stableTag }}
condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
inputs:
containerRegistry: ${{ parameters.ContainerRegistry }}
repository: ${{ config.dockerRepo }}
command: push
tags: ${{ stableTag }}

- ${{ if and(parameters.ManifestDeployment, parameters.Publish) }}:
- ${{ each deployment in parameters.ManifestDeployment }}:
- job: container_build_${{ deployment.name }}
displayName: Build ${{ deployment.name }} Manifest
${{ if gt(length(parameters.DockerDeployments), 0) }}:
dependsOn:
- ${{ each config in parameters.DockerDeployments }}:
- container_build_${{ config.name }}
pool:
name: azsdk-pool-mms-ubuntu-2204-general
vmImage: ubuntu-22.04

steps:
- task: Docker@2
displayName: Login to ACR
inputs:
command: login
containerRegistry: ${{ parameters.ContainerRegistry }}

- pwsh: |
$configurations = '${{ convertToJson(parameters.DockerDeployments) }}' -replace '\\', '/'
$assembledDependentTags = $(Build.SourcesDirectory)/eng/pipelines/templates/scripts/get-docker-manifest-input.ps1 `
-DockerDeploymentJson $configurations `
-ContainerRegistry "${{ parameters.ContainerRegistry }}" `
-ImageTag "${{ parameters.ImageTag }}" `
Write-Host "##vso[task.setvariable variable=ManifestVariable]$assembledDependentTags"
displayName: Generate Manifest Variable
- pwsh: |
docker manifest create ${{ parameters.ContainerRegistry }}.azurecr.io/${{ deployment.dockerRepo }}:${{ parameters.ImageTag }} $(ManifestVariable)
displayName: Generate Manifest
- pwsh: |
docker manifest push ${{ parameters.ContainerRegistry }}.azurecr.io/${{ deployment.dockerRepo }}:${{ parameters.ImageTag }}
displayName: Upload Manifest
- ${{ if deployment.stableTags }}:
- ${{ each stableTag in deployment.stableTags }}:
- pwsh: |
$configurations = '${{ convertToJson(parameters.DockerDeployments) }}' -replace '\\', '/'
$assembledDependentTags = $(Build.SourcesDirectory)/eng/pipelines/templates/scripts/get-docker-manifest-input.ps1 `
-DockerDeploymentJson $configurations `
-ContainerRegistry "${{ parameters.ContainerRegistry }}" `
-ImageTag "${{ stableTag }}" `
Write-Host "##vso[task.setvariable variable=ManifestVariable]$assembledDependentTags"
displayName: Generate Manifest Variable
- pwsh: |
docker manifest create ${{ parameters.ContainerRegistry }}.azurecr.io/${{ deployment.dockerRepo }}:${{ stableTag }} $(ManifestVariable)
displayName: Generate Manifest
- pwsh: |
docker manifest push ${{ parameters.ContainerRegistry }}.azurecr.io/${{ deployment.dockerRepo }}:${{ stableTag }}
displayName: Upload Manifest
extends:
template: /eng/pipelines/templates/stages/1es-redirect.yml
parameters:
stages:
- template: /eng/pipelines/publish-docker-image-isolated.yml
parameters: ${{ parameters }}
Loading

0 comments on commit 7184890

Please sign in to comment.