diff --git a/eng/pipelines/templates/jobs/vmr-build.yml b/eng/pipelines/templates/jobs/vmr-build.yml index 09910e116159..5f0e9376a14d 100644 --- a/eng/pipelines/templates/jobs/vmr-build.yml +++ b/eng/pipelines/templates/jobs/vmr-build.yml @@ -18,8 +18,10 @@ parameters: default: 'Release' - name: container - type: string - default: '' + type: object + default: + image: '' + name: '' - name: crossRootFs type: string @@ -115,6 +117,22 @@ jobs: - job: ${{ parameters.buildName }}_${{ parameters.targetArchitecture }}${{ replace(format('_BuildPass{0}', coalesce(parameters.buildPass, '1')), '_BuildPass1', '') }} pool: ${{ parameters.pool }} + # 1ES pipeline template requires that the container is specified in the job level. + # If we are using a container image, we set the container to correct name. + # Otherwise, we set the container to host so that the job runs on the host agent. + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + ${{ if ne(parameters.container.name, '') }}: + container: ${{ parameters.container.name }} + ${{ else }}: + container: host + + # For public projects, we always use the container image if it is specified. + ${{ else }}: + ${{ if ne(parameters.container.image, '') }}: + container: + image: ${{ parameters.container.image }} + options: --privileged + # Currently, CodeQL slows the build down too much # https://github.com/dotnet/source-build/issues/4276 ${{ if and(parameters.isBuiltFromVmr, startswith(parameters.buildName, 'Windows'), eq(variables['System.TeamProject'], 'internal'), ne(variables['Build.Reason'], 'PullRequest')) }}: @@ -328,21 +346,25 @@ jobs: eval tar -ozxf "$previousSdkPath" -C "$(sourcesPath)/.dotnet" eval rm -f "$previousSdkPath" - echo "##vso[task.setvariable variable=additionalBuildArgs]--with-sdk /vmr/.dotnet" + echo "##vso[task.setvariable variable=additionalBuildArgs]--with-sdk $(sourcesPath)/.dotnet" fi - docker run --rm -v "$(sourcesPath):/vmr" -w /vmr ${{ parameters.container }} ./prep-source-build.sh $customPrepArgs + ./prep-source-build.sh $customPrepArgs displayName: Prep the Build + workingDirectory: $(sourcesPath) - script: | set -ex df -h customEnvVars="" + customPreBuildArgs="" customBuildArgs="--ci --clean-while-building --prepareMachine -c ${{ parameters.configuration }}" if [[ '${{ parameters.runOnline }}' == 'True' ]]; then customBuildArgs="$customBuildArgs --online" + else + customPreBuildArgs="$customPreBuildArgs sudo unshare -n" fi if [[ '${{ parameters.enablePoison }}' == 'True' ]]; then @@ -402,32 +424,14 @@ jobs: buildArgs="$(additionalBuildArgs) $customBuildArgs $extraBuildProperties" - # Only use Docker when a container is specified - if [[ -n "${{ parameters.container }}" ]]; then - # Allows Arcade to have access to the commit for the build, pass it through to the container - customEnvVars="$customEnvVars BUILD_SOURCEVERSION=$BUILD_SOURCEVERSION" - - customEnvVars="$customEnvVars BUILD_BUILDNUMBER=$(Build.BuildNumber)" + for envVar in $customEnvVars; do + customEnvVarsWithBashSyntax="$customEnvVarsWithBashSyntax export $envVar;" + done - customDockerRunArgs="" - for envVar in $customEnvVars; do - customDockerRunArgs="$customDockerRunArgs -e $envVar" - done - - if [[ '${{ parameters.runOnline }}' == 'False' ]]; then - customDockerRunArgs="$customDockerRunArgs --network none" - fi - - docker run --rm -v "$(sourcesPath):/vmr" -w /vmr $customDockerRunArgs ${{ parameters.container }} ./build.sh $buildArgs - else - for envVar in $customEnvVars; do - customEnvVarsWithBashSyntax="$customEnvVarsWithBashSyntax export $envVar;" - done - cd $(sourcesPath) - eval $customEnvVarsWithBashSyntax - ./build.sh $buildArgs - fi + eval $customEnvVarsWithBashSyntax + $customPreBuildArgs ./build.sh $buildArgs displayName: Build + workingDirectory: $(sourcesPath) # Only run tests if enabled - ${{ if eq(parameters.runTests, 'True') }}: @@ -448,10 +452,14 @@ jobs: - script: | set -ex - dockerVolumeArgs="-v $(sourcesPath):/vmr" + customPreBuildArgs='' customBuildArgs='' extraBuildProperties='' + if [[ '${{ parameters.runOnline }}' == 'False' ]]; then + customPreBuildArgs="$customPreBuildArgs sudo" + fi + if [[ ! -z '${{ parameters.targetOS }}' ]]; then extraBuildProperties="$extraBuildProperties /p:TargetOS=${{ parameters.targetOS }}" fi @@ -481,13 +489,8 @@ jobs: extraBuildProperties="$extraBuildProperties ${{ parameters.extraProperties }}" fi - # Only use Docker when a container is specified - if [[ -n "${{ parameters.container }}" ]]; then - docker run --rm $dockerVolumeArgs -w /vmr ${{ parameters.container }} ./build.sh --test --excludeCIBinarylog /bl:artifacts/log/Release/Test.binlog -c ${{ parameters.configuration }} $customBuildArgs $extraBuildProperties $(additionalBuildArgs) - else - cd $(sourcesPath) - ./build.sh --test --excludeCIBinarylog /bl:artifacts/log/Release/Test.binlog -c ${{ parameters.configuration }} $customBuildArgs $extraBuildProperties $(additionalBuildArgs) - fi + cd $(sourcesPath) + $customPreBuildArgs ./build.sh --test --excludeCIBinarylog /bl:artifacts/log/Release/Test.binlog $customBuildArgs $extraBuildProperties $(additionalBuildArgs) displayName: Run Tests timeoutInMinutes: ${{ variables.runTestsTimeout }} @@ -536,30 +539,24 @@ jobs: targetFolder=$(Build.ArtifactStagingDirectory)/BuildLogs/ mkdir -p ${targetFolder} - # Download rsync if using mariner - if grep -qE "CBL-Mariner|Azure Linux" /etc/os-release; then - sudo tdnf install -y rsync - sudo tdnf clean all - fi - cd "$(sourcesPath)" - find artifacts/log/ -exec rsync -R {} -t ${targetFolder} \; + find artifacts/log/ -exec cp --parents {} ${targetFolder} \; if [ -d "artifacts/scenario-tests/" ]; then - find artifacts/scenario-tests/ -type f -name "*.binlog" -exec rsync -R {} -t ${targetFolder} \; + find artifacts/scenario-tests/ -type f -name "*.binlog" -exec cp --parents {} ${targetFolder} \; echo "##vso[task.setvariable variable=hasScenarioTestResults]true" fi if [ -d "artifacts/TestResults/" ]; then - find artifacts/TestResults/ -type f -name "*.binlog" -exec rsync -R {} -t ${targetFolder} \; - find artifacts/TestResults/ -type f -name "*.diff" -exec rsync -R {} -t ${targetFolder} \; - find artifacts/TestResults/ -type f -name "Updated*.txt" -exec rsync -R {} -t ${targetFolder} \; - find artifacts/TestResults/ -type f -name "*.trx" -exec rsync -R {} -t ${targetFolder} \; + find artifacts/TestResults/ -type f -name "*.binlog" -exec cp --parents {} ${targetFolder} \; + find artifacts/TestResults/ -type f -name "*.diff" -exec cp --parents {} ${targetFolder} \; + find artifacts/TestResults/ -type f -name "Updated*.txt" -exec cp --parents {} ${targetFolder} \; + find artifacts/TestResults/ -type f -name "*.trx" -exec cp --parents {} ${targetFolder} \; fi if [[ "${{ parameters.buildSourceOnly }}" == "True" ]]; then - find artifacts/prebuilt-report/ -exec rsync -R {} -t ${targetFolder} \; + find artifacts/prebuilt-report/ -exec cp --parents {} ${targetFolder} \; fi displayName: Prepare BuildLogs staging directory and check assets continueOnError: true diff --git a/eng/pipelines/templates/stages/vmr-build.yml b/eng/pipelines/templates/stages/vmr-build.yml index f6b491b6fefb..83caf368af7d 100644 --- a/eng/pipelines/templates/stages/vmr-build.yml +++ b/eng/pipelines/templates/stages/vmr-build.yml @@ -92,7 +92,9 @@ stages: vmrBranch: ${{ variables.VmrBranch }} targetArchitecture: x64 pool: ${{ parameters.pool_Linux }} - container: ${{ variables.centOSStreamContainer }} + container: + name: ${{ variables.centOSStreamContainerName }} + image: ${{ variables.centOSStreamContainerImage }} buildFromArchive: false # 🚫 buildSourceOnly: true # ✅ enablePoison: false # 🚫 @@ -113,7 +115,9 @@ stages: # vmrBranch: ${{ variables.VmrBranch }} # targetArchitecture: x64 # pool: ${{ parameters.pool_Linux }} - # container: ${{ variables.centOSStreamContainer }} + # container: + # name: ${{ variables.centOSStreamContainerName }} + # image: ${{ variables.centOSStreamContainerImage }} # buildFromArchive: false # 🚫 # buildSourceOnly: true # ✅ # enablePoison: false # 🚫 @@ -134,7 +138,9 @@ stages: # targetArchitecture: x64 # artifactsRid: ${{ variables.alpinePreviousX64Rid }} # pool: ${{ parameters.pool_Linux }} - # container: ${{ variables.alpinePreviousContainer }} + # container: + # name: ${{ variables.alpinePreviousContainerName }} + # image: ${{ variables.alpinePreviousContainerImage }} # targetRid: ${{ variables.alpinePreviousX64Rid }} # buildFromArchive: false # 🚫 # buildSourceOnly: true # ✅ @@ -152,7 +158,9 @@ stages: vmrBranch: ${{ variables.VmrBranch }} targetArchitecture: x64 pool: ${{ parameters.pool_Linux }} - container: ${{ variables.ubuntuContainer }} + container: + name: ${{ variables.ubuntuContainerName }} + image: ${{ variables.ubuntuContainerImage }} buildFromArchive: false # 🚫 buildSourceOnly: true # ✅ enablePoison: false # 🚫 @@ -173,7 +181,9 @@ stages: vmrBranch: ${{ variables.VmrBranch }} targetArchitecture: x64 pool: ${{ parameters.pool_Linux }} - container: ${{ variables.almaLinuxContainer }} + container: + name: ${{ variables.almaLinuxContainerName }} + image: ${{ variables.almaLinuxContainerImage }} targetRid: ${{ variables.almaLinuxX64Rid }} buildFromArchive: false # 🚫 buildSourceOnly: true # ✅ @@ -191,7 +201,9 @@ stages: vmrBranch: ${{ variables.VmrBranch }} targetArchitecture: x64 pool: ${{ parameters.pool_Linux }} - container: ${{ variables.alpineLatestContainer }} + container: + name: ${{ variables.alpineLatestContainerName }} + image: ${{ variables.alpineLatestContainerImage }} targetRid: ${{ variables.alpineLatestX64Rid }} buildFromArchive: false # 🚫 buildSourceOnly: true # ✅ @@ -209,7 +221,9 @@ stages: vmrBranch: ${{ variables.VmrBranch }} targetArchitecture: x64 pool: ${{ parameters.pool_Linux }} - container: ${{ variables.centOSStreamContainer }} + container: + name: ${{ variables.centOSStreamContainerName }} + image: ${{ variables.centOSStreamContainerImage }} buildFromArchive: true # ✅ buildSourceOnly: true # ✅ enablePoison: false # 🚫 @@ -228,7 +242,9 @@ stages: # targetArchitecture: x64 # artifactsRid: ${{ variables.centOSStreamX64Rid }} # pool: ${{ parameters.pool_Linux }} - # container: ${{ variables.centOSStreamContainer }} + # container: + # name: ${{ variables.centOSStreamContainerName }} + # image: ${{ variables.centOSStreamContainerImage }} # buildFromArchive: false # 🚫 # buildSourceOnly: true # ✅ # enablePoison: false # 🚫 @@ -247,7 +263,9 @@ stages: # targetArchitecture: x64 # artifactsRid: ${{ variables.centOSStreamX64Rid }} # pool: ${{ parameters.pool_Linux }} - # container: ${{ variables.centOSStreamContainer }} + # container: + # name: ${{ variables.centOSStreamContainerName }} + # image: ${{ variables.centOSStreamContainerImage }} # buildFromArchive: false # 🚫 # buildSourceOnly: true # ✅ # enablePoison: false # 🚫 @@ -264,7 +282,9 @@ stages: vmrBranch: ${{ variables.VmrBranch }} targetArchitecture: x64 pool: ${{ parameters.pool_Linux }} - container: ${{ variables.centOSStreamContainer }} + container: + name: ${{ variables.centOSStreamContainerName }} + image: ${{ variables.centOSStreamContainerImage }} buildFromArchive: true # ✅ buildSourceOnly: true # ✅ enablePoison: false # 🚫 @@ -281,7 +301,9 @@ stages: vmrBranch: ${{ variables.VmrBranch }} targetArchitecture: x64 pool: ${{ parameters.pool_Linux }} - container: ${{ variables.fedoraContainer }} + container: + name: ${{ variables.fedoraContainerName }} + image: ${{ variables.fedoraContainerImage }} buildFromArchive: true # ✅ buildSourceOnly: true # ✅ enablePoison: false # 🚫 @@ -298,7 +320,9 @@ stages: vmrBranch: ${{ variables.VmrBranch }} targetArchitecture: arm64 pool: ${{ parameters.pool_LinuxArm64 }} - container: ${{ variables.ubuntuArmContainer }} + container: + name: ${{ variables.ubuntuArmContainerName }} + image: ${{ variables.ubuntuArmContainerImage }} buildFromArchive: false # 🚫 buildSourceOnly: true # ✅ enablePoison: false # 🚫 @@ -316,7 +340,9 @@ stages: # vmrBranch: ${{ variables.VmrBranch }} # targetArchitecture: x64 # pool: ${{ parameters.pool_Linux }} - # container: ${{ variables.fedoraContainer }} + # container: + # name: ${{ variables.fedoraContainerName }} + # image: ${{ variables.fedoraContainerImage }} # buildFromArchive: false # 🚫 # buildSourceOnly: true # ✅ # enablePoison: false # 🚫 @@ -336,7 +362,9 @@ stages: # vmrBranch: ${{ variables.VmrBranch }} # targetArchitecture: x64 # pool: ${{ parameters.pool_Linux }} - # container: ${{ variables.centOSStreamContainer }} + # container: + # name: ${{ variables.centOSStreamContainerName }} + # image: ${{ variables.centOSStreamContainerImage }} # buildFromArchive: true # ✅ # buildSourceOnly: true # ✅ # enablePoison: false # 🚫 @@ -364,7 +392,9 @@ stages: isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} pool: ${{ parameters.pool_Linux }} - container: ${{ variables.ubuntuContainer }} + container: + name: ${{ variables.ubuntuContainerName }} + image: ${{ variables.ubuntuContainerImage }} targetOS: linux targetArchitecture: x64 useDevVersions: true # Use dev versions for CI validation of the experience. @@ -407,7 +437,9 @@ stages: isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} pool: ${{ parameters.pool_Linux_Shortstack }} - container: ${{ variables.androidCrossContainer }} + container: + name: ${{ variables.androidCrossContainerName }} + image: ${{ variables.androidCrossContainerImage }} targetOS: android targetArchitecture: arm64 runTests: false @@ -418,7 +450,9 @@ stages: isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} pool: ${{ parameters.pool_Linux_Shortstack }} - container: ${{ variables.browserCrossContainer }} + container: + name: ${{ variables.browserCrossContainerName }} + image: ${{ variables.browserCrossContainerImage }} crossRootFs: '/crossrootfs/x64' targetOS: browser targetArchitecture: wasm @@ -443,7 +477,9 @@ stages: isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} pool: ${{ parameters.pool_Linux_Shortstack }} - container: ${{ variables.androidCrossContainer }} + container: + name: ${{ variables.androidCrossContainerName }} + image: ${{ variables.androidCrossContainerImage }} targetOS: android targetArchitecture: arm runTests: false @@ -454,7 +490,9 @@ stages: isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} pool: ${{ parameters.pool_Linux_Shortstack }} - container: ${{ variables.androidCrossContainer }} + container: + name: ${{ variables.androidCrossContainerName }} + image: ${{ variables.androidCrossContainerImage }} targetOS: android targetArchitecture: x64 runTests: false @@ -465,7 +503,9 @@ stages: isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} pool: ${{ parameters.pool_Linux_Shortstack }} - container: ${{ variables.androidCrossContainer }} + container: + name: ${{ variables.androidCrossContainerName }} + image: ${{ variables.androidCrossContainerImage }} targetOS: android targetArchitecture: x86 runTests: false @@ -476,7 +516,9 @@ stages: isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} pool: ${{ parameters.pool_Linux_Shortstack }} - container: ${{ variables.browserCrossContainer }} + container: + name: ${{ variables.browserCrossContainerName }} + image: ${{ variables.browserCrossContainerImage }} crossRootFs: '/crossrootfs/x64' targetOS: browser targetArchitecture: wasm @@ -489,7 +531,9 @@ stages: isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} pool: ${{ parameters.pool_Linux_Shortstack }} - container: ${{ variables.linuxBionicCrossContainer }} + container: + name: ${{ variables.linuxBionicCrossContainerName }} + image: ${{ variables.linuxBionicCrossContainerImage }} crossRootFs: '/crossrootfs/x64' targetOS: linux-bionic targetArchitecture: arm64 @@ -501,7 +545,9 @@ stages: isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} pool: ${{ parameters.pool_Linux_Shortstack }} - container: ${{ variables.linuxBionicCrossContainer }} + container: + name: ${{ variables.linuxBionicCrossContainerName }} + image: ${{ variables.linuxBionicCrossContainerImage }} crossRootFs: '/crossrootfs/x64' targetOS: linux-bionic targetArchitecture: arm64 @@ -514,7 +560,9 @@ stages: isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} pool: ${{ parameters.pool_Linux_Shortstack }} - container: ${{ variables.linuxBionicCrossContainer }} + container: + name: ${{ variables.linuxBionicCrossContainerName }} + image: ${{ variables.linuxBionicCrossContainerImage }} crossRootFs: '/crossrootfs/x64' targetOS: linux-bionic targetArchitecture: arm @@ -527,7 +575,9 @@ stages: isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} pool: ${{ parameters.pool_Linux_Shortstack }} - container: ${{ variables.linuxBionicCrossContainer }} + container: + name: ${{ variables.linuxBionicCrossContainerName }} + image: ${{ variables.linuxBionicCrossContainerImage }} crossRootFs: '/crossrootfs/x64' targetOS: linux-bionic targetArchitecture: x64 @@ -539,7 +589,9 @@ stages: isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} pool: ${{ parameters.pool_Linux_Shortstack }} - container: ${{ variables.linuxBionicCrossContainer }} + container: + name: ${{ variables.linuxBionicCrossContainerName }} + image: ${{ variables.linuxBionicCrossContainerImage }} crossRootFs: '/crossrootfs/x64' targetOS: linux-bionic targetArchitecture: x64 @@ -710,7 +762,9 @@ stages: isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} pool: ${{ parameters.pool_Linux_Shortstack }} - container: ${{ variables.wasiCrossContainer }} + container: + name: ${{ variables.wasiCrossContainerName }} + image: ${{ variables.wasiCrossContainerImage }} crossRootFs: '/crossrootfs/x64' targetOS: wasi targetArchitecture: wasm @@ -741,7 +795,9 @@ stages: isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} pool: ${{ parameters.pool_Linux }} - container: ${{ variables.azurelinuxX64CrossContainer }} + container: + name: ${{ variables.azurelinuxX64CrossContainerName }} + image: ${{ variables.azurelinuxX64CrossContainerImage }} crossRootFs: '/crossrootfs/x64' targetOS: linux targetArchitecture: x64 @@ -752,7 +808,9 @@ stages: isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} pool: ${{ parameters.pool_Linux }} - container: ${{ variables.azurelinuxX64CrossContainer }} + container: + name: ${{ variables.azurelinuxX64CrossContainerName }} + image: ${{ variables.azurelinuxX64CrossContainerImage }} crossRootFs: '/crossrootfs/x64' targetOS: linux targetArchitecture: x64 @@ -766,7 +824,9 @@ stages: isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} pool: ${{ parameters.pool_Linux }} - container: ${{ variables.azurelinuxX64CrossContainer }} + container: + name: ${{ variables.azurelinuxX64CrossContainerName }} + image: ${{ variables.azurelinuxX64CrossContainerImage }} crossRootFs: '/crossrootfs/x64' useMonoRuntime: true targetOS: linux @@ -779,7 +839,9 @@ stages: isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} pool: ${{ parameters.pool_Linux_Shortstack }} - container: ${{ variables.azurelinuxX64CrossContainer }} + container: + name: ${{ variables.azurelinuxX64CrossContainerName }} + image: ${{ variables.azurelinuxX64CrossContainerImage }} crossRootFs: '/crossrootfs/x64' useMonoRuntime: true targetOS: linux @@ -793,7 +855,9 @@ stages: isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} pool: ${{ parameters.pool_Linux_Shortstack }} - container: ${{ variables.azurelinuxX64CrossContainer }} + container: + name: ${{ variables.azurelinuxX64CrossContainerName }} + image: ${{ variables.azurelinuxX64CrossContainerImage }} crossRootFs: '/crossrootfs/x64' useMonoRuntime: true targetOS: linux @@ -807,7 +871,9 @@ stages: isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} pool: ${{ parameters.pool_Linux }} - container: ${{ variables.azurelinuxArmCrossContainer }} + container: + name: ${{ variables.azurelinuxArmCrossContainerName }} + image: ${{ variables.azurelinuxArmCrossContainerImage }} crossRootFs: '/crossrootfs/arm' targetOS: linux targetArchitecture: arm @@ -818,7 +884,9 @@ stages: isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} pool: ${{ parameters.pool_Linux }} - container: ${{ variables.azurelinuxArmCrossContainer }} + container: + name: ${{ variables.azurelinuxArmCrossContainerName }} + image: ${{ variables.azurelinuxArmCrossContainerImage }} crossRootFs: '/crossrootfs/arm' useMonoRuntime: true targetOS: linux @@ -830,7 +898,9 @@ stages: isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} pool: ${{ parameters.pool_Linux }} - container: ${{ variables.azurelinuxArmCrossContainer }} + container: + name: ${{ variables.azurelinuxArmCrossContainerName }} + image: ${{ variables.azurelinuxArmCrossContainerImage }} crossRootFs: '/crossrootfs/arm' targetOS: linux targetArchitecture: arm @@ -843,7 +913,9 @@ stages: isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} pool: ${{ parameters.pool_Linux }} - container: ${{ variables.azurelinuxArm64CrossContainer }} + container: + name: ${{ variables.azurelinuxArm64CrossContainerName }} + image: ${{ variables.azurelinuxArm64CrossContainerImage }} crossRootFs: '/crossrootfs/arm64' targetOS: linux targetArchitecture: arm64 @@ -854,7 +926,9 @@ stages: isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} pool: ${{ parameters.pool_Linux }} - container: ${{ variables.azurelinuxArm64CrossContainer }} + container: + name: ${{ variables.azurelinuxArm64CrossContainerName }} + image: ${{ variables.azurelinuxArm64CrossContainerImage }} crossRootFs: '/crossrootfs/arm64' targetOS: linux targetArchitecture: arm64 @@ -868,7 +942,9 @@ stages: isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} pool: ${{ parameters.pool_Linux }} - container: ${{ variables.azurelinuxArm64CrossContainer }} + container: + name: ${{ variables.azurelinuxArm64CrossContainerName }} + image: ${{ variables.azurelinuxArm64CrossContainerImage }} crossRootFs: '/crossrootfs/arm64' useMonoRuntime: true targetOS: linux @@ -880,7 +956,9 @@ stages: isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} pool: ${{ parameters.pool_Linux }} - container: ${{ variables.azurelinuxX64AlpineCrossContainer }} + container: + name: ${{ variables.azurelinuxX64AlpineCrossContainerName }} + image: ${{ variables.azurelinuxX64AlpineCrossContainerImage }} crossRootFs: '/crossrootfs/x64' targetOS: linux-musl targetArchitecture: x64 @@ -892,7 +970,9 @@ stages: isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} pool: ${{ parameters.pool_Linux }} - container: ${{ variables.azurelinuxX64AlpineCrossContainer }} + container: + name: ${{ variables.azurelinuxX64AlpineCrossContainerName }} + image: ${{ variables.azurelinuxX64AlpineCrossContainerImage }} crossRootFs: '/crossrootfs/x64' useMonoRuntime: true targetOS: linux-musl @@ -905,7 +985,9 @@ stages: isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} pool: ${{ parameters.pool_Linux }} - container: ${{ variables.azurelinuxArmAlpineCrossContainer }} + container: + name: ${{ variables.azurelinuxArmAlpineCrossContainerName }} + image: ${{ variables.azurelinuxArmAlpineCrossContainerImage }} crossRootFs: '/crossrootfs/arm' targetOS: linux-musl targetArchitecture: arm @@ -917,7 +999,9 @@ stages: isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} pool: ${{ parameters.pool_Linux }} - container: ${{ variables.azurelinuxArm64AlpineCrossContainer }} + container: + name: ${{ variables.azurelinuxArm64AlpineCrossContainerName }} + image: ${{ variables.azurelinuxArm64AlpineCrossContainerImage }} crossRootFs: '/crossrootfs/arm64' targetOS: linux-musl targetArchitecture: arm64 @@ -929,7 +1013,9 @@ stages: isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} pool: ${{ parameters.pool_Linux }} - container: ${{ variables.azurelinuxX64AlpineCrossContainer }} + container: + name: ${{ variables.azurelinuxX64AlpineCrossContainerName }} + image: ${{ variables.azurelinuxX64AlpineCrossContainerImage }} crossRootFs: '/crossrootfs/x64' targetOS: linux-musl targetArchitecture: x64 @@ -943,7 +1029,9 @@ stages: isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} pool: ${{ parameters.pool_Linux }} - container: ${{ variables.azurelinuxArm64AlpineCrossContainer }} + container: + name: ${{ variables.azurelinuxArm64AlpineCrossContainerName }} + image: ${{ variables.azurelinuxArm64AlpineCrossContainerImage }} crossRootFs: '/crossrootfs/arm64' targetOS: linux-musl targetArchitecture: arm64 @@ -957,7 +1045,9 @@ stages: isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} pool: ${{ parameters.pool_Linux }} - container: ${{ variables.azurelinuxX64AlpineCrossContainer }} + container: + name: ${{ variables.azurelinuxX64AlpineCrossContainerName }} + image: ${{ variables.azurelinuxX64AlpineCrossContainerImage }} crossRootFs: '/crossrootfs/x64' targetOS: linux-musl targetArchitecture: x64 @@ -971,7 +1061,9 @@ stages: isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} pool: ${{ parameters.pool_Linux }} - container: ${{ variables.azurelinuxArmAlpineCrossContainer }} + container: + name: ${{ variables.azurelinuxArmAlpineCrossContainerName }} + image: ${{ variables.azurelinuxArmAlpineCrossContainerImage }} crossRootFs: '/crossrootfs/arm' targetOS: linux-musl targetArchitecture: arm @@ -985,7 +1077,9 @@ stages: isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} pool: ${{ parameters.pool_Linux }} - container: ${{ variables.azurelinuxArm64AlpineCrossContainer }} + container: + name: ${{ variables.azurelinuxArm64AlpineCrossContainerName }} + image: ${{ variables.azurelinuxArm64AlpineCrossContainerImage }} crossRootFs: '/crossrootfs/arm64' targetOS: linux-musl targetArchitecture: arm64 @@ -999,7 +1093,9 @@ stages: isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} pool: ${{ parameters.pool_Linux_Shortstack }} - container: ${{ variables.azurelinuxX64CrossContainer }} + container: + name: ${{ variables.azurelinuxX64CrossContainerName }} + image: ${{ variables.azurelinuxX64CrossContainerImage }} crossRootFs: '/crossrootfs/x64' targetOS: linux targetArchitecture: x64 @@ -1012,7 +1108,9 @@ stages: isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} pool: ${{ parameters.pool_Linux_Shortstack }} - container: ${{ variables.azurelinuxX64CrossContainer }} + container: + name: ${{ variables.azurelinuxX64CrossContainerName }} + image: ${{ variables.azurelinuxX64CrossContainerImage }} crossRootFs: '/crossrootfs/x64' targetOS: linux targetArchitecture: x64 @@ -1025,7 +1123,9 @@ stages: isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} pool: ${{ parameters.pool_Linux_Shortstack }} - container: ${{ variables.azurelinuxArm64CrossContainer }} + container: + name: ${{ variables.azurelinuxArm64CrossContainerName }} + image: ${{ variables.azurelinuxArm64CrossContainerImage }} crossRootFs: '/crossrootfs/arm64' targetOS: linux targetArchitecture: arm64 @@ -1038,7 +1138,9 @@ stages: isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} pool: ${{ parameters.pool_Linux_Shortstack }} - container: ${{ variables.azurelinuxArm64CrossContainer }} + container: + name: ${{ variables.azurelinuxArm64CrossContainerName }} + image: ${{ variables.azurelinuxArm64CrossContainerImage }} crossRootFs: '/crossrootfs/arm64' useMonoRuntime: true targetOS: linux @@ -1052,7 +1154,9 @@ stages: isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} pool: ${{ parameters.pool_Linux_Shortstack }} - container: ${{ variables.azurelinuxArm64CrossContainer }} + container: + name: ${{ variables.azurelinuxArm64CrossContainerName }} + image: ${{ variables.azurelinuxArm64CrossContainerImage }} crossRootFs: '/crossrootfs/arm64' useMonoRuntime: true targetOS: linux @@ -1066,7 +1170,9 @@ stages: isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} pool: ${{ parameters.pool_Linux_Shortstack }} - container: ${{ variables.azurelinuxArm64CrossContainer }} + container: + name: ${{ variables.azurelinuxArm64CrossContainerName }} + image: ${{ variables.azurelinuxArm64CrossContainerImage }} crossRootFs: '/crossrootfs/arm64' targetOS: linux targetArchitecture: arm64 diff --git a/eng/pipelines/templates/variables/vmr-build.yml b/eng/pipelines/templates/variables/vmr-build.yml index 584fbd97ab86..b2960ffdca31 100644 --- a/eng/pipelines/templates/variables/vmr-build.yml +++ b/eng/pipelines/templates/variables/vmr-build.yml @@ -12,39 +12,89 @@ variables: - name: VmrBranch value: ${{ replace(replace(variables['Build.SourceBranch'], 'refs/heads/', ''), 'refs/pull/', '') }} -- name: almaLinuxContainer +- name: almaLinuxContainerName + value: almaLinuxContainer +- name: almaLinuxContainerImage value: mcr.microsoft.com/dotnet-buildtools/prereqs:almalinux-8-source-build -- name: alpineLatestContainer + +- name: alpineLatestContainerName + value: alpineLatestContainer +- name: alpineLatestContainerImage value: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.20-withnode -- name: alpinePreviousContainer + +- name: alpinePreviousContainerName + value: alpinePreviousContainer +- name: alpinePreviousContainerImage value: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.19-WithNode -- name: centOSStreamContainer + +- name: centOSStreamContainerName + value: centOSStreamContainer +- name: centOSStreamContainerImage value: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9 -- name: fedoraContainer + +- name: fedoraContainerName + value: fedoraContainer +- name: fedoraContainerImage value: mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-41 -- name: ubuntuContainer + +- name: ubuntuContainerName + value: ubuntuContainer +- name: ubuntuContainerImage value: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-24.04 -- name: ubuntuArmContainer + +- name: ubuntuArmContainerName + value: ubuntuArmContainer +- name: ubuntuArmContainerImage value: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-24.04-arm64 -- name: azurelinuxX64CrossContainer + +- name: azurelinuxX64CrossContainerName + value: azurelinuxX64CrossContainer +- name: azurelinuxX64CrossContainerImage value: mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux-3.0-net9.0-cross-amd64 -- name: azurelinuxArmCrossContainer + +- name: azurelinuxArmCrossContainerName + value: azurelinuxArmCrossContainer +- name: azurelinuxArmCrossContainerImage value: mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux-3.0-net9.0-cross-arm -- name: azurelinuxArm64CrossContainer + +- name: azurelinuxArm64CrossContainerName + value: azurelinuxArm64CrossContainer +- name: azurelinuxArm64CrossContainerImage value: mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux-3.0-net9.0-cross-arm64 -- name: azurelinuxX64AlpineCrossContainer + +- name: azurelinuxX64AlpineCrossContainerName + value: azurelinuxX64AlpineCrossContainer +- name: azurelinuxX64AlpineCrossContainerImage value: mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux-3.0-net9.0-cross-amd64-alpine -- name: azurelinuxArmAlpineCrossContainer + +- name: azurelinuxArmAlpineCrossContainerName + value: azurelinuxArmAlpineCrossContainer +- name: azurelinuxArmAlpineCrossContainerImage value: mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux-3.0-net9.0-cross-arm-alpine -- name: azurelinuxArm64AlpineCrossContainer + +- name: azurelinuxArm64AlpineCrossContainerName + value: azurelinuxArm64AlpineCrossContainer +- name: azurelinuxArm64AlpineCrossContainerImage value: mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux-3.0-net9.0-cross-arm64-alpine -- name: androidCrossContainer + +- name: androidCrossContainerName + value: androidCrossContainer +- name: androidCrossContainerImage value: mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux-3.0-net9.0-cross-android-amd64 -- name: linuxBionicCrossContainer + +- name: linuxBionicCrossContainerName + value: linuxBionicCrossContainer +- name: linuxBionicCrossContainerImage value: mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux-3.0-net9.0-android-openssl -- name: browserCrossContainer + +- name: browserCrossContainerName + value: browserCrossContainer +- name: browserCrossContainerImage value: mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux-3.0-net9.0-webassembly-amd64 -- name: wasiCrossContainer + +- name: wasiCrossContainerName + value: wasiCrossContainer +- name: wasiCrossContainerImage value: mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux-3.0-net9.0-webassembly-amd64 - name: almaLinuxName diff --git a/src/SourceBuild/content/build.sh b/src/SourceBuild/content/build.sh index ded752aa09c0..7a73e69f1be6 100755 --- a/src/SourceBuild/content/build.sh +++ b/src/SourceBuild/content/build.sh @@ -275,10 +275,10 @@ function Build { initSourceOnlyBinaryLog="/bl:\"$log_dir/init-source-only.binlog\"" fi - "$CLI_ROOT/dotnet" build-server shutdown + "$CLI_ROOT/dotnet" build-server shutdown --msbuild "$CLI_ROOT/dotnet" msbuild "$scriptroot/eng/init-source-only.proj" $initSourceOnlyBinaryLog "${properties[@]}" # kill off the MSBuild server so that on future invocations we pick up our custom SDK Resolver - "$CLI_ROOT/dotnet" build-server shutdown + "$CLI_ROOT/dotnet" build-server shutdown --msbuild fi # Point MSBuild to the custom SDK resolvers folder, so it will pick up our custom SDK Resolver diff --git a/src/SourceBuild/content/eng/pipelines/ci.yml b/src/SourceBuild/content/eng/pipelines/ci.yml index 33b41f2c9163..db2364ebf02c 100644 --- a/src/SourceBuild/content/eng/pipelines/ci.yml +++ b/src/SourceBuild/content/eng/pipelines/ci.yml @@ -77,6 +77,59 @@ extends: baseline: baselineFile: $(Build.SourcesDirectory)\.config\guardian\.gdnbaselines + containers: + almaLinuxContainer: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:almalinux-8-source-build + options: --privileged + alpineLatestContainer: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.20-withnode + options: --privileged + alpinePreviousContainer: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.19-WithNode + options: --privileged + centOSStreamContainer: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9 + options: --privileged + fedoraContainer: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-41 + options: --privileged + ubuntuContainer: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-24.04 + options: --privileged + ubuntuArmContainer: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-24.04-arm64 + options: --privileged + azurelinuxX64CrossContainer: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux-3.0-net9.0-cross-amd64 + options: --privileged + azurelinuxArmCrossContainer: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux-3.0-net9.0-cross-arm + options: --privileged + azurelinuxArm64CrossContainer: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux-3.0-net9.0-cross-arm64 + options: --privileged + azurelinuxX64AlpineCrossContainer: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux-3.0-net9.0-cross-amd64-alpine + options: --privileged + azurelinuxArmAlpineCrossContainer: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux-3.0-net9.0-cross-arm-alpine + options: --privileged + azurelinuxArm64AlpineCrossContainer: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux-3.0-net9.0-cross-arm64-alpine + options: --privileged + androidCrossContainer: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux-3.0-net9.0-cross-android-amd64 + options: --privileged + linuxBionicCrossContainer: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux-3.0-net9.0-android-openssl + options: --privileged + browserCrossContainer: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux-3.0-net9.0-webassembly-amd64 + options: --privileged + wasiCrossContainer: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux-3.0-net9.0-webassembly-amd64 + options: --privileged + stages: - ${{ if and(ne(variables.isPRTrigger, 'true'), eq(variables['System.TeamProject'], 'internal')) }}: - template: /eng/pipelines/templates/stages/vmr-scan.yml@self diff --git a/src/SourceBuild/content/repo-projects/Directory.Build.targets b/src/SourceBuild/content/repo-projects/Directory.Build.targets index a835f97ba84e..c705ea6d8f7c 100644 --- a/src/SourceBuild/content/repo-projects/Directory.Build.targets +++ b/src/SourceBuild/content/repo-projects/Directory.Build.targets @@ -633,8 +633,11 @@ but the required features to read in binary files as input to a source generator don't exist. To work around these source generators, shut down the compiler server so we can delete the obj directories. Don't fail the build when shutdown returns a failure. + + Don't run when source building to prevent the build from hanging indefinitely - https://github.com/dotnet/source-build/issues/4796 -->