Skip to content

Commit

Permalink
Update azure-sdk-build-tools Repository Resource Refs in Yaml files (#…
Browse files Browse the repository at this point in the history
…5174)

* now produce standalone binaries on mac and linux agents as two different builds in the build stage
* add entitlements file, sign during production of the binary
* publish individual artifacts from the job, allowing us to download them individually as well
* Update azure-sdk-build-tools Repository Resource Refs in Yaml files

Co-authored-by: scbedd <[email protected]>
Co-authored-by: semick-dev <[email protected]>
Co-authored-by: Ben Broderick Phillips <[email protected]>
  • Loading branch information
4 people authored Jan 26, 2023
1 parent 32847c3 commit d4c712d
Show file tree
Hide file tree
Showing 6 changed files with 181 additions and 56 deletions.
12 changes: 12 additions & 0 deletions eng/dotnet-executable-entitlements.plist
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "https://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>com.apple.security.cs.allow-dyld-environment-variables</key><true/>
<key>com.apple.security.cs.allow-jit</key><true/>
<key>com.apple.security.cs.debugger</key><true/>
<key>com.apple.security.cs.disable-library-validation</key><true/>
<key>com.apple.security.get-task-allow</key><true/>
</dict>
</plist>
</xml>
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ resources:
- repository: azure-sdk-build-tools
type: git
name: internal/azure-sdk-build-tools
ref: refs/tags/azure-sdk-build-tools_20230112.1
ref: refs/tags/azure-sdk-build-tools_20230126.1

parameters:
- name: BuildToolsRepoPath
Expand Down
92 changes: 92 additions & 0 deletions eng/pipelines/templates/scripts/assemble-dotnet-standalone-exe.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
<#
.SYNOPSIS
Publishes a standalone dotnet executable to an artifact staging directory.
.DESCRIPTION
Assembles a standalone executable and places it within the given staging directory. This script takes care of any additional minutae that is required to
enable a usable binary down the line after signing or notarization.
.PARAMETER Rid
The target platform. Takes the form of "osx-x64", "win-arm64", "linux-x64", etc. A full list is available here: https://learn.microsoft.com/en-us/dotnet/core/rid-catalog
.PARAMETER ArtifactStagingDirectory
The root directory which will receive the compressed standalone executable.
.PARAMETER Target
The targeted folder that should be built and assembled into a standalone executable.
.PARAMETER Framework
The targeted .NET framework. Defaults to "net6.0."
#>
param(
[Parameter(mandatory=$true)]
[string] $Rid,
[Parameter(mandatory=$true)]
[string] $Target,
[Parameter(mandatory=$true)]
[string] $ArtifactStagingDirectory,
[Parameter(mandatory=$false)]
[string] $Framework = "net6.0"
)

# resolves to <artifactfolder>/win-x64
$destinationArtifactFolder = Join-Path $ArtifactStagingDirectory $Rid

# resolves to <artifactfolder>/win-x64/test-proxy-standalone-win-x64 (.zip or .tar.gz will be added as appropriate for platform)
$destinationPathSegment = Join-Path $destinationArtifactFolder "$(Split-Path -Leaf "$Target")-standalone-$Rid"

# resolves to tools/test-proxy/win-x64
$outputPath = Join-Path $Target $Rid

# ensure the destination artifact directory exists
if (!(Test-Path $destinationArtifactFolder)){
New-Item -Force -Path $destinationArtifactFolder -ItemType directory
}

Write-Host "dotnet publish -f $Framework -c Release -r $Rid -p:PublishSingleFile=true --self-contained --output $outputPath $Target"
dotnet publish -f $Framework -c Release -r $Rid -p:PublishSingleFile=true --self-contained --output $outputPath $Target

if ($LASTEXITCODE -ne 0) {
Write-Error "dotnet publish failed with exit code $LASTEXITCODE."
exit $LASTEXITCODE
}

# produce a tar.gz only for linux
if ("$($Rid)".Contains("linux")){
# tar on powershell in linux has some weirdness. For instance, this is a proper call to tar when we don't want to include the relative path to the target folder
# tar -cvzf -C tools/test-proxy/linux-arm64 blah.tar.gz tools/test-proxy/linux-arm64
# however when we use this, we actually get an error. To avoid this, we simply CD into the target directory before tar-ing it.
Push-Location "$outputPath"
# The sum contents within this folder will be: `appSettings.json`, `test-proxy.pdb`, `test-proxy` (the binary), and a certificate.
# This statement grabs the first extensionless file within the produced binary folder, which will always be the binary we need to set the executable bit on.
$binaryFile = (Get-ChildItem -Path . | Where-Object { !([System.IO.Path]::HasExtension($_)) } | Select-Object -First 1).ToString().Replace("`\","/")
bash -c "chmod +x $binaryFile"
tar -cvzf "$($destinationPathSegment).tar.gz" .
Pop-Location
}
elseif("$($Rid)".Contains("osx")){
# need to codesign the binary with an entitlements file such that the signed and notarized binary will properly invoke on
# a mac system. However, the `codesign` command is only available on a MacOS agent. With that being the case, we simply special case
# this function here to ensure that the script does not fail outside of a MacOS agent.
if ($IsMacOS) {
$binaryFile = Get-ChildItem -Path $outputPath | Where-Object { !([System.IO.Path]::hasExtension($_)) } | Select-Object -First 1
$binaryFileBash = $binaryFile.ToString().Replace("`\","/")

$entitlements = (Resolve-Path -Path (Join-Path $PSScriptRoot ".." ".." ".." "dotnet-executable-entitlements.plist")).ToString().Replace("`\", "/")

bash -c "codesign --deep -s - -f --options runtime --entitlements $($entitlements) $($binaryFileBash)"
bash -c "codesign -d --entitlements :- $($binaryFileBash)"
}

Compress-Archive -Path "$($outputPath)/*" -DestinationPath "$($destinationPathSegment).zip"
}
else {
Compress-Archive -Path "$($outputPath)/*" -DestinationPath "$($destinationPathSegment).zip"
}

# clean up the uncompressed artifact directory
Remove-Item -Recurse -Force -Path $outputPath



40 changes: 28 additions & 12 deletions eng/pipelines/templates/stages/archetype-sdk-tool-dotnet.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ resources:
- repository: azure-sdk-build-tools
type: git
name: internal/azure-sdk-build-tools
ref: refs/tags/azure-sdk-build-tools_20230112.1
ref: refs/tags/azure-sdk-build-tools_20230126.1

parameters:
- name: ToolDirectory
Expand Down Expand Up @@ -76,12 +76,6 @@ stages:
DOTNET_CLI_TELEMETRY_OPTOUT: 1
DOTNET_MULTILEVEL_LOOKUP: 0

- template: /eng/pipelines/templates/steps/produce-net-standalone-packs.yml
parameters:
StagingDirectory: $(Build.ArtifactStagingDirectory)
BuildMatrix: ${{ parameters.StandaloneExeMatrix }}
TargetDirectory: '${{ coalesce(parameters.PackageDirectory, parameters.ToolDirectory) }}'

# This step creates "$(packagesToPublishDir)" directory if it doesn't exist.
# This step is necessary since migration to net6.0. This is because since net6.0,
# in case the "Build and Package" above would not output any packages to this directory,
Expand All @@ -102,11 +96,33 @@ stages:
artifact: packages
condition: succeededOrFailed()

- ${{ if gt(length(parameters.StandaloneExeMatrix), 0) }}:
- publish: $(Build.ArtifactStagingDirectory)/binaries
displayName: Publish executables to binaries artifact
artifact: binaries
condition: succeededOrFailed()
- job: Produce_Executables

strategy:
matrix:
linux:
imageName: 'ubuntu-22.04'
poolName: 'azsdk-pool-mms-ubuntu-2204-general'
artifactName: 'linux_windows'
mac:
imageName: 'macos-11'
poolName: 'Azure Pipelines'
artifactName: 'mac'

pool:
name: $(poolName)
vmImage: $(imageName)

steps:
- template: /eng/pipelines/templates/steps/install-dotnet.yml
parameters:
DotNetCoreVersion: ${{ parameters.DotNetCoreVersion }}

- template: /eng/pipelines/templates/steps/produce-net-standalone-packs.yml
parameters:
StagingDirectory: $(Build.ArtifactStagingDirectory)
BuildMatrix: ${{ parameters.StandaloneExeMatrix }}
TargetDirectory: '${{ coalesce(parameters.PackageDirectory, parameters.ToolDirectory) }}'

- job: Test

Expand Down
89 changes: 47 additions & 42 deletions eng/pipelines/templates/steps/produce-net-standalone-packs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ parameters:
type: object
default: []

# A `BuildMatrix` is merely a list of possible targeted platforms. .NET 6 can build for any target from any other target.
# A `BuildMatrix` is merely a list of possible targeted platforms. .NET 6+ can build for any target from any other target.
# - rid: win-x64
# framework: net6.0
# - rid: linux-x64
Expand All @@ -17,48 +17,53 @@ parameters:
# framework: net6.0

steps:
- pwsh: |
New-Item -ItemType Directory -Path "$(Build.ArtifactStagingDirectory)/binaries" -Force
Write-Host "Created directory ""$(Build.ArtifactStagingDirectory)/binaries"""
displayName: Create .NET standalone packs destination directory
- ${{ each target in parameters.BuildMatrix }}:
- pwsh: |
$destinationPathSegment = "$(Build.ArtifactStagingDirectory)/binaries/$((Split-Path -Leaf "${{ parameters.TargetDirectory }}"))-standalone-${{ target.rid }}"
$sourcePath = "${{ parameters.TargetDirectory }}/${{ target.rid }}"
Write-Host "dotnet publish -f ${{ target.framework }} -c Release -r ${{ target.rid }} --self-contained -p:PublishSingleFile=true --output ${{ parameters.TargetDirectory }}/${{ target.rid }}"
dotnet publish -f ${{ target.framework }} -c Release -r ${{ target.rid }} --self-contained -p:PublishSingleFile=true --output ${{ parameters.TargetDirectory }}/${{ target.rid }}
if ($LASTEXITCODE -ne 0) {
Write-Error "dotnet publish failed with exit code $LASTEXITCODE."
exit $LASTEXITCODE
}
# produce a tar.gz only for linux
if ("${{ target.rid }}".Contains("linux")){
# tar on powershell in linux has some weirdness. For instance, this is a proper call to tar when we don't want to include the relative path to the target folder
# tar -cvzf -C tools/test-proxy/linux-arm64 blah.tar.gz tools/test-proxy/linux-arm64
# however when we use this, we actually get an error. To avoid this, we simply CD into the target directory before tar-ing it.
Push-Location "$sourcePath"
# The sum contents within this folder will be: `appSettings.json`, `test-proxy.pdb`, `test-proxy` (the binary), and a certificate.
# This statement grabs the first extensionless file within the produced binary folder, which will always be the binary we need to set the executable bit on.
$binaryFile = (Get-ChildItem -Path . | Where-Object { !([System.IO.Path]::HasExtension($_)) } | Select-Object -First 1).ToString().Replace("`\","/")
bash -c "chmod +x $binaryFile"
tar -cvzf "$($destinationPathSegment).tar.gz" .
Pop-Location
}
else {
Compress-Archive -Path "$($sourcePath)/*" -DestinationPath "$($destinationPathSegment).zip"
}
# ensure that we only build the mac standalone executables on a mac agent, everything else on not mac
- ${{ if startsWith(target.rid, 'osx') }}:
- task: Powershell@2
inputs:
workingDirectory: "$(Build.SourcesDirectory)"
filePath: $(Build.SourcesDirectory)/eng/pipelines/templates/scripts/assemble-dotnet-standalone-exe.ps1
arguments: >
-Rid "${{ target.rid }}"
-Target "${{ parameters.TargetDirectory }}"
-ArtifactStagingDirectory "$(Build.ArtifactStagingDirectory)"
-Framework "${{ target.framework }}"
pwsh: true
displayName: 'Produce Executable for ${{ target.rid }}'
condition: and(succeeded(), eq(variables['Agent.OS'], 'Darwin'))
env:
DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1
DOTNET_CLI_TELEMETRY_OPTOUT: 1
DOTNET_MULTILEVEL_LOOKUP: 0

# clean up the uncompressed artifact directory
Remove-Item -Recurse -Force -Path $sourcePath
displayName: 'Produce Executable for ${{ target.rid }}'
workingDirectory: "${{ parameters.TargetDirectory }}"
env:
DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1
DOTNET_CLI_TELEMETRY_OPTOUT: 1
DOTNET_MULTILEVEL_LOOKUP: 0
- template: /eng/common/pipelines/templates/steps/publish-artifact.yml
parameters:
ArtifactName: "standalone-${{ target.rid }}"
ArtifactPath: "$(Build.ArtifactStagingDirectory)/${{ target.rid }}"
customCondition: eq(variables['Agent.OS'], 'Darwin')

- ${{ if not(startswith(target.rid, 'osx')) }}:
- task: Powershell@2
inputs:
workingDirectory: "$(Build.SourcesDirectory)"
filePath: $(Build.SourcesDirectory)/eng/pipelines/templates/scripts/assemble-dotnet-standalone-exe.ps1
arguments: >
-Rid "${{ target.rid }}"
-Target "${{ parameters.TargetDirectory }}"
-ArtifactStagingDirectory "$(Build.ArtifactStagingDirectory)"
-Framework "${{ target.framework }}"
pwsh: true
displayName: 'Produce Executable for ${{ target.rid }}'
condition: and(succeeded(), not(eq(variables['Agent.OS'], 'Darwin')))
env:
DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1
DOTNET_CLI_TELEMETRY_OPTOUT: 1
DOTNET_MULTILEVEL_LOOKUP: 0

- template: /eng/common/pipelines/templates/steps/publish-artifact.yml
parameters:
ArtifactName: "standalone-${{ target.rid }}"
ArtifactPath: "$(Build.ArtifactStagingDirectory)/${{ target.rid }}"
CustomCondition: not(eq(variables['Agent.OS'], 'Darwin'))

2 changes: 1 addition & 1 deletion src/dotnet/Mgmt.CI.BuildTools/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ resources:
- repository: azure-sdk-build-tools
type: git
name: internal/azure-sdk-build-tools
ref: refs/tags/azure-sdk-build-tools_20230112.1
ref: refs/tags/azure-sdk-build-tools_20230126.1

variables:
- template: /eng/pipelines/templates/variables/globals.yml
Expand Down

0 comments on commit d4c712d

Please sign in to comment.