From a4e98b52792771e839c65957a0079ba9a5cfdf7a Mon Sep 17 00:00:00 2001 From: Matthew Leibowitz Date: Wed, 27 Mar 2024 16:15:21 +0200 Subject: [PATCH] [release/2.x] Clean up APIScan and other compliance yaml (#2805) --- VERSIONS.txt | 1 + build.cake | 1 + .../SkiaSharp.Views.Interop.UWP.vcxproj | 8 + native/uwp/build.cake | 3 +- .../libHarfBuzzSharp/libHarfBuzzSharp.vcxproj | 8 + native/windows/build.cake | 2 +- .../libHarfBuzzSharp/libHarfBuzzSharp.vcxproj | 6 + nuget/SkiaSharp.Views.NativeAssets.UWP.nuspec | 54 +++ nuget/SkiaSharp.Views.nuspec | 7 +- SignList.xml => scripts/SignList.xml | 0 scripts/azure-pipelines-complete-internal.yml | 83 +++- scripts/azure-pipelines-complete.yml | 42 +- scripts/azure-pipelines-tests.yml | 34 +- scripts/azure-pipelines-variables.yml | 1 + scripts/azure-pipelines.yml | 85 +++- scripts/azure-template.yml | 31 ++ scripts/azure-templates-bootstrapper.yml | 190 +++++---- .../azure-templates-download-artifacts.yml | 3 +- scripts/azure-templates-linux-matrix.yml | 8 +- scripts/azure-templates-merger.yml | 53 +++ scripts/azure-templates-stages.yml | 369 ++++++++++++------ scripts/azure-templates-wasm-matrix.yml | 18 +- scripts/guardian/APIScanSurrogates.in.xml | 106 +++++ scripts/install-python.ps1 | 9 +- .../SkiaSharp.Vulkan.NetCore.Tests.csproj | 5 + .../xunit.runner.json | 6 + 26 files changed, 824 insertions(+), 309 deletions(-) create mode 100644 nuget/SkiaSharp.Views.NativeAssets.UWP.nuspec rename SignList.xml => scripts/SignList.xml (100%) create mode 100644 scripts/azure-template.yml create mode 100644 scripts/azure-templates-merger.yml create mode 100644 scripts/guardian/APIScanSurrogates.in.xml create mode 100644 tests/SkiaSharp.Vulkan.NetCore.Tests/xunit.runner.json diff --git a/VERSIONS.txt b/VERSIONS.txt index a2bc3dca44..a738731655 100644 --- a/VERSIONS.txt +++ b/VERSIONS.txt @@ -69,6 +69,7 @@ SkiaSharp.NativeAssets.UWP nuget 2.88.8 SkiaSharp.NativeAssets.watchOS nuget 2.88.8 SkiaSharp.NativeAssets.Win32 nuget 2.88.8 SkiaSharp.Views nuget 2.88.8 +SkiaSharp.Views.NativeAssets.UWP nuget 2.88.8 SkiaSharp.Views.Desktop.Common nuget 2.88.8 SkiaSharp.Views.Gtk2 nuget 2.88.8 SkiaSharp.Views.Gtk3 nuget 2.88.8 diff --git a/build.cake b/build.cake index 170dcb2e58..a6e2ffc0ca 100644 --- a/build.cake +++ b/build.cake @@ -81,6 +81,7 @@ var TRACKED_NUGETS = new Dictionary { { "SkiaSharp.NativeAssets.watchOS", new Version (1, 60, 0) }, { "SkiaSharp.NativeAssets.Win32", new Version (1, 60, 0) }, { "SkiaSharp.Views", new Version (1, 60, 0) }, + { "SkiaSharp.Views.NativeAssets.UWP", new Version (1, 60, 0) }, { "SkiaSharp.Views.Desktop.Common", new Version (1, 60, 0) }, { "SkiaSharp.Views.Gtk2", new Version (1, 60, 0) }, { "SkiaSharp.Views.Gtk3", new Version (1, 60, 0) }, diff --git a/native/uwp/SkiaSharp.Views.Interop.UWP/SkiaSharp.Views.Interop.UWP.vcxproj b/native/uwp/SkiaSharp.Views.Interop.UWP/SkiaSharp.Views.Interop.UWP.vcxproj index 38649ec730..4019f0513c 100644 --- a/native/uwp/SkiaSharp.Views.Interop.UWP/SkiaSharp.Views.Interop.UWP.vcxproj +++ b/native/uwp/SkiaSharp.Views.Interop.UWP/SkiaSharp.Views.Interop.UWP.vcxproj @@ -175,6 +175,7 @@ Console false + /DEBUGTYPE:CV,FIXUP @@ -190,6 +191,7 @@ Console false + /DEBUGTYPE:CV,FIXUP @@ -205,6 +207,7 @@ Console false + /DEBUGTYPE:CV,FIXUP @@ -220,6 +223,7 @@ Console false + /DEBUGTYPE:CV,FIXUP @@ -235,6 +239,7 @@ Console false + /DEBUGTYPE:CV,FIXUP @@ -250,6 +255,7 @@ Console false + /DEBUGTYPE:CV,FIXUP @@ -265,6 +271,7 @@ Console false + /DEBUGTYPE:CV,FIXUP @@ -280,6 +287,7 @@ Console false + /DEBUGTYPE:CV,FIXUP diff --git a/native/uwp/build.cake b/native/uwp/build.cake index afbf146ff0..7a5730b88d 100644 --- a/native/uwp/build.cake +++ b/native/uwp/build.cake @@ -42,7 +42,7 @@ Task("libSkiaSharp") $"extra_cflags=[ " + $" '-DSKIA_C_DLL', '/MD{d}', '/EHsc', '/Z7', " + $" '-DSK_HAS_DWRITE_1_H', '-DSK_HAS_DWRITE_2_H', '-DNO_GETENV', '-D_HAS_AUTO_PTR_ETC=1' ] " + - $"extra_ldflags=[ '/DEBUG:FULL' ]"); + $"extra_ldflags=[ '/DEBUG:FULL', '/DEBUGTYPE:CV,FIXUP' ]"); var outDir = OUTPUT_PATH.Combine(dir); EnsureDirectoryExists(outDir); @@ -136,6 +136,7 @@ Task("ANGLE") System.IO.File.AppendAllLines(cmake.FullPath, new [] { $"set(VCPKG_PLATFORM_TOOLSET \"{platform_toolset}\")", $"set(VCPKG_DEP_INFO_OVERRIDE_VARS \"{platform_toolset}\")", + $"set(VCPKG_LINKER_FLAGS \"/DEBUG:FULL /DEBUGTYPE:CV,FIXUP\")", }); } } diff --git a/native/uwp/libHarfBuzzSharp/libHarfBuzzSharp.vcxproj b/native/uwp/libHarfBuzzSharp/libHarfBuzzSharp.vcxproj index 9b18926365..b32d5d21b7 100644 --- a/native/uwp/libHarfBuzzSharp/libHarfBuzzSharp.vcxproj +++ b/native/uwp/libHarfBuzzSharp/libHarfBuzzSharp.vcxproj @@ -186,6 +186,7 @@ Console false false + /DEBUGTYPE:CV,FIXUP @@ -201,6 +202,7 @@ Console false false + /DEBUGTYPE:CV,FIXUP @@ -216,6 +218,7 @@ Console false false + /DEBUGTYPE:CV,FIXUP @@ -231,6 +234,7 @@ Console false false + /DEBUGTYPE:CV,FIXUP @@ -246,6 +250,7 @@ Console false false + /DEBUGTYPE:CV,FIXUP @@ -261,6 +266,7 @@ Console false false + /DEBUGTYPE:CV,FIXUP @@ -276,6 +282,7 @@ Console false false + /DEBUGTYPE:CV,FIXUP @@ -291,6 +298,7 @@ Console false false + /DEBUGTYPE:CV,FIXUP diff --git a/native/windows/build.cake b/native/windows/build.cake index 0cda591bd3..9fcadf2b71 100644 --- a/native/windows/build.cake +++ b/native/windows/build.cake @@ -53,7 +53,7 @@ Task("libSkiaSharp") clang + win_vcvars_version + $"extra_cflags=[ '-DSKIA_C_DLL', '/MT{d}', '/EHsc', '/Z7', '-D_HAS_AUTO_PTR_ETC=1' ] " + - $"extra_ldflags=[ '/DEBUG:FULL' ] " + + $"extra_ldflags=[ '/DEBUG:FULL', '/DEBUGTYPE:CV,FIXUP' ] " + ADDITIONAL_GN_ARGS); var outDir = OUTPUT_PATH.Combine($"{VARIANT}/{dir}"); diff --git a/native/windows/libHarfBuzzSharp/libHarfBuzzSharp.vcxproj b/native/windows/libHarfBuzzSharp/libHarfBuzzSharp.vcxproj index 8072b9d1e8..f97c3656e5 100644 --- a/native/windows/libHarfBuzzSharp/libHarfBuzzSharp.vcxproj +++ b/native/windows/libHarfBuzzSharp/libHarfBuzzSharp.vcxproj @@ -148,6 +148,7 @@ Windows + /DEBUGTYPE:CV,FIXUP @@ -163,6 +164,7 @@ Windows + /DEBUGTYPE:CV,FIXUP @@ -178,6 +180,7 @@ Windows + /DEBUGTYPE:CV,FIXUP @@ -197,6 +200,7 @@ Windows true true + /DEBUGTYPE:CV,FIXUP @@ -216,6 +220,7 @@ Windows true true + /DEBUGTYPE:CV,FIXUP @@ -235,6 +240,7 @@ Windows true true + /DEBUGTYPE:CV,FIXUP diff --git a/nuget/SkiaSharp.Views.NativeAssets.UWP.nuspec b/nuget/SkiaSharp.Views.NativeAssets.UWP.nuspec new file mode 100644 index 0000000000..b5bd2f9c94 --- /dev/null +++ b/nuget/SkiaSharp.Views.NativeAssets.UWP.nuspec @@ -0,0 +1,54 @@ + + + + + + SkiaSharp.Views.NativeAssets.UWP + SkiaSharp Views & Layers - Native Assets for UWP + 1.0.0 + +SkiaSharp Views & Layers are a set of platform-specific views and containers that can be used to draw on the screen. + + +SkiaSharp Views & Layers are a set of platform-specific views and containers that can be used to draw on the screen. + + +Please visit https://go.microsoft.com/fwlink/?linkid=868517 to view the release notes. + + https://go.microsoft.com/fwlink/?linkid=868515 + https://go.microsoft.com/fwlink/?linkid=2130524 + ui xamarin graphics ios android linux windows uwp tvos watchos macos tizen cross-platform skiasharp + + + https://go.microsoft.com/fwlink/?linkid=868514 + Microsoft + Microsoft + true + © Microsoft Corporation. All rights reserved. + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/nuget/SkiaSharp.Views.nuspec b/nuget/SkiaSharp.Views.nuspec index 5ff1dad4fa..9563a3b2c7 100644 --- a/nuget/SkiaSharp.Views.nuspec +++ b/nuget/SkiaSharp.Views.nuspec @@ -36,6 +36,7 @@ Please visit https://go.microsoft.com/fwlink/?linkid=868517 to view the release + @@ -123,12 +124,6 @@ Please visit https://go.microsoft.com/fwlink/?linkid=868517 to view the release - - - - - - diff --git a/SignList.xml b/scripts/SignList.xml similarity index 100% rename from SignList.xml rename to scripts/SignList.xml diff --git a/scripts/azure-pipelines-complete-internal.yml b/scripts/azure-pipelines-complete-internal.yml index eb67d50b97..b168afe2ba 100644 --- a/scripts/azure-pipelines-complete-internal.yml +++ b/scripts/azure-pipelines-complete-internal.yml @@ -4,40 +4,53 @@ pr: none parameters: - name: buildExternals - displayName: 'The specific native artifacts to use for this build.' + displayName: 'The Build ID containing the specific native artifacts to use:' type: string default: 'latest' - name: VM_IMAGE_HOST + displayName: 'The generic host build agent configuration:' type: object default: pool: name: Azure Pipelines vmImage: ubuntu-20.04 + os: windows - name: VM_IMAGE_WINDOWS + displayName: 'The Windows build agent configuration:' type: object default: pool: name: Azure Pipelines vmImage: windows-2022 + os: windows - name: VM_IMAGE_MAC + displayName: 'The macOS build agent configuration:' type: object default: pool: name: Azure Pipelines vmImage: macos-12 + os: macos - name: VM_IMAGE_LINUX + displayName: 'The Linux build agent configuration:' type: object default: pool: name: Azure Pipelines vmImage: ubuntu-20.04 + os: linux + - name: enableSigning + displayName: 'Enable package signing (Test signing)' + type: boolean + default: false - name: runCompliance + displayName: 'Run post-build compliance tasks (such as API Scan and PoliCheck)' + type: boolean + default: false + - name: use1ESPipelineTemplates + displayName: 'Run the build using the internal 1ES Pipeline Templates' type: boolean default: false - -pool: - name: Azure Pipelines - vmImage: ubuntu-20.04 variables: - template: /scripts/azure-pipelines-variables.yml@self @@ -49,17 +62,51 @@ resources: name: xamarin/yaml-templates endpoint: xamarin ref: refs/heads/main + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release -stages: - - template: /scripts/azure-templates-stages.yml@self - parameters: - buildPipelineType: 'both' - buildExternals: ${{ parameters.buildExternals }} - runCompliance: ${{ parameters.runCompliance }} - VM_IMAGE_HOST: ${{ parameters.VM_IMAGE_HOST }} - VM_IMAGE_WINDOWS: ${{ parameters.VM_IMAGE_WINDOWS }} - VM_IMAGE_WINDOWS_NATIVE: ${{ parameters.VM_IMAGE_WINDOWS }} - VM_IMAGE_MAC: ${{ parameters.VM_IMAGE_MAC }} - VM_IMAGE_MAC_NATIVE: ${{ parameters.VM_IMAGE_MAC }} - VM_IMAGE_LINUX: ${{ parameters.VM_IMAGE_LINUX }} - VM_IMAGE_LINUX_NATIVE: ${{ parameters.VM_IMAGE_LINUX }} +extends: + ${{ if eq('${{ parameters.use1ESPipelineTemplates }}', 'true') }}: + template: v1/1ES.Unofficial.PipelineTemplate.yml@1ESPipelineTemplates + ${{ if ne('${{ parameters.use1ESPipelineTemplates }}', 'true') }}: + template: /scripts/azure-template.yml@self + parameters: + pool: ${{ parameters.VM_IMAGE_HOST.pool }} + customBuildTags: + - ES365AIMigrationTooling + stages: + - template: /scripts/azure-templates-stages.yml@self + parameters: + buildPipelineType: 'both' + buildExternals: ${{ parameters.buildExternals }} + enableSigning: ${{ parameters.enableSigning }} + ${{ if eq(parameters.runCompliance, 'true') }}: + sdl: + apiscan: + enabled: true + binskim: + break: false + codeInspector: + enabled: true + credscan: + suppressionsFile: $(Build.SourcesDirectory)\scripts\guardian\CredScanSuppressions.json + policheck: + enabled: true + exclusionsFile: $(Build.SourcesDirectory)\scripts\guardian\PoliCheckExclusions.xml + spotBugs: + enabled: false + suppression: + suppressionFile: $(Build.SourcesDirectory)\scripts\guardian\source.gdnsuppress + tsa: + enabled: true + configFile: $(Build.SourcesDirectory)\scripts\guardian\tsaoptions-v2.json + use1ESPipelineTemplates: ${{ parameters.use1ESPipelineTemplates }} + VM_IMAGE_HOST: ${{ parameters.VM_IMAGE_HOST }} + VM_IMAGE_WINDOWS: ${{ parameters.VM_IMAGE_WINDOWS }} + VM_IMAGE_WINDOWS_NATIVE: ${{ parameters.VM_IMAGE_WINDOWS }} + VM_IMAGE_MAC: ${{ parameters.VM_IMAGE_MAC }} + VM_IMAGE_MAC_NATIVE: ${{ parameters.VM_IMAGE_MAC }} + VM_IMAGE_LINUX: ${{ parameters.VM_IMAGE_LINUX }} + VM_IMAGE_LINUX_NATIVE: ${{ parameters.VM_IMAGE_LINUX }} \ No newline at end of file diff --git a/scripts/azure-pipelines-complete.yml b/scripts/azure-pipelines-complete.yml index f03e51c988..9afe1841c6 100644 --- a/scripts/azure-pipelines-complete.yml +++ b/scripts/azure-pipelines-complete.yml @@ -10,50 +10,58 @@ pr: parameters: - name: buildExternals - displayName: 'The specific native artifacts to use for this build.' + displayName: 'The Build ID containing the specific native artifacts to use:' type: string default: 'latest' - name: VM_IMAGE_HOST + displayName: 'The generic host build agent configuration:' type: object default: pool: name: Azure Pipelines vmImage: ubuntu-20.04 + os: windows - name: VM_IMAGE_WINDOWS + displayName: 'The Windows build agent configuration:' type: object default: pool: name: Azure Pipelines vmImage: windows-2022 + os: windows - name: VM_IMAGE_MAC + displayName: 'The macOS build agent configuration:' type: object default: pool: name: Azure Pipelines vmImage: macos-12 + os: macos - name: VM_IMAGE_LINUX + displayName: 'The Linux build agent configuration:' type: object default: pool: name: Azure Pipelines vmImage: ubuntu-20.04 - -pool: - name: Azure Pipelines - vmImage: ubuntu-20.04 + os: linux variables: - template: /scripts/azure-pipelines-variables.yml@self -stages: - - template: /scripts/azure-templates-stages.yml@self - parameters: - buildPipelineType: 'both' - buildExternals: ${{ parameters.buildExternals }} - VM_IMAGE_HOST: ${{ parameters.VM_IMAGE_HOST }} - VM_IMAGE_WINDOWS: ${{ parameters.VM_IMAGE_WINDOWS }} - VM_IMAGE_WINDOWS_NATIVE: ${{ parameters.VM_IMAGE_WINDOWS }} - VM_IMAGE_MAC: ${{ parameters.VM_IMAGE_MAC }} - VM_IMAGE_MAC_NATIVE: ${{ parameters.VM_IMAGE_MAC }} - VM_IMAGE_LINUX: ${{ parameters.VM_IMAGE_LINUX }} - VM_IMAGE_LINUX_NATIVE: ${{ parameters.VM_IMAGE_LINUX }} +extends: + template: /scripts/azure-template.yml@self + parameters: + pool: ${{ parameters.VM_IMAGE_HOST.pool }} + stages: + - template: /scripts/azure-templates-stages.yml@self + parameters: + buildPipelineType: 'both' + buildExternals: ${{ parameters.buildExternals }} + VM_IMAGE_HOST: ${{ parameters.VM_IMAGE_HOST }} + VM_IMAGE_WINDOWS: ${{ parameters.VM_IMAGE_WINDOWS }} + VM_IMAGE_WINDOWS_NATIVE: ${{ parameters.VM_IMAGE_WINDOWS }} + VM_IMAGE_MAC: ${{ parameters.VM_IMAGE_MAC }} + VM_IMAGE_MAC_NATIVE: ${{ parameters.VM_IMAGE_MAC }} + VM_IMAGE_LINUX: ${{ parameters.VM_IMAGE_LINUX }} + VM_IMAGE_LINUX_NATIVE: ${{ parameters.VM_IMAGE_LINUX }} diff --git a/scripts/azure-pipelines-tests.yml b/scripts/azure-pipelines-tests.yml index d5c904f4ae..ea118bd2ab 100644 --- a/scripts/azure-pipelines-tests.yml +++ b/scripts/azure-pipelines-tests.yml @@ -4,29 +4,37 @@ pr: none parameters: - name: VM_IMAGE_HOST + displayName: 'The generic host build agent configuration:' type: object default: pool: name: Azure Pipelines vmImage: ubuntu-20.04 + os: windows - name: VM_IMAGE_WINDOWS + displayName: 'The Windows build agent configuration:' type: object default: pool: name: Azure Pipelines vmImage: windows-2022 + os: windows - name: VM_IMAGE_MAC + displayName: 'The macOS build agent configuration:' type: object default: pool: name: Azure Pipelines vmImage: macos-12 + os: macos - name: VM_IMAGE_LINUX + displayName: 'The Linux build agent configuration:' type: object default: pool: name: Azure Pipelines vmImage: ubuntu-20.04 + os: linux variables: - template: /scripts/azure-pipelines-variables.yml@self @@ -43,14 +51,18 @@ resources: source: SkiaSharp trigger: true -stages: - - template: /scripts/azure-templates-stages.yml@self - parameters: - buildPipelineType: 'tests' - VM_IMAGE_HOST: ${{ parameters.VM_IMAGE_HOST }} - VM_IMAGE_WINDOWS: ${{ parameters.VM_IMAGE_WINDOWS }} - VM_IMAGE_WINDOWS_NATIVE: ${{ parameters.VM_IMAGE_WINDOWS }} - VM_IMAGE_MAC: ${{ parameters.VM_IMAGE_MAC }} - VM_IMAGE_MAC_NATIVE: ${{ parameters.VM_IMAGE_MAC }} - VM_IMAGE_LINUX: ${{ parameters.VM_IMAGE_LINUX }} - VM_IMAGE_LINUX_NATIVE: ${{ parameters.VM_IMAGE_LINUX }} +extends: + template: /scripts/azure-template.yml@self + parameters: + pool: ${{ parameters.VM_IMAGE_HOST.pool }} + stages: + - template: /scripts/azure-templates-stages.yml@self + parameters: + buildPipelineType: 'tests' + VM_IMAGE_HOST: ${{ parameters.VM_IMAGE_HOST }} + VM_IMAGE_WINDOWS: ${{ parameters.VM_IMAGE_WINDOWS }} + VM_IMAGE_WINDOWS_NATIVE: ${{ parameters.VM_IMAGE_WINDOWS }} + VM_IMAGE_MAC: ${{ parameters.VM_IMAGE_MAC }} + VM_IMAGE_MAC_NATIVE: ${{ parameters.VM_IMAGE_MAC }} + VM_IMAGE_LINUX: ${{ parameters.VM_IMAGE_LINUX }} + VM_IMAGE_LINUX_NATIVE: ${{ parameters.VM_IMAGE_LINUX }} \ No newline at end of file diff --git a/scripts/azure-pipelines-variables.yml b/scripts/azure-pipelines-variables.yml index 11992c3323..d0197a278c 100644 --- a/scripts/azure-pipelines-variables.yml +++ b/scripts/azure-pipelines-variables.yml @@ -1,4 +1,5 @@ variables: + SKIASHARP_MAJOR_VERSION: 2 SKIASHARP_VERSION: 2.88.8 FEATURE_NAME_PREFIX: 'feature/' VERBOSITY: normal diff --git a/scripts/azure-pipelines.yml b/scripts/azure-pipelines.yml index 0d78629c1e..33669b4812 100644 --- a/scripts/azure-pipelines.yml +++ b/scripts/azure-pipelines.yml @@ -10,33 +10,45 @@ pr: parameters: - name: buildExternals - displayName: 'The specific native artifacts to use for this build.' + displayName: 'The Build ID containing the specific native artifacts to use:' type: string default: 'latest' - name: VM_IMAGE_HOST + displayName: 'The generic host build agent configuration:' type: object default: pool: - name: Azure Pipelines - vmImage: ubuntu-20.04 + name: Maui-1ESPT + image: 1ESPT-Windows2022 + os: windows - name: VM_IMAGE_WINDOWS + displayName: 'The Windows build agent configuration:' type: object default: pool: - name: Azure Pipelines - vmImage: windows-2022 + name: Maui-1ESPT + image: 1ESPT-Windows2022 + os: windows - name: VM_IMAGE_MAC + displayName: 'The macOS build agent configuration:' type: object default: pool: name: Azure Pipelines vmImage: macos-12 + os: macos - name: VM_IMAGE_LINUX + displayName: 'The Linux build agent configuration:' type: object default: pool: - name: Azure Pipelines - vmImage: ubuntu-20.04 + name: Maui-1ESPT + image: 1ESPT-Ubuntu20.04 + os: linux + - name: runCompliance + displayName: 'Run post-build compliance tasks (such as API Scan and PoliCheck)' + type: boolean + default: false variables: - template: /scripts/azure-pipelines-variables.yml@self @@ -48,16 +60,51 @@ resources: name: xamarin/yaml-templates endpoint: xamarin ref: refs/heads/main + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release -stages: - - template: /scripts/azure-templates-stages.yml@self - parameters: - buildPipelineType: 'build' - buildExternals: ${{ parameters.buildExternals }} - VM_IMAGE_HOST: ${{ parameters.VM_IMAGE_HOST }} - VM_IMAGE_WINDOWS: ${{ parameters.VM_IMAGE_WINDOWS }} - VM_IMAGE_WINDOWS_NATIVE: ${{ parameters.VM_IMAGE_WINDOWS }} - VM_IMAGE_MAC: ${{ parameters.VM_IMAGE_MAC }} - VM_IMAGE_MAC_NATIVE: ${{ parameters.VM_IMAGE_MAC }} - VM_IMAGE_LINUX: ${{ parameters.VM_IMAGE_LINUX }} - VM_IMAGE_LINUX_NATIVE: ${{ parameters.VM_IMAGE_LINUX }} +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + pool: ${{ parameters.VM_IMAGE_HOST.pool }} + customBuildTags: + - ES365AIMigrationTooling + stages: + - template: /scripts/azure-templates-stages.yml@self + parameters: + buildPipelineType: 'build' + buildExternals: ${{ parameters.buildExternals }} + ${{ if and(eq(variables['System.TeamProject'], 'devdiv'), ne(variables['System.PullRequest.IsFork'], 'true')) }}: + enableSigning: true + ${{ if or(parameters.runCompliance, and(eq(variables['Build.Reason'], 'Schedule'), or(eq(variables['Build.SourceBranch'], 'refs/heads/main'), startsWith(variables['Build.SourceBranch'], 'refs/heads/release/')))) }}: + sdl: + apiscan: + enabled: true + binskim: + enabled: true + break: false + codeInspector: + enabled: true + credscan: + enabled: true + # suppressionsFile: $(Build.SourcesDirectory)\scripts\guardian\CredScanSuppressions.json + policheck: + enabled: true + exclusionsFile: $(Build.SourcesDirectory)\scripts\guardian\PoliCheckExclusions.xml + spotBugs: + enabled: false + suppression: + suppressionFile: $(Build.SourcesDirectory)\scripts\guardian\source.gdnsuppress + tsa: + enabled: true + configFile: $(Build.SourcesDirectory)\scripts\guardian\tsaoptions-v2.json + use1ESPipelineTemplates: true + VM_IMAGE_HOST: ${{ parameters.VM_IMAGE_HOST }} + VM_IMAGE_WINDOWS: ${{ parameters.VM_IMAGE_WINDOWS }} + VM_IMAGE_WINDOWS_NATIVE: ${{ parameters.VM_IMAGE_WINDOWS }} + VM_IMAGE_MAC: ${{ parameters.VM_IMAGE_MAC }} + VM_IMAGE_MAC_NATIVE: ${{ parameters.VM_IMAGE_MAC }} + VM_IMAGE_LINUX: ${{ parameters.VM_IMAGE_LINUX }} + VM_IMAGE_LINUX_NATIVE: ${{ parameters.VM_IMAGE_LINUX }} \ No newline at end of file diff --git a/scripts/azure-template.yml b/scripts/azure-template.yml new file mode 100644 index 0000000000..60a811b5e7 --- /dev/null +++ b/scripts/azure-template.yml @@ -0,0 +1,31 @@ +parameters: + - name: stages + type: stageList + default: [] + - name: pool + type: object + default: {} + - name: customBuildTags + type: object + default: null + +stages: + - ${{ each stage in parameters.stages }}: + - ${{ each stageProperty in stage }}: + ${{ if notIn(stageProperty.key, 'jobs', 'pool') }}: + ${{ stageProperty.key }}: ${{ stageProperty.value }} + pool: ${{ parameters.pool }} + jobs: + - ${{ each job in stage.jobs }}: + - ${{ each jobProperty in job }}: + ${{ if notIn(jobProperty.key, 'steps', 'templateContext') }}: + ${{ jobProperty.key }}: ${{ jobProperty.value }} + steps: + - ${{ job.steps }} + - ${{ each output in job.templateContext.outputs }}: + - task: PublishPipelineArtifact@1 + displayName: ${{ output.displayName }} + condition: ${{ coalesce(output.condition, 'succeeded()') }} + inputs: + artifactName: ${{ output.artifactName }} + targetPath: ${{ output.targetPath }} diff --git a/scripts/azure-templates-bootstrapper.yml b/scripts/azure-templates-bootstrapper.yml index e7edb4ae98..285ba51e4e 100644 --- a/scripts/azure-templates-bootstrapper.yml +++ b/scripts/azure-templates-bootstrapper.yml @@ -1,7 +1,7 @@ parameters: name: '' # in the form type_platform_host displayName: '' # the human name - vmImage: '' # the VM image + vmImage: '' # the configuration for the build agent packages: '' # any additional packages target: '' # the bootstrapper target dependsOn: [] # the dependiencies @@ -29,8 +29,12 @@ parameters: installDotNet: true # whether or not to install the dotnet SDK installLlvm: true # whether or not to install the LLVM compiler installEmsdk: false # whether or not to install the Emscripten SDK - artifactName: '' # the name of the artifact to merge this run into + publishArtifacts: [] # the additional artifacts to publish tools: [] # any additional .net global tools + skipInstall: false # whether or not to install any tools + skipSteps: false # whether or not to run any steps + use1ESPipelineTemplates: false # whether or not we are building using the internal 1ES Pipeline Templates + sdl: [] # the SDL properties to use for this job jobs: - job: ${{ parameters.name }} @@ -44,10 +48,38 @@ jobs: ${{ parameters.vmImage.variables }} ${{ if ne(length(parameters.variables), 0) }}: ${{ parameters.variables }} + templateContext: + sdl: ${{ parameters.sdl }} + outputParentDirectory: 'output' + outputs: + - ${{ if eq(parameters.shouldPublish, 'true') }}: + - output: pipelineArtifact + displayName: 'Publish the ${{ parameters.name }} artifacts' + artifactName: ${{ parameters.name }} + targetPath: 'output' + - output: pipelineArtifact + displayName: 'Publish the failed ${{ parameters.name }} artifacts' + condition: failed() + artifactName: ${{ parameters.name }}_failed_$(System.JobAttempt) + targetPath: 'output' + - ${{ each additionalArtifact in parameters.publishArtifacts }}: + - output: pipelineArtifact + displayName: 'Publish the ${{ additionalArtifact.name }} artifacts' + artifactName: ${{ additionalArtifact.name }} + targetPath: ${{ additionalArtifact.path }} + - ${{ if eq(additionalArtifact.always, 'true') }}: + - output: pipelineArtifact + displayName: 'Publish the failed ${{ additionalArtifact.name }} artifacts' + condition: failed() + artifactName: ${{ additionalArtifact.name }}_failed_$(System.JobAttempt) + targetPath: ${{ additionalArtifact.path }} + steps: # prepare - checkout: self submodules: recursive + - pwsh: Write-Host '${{ convertToJson(parameters.sdl) }}' + displayName: Print current SDL input - template: /scripts/azure-templates-variables.yml@self # checkout required skia PR @@ -71,7 +103,7 @@ jobs: - ${{ parameters.provisioningSteps }} # install any packages on linux - - ${{ if and(eq(parameters.docker, ''), endsWith(parameters.name, '_linux')) }}: + - ${{ if and(eq(parameters.docker, ''), endsWith(parameters.name, '_linux'), ne(parameters.skipInstall, 'true')) }}: - bash: | sudo apt update sudo apt install -y ${{ parameters.packages }} @@ -80,24 +112,20 @@ jobs: condition: and(succeeded(), eq(variables['DOWNLOAD_EXTERNALS'], ''), ne('${{ parameters.packages }}', '')) # make sure mono/msbuild is the correct version - - ${{ if and(eq(parameters.docker, ''), endsWith(parameters.name, '_linux')) }}: + - ${{ if and(eq(parameters.docker, ''), endsWith(parameters.name, '_linux'), ne(parameters.skipInstall, 'true')) }}: - bash: ./scripts/install-mono.sh $(MONO_VERSION_LINUX) displayName: Install Mono and MSBuild retryCountOnTaskFailure: 3 condition: and(succeeded(), eq(variables['DOWNLOAD_EXTERNALS'], ''), ne(variables['MONO_VERSION_LINUX'], '')) - - ${{ if endsWith(parameters.name, '_macos') }}: + - ${{ if and(endsWith(parameters.name, '_macos'), ne(parameters.skipInstall, 'true')) }}: - bash: sudo ./scripts/select-xamarin.sh $(MONO_VERSION_MACOS) displayName: Switch to the latest Xamarin SDK retryCountOnTaskFailure: 3 condition: and(succeeded(), eq(variables['DOWNLOAD_EXTERNALS'], '')) # install extra bits for the native builds - - ${{ if startsWith(parameters.name, 'native_') }}: + - ${{ if and(startsWith(parameters.name, 'native_'), ne(parameters.skipInstall, 'true')) }}: # switch to the correct Python version - - pwsh: .\scripts\install-python.ps1 - displayName: Install Python - retryCountOnTaskFailure: 3 - condition: and(succeeded(), eq(variables['DOWNLOAD_EXTERNALS'], '')) - task: UsePythonVersion@0 displayName: Switch to the correct Python version retryCountOnTaskFailure: 3 @@ -130,7 +158,7 @@ jobs: condition: and(succeeded(), eq(variables['DOWNLOAD_EXTERNALS'], '')) # install extra bits for the managed builds - - ${{ if not(startsWith(parameters.name, 'native_')) }}: + - ${{ if and(not(startsWith(parameters.name, 'native_')), ne(parameters.skipInstall, 'true')) }}: # install the bits needed for Android on macOS and Windows - ${{ if and(eq(parameters.installAndroidSdk, 'true'), not(endsWith(parameters.name, '_linux'))) }}: # install the correct version of the JDK for .NET 6 @@ -205,32 +233,32 @@ jobs: displayName: Install the .NET Core workloads # select the correct/latest version of Visual Studio - - ${{ if endsWith(parameters.name, '_windows') }}: + - ${{ if and(endsWith(parameters.name, '_windows'), ne(parameters.skipInstall, 'true')) }}: - pwsh: .\scripts\select-vs.ps1 displayName: Select Visual Studio condition: and(succeeded(), eq(variables['DOWNLOAD_EXTERNALS'], '')) # install nuget - - ${{ if endsWith(parameters.name, '_windows') }}: + - ${{ if and(endsWith(parameters.name, '_windows'), ne(parameters.skipInstall, 'true')) }}: - pwsh: .\scripts\install-nuget.ps1 displayName: Install NuGet condition: and(succeeded(), eq(variables['DOWNLOAD_EXTERNALS'], '')) # install the mac tools - - ${{ if endsWith(parameters.name, '_macos') }}: + - ${{ if and(endsWith(parameters.name, '_macos'), ne(parameters.skipInstall, 'true')) }}: - bash: sudo ./scripts/select-xcode.sh $(XCODE_VERSION) displayName: Switch to the latest Xcode condition: and(succeeded(), eq(variables['DOWNLOAD_EXTERNALS'], '')) # install the linux tools - - ${{ if and(eq(parameters.installEmsdk, 'true'), endsWith(parameters.name, '_linux')) }}: + - ${{ if and(eq(parameters.installEmsdk, 'true'), endsWith(parameters.name, '_linux'), ne(parameters.skipInstall, 'true')) }}: - bash: ./scripts/install-emsdk.sh $(EMSCRIPTEN_VERSION) displayName: Install the Emscripten SDK retryCountOnTaskFailure: 3 condition: and(succeeded(), eq(variables['DOWNLOAD_EXTERNALS'], '')) # install the Windows tools - - ${{ if endsWith(parameters.name, '_windows') }}: + - ${{ if and(endsWith(parameters.name, '_windows'), ne(parameters.skipInstall, 'true')) }}: - ${{ if not(contains(parameters.name, '_checks_')) }}: # install the older Windows SDKs - ${{ if eq(parameters.installWindowsSdk, 'true') }}: @@ -252,11 +280,12 @@ jobs: condition: and(succeeded(), eq(variables['DOWNLOAD_EXTERNALS'], '')) # install any .NET Core global tools - - ${{ each tool in parameters.tools }}: - - pwsh: dotnet tool install -g ${{ tool }} - displayName: Install ${{ tool }} - retryCountOnTaskFailure: 3 - condition: and(succeeded(), eq(variables['DOWNLOAD_EXTERNALS'], '')) + - ${{ if ne(parameters.skipInstall, 'true') }}: + - ${{ each tool in parameters.tools }}: + - pwsh: dotnet tool install -g ${{ tool }} + displayName: Install ${{ tool }} + retryCountOnTaskFailure: 3 + condition: and(succeeded(), eq(variables['DOWNLOAD_EXTERNALS'], '')) # download artifacts - template: /scripts/azure-templates-download-artifacts.yml@self @@ -277,77 +306,68 @@ jobs: - ${{ parameters.preBuildSteps }} # build - - ${{ if eq(parameters.docker, '') }}: - - ${{ if endsWith(parameters.name, '_windows') }}: - - pwsh: | - dotnet tool restore - ${{ parameters.initScript }} - dotnet cake --target=${{ parameters.target }} --verbosity=${{ parameters.verbosity }} --configuration=${{ coalesce(parameters.configuration, 'Release') }} ${{ parameters.additionalArgs }} - env: - JavaSdkDirectory: $(JAVA_HOME) - LLVM_HOME: $(LLVM_HOME) - # There seems to be a bug in some verions of mspdbcmf.exe. This looks to be fixed in a VS preview. - AppxSymbolPackageEnabled: false - displayName: Run the bootstrapper for ${{ parameters.target }} - retryCountOnTaskFailure: ${{ parameters.retryCount }} + - ${{ if ne(parameters.skipSteps, 'true') }}: + - ${{ if eq(parameters.docker, '') }}: + - ${{ if endsWith(parameters.name, '_windows') }}: + - pwsh: | + Get-Content $PSCommandPath + dotnet tool restore + ${{ parameters.initScript }} + dotnet cake --target=${{ parameters.target }} --verbosity=${{ parameters.verbosity }} --configuration=${{ coalesce(parameters.configuration, 'Release') }} ${{ parameters.additionalArgs }} + env: + JavaSdkDirectory: $(JAVA_HOME) + LLVM_HOME: $(LLVM_HOME) + # There seems to be a bug in some verions of mspdbcmf.exe. This looks to be fixed in a VS preview. + AppxSymbolPackageEnabled: false + displayName: Run the bootstrapper for ${{ parameters.target }} + retryCountOnTaskFailure: ${{ parameters.retryCount }} + condition: and(succeeded(), eq(variables['DOWNLOAD_EXTERNALS'], '')) + - ${{ if not(endsWith(parameters.name, '_windows')) }}: + - bash: | + cat ${BASH_SOURCE[0]} + dotnet tool restore + ${{ parameters.initScript }} + dotnet cake --target=${{ parameters.target }} --verbosity=${{ parameters.verbosity }} --configuration=${{ coalesce(parameters.configuration, 'Release') }} ${{ parameters.additionalArgs }} + env: + JavaSdkDirectory: $(JAVA_HOME) + displayName: Run the bootstrapper for ${{ parameters.target }} + retryCountOnTaskFailure: ${{ parameters.retryCount }} + condition: and(succeeded(), eq(variables['DOWNLOAD_EXTERNALS'], '')) + - ${{ if ne(parameters.docker, '') }}: + - ${{ if eq(parameters.use1ESPipelineTemplates, 'true') }}: + - task: 1ES.BuildContainerImage@1 + displayName: Build the Docker image for ${{ parameters.docker }} + condition: and(succeeded(), eq(variables['DOWNLOAD_EXTERNALS'], '')) + inputs: + dockerfile: ${{ parameters.docker }}/Dockerfile + context: ${{ parameters.docker }} + image: skiasharp:skiasharp + buildArguments: --tag skiasharp ${{ parameters.dockerArgs }} + enableNetwork: true + - ${{ if ne(parameters.use1ESPipelineTemplates, 'true') }}: + - task: Docker@2 + displayName: Build the Docker image for ${{ parameters.docker }} + condition: and(succeeded(), eq(variables['DOWNLOAD_EXTERNALS'], '')) + inputs: + command: build + buildContext: ${{ parameters.docker }} + dockerfile: ${{ parameters.docker }}/Dockerfile + arguments: --tag skiasharp ${{ parameters.dockerArgs }} + - bash: | + echo dotnet tool restore > cmd.sh + echo dotnet cake --target=${{ parameters.target }} --verbosity=${{ parameters.verbosity }} --configuration=${{ coalesce(parameters.configuration, 'Release') }} ${{ parameters.additionalArgs }} >> cmd.sh + sed -i 's/--gnArgs=\" \"//' cmd.sh + cat cmd.sh + displayName: Generate the script for the Docker image condition: and(succeeded(), eq(variables['DOWNLOAD_EXTERNALS'], '')) - - ${{ if not(endsWith(parameters.name, '_windows')) }}: - bash: | - dotnet tool restore - ${{ parameters.initScript }} - dotnet cake --target=${{ parameters.target }} --verbosity=${{ parameters.verbosity }} --configuration=${{ coalesce(parameters.configuration, 'Release') }} ${{ parameters.additionalArgs }} - env: - JavaSdkDirectory: $(JAVA_HOME) - displayName: Run the bootstrapper for ${{ parameters.target }} + docker run --rm --name skiasharp --volume $(pwd):/work skiasharp /bin/bash /work/cmd.sh + displayName: Run the bootstrapper for ${{ parameters.target }} using the Docker image retryCountOnTaskFailure: ${{ parameters.retryCount }} condition: and(succeeded(), eq(variables['DOWNLOAD_EXTERNALS'], '')) - - ${{ if ne(parameters.docker, '') }}: - - task: Docker@2 - displayName: Build the Docker image for ${{ parameters.docker }} - condition: and(succeeded(), eq(variables['DOWNLOAD_EXTERNALS'], '')) - inputs: - command: build - buildContext: ${{ parameters.docker }} - dockerfile: ${{ parameters.docker }}/Dockerfile - arguments: --tag skiasharp ${{ parameters.dockerArgs }} - - bash: | - echo dotnet tool restore > cmd.sh - echo dotnet cake --target=${{ parameters.target }} --verbosity=${{ parameters.verbosity }} --configuration=${{ coalesce(parameters.configuration, 'Release') }} ${{ parameters.additionalArgs }} >> cmd.sh - sed -i 's/--gnArgs=\" \"//' cmd.sh - displayName: Generate the script for the Docker image - condition: and(succeeded(), eq(variables['DOWNLOAD_EXTERNALS'], '')) - - bash: | - docker run --rm --name skiasharp --volume $(pwd):/work skiasharp /bin/bash /work/cmd.sh - displayName: Run the bootstrapper for ${{ parameters.target }} using the Docker image - retryCountOnTaskFailure: ${{ parameters.retryCount }} - condition: and(succeeded(), eq(variables['DOWNLOAD_EXTERNALS'], '')) # post-build steps - ${{ parameters.postBuildSteps }} - # publish artifacts - - task: PublishBuildArtifacts@1 - displayName: Publish the ${{ parameters.name }} artifacts - condition: or(${{ parameters.shouldPublish }}, failed()) - retryCountOnTaskFailure: 3 - inputs: - artifactName: ${{ parameters.name }} - pathToPublish: 'output' - - ${{ if ne(parameters.artifactName, '') }}: - - task: PublishBuildArtifacts@1 - displayName: Publish the combined ${{ parameters.artifactName }} artifacts - retryCountOnTaskFailure: 3 - inputs: - artifactName: ${{ parameters.artifactName }} - pathToPublish: 'output' - - ${{ if eq(variables['System.TeamProject'], 'devdiv') }}: - - task: ComponentGovernanceComponentDetection@0 - displayName: Run component detection - condition: always() - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' - - ${{ if eq(parameters.buildPipelineType, 'tests') }}: - template: /scripts/azure-templates-github-status.yml@self diff --git a/scripts/azure-templates-download-artifacts.yml b/scripts/azure-templates-download-artifacts.yml index db94dadcbe..2300409591 100644 --- a/scripts/azure-templates-download-artifacts.yml +++ b/scripts/azure-templates-download-artifacts.yml @@ -54,9 +54,10 @@ steps: downloadType: 'single' allowPartiallySucceededBuilds: true artifactName: ${{ artifact.name }} - downloadPath: 'download-temp' + downloadPath: 'download-temp/${{ artifact.name }}' - pwsh: | + Get-ChildItem '.\download-temp\' New-Item '.\output\${{ artifact.dir }}\' -Type Directory -Force | Out-Null Get-ChildItem '.\download-temp\${{ artifact.name }}\' | Copy-Item -Destination '.\output\${{ artifact.dir }}\' -Recurse -Force Remove-Item '.\download-temp\${{ artifact.name }}\' -Recurse -Force diff --git a/scripts/azure-templates-linux-matrix.yml b/scripts/azure-templates-linux-matrix.yml index 7750761786..590bb7f98d 100644 --- a/scripts/azure-templates-linux-matrix.yml +++ b/scripts/azure-templates-linux-matrix.yml @@ -1,8 +1,9 @@ parameters: - artifactName: '' # the name of the artifact to merge this run into buildExternals: '' # the build number to download externals from buildPipelineType: 'both' # the type of build pipeline setup - vmImage: '' # the VM image + vmImage: '' # the configuration for the build agent + use1ESPipelineTemplates: false # whether or not we are building using the internal 1ES Pipeline Templates + sdl: [] # the SDL properties to use for this job builds: - name: '' desc: '' @@ -24,11 +25,12 @@ jobs: parameters: name: ${{ replace(replace(format('native_linux_{0}_{1}_{2}_{3}_linux', item.arch, item.variant, build.name, item.alt), '__', '_'), '__', '_') }} displayName: Linux ${{ replace(replace(replace(replace(replace(format('({0}|{1}|{2}|{3})', item.arch, item.variant, build.name, item.alt), '||', '|'), '||', '|'), '(|', '('), '|)', ')'), '|', ', ') }} + sdl: ${{ parameters.sdl }} buildExternals: ${{ parameters.buildExternals }} buildPipelineType: ${{ parameters.buildPipelineType }} vmImage: ${{ parameters.vmImage }} + use1ESPipelineTemplates: ${{ parameters.use1ESPipelineTemplates }} docker: ${{ item.docker }} dockerArgs: ${{ item.dockerArgs }} target: ${{ coalesce(item.target, 'externals-linux') }} additionalArgs: --buildarch=${{ item.arch }} --variant=${{ coalesce(item.variant, 'linux') }}${{ build.name }} --gnArgs="\"${{ build.gnArgs }} ${{ item.gnArgs }}\"" ${{ build.additionalArgs }} ${{ item.additionalArgs }} - artifactName: ${{ parameters.artifactName }} diff --git a/scripts/azure-templates-merger.yml b/scripts/azure-templates-merger.yml new file mode 100644 index 0000000000..1aeab8956f --- /dev/null +++ b/scripts/azure-templates-merger.yml @@ -0,0 +1,53 @@ +parameters: + name: '' # in the form type_platform_host + displayName: '' # the human name + dependsOn: [] # the dependiencies + vmImage: '' # the configuration for the build agent + buildPipelineType: 'both' # the type of build pipeline setup + requiredArtifacts: [] # the artifacts that this build needs to download + matrixArtifacts: [] # the artifacts that this build needs to download + sdl: [] # the SDL properties to use for this job + +jobs: + - template: /scripts/azure-templates-bootstrapper.yml@self + parameters: + name: ${{ parameters.name }} + displayName: ${{ parameters.displayName }} + dependsOn: ${{ parameters.dependsOn }} + sdl: ${{ parameters.sdl }} + buildPipelineType: ${{ parameters.buildPipelineType }} + vmImage: ${{ parameters.vmImage }} + skipInstall: true + skipSteps: true + requiredArtifacts: ${{ parameters.requiredArtifacts }} + preBuildSteps: + - pwsh: az devops configure --defaults organization=$(System.TeamFoundationCollectionUri) project=$(System.TeamProject) --use-git-aliases true + displayName: Configure the az CLI tool + - ${{ each artifact in parameters.matrixArtifacts }}: + - pwsh: | + $artifactJson=@' + ${{ artifact.jobs }} + '@ + + echo $artifactJson + + $json = ConvertFrom-Json $artifactJson + $objects = $json | Get-Member -MemberType NoteProperty + $names = $objects | ForEach-Object { $json."$($_.Name)".name } + + Write-Host "Found $($names.Length) items:" + $names | ForEach-Object { Write-Host " - $_" } + + $dir = "$(Build.SourcesDirectory)/output" + New-Item "$dir" -Type Directory -Force | Out-Null + + $id = "$(Build.BuildId)" + foreach ($name in $names) { + Write-Host "Downloading '$name'..." + az pipelines runs artifact download --artifact-name "$name" --path "$dir" --run-id "$id" --verbose + } + Write-Host "Downloads complete." + Get-ChildItem "$dir" + env: + AZURE_DEVOPS_EXT_PAT: $(System.AccessToken) + displayName: Download the pre-built ${{ artifact.name }} artifacts diff --git a/scripts/azure-templates-stages.yml b/scripts/azure-templates-stages.yml index c935a8871e..6228246f7f 100644 --- a/scripts/azure-templates-stages.yml +++ b/scripts/azure-templates-stages.yml @@ -19,7 +19,19 @@ parameters: type: object - name: VM_IMAGE_LINUX_NATIVE type: object - - name: runCompliance + - name: sdl + type: object + default: + apiscan: + enabled: false + binskim: + break: false + spotBugs: + enabled: false + - name: use1ESPipelineTemplates + type: boolean + default: false + - name: enableSigning type: boolean default: false @@ -53,175 +65,175 @@ stages: parameters: name: native_android_x86_windows displayName: Android x86 + sdl: ${{ parameters.sdl }} buildExternals: ${{ parameters.buildExternals }} buildPipelineType: ${{ parameters.buildPipelineType }} vmImage: ${{ parameters.VM_IMAGE_WINDOWS_NATIVE }} target: externals-android additionalArgs: --buildarch=x86 installWindowsSdk: false - artifactName: native - template: /scripts/azure-templates-bootstrapper.yml@self # Build Native Android|x64 (Win) parameters: name: native_android_x64_windows displayName: Android x64 + sdl: ${{ parameters.sdl }} buildExternals: ${{ parameters.buildExternals }} buildPipelineType: ${{ parameters.buildPipelineType }} vmImage: ${{ parameters.VM_IMAGE_WINDOWS_NATIVE }} target: externals-android additionalArgs: --buildarch=x64 installWindowsSdk: false - artifactName: native - template: /scripts/azure-templates-bootstrapper.yml@self # Build Native Android|arm (Win) parameters: name: native_android_arm_windows displayName: Android arm + sdl: ${{ parameters.sdl }} buildExternals: ${{ parameters.buildExternals }} buildPipelineType: ${{ parameters.buildPipelineType }} vmImage: ${{ parameters.VM_IMAGE_WINDOWS_NATIVE }} target: externals-android additionalArgs: --buildarch=arm installWindowsSdk: false - artifactName: native - template: /scripts/azure-templates-bootstrapper.yml@self # Build Native Android|arm64 (Win) parameters: name: native_android_arm64_windows displayName: Android arm64 + sdl: ${{ parameters.sdl }} buildExternals: ${{ parameters.buildExternals }} buildPipelineType: ${{ parameters.buildPipelineType }} vmImage: ${{ parameters.VM_IMAGE_WINDOWS_NATIVE }} target: externals-android additionalArgs: --buildarch=arm64 installWindowsSdk: false - artifactName: native - template: /scripts/azure-templates-bootstrapper.yml@self # Build Native Tizen (Win) parameters: name: native_tizen_windows displayName: Tizen + sdl: ${{ parameters.sdl }} buildExternals: ${{ parameters.buildExternals }} buildPipelineType: ${{ parameters.buildPipelineType }} vmImage: ${{ parameters.VM_IMAGE_WINDOWS_NATIVE }} target: externals-tizen installWindowsSdk: false - artifactName: native - template: /scripts/azure-templates-bootstrapper.yml@self # Build ANGLE UWP|x86 (Win) parameters: name: native_uwp_angle_x86_windows displayName: ANGLE x86 + sdl: ${{ parameters.sdl }} buildExternals: ${{ parameters.buildExternals }} buildPipelineType: ${{ parameters.buildPipelineType }} vmImage: ${{ parameters.VM_IMAGE_WINDOWS_NATIVE }} target: ANGLE additionalArgs: -Script .\native\uwp\build.cake --buildarch=x86 - artifactName: native - template: /scripts/azure-templates-bootstrapper.yml@self # Build ANGLE UWP|x64 (Win) parameters: name: native_uwp_angle_x64_windows displayName: ANGLE x64 + sdl: ${{ parameters.sdl }} buildExternals: ${{ parameters.buildExternals }} buildPipelineType: ${{ parameters.buildPipelineType }} vmImage: ${{ parameters.VM_IMAGE_WINDOWS_NATIVE }} target: ANGLE additionalArgs: -Script .\native\uwp\build.cake --buildarch=x64 - artifactName: native - template: /scripts/azure-templates-bootstrapper.yml@self # Build ANGLE UWP|arm (Win) parameters: name: native_uwp_angle_arm_windows displayName: ANGLE arm + sdl: ${{ parameters.sdl }} buildExternals: ${{ parameters.buildExternals }} buildPipelineType: ${{ parameters.buildPipelineType }} vmImage: ${{ parameters.VM_IMAGE_WINDOWS_NATIVE }} target: ANGLE additionalArgs: -Script .\native\uwp\build.cake --buildarch=arm - artifactName: native - template: /scripts/azure-templates-bootstrapper.yml@self # Build ANGLE UWP|arm64 (Win) parameters: name: native_uwp_angle_arm64_windows displayName: ANGLE arm64 + sdl: ${{ parameters.sdl }} buildExternals: ${{ parameters.buildExternals }} buildPipelineType: ${{ parameters.buildPipelineType }} vmImage: ${{ parameters.VM_IMAGE_WINDOWS_NATIVE }} target: ANGLE additionalArgs: -Script .\native\uwp\build.cake --buildarch=arm64 - artifactName: native - template: /scripts/azure-templates-bootstrapper.yml@self # Build Native UWP|x86 (Win) parameters: name: native_uwp_x86_windows displayName: UWP x86 + sdl: ${{ parameters.sdl }} buildExternals: ${{ parameters.buildExternals }} buildPipelineType: ${{ parameters.buildPipelineType }} vmImage: ${{ parameters.VM_IMAGE_WINDOWS_NATIVE }} target: externals-uwp additionalArgs: --buildarch=x86 --skipAngle=true - artifactName: native - template: /scripts/azure-templates-bootstrapper.yml@self # Build Native UWP|x64 (Win) parameters: name: native_uwp_x64_windows displayName: UWP x64 + sdl: ${{ parameters.sdl }} buildExternals: ${{ parameters.buildExternals }} vmImage: ${{ parameters.VM_IMAGE_WINDOWS_NATIVE }} target: externals-uwp additionalArgs: --buildarch=x64 --skipAngle=true - artifactName: native - template: /scripts/azure-templates-bootstrapper.yml@self # Build Native UWP|arm (Win) parameters: name: native_uwp_arm_windows displayName: UWP arm + sdl: ${{ parameters.sdl }} buildExternals: ${{ parameters.buildExternals }} buildPipelineType: ${{ parameters.buildPipelineType }} vmImage: ${{ parameters.VM_IMAGE_WINDOWS_NATIVE }} target: externals-uwp additionalArgs: --buildarch=arm --skipAngle=true - artifactName: native - template: /scripts/azure-templates-bootstrapper.yml@self # Build Native UWP|arm64 (Win) parameters: name: native_uwp_arm64_windows displayName: UWP arm64 + sdl: ${{ parameters.sdl }} buildExternals: ${{ parameters.buildExternals }} buildPipelineType: ${{ parameters.buildPipelineType }} vmImage: ${{ parameters.VM_IMAGE_WINDOWS_NATIVE }} target: externals-uwp additionalArgs: --buildarch=arm64 --skipAngle=true - artifactName: native - template: /scripts/azure-templates-bootstrapper.yml@self # Build Native Win32|x86 (Win) parameters: name: native_win32_x86_windows displayName: Win32 x86 + sdl: ${{ parameters.sdl }} buildExternals: ${{ parameters.buildExternals }} buildPipelineType: ${{ parameters.buildPipelineType }} vmImage: ${{ parameters.VM_IMAGE_WINDOWS_NATIVE }} target: externals-windows additionalArgs: --buildarch=x86 - artifactName: native - template: /scripts/azure-templates-bootstrapper.yml@self # Build Native Win32|x64 (Win) parameters: name: native_win32_x64_windows displayName: Win32 x64 + sdl: ${{ parameters.sdl }} buildExternals: ${{ parameters.buildExternals }} buildPipelineType: ${{ parameters.buildPipelineType }} vmImage: ${{ parameters.VM_IMAGE_WINDOWS_NATIVE }} target: externals-windows additionalArgs: --buildarch=x64 - artifactName: native - template: /scripts/azure-templates-bootstrapper.yml@self # Build Native Win32|arm64 (Win) parameters: name: native_win32_arm64_windows displayName: Win32 arm64 + sdl: ${{ parameters.sdl }} buildExternals: ${{ parameters.buildExternals }} buildPipelineType: ${{ parameters.buildPipelineType }} vmImage: ${{ parameters.VM_IMAGE_WINDOWS_NATIVE }} target: externals-windows additionalArgs: --buildarch=arm64 - artifactName: native - template: /scripts/azure-templates-bootstrapper.yml@self # Build Native NanoServer|x64 (Win) parameters: name: native_win32_x64_nanoserver_windows displayName: Nano Server x64 + sdl: ${{ parameters.sdl }} buildExternals: ${{ parameters.buildExternals }} buildPipelineType: ${{ parameters.buildPipelineType }} vmImage: ${{ parameters.VM_IMAGE_WINDOWS_NATIVE }} target: externals-nanoserver additionalArgs: --buildarch=x64 - artifactName: native - ${{ if ne(parameters.buildPipelineType, 'tests') }}: - stage: native_macos @@ -232,6 +244,7 @@ stages: parameters: name: native_android_x86_macos displayName: Android x86 + sdl: ${{ parameters.sdl }} buildExternals: ${{ parameters.buildExternals }} buildPipelineType: ${{ parameters.buildPipelineType }} vmImage: ${{ parameters.VM_IMAGE_MAC_NATIVE }} @@ -241,6 +254,7 @@ stages: parameters: name: native_android_x64_macos displayName: Android x64 + sdl: ${{ parameters.sdl }} buildExternals: ${{ parameters.buildExternals }} buildPipelineType: ${{ parameters.buildPipelineType }} vmImage: ${{ parameters.VM_IMAGE_MAC_NATIVE }} @@ -250,6 +264,7 @@ stages: parameters: name: native_android_arm_macos displayName: Android arm + sdl: ${{ parameters.sdl }} buildExternals: ${{ parameters.buildExternals }} buildPipelineType: ${{ parameters.buildPipelineType }} vmImage: ${{ parameters.VM_IMAGE_MAC_NATIVE }} @@ -259,6 +274,7 @@ stages: parameters: name: native_android_arm64_macos displayName: Android arm64 + sdl: ${{ parameters.sdl }} buildExternals: ${{ parameters.buildExternals }} buildPipelineType: ${{ parameters.buildPipelineType }} vmImage: ${{ parameters.VM_IMAGE_MAC_NATIVE }} @@ -268,38 +284,38 @@ stages: parameters: name: native_ios_macos displayName: iOS + sdl: ${{ parameters.sdl }} buildExternals: ${{ parameters.buildExternals }} buildPipelineType: ${{ parameters.buildPipelineType }} vmImage: ${{ parameters.VM_IMAGE_MAC_NATIVE }} target: externals-ios - artifactName: native - template: /scripts/azure-templates-bootstrapper.yml@self # Build Native Mac Catalyst (macOS) parameters: name: native_maccatalyst_macos displayName: Mac Catalyst + sdl: ${{ parameters.sdl }} buildExternals: ${{ parameters.buildExternals }} buildPipelineType: ${{ parameters.buildPipelineType }} vmImage: ${{ parameters.VM_IMAGE_MAC_NATIVE }} target: externals-maccatalyst - artifactName: native - template: /scripts/azure-templates-bootstrapper.yml@self # Build Native macOS (macOS) parameters: name: native_macos_macos displayName: macOS + sdl: ${{ parameters.sdl }} buildExternals: ${{ parameters.buildExternals }} buildPipelineType: ${{ parameters.buildPipelineType }} vmImage: ${{ parameters.VM_IMAGE_MAC_NATIVE }} target: externals-macos - artifactName: native - template: /scripts/azure-templates-bootstrapper.yml@self # Build Native tvOS (macOS) parameters: name: native_tvos_macos displayName: tvOS + sdl: ${{ parameters.sdl }} buildExternals: ${{ parameters.buildExternals }} buildPipelineType: ${{ parameters.buildPipelineType }} vmImage: ${{ parameters.VM_IMAGE_MAC_NATIVE }} target: externals-tvos - artifactName: native - template: /scripts/azure-templates-bootstrapper.yml@self # Build Native watchOS (macOS) parameters: name: native_watchos_macos @@ -308,11 +324,11 @@ stages: buildPipelineType: ${{ parameters.buildPipelineType }} vmImage: ${{ parameters.VM_IMAGE_MAC_NATIVE }} target: externals-watchos - artifactName: native - template: /scripts/azure-templates-bootstrapper.yml@self # Build Native Tizen (macOS) parameters: name: native_tizen_macos displayName: Tizen + sdl: ${{ parameters.sdl }} buildExternals: ${{ parameters.buildExternals }} buildPipelineType: ${{ parameters.buildPipelineType }} vmImage: ${{ parameters.VM_IMAGE_MAC_NATIVE }} @@ -326,10 +342,11 @@ stages: jobs: - template: /scripts/azure-templates-linux-matrix.yml@self # Build Native Linux (Linux) parameters: - artifactName: native + sdl: ${{ parameters.sdl }} buildExternals: ${{ parameters.buildExternals }} buildPipelineType: ${{ parameters.buildPipelineType }} vmImage: ${{ parameters.VM_IMAGE_LINUX_NATIVE }} + use1ESPipelineTemplates: ${{ parameters.use1ESPipelineTemplates }} builds: - name: '' - name: nodeps @@ -366,10 +383,11 @@ stages: jobs: - template: /scripts/azure-templates-wasm-matrix.yml@self # Build Native WASM (Linux) parameters: + sdl: ${{ parameters.sdl }} buildExternals: ${{ parameters.buildExternals }} buildPipelineType: ${{ parameters.buildPipelineType }} vmImage: ${{ parameters.VM_IMAGE_LINUX_NATIVE }} - artifactName: native + use1ESPipelineTemplates: ${{ parameters.use1ESPipelineTemplates }} emscripten: - 2.0.6: displayName: 2.0.6 @@ -423,18 +441,86 @@ stages: features: _wasmeh,simd,mt - ${{ if ne(parameters.buildPipelineType, 'tests') }}: - - stage: managed - displayName: Build Managed + - stage: native + displayName: Native + variables: + nativeLinuxJobs: $[ convertToJson(stageDependencies.native_linux) ] + nativeWasmJobs: $[ convertToJson(stageDependencies.native_wasm) ] dependsOn: - native_windows - native_macos - native_linux - native_wasm + jobs: + - template: /scripts/azure-templates-merger.yml@self # Merge Native Artifacts + parameters: + name: native + displayName: Merge Native Artifacts + sdl: ${{ parameters.sdl }} + buildPipelineType: ${{ parameters.buildPipelineType }} + vmImage: ${{ parameters.VM_IMAGE_HOST }} + requiredArtifacts: + # Android + - name: native_android_x86_windows + - name: native_android_x64_windows + - name: native_android_arm_windows + - name: native_android_arm64_windows + # Tizen + - name: native_tizen_windows + # UWP + - name: native_uwp_angle_x86_windows + - name: native_uwp_angle_x64_windows + - name: native_uwp_angle_arm_windows + - name: native_uwp_angle_arm64_windows + - name: native_uwp_x86_windows + - name: native_uwp_x64_windows + - name: native_uwp_arm_windows + - name: native_uwp_arm64_windows + # Win32 + - name: native_win32_x86_windows + - name: native_win32_x64_windows + - name: native_win32_arm64_windows + # Nano Server + - name: native_win32_x64_nanoserver_windows + # iOS + - name: native_ios_macos + # Mac Catalyst + - name: native_maccatalyst_macos + # macOS + - name: native_macos_macos + # tvOS + - name: native_tvos_macos + # tvOS + - name: native_watchos_macos + matrixArtifacts: + - name: native_linux + jobs: $(nativeLinuxJobs) + - name: native_wasm + jobs: $(nativeWasmJobs) + - template: /scripts/azure-templates-merger.yml@self # Merge Native WASM Artifacts + parameters: + name: native_wasm_linux + displayName: Merge Native WASM Artifacts + sdl: ${{ parameters.sdl }} + buildPipelineType: ${{ parameters.buildPipelineType }} + vmImage: ${{ parameters.VM_IMAGE_HOST }} + matrixArtifacts: + - name: native_wasm + jobs: $(nativeWasmJobs) + + - ${{ if ne(parameters.buildPipelineType, 'tests') }}: + - stage: managed + displayName: Build Managed + ${{ if eq(parameters.buildPipelineType, 'tests') }}: + dependsOn: prepare + ${{ if eq(parameters.buildPipelineType, 'both') }}: + dependsOn: native jobs: - template: /scripts/azure-templates-bootstrapper.yml@self # Build Managed .NET Framework (Windows) parameters: name: managed_netfx_windows displayName: Managed .NET Framework (Windows) + sdl: ${{ parameters.sdl }} buildPipelineType: ${{ parameters.buildPipelineType }} vmImage: ${{ parameters.VM_IMAGE_WINDOWS}} target: libs @@ -442,7 +528,6 @@ stages: installPreviewVs: true requiredArtifacts: - name: native - artifactName: managed postBuildSteps: - pwsh: Remove-Item ./output/native/ -Recurse -Force -ErrorAction Continue displayName: Delete the native folder @@ -450,6 +535,7 @@ stages: parameters: name: managed_net_windows displayName: Managed .NET (Windows) + sdl: ${{ parameters.sdl }} buildPipelineType: ${{ parameters.buildPipelineType }} vmImage: ${{ parameters.VM_IMAGE_WINDOWS}} target: libs @@ -457,7 +543,6 @@ stages: installPreviewVs: true requiredArtifacts: - name: native - artifactName: managed postBuildSteps: - pwsh: Remove-Item ./output/native/ -Recurse -Force -ErrorAction Continue displayName: Delete the native folder @@ -465,13 +550,13 @@ stages: parameters: name: managed_macos displayName: Managed (macOS) + sdl: ${{ parameters.sdl }} buildPipelineType: ${{ parameters.buildPipelineType }} vmImage: ${{ parameters.VM_IMAGE_MAC }} target: libs additionalArgs: --skipExternals="all" requiredArtifacts: - name: native - artifactName: managed postBuildSteps: - pwsh: Remove-Item ./output/native/ -Recurse -Force -ErrorAction Continue displayName: Delete the native folder @@ -479,6 +564,7 @@ stages: parameters: name: managed_linux displayName: Managed (Linux) + sdl: ${{ parameters.sdl }} buildPipelineType: ${{ parameters.buildPipelineType }} vmImage: ${{ parameters.VM_IMAGE_LINUX }} packages: $(MANAGED_LINUX_PACKAGES) @@ -486,10 +572,26 @@ stages: additionalArgs: --skipExternals="all" requiredArtifacts: - name: native - artifactName: managed postBuildSteps: - pwsh: Remove-Item ./output/native/ -Recurse -Force -ErrorAction Continue displayName: Delete the native folder + - template: /scripts/azure-templates-merger.yml@self # Merge Managed Artifacts + parameters: + name: managed + displayName: Merge Managed Artifacts + sdl: ${{ parameters.sdl }} + dependsOn: + - managed_netfx_windows + - managed_net_windows + - managed_macos + - managed_linux + buildPipelineType: ${{ parameters.buildPipelineType }} + vmImage: ${{ parameters.VM_IMAGE_HOST }} + requiredArtifacts: + - name: managed_netfx_windows + - name: managed_net_windows + - name: managed_macos + - name: managed_linux - ${{ if ne(parameters.buildPipelineType, 'tests') }}: - stage: package @@ -500,10 +602,12 @@ stages: parameters: name: package_windows displayName: Package NuGets + sdl: ${{ parameters.sdl }} buildPipelineType: ${{ parameters.buildPipelineType }} vmImage: ${{ parameters.VM_IMAGE_WINDOWS}} target: nuget additionalArgs: --packall=true --skipbuild=true + shouldPublish: false installWindowsSdk: false installAndroidSdk: false installDotNet: false @@ -511,32 +615,42 @@ stages: - name: managed - name: native postBuildSteps: - - task: PublishBuildArtifacts@1 - displayName: Publish the nuget artifacts - inputs: - artifactName: nuget - pathToPublish: 'output/nugets' - - task: PublishBuildArtifacts@1 - displayName: Publish the special nuget artifacts - inputs: - artifactName: nuget_special - pathToPublish: 'output/nugets-special' - - task: PublishBuildArtifacts@1 - displayName: Publish the special nuget artifacts - inputs: - artifactName: nuget_symbols - pathToPublish: 'output/nugets-symbols' - - task: PublishBuildArtifacts@1 - displayName: Publish the SignList.xml into nuget artifacts - inputs: - artifactName: nuget - pathToPublish: 'SignList.xml' - pwsh: | - Remove-Item ./output/native/ -Recurse -Force - Remove-Item ./output/nugets/ -Recurse -Force - Remove-Item ./output/nugets-special/ -Recurse -Force - Remove-Item ./output/nugets-symbols/ -Recurse -Force - displayName: Delete the pre-published folders + Remove-Item ./output/native/ -Recurse -Force -ErrorAction Continue + Move-Item -Path '.\output\' -Destination '$(Build.ArtifactStagingDirectory)\output\' + New-Item '.\output\' -Type Directory -Force | Out-Null + displayName: Re-organize the output folder for publishing + - pwsh: | + Move-Item -Path '$(Build.ArtifactStagingDirectory)\output\nugets\' -Destination '.\output\' + Copy-Item -Path '.\scripts\SignList.xml' -Destination '.\output\nugets\' + displayName: Prepare the nugets artifact for publishing + - pwsh: | + Move-Item -Path '$(Build.ArtifactStagingDirectory)\output\nugets-symbols\' -Destination '.\output\' + displayName: Prepare the nugets-symbols artifact for publishing + - pwsh: | + $nupkgs = (Get-ChildItem ".\output\nugets*\*.*nupkg") + foreach ($nupkg in $nupkgs) { + $filename = $nupkg.Name.TrimEnd('.nupkg') + $dest = ".\output\extracted_nugets\$filename" + Write-Host "Extracting '$nupkg' to '$dest'..." + Expand-Archive $nupkg $dest + } + displayName: Extract all the .nupkg files for scanning + - pwsh: | + Move-Item -Path '$(Build.ArtifactStagingDirectory)\output\nugets-special\' -Destination '.\output\' + displayName: Prepare the nugets-special artifact for publishing + - pwsh: | + Move-Item -Path '$(Build.ArtifactStagingDirectory)\output\' -Destination '.\output\' + displayName: Prepare the build artifact for publishing + publishArtifacts: + - name: package_windows + path: '.\output\output\' + - name: nuget + path: '.\output\nugets' + - name: nuget_special + path: '.\output\nugets-special' + - name: nuget_symbols + path: '.\output\nugets-symbols' - ${{ if ne(parameters.buildPipelineType, 'build') }}: - stage: api_diff @@ -562,15 +676,12 @@ stages: preBuildSteps: - pwsh: .\scripts\install-gtk.ps1 displayName: Install GTK# 2.12 - postBuildSteps: - - task: PublishBuildArtifacts@1 - displayName: Publish the API diffs - condition: always() - inputs: - artifactName: api-diff - pathToPublish: '$(Build.SourcesDirectory)\output\api-diff' + publishArtifacts: + - name: api-diff + always: true + path: '$(Build.SourcesDirectory)\output\api-diff' - - ${{ if and(eq(variables['System.TeamProject'], 'devdiv'), ne(parameters.buildPipelineType, 'tests'), ne(variables['System.PullRequest.IsFork'], 'true')) }}: + - ${{ if eq(parameters.enableSigning, 'true') }}: - stage: signing displayName: Sign NuGets dependsOn: package @@ -581,17 +692,8 @@ stages: signType: 'Real' ${{ if not(or(eq(variables['Build.SourceBranch'], 'refs/heads/main'), startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'))) }}: signType: 'Test' - - - ${{ if and(eq(variables['System.TeamProject'], 'devdiv'), ne(parameters.buildPipelineType, 'tests'), ne(variables['System.PullRequest.IsFork'], 'true')) }}: - - stage: sbom - displayName: 'Software Bill of Materials' - dependsOn: signing - jobs: - - template: compliance/sbom/job.v1.yml@xamarin-templates # Software Bill of Materials (SBOM): https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/secure-supply-chain/ado-sbom-generator - parameters: - artifactNames: ['nuget'] - packageName: 'SkiaSharp' - packageFilter: '*.nupkg' + use1ESTemplate: ${{ parameters.use1ESPipelineTemplates }} + usePipelineArtifactTasks: true - ${{ if ne(parameters.buildPipelineType, 'build') }}: - stage: tests @@ -599,11 +701,7 @@ stages: ${{ if eq(parameters.buildPipelineType, 'tests') }}: dependsOn: prepare ${{ if eq(parameters.buildPipelineType, 'both') }}: - dependsOn: - - native_windows - - native_macos - - native_linux - - native_wasm + dependsOn: native jobs: - template: /scripts/azure-templates-bootstrapper.yml@self # Tests|netfx (Windows) parameters: @@ -619,6 +717,7 @@ stages: requiredArtifacts: - name: native_win32_x86_windows - name: native_win32_x64_windows + - name: native_win32_arm64_windows postBuildSteps: - task: PublishTestResults@2 displayName: Publish the .NET Framework test results @@ -640,6 +739,7 @@ stages: requiredArtifacts: - name: native_win32_x86_windows - name: native_win32_x64_windows + - name: native_win32_arm64_windows postBuildSteps: - task: PublishTestResults@2 displayName: Publish the .NET Core test results @@ -648,11 +748,9 @@ stages: testResultsFormat: xUnit testResultsFiles: 'tests/SkiaSharp*.NetCore.Tests/**/TestResults.xml' testRunTitle: 'Windows .NET Core Tests' - - task: PublishBuildArtifacts@1 - displayName: 'Publish the code coverage results' - inputs: - artifactName: coverage_netcore_windows - pathToPublish: 'output/coverage' + publishArtifacts: + - name: coverage_netcore_windows + path: 'output/coverage' - template: /scripts/azure-templates-bootstrapper.yml@self # Tests|netfx (macOS) parameters: name: tests_netfx_macos @@ -691,11 +789,9 @@ stages: testResultsFormat: xUnit testResultsFiles: 'tests/SkiaSharp*.NetCore.Tests/**/TestResults.xml' testRunTitle: 'macOS .NET Core Tests' - - task: PublishBuildArtifacts@1 - displayName: 'Publish the code coverage results' - inputs: - artifactName: coverage_netcore_macos - pathToPublish: 'output/coverage' + publishArtifacts: + - name: coverage_netcore_macos + path: 'output/coverage' - template: /scripts/azure-templates-bootstrapper.yml@self # Tests|android (macOS) parameters: name: tests_android_macos @@ -723,12 +819,10 @@ stages: testResultsFormat: xUnit testResultsFiles: 'output/logs/testlogs/SkiaSharp.Android.Tests/**/TestResults.xml' testRunTitle: 'Android Tests' - - task: PublishBuildArtifacts@1 - displayName: Publish the test logs - condition: always() - inputs: - artifactName: testlogs_android - pathToPublish: 'output/logs/testlogs' + publishArtifacts: + - name: testlogs_android + always: true + path: 'output/logs/testlogs' - template: /scripts/azure-templates-bootstrapper.yml@self # Tests|ios (macOS) parameters: name: tests_ios_macos @@ -750,12 +844,10 @@ stages: testResultsFormat: xUnit testResultsFiles: 'output/logs/testlogs/SkiaSharp.iOS.Tests/**/TestResults.xml' testRunTitle: 'iOS Tests' - - task: PublishBuildArtifacts@1 - displayName: Publish the test logs - condition: always() - inputs: - artifactName: testlogs_ios - pathToPublish: 'output/logs/testlogs' + publishArtifacts: + - name: testlogs_ios + always: true + path: 'output/logs/testlogs' - template: /scripts/azure-templates-bootstrapper.yml@self # Tests|netfx (Linux) parameters: name: tests_netfx_linux @@ -768,6 +860,7 @@ stages: shouldPublish: false requiredArtifacts: - name: native_linux_x64_linux + - name: native_linux_arm64_linux postBuildSteps: - task: PublishTestResults@2 displayName: Publish the Mono test results @@ -788,6 +881,7 @@ stages: shouldPublish: false requiredArtifacts: - name: native_linux_x64_linux + - name: native_linux_arm64_linux postBuildSteps: - task: PublishTestResults@2 displayName: Publish the .NET Core test results @@ -796,11 +890,9 @@ stages: testResultsFormat: xUnit testResultsFiles: 'tests/SkiaSharp*.NetCore.Tests/**/TestResults.xml' testRunTitle: 'Linux .NET Core Tests' - - task: PublishBuildArtifacts@1 - displayName: 'Publish the code coverage results' - inputs: - artifactName: coverage_netcore_linux - pathToPublish: 'output/coverage' + publishArtifacts: + - name: coverage_netcore_linux + path: 'output/coverage' - template: /scripts/azure-templates-bootstrapper.yml@self # Tests [WASM] (Linux) parameters: name: tests_wasm_linux @@ -913,33 +1005,50 @@ stages: installEmsdk: true initScript: source ~/emsdk/emsdk_env.sh - - ${{ if and(eq(variables['System.TeamProject'], 'devdiv'), ne(parameters.buildPipelineType, 'tests'), ne(variables['System.PullRequest.IsFork'], 'true'), or(and(eq(variables['Build.Reason'], 'Schedule'), or(eq(variables['Build.SourceBranch'], 'refs/heads/main'), startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'))), parameters.runCompliance)) }}: - - template: security/full/v1.yml@xamarin-templates + - ${{ if eq(parameters.sdl.apiscan.enabled, 'true') }}: + - template: security/apiscan/v0.yml@xamarin-templates parameters: stageDependsOn: - - managed - - native_windows - - native_macos - - native_linux - - native_wasm - complianceEnabled: true - complianceTimeoutInMinutes: 480 + - package + timeoutInMinutes: 480 + windowsPoolName: ${{ parameters.VM_IMAGE_HOST.pool.name }} + windowsImageOverride: ${{ parameters.VM_IMAGE_HOST.pool.image }} scanArtifacts: - - managed - - native - antiMalwareEnabled: true - binSkimEnabled: true - policheckExclusionFile: $(Build.SourcesDirectory)\scripts\guardian\PoliCheckExclusions.xml - policheckGdnSuppressionFilesFolder: $(Build.SourcesDirectory)\scripts\guardian - credScanEnabled: true - credScanSuppressionFile: $(Build.SourcesDirectory)\scripts\guardian\CredScanSuppressions.json - sourceGdnSuppressionFile: $(Build.SourcesDirectory)\scripts\guardian\source.gdnsuppress - tsaConfigFile: $(Build.SourcesDirectory)\scripts\guardian\tsaoptions-v2.json - tsaReportBranch: $(Build.SourceBranch) - enableCodeInspector: true - apiScanEnabled: true + - nuget + - nuget_symbols apiScanSoftwareName: 'SkiaSharp' - apiScanSoftwareVersionNum: $(Build.BuildNumber) + apiScanSoftwareVersionNum: $(SKIASHARP_MAJOR_VERSION) + apiScanPreserveLogsFolder: true + apiScanAuthConnectionString: 'runAs=App;AppId=$(ApiScanClientId)' + apiScanSurrogateConfigurationFolder: $(Build.ArtifactStagingDirectory)\APIScanSurrogates + preScanSteps: + - pwsh: | + $nupkgs = (Get-ChildItem "$(Build.ArtifactStagingDirectory)\binaries-to-scan\*\*.*nupkg") + foreach ($nupkg in $nupkgs) { + $filename = $nupkg.Name.TrimEnd('.nupkg') + $dest = "$(Build.ArtifactStagingDirectory)\binaries-to-scan\nuget_symbols-extracted\$filename" + Write-Host "Extracting '$nupkg' to '$dest'..." + Expand-Archive $nupkg $dest + Remove-Item $nupkg + } + displayName: Extract all the .nupkg files + - pwsh: | + $softwareFolder = "$(Build.ArtifactStagingDirectory)\binaries-to-scan" + $surrogateFile = "$(Build.SourcesDirectory)\scripts\guardian\APIScanSurrogates.in.xml" + $destFolder = "$(Build.ArtifactStagingDirectory)\APIScanSurrogates" + $destFile = "$destFolder\APIScanSurrogates.xml" + New-Item -ItemType Directory -Force -Path $destFolder | Out-Null + $surrogateContents = (Get-Content $surrogateFile) + $surrogateContents = $surrogateContents.Replace("{SOFTWARE_FOLDER}", $softwareFolder) + $nuspecs = (Get-ChildItem "$(Build.ArtifactStagingDirectory)\binaries-to-scan\nuget_symbols-extracted\*\*.nuspec") + foreach ($nuspec in $nuspecs) { + [xml] $nuspecContent = (Get-Content $nuspec) + $nuspecId = $nuspecContent.package.metadata.id + $nuspecVersion = $nuspecContent.package.metadata.version + $surrogateContents = $surrogateContents.Replace("{$nuspecId}", "$nuspecId.$nuspecVersion", 'InvariantCultureIgnoreCase') + } + $surrogateContents | Set-Content $destFile + displayName: Generate the surrogate files - ${{ if eq(parameters.buildPipelineType, 'tests') }}: - stage: finalize diff --git a/scripts/azure-templates-wasm-matrix.yml b/scripts/azure-templates-wasm-matrix.yml index 30b0a98cb8..720a01e699 100644 --- a/scripts/azure-templates-wasm-matrix.yml +++ b/scripts/azure-templates-wasm-matrix.yml @@ -1,8 +1,9 @@ parameters: - artifactName: '' # the name of the artifact to merge this run into buildExternals: '' # the build number to download externals from - buildPipelineType: false - vmImage: '' # the VM image + buildPipelineType: 'both' # the type of build pipeline setup + vmImage: '' # the configuration for the build agent + use1ESPipelineTemplates: false # whether or not we are building using the internal 1ES Pipeline Templates + sdl: [] # the SDL properties to use for this job emscripten: [ ] jobs: @@ -11,17 +12,12 @@ jobs: parameters: name: native_wasm_${{ replace(version.displayName, '.', '_') }}_linux displayName: WASM (${{ version.displayName }}) + sdl: ${{ parameters.sdl }} buildExternals: ${{ parameters.buildExternals }} buildPipelineType: ${{ parameters.buildPipelineType }} vmImage: ${{ parameters.vmImage }} + use1ESPipelineTemplates: ${{ parameters.use1ESPipelineTemplates }} docker: scripts/Docker/wasm target: externals-wasm dockerArgs: --build-arg EMSCRIPTEN_VERSION=${{ version.version }} - additionalArgs: --emscriptenVersion=${{ version.version }} --emscriptenFeatures="${{ version.features }}" - artifactName: ${{ parameters.artifactName }} - postBuildSteps: - - task: PublishBuildArtifacts@1 - displayName: Publish the native_wasm_linux artifacts - inputs: - artifactName: native_wasm_linux - pathToPublish: 'output' \ No newline at end of file + additionalArgs: --emscriptenVersion=${{ version.version }} --emscriptenFeatures="${{ version.features }}" \ No newline at end of file diff --git a/scripts/guardian/APIScanSurrogates.in.xml b/scripts/guardian/APIScanSurrogates.in.xml new file mode 100644 index 0000000000..3f769fcebd --- /dev/null +++ b/scripts/guardian/APIScanSurrogates.in.xml @@ -0,0 +1,106 @@ + + + + + + + + + {SOFTWARE_FOLDER} + SRV*https://symweb + + + + + + + + + + + + + {SOFTWARE_FOLDER} + SRV*https://symweb + + + + + + + + + + + + + + {SOFTWARE_FOLDER} + SRV*https://symweb + + + + + + + + + + + + + {SOFTWARE_FOLDER} + SRV*https://symweb + + + + + + + + + + + + + {SOFTWARE_FOLDER} + SRV*https://symweb + + + + + + + + + + + + + + {SOFTWARE_FOLDER} + SRV*https://symweb + + + + + + + + + + + + + {SOFTWARE_FOLDER} + SRV*https://symweb + + + + + + + + + + \ No newline at end of file diff --git a/scripts/install-python.ps1 b/scripts/install-python.ps1 index 029a0869c7..6002b96e5e 100644 --- a/scripts/install-python.ps1 +++ b/scripts/install-python.ps1 @@ -31,12 +31,9 @@ if ($IsMacOS) { } else { $platform = "win32" } - -$downloadUrl = (($pythonManifest - | Where-Object { $_.version -eq $Version } - | Select-Object -First 1).files - | Where-Object { $_.platform -eq $platform -and $_.arch -eq $Arch } - | Select-Object -First 1).download_url +$manifestFileVersion = $pythonManifest | Where-Object { $_.version -eq $Version } | Select-Object -First 1 +$manifestFileItem = $manifestFileVersion.files | Where-Object { $_.platform -eq $platform -and $_.arch -eq $Arch } | Select-Object -First 1 +$downloadUrl = $manifestFileItem.download_url # download $tempDir = Join-Path "$HOME_DIR" "python-temp" diff --git a/tests/SkiaSharp.Vulkan.NetCore.Tests/SkiaSharp.Vulkan.NetCore.Tests.csproj b/tests/SkiaSharp.Vulkan.NetCore.Tests/SkiaSharp.Vulkan.NetCore.Tests.csproj index f0ca0dd9a9..bfc0da1c0e 100644 --- a/tests/SkiaSharp.Vulkan.NetCore.Tests/SkiaSharp.Vulkan.NetCore.Tests.csproj +++ b/tests/SkiaSharp.Vulkan.NetCore.Tests/SkiaSharp.Vulkan.NetCore.Tests.csproj @@ -33,4 +33,9 @@ + + + PreserveNewest + + \ No newline at end of file diff --git a/tests/SkiaSharp.Vulkan.NetCore.Tests/xunit.runner.json b/tests/SkiaSharp.Vulkan.NetCore.Tests/xunit.runner.json new file mode 100644 index 0000000000..5396be559e --- /dev/null +++ b/tests/SkiaSharp.Vulkan.NetCore.Tests/xunit.runner.json @@ -0,0 +1,6 @@ +{ + "$schema": "https://xunit.net/schema/current/xunit.runner.schema.json", + "diagnosticMessages": true, + "internalDiagnosticMessages": true, + "longRunningTestSeconds": 5 +} \ No newline at end of file