From 9221e49396631f61d69c385769f731d1313a3d34 Mon Sep 17 00:00:00 2001 From: michael-hawker <24302614+michael-hawker@users.noreply.github.com> Date: Wed, 9 Dec 2020 16:32:03 -0800 Subject: [PATCH 1/7] Split out Unit Tests and Smoke Tests as separate Jobs in the Pipeline for #3560 --- SmokeTests/SmokeTests.proj | 2 +- azure-pipelines.yml | 40 ++++++++++++++++++++++++++++---------- 2 files changed, 31 insertions(+), 11 deletions(-) diff --git a/SmokeTests/SmokeTests.proj b/SmokeTests/SmokeTests.proj index 4ae1f8184b2..0da3dd81c36 100644 --- a/SmokeTests/SmokeTests.proj +++ b/SmokeTests/SmokeTests.proj @@ -12,7 +12,7 @@ Inputs="@(ProjectsToBuild)" Outputs="%(Filename)"> - + Date: Fri, 11 Dec 2020 01:24:39 -0800 Subject: [PATCH 2/7] Add a UWP Baseline project to the Smoke Tests for size comparison #3600 --- SmokeTests/SmokeTest.csproj | 6 +++--- SmokeTests/SmokeTests.proj | 2 +- SmokeTests/UWPBaseline/MainPage.xaml | 12 ++++++++++++ SmokeTests/UWPBaseline/MainPage.xaml.cs | 14 ++++++++++++++ 4 files changed, 30 insertions(+), 4 deletions(-) create mode 100644 SmokeTests/UWPBaseline/MainPage.xaml create mode 100644 SmokeTests/UWPBaseline/MainPage.xaml.cs diff --git a/SmokeTests/SmokeTest.csproj b/SmokeTests/SmokeTest.csproj index bbcfbdd57ab..3c197172d24 100644 --- a/SmokeTests/SmokeTest.csproj +++ b/SmokeTests/SmokeTest.csproj @@ -112,7 +112,7 @@ 2.4.3 - + @@ -120,10 +120,10 @@ - + - + \ No newline at end of file diff --git a/SmokeTests/SmokeTests.proj b/SmokeTests/SmokeTests.proj index 0da3dd81c36..18b9aee3a8b 100644 --- a/SmokeTests/SmokeTests.proj +++ b/SmokeTests/SmokeTests.proj @@ -4,7 +4,7 @@ x86 Release - Microsoft.Toolkit;Microsoft.Toolkit.HighPerformance;Microsoft.Toolkit.Parsers;Microsoft.Toolkit.Mvvm;Microsoft.Toolkit.Services;Microsoft.Toolkit.Uwp;Microsoft.Toolkit.Uwp.Connectivity;Microsoft.Toolkit.Uwp.DeveloperTools;Microsoft.Toolkit.Uwp.Input.GazeInteraction;Microsoft.Toolkit.Uwp.Notifications;Microsoft.Toolkit.Uwp.UI;Microsoft.Toolkit.Uwp.UI.Animations;Microsoft.Toolkit.Uwp.UI.Controls;Microsoft.Toolkit.Uwp.UI.Controls.DataGrid;Microsoft.Toolkit.Uwp.UI.Controls.Layout;Microsoft.Toolkit.Uwp.UI.Media;Microsoft.Toolkit.Uwp.UI.Controls.Markdown + UWPBaseline;Microsoft.Toolkit;Microsoft.Toolkit.HighPerformance;Microsoft.Toolkit.Parsers;Microsoft.Toolkit.Mvvm;Microsoft.Toolkit.Services;Microsoft.Toolkit.Uwp;Microsoft.Toolkit.Uwp.Connectivity;Microsoft.Toolkit.Uwp.DeveloperTools;Microsoft.Toolkit.Uwp.Input.GazeInteraction;Microsoft.Toolkit.Uwp.Notifications;Microsoft.Toolkit.Uwp.UI;Microsoft.Toolkit.Uwp.UI.Animations;Microsoft.Toolkit.Uwp.UI.Controls;Microsoft.Toolkit.Uwp.UI.Controls.DataGrid;Microsoft.Toolkit.Uwp.UI.Controls.Layout;Microsoft.Toolkit.Uwp.UI.Media;Microsoft.Toolkit.Uwp.UI.Controls.Markdown + + + diff --git a/SmokeTests/UWPBaseline/MainPage.xaml.cs b/SmokeTests/UWPBaseline/MainPage.xaml.cs new file mode 100644 index 00000000000..65e09322217 --- /dev/null +++ b/SmokeTests/UWPBaseline/MainPage.xaml.cs @@ -0,0 +1,14 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace SmokeTest +{ + public sealed partial class MainPage + { + public MainPage() + { + InitializeComponent(); + } + } +} From 367a09abc28dede44d6e2c5d9ceed8feecefe446 Mon Sep 17 00:00:00 2001 From: michael-hawker <24302614+michael-hawker@users.noreply.github.com> Date: Fri, 11 Dec 2020 02:55:50 -0800 Subject: [PATCH 3/7] Move to Publish/DownloadPipelineArtifact --- azure-pipelines.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ffe036106d5..fca160e1722 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -87,12 +87,12 @@ jobs: ArtifactDirectory: bin\nupkg condition: and(succeeded(), not(eq(variables['build.reason'], 'PullRequest')), not(eq(variables['SignClientSecret'], '')), not(eq(variables['SignClientUser'], ''))) - - task: PublishBuildArtifacts@1 + - task: PublishPipelineArtifact@1 displayName: Publish Package Artifacts inputs: - pathToPublish: .\bin\nupkg - artifactType: container + targetPath: .\bin\nupkg artifactName: Packages + ### Smoke Tests ### @@ -101,14 +101,14 @@ jobs: timeoutInMinutes: 40 steps: - - task: DownloadBuildArtifacts@0 + - task: DownloadPipelineArtifact@2 displayName: Download NuGet Packages Artifact inputs: - artifactName: Packages - downloadPath: .\bin + artifact: Packages + path: .\bin\nupkg - - powershell: Expand-Archive -Path '.\bin\Packages.zip' -DestinationPath '.\bin\nupkg' - displayName: Expand NuGet Archive + - powershell: dir .\bin\nupkg + displayName: List NuGet Archive - powershell: .\build\build.ps1 -target=SmokeTest displayName: SmokeTest From 36ccb532d634d74b2bb9a3f67cbefa5ad69df113 Mon Sep 17 00:00:00 2001 From: michael-hawker <24302614+michael-hawker@users.noreply.github.com> Date: Fri, 11 Dec 2020 11:50:55 -0800 Subject: [PATCH 4/7] Split out UI Tests as their own step Makes it easier to see output from only UI tests without knowing a scroll point in the log --- azure-pipelines.yml | 24 +++++++++++++----------- build/build.cake | 9 +++++++-- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index fca160e1722..8fdc3e5f30d 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -58,20 +58,25 @@ jobs: displayName: Publish Test Results condition: always() - - task: PublishPipelineArtifact@1 - displayName: Publish UI Test Results - inputs: - targetPath: .\build\UITestResults.wtl - artifactName: WexLogFileOutput - condition: always() - - task: PublishPipelineArtifact@1 displayName: Publish Test WexLogFileOutput inputs: targetPath: .\build\WexLogFileOutput - artifactName: WexErrorLogFileOutput + artifactName: WexUnitTestErrorLogFileOutput condition: failed() + ### UI Integration Tests ### + + - powershell: .\build\build.ps1 -target=UITest + displayName: UI Integration Tests + + - task: PublishPipelineArtifact@1 + displayName: Publish UI Test Results + inputs: + targetPath: .\build\UITestResults.wtl + artifactName: WexUITestLogFileOutput + condition: always() + ### Package ### - powershell: .\build\build.ps1 -target=Package @@ -107,9 +112,6 @@ jobs: artifact: Packages path: .\bin\nupkg - - powershell: dir .\bin\nupkg - displayName: List NuGet Archive - - powershell: .\build\build.ps1 -target=SmokeTest displayName: SmokeTest diff --git a/build/build.cake b/build/build.cake index edefaa1eea7..4369b25d3d3 100644 --- a/build/build.cake +++ b/build/build.cake @@ -243,7 +243,7 @@ public string getMSTestAdapterPath(){ } Task("Test") - .Description("Runs all Tests") + .Description("Runs all Unit Tests") .Does(() => { Information("\nRunning Unit Tests"); @@ -272,7 +272,11 @@ Task("Test") ArgumentCustomization = arg => arg.Append($"-s {baseDir}/.runsettings"), }; DotNetCoreTest(file.FullPath, testSettings); -}).DoesForEach(GetFiles(taefBinDir + "/**/UITests.Tests.TAEF.dll"), (file) => +}).DeferOnError(); + +Task("UITest") + .Description("Runs all UI Tests") + .DoesForEach(GetFiles(taefBinDir + "/**/UITests.Tests.TAEF.dll"), (file) => { Information("\nRunning TAEF Interaction Tests"); @@ -315,6 +319,7 @@ Task("MSTestUITest") Task("Default") .IsDependentOn("Build") .IsDependentOn("Test") + .IsDependentOn("UITest") .IsDependentOn("Package"); Task("UpdateHeaders") From b68d903a216e441d82736be49a56252018ba33fb Mon Sep 17 00:00:00 2001 From: michael-hawker <24302614+michael-hawker@users.noreply.github.com> Date: Fri, 11 Dec 2020 16:02:53 -0800 Subject: [PATCH 5/7] Add initial script and step to SmokeTest job to analyze dependencies and package sizes Provides a more human readible way to understand our package sizes and dependencies against a baseline blank application See more info in #3600 for potential improvements --- SmokeTests/SmokeTestAnalysis.ps1 | 141 +++++++++++++++++++++++++++++++ azure-pipelines.yml | 3 + 2 files changed, 144 insertions(+) create mode 100644 SmokeTests/SmokeTestAnalysis.ps1 diff --git a/SmokeTests/SmokeTestAnalysis.ps1 b/SmokeTests/SmokeTestAnalysis.ps1 new file mode 100644 index 00000000000..f65f505f38d --- /dev/null +++ b/SmokeTests/SmokeTestAnalysis.ps1 @@ -0,0 +1,141 @@ +$ErrorActionPreference = "Stop" +$ProgressPreference="SilentlyContinue" + +Write-Host "Running Smoke Test Package Analyis" +Write-Host "----------------------------------" + +# Our script is at our SmokeTest root, we want to look for the AppPackages folder +$PackagePath = $PSScriptRoot + "\AppPackages\" +$FilePattern = "SmokeTest_{0}_x86_bundle.msixupload" +$BaselineName = $FilePattern -f "UWPBaseline" +$TempFolder = "ExplodedArchive" + +function Expand-MsixUploadPackage { + param ( + [string]$PackageFile, + [string]$Destination + ) + + $ZipUpload = $PackageFile.Replace("msixupload", "zip") + + Move-Item $PackageFile -Destination $ZipUpload + + Expand-Archive $ZipUpload -DestinationPath $Destination + + Move-Item $ZipUpload -Destination $PackageFile + + Push-Location $Destination + + $Bundle = (Get-ChildItem "*.msixbundle").Name + $ZipBundle = $Bundle.Replace("msixbundle", "zip") + + Move-Item $Bundle -Destination $ZipBundle + + Expand-Archive $ZipBundle -DestinationPath . + + Remove-Item $ZipBundle + + $msix = (Get-ChildItem "*.msix").Name + $ZipMSIX = $msix.Replace("msix", "zip") + + Move-Item $msix -Destination $ZipMSIX + + Expand-Archive $ZipMSIX -DestinationPath . -Force # Force here as we have some duplicate file names we don't really care about from parent archives + + Remove-Item $ZipMSIX + + Pop-Location +} + +if (Test-Path $PackagePath) +{ + Push-Location $PackagePath + + Write-Host "Extracting Baseline..." + + # TODO: Theoretically we could grab bits from the bin directory instead of having to expand each package, not sure about what we ignore though + Expand-MsixUploadPackage $BaselineName -Destination $TempFolder + + # Get all the base file info only (grab stuff in directories but not the directories themselves) + $BaselineFiles = Get-ChildItem $TempFolder -Recurse -Attributes !Directory -Exclude "SmokeTest*" + $SmokeTestFiles = Get-ChildItem $TempFolder -Recurse -Attributes !Directory -Include "SmokeTest*" + + $BaselineFootprint = ($BaselineFiles | Measure-Object -Property Length -sum).Sum + ($SmokeTestFiles | Measure-Object -Property Length -sum).Sum + Write-Host ("Baseline Footprint: {0:n0} bytes" -f $BaselineFootprint) + Write-Host "-----------------------------------------" + + $PackageList = Get-ChildItem "$PackagePath*.msixupload" -Exclude $BaselineName + + #$i = 0 + foreach ($Package in $PackageList) + { + #Write-Progress -Id 0 -Activity "Comparing Against Baseline..." -Status "Prepping Package" -PercentComplete (($i++ / $PackageList.count)*100) -CurrentOperation $Package.Name + + # Make sure we've cleaned-up the last archive + Remove-Item $TempFolder -Recurse -Force + + #$ProgressPreference="SilentlyContinue" + Expand-MsixUploadPackage $Package.Name -Destination $TempFolder + #$ProgressPreference="Continue" + + [System.Collections.ArrayList]$PackageFiles = Get-ChildItem $TempFolder -Recurse -Attributes !Directory -Exclude "SmokeTest*" + $PackageSmokeTestFiles = Get-ChildItem $TempFolder -Recurse -Attributes !Directory -Include "SmokeTest*" + + # TODO: Make function or regex better to extra package name more easily based on a template string at the top or something... + Write-Host ("{0} Additional Footprint: {1:n0} bytes" -f $Package.Name.substring(10, $Package.Name.Length - 32), (($PackageFiles | Measure-Object -Property Length -sum).Sum + ($PackageSmokeTestFiles | Measure-Object -Property Length -sum).Sum - $BaselineFootprint)) + + # Quick check on the base exe file/symbols differences + foreach ($file in $SmokeTestFiles) + { + $match = $null + $match = $PackageSmokeTestFiles | Where-Object {$_.Extension -eq $file.Extension} + if ($null -ne $match) + { + Write-Host (" App Diff: ({0}) = {1:n0}" -f $file.Extension, ($match.Length - $file.Length)) -ForegroundColor DarkCyan + } + } + + #$j = 0 + foreach ($file in $BaselineFiles) + { + #Write-Progress -Id 1 -ParentId 0 -Activity "Comparing Against Baseline..." -Status "Comparing Package" -PercentComplete (($j++ / $BaselineFiles.count)*100) -CurrentOperation $file.Name + + $match = $null + $match = $PackageFiles | Where-Object {$_.Name -eq $file.Name} + if ($null -ne $match) + { + # File was in baseline, but has a different size + if ($match.Length -ne $file.Length) + { + Write-Host (" Size Diff: {0} = {1:n0}" -f $file.Name, ($match.Length - $file.Length)) -ForegroundColor Magenta + } + + # Remove checked files (performance) and also remaining are new + $PackageFiles.Remove($match) + } + } + + # List remaining (new) files to this package + foreach ($file in $PackageFiles) + { + Write-Host (" Additional: {0} = {1:n0}" -f $file.Name, $file.Length) -ForegroundColor Yellow + } + + # TODO: Especially if we add comparison to the main branch, we should format as an actual table and colorize via VT: https://stackoverflow.com/a/49038815/8798708 + + #Write-Progress -Id 1 -ParentId 0 -Activity "Comparing Against Baseline..." -Completed + Write-Host "-----------------------------------------" + Write-Host + } + + #Write-Progress -Id 0 -Activity "Comparing Against Baseline..." -Completed + + # Clean-up + Remove-Item $TempFolder -Recurse -Force + + Pop-Location +} +else +{ + Write-Error "Path $PackagePath not found for analysis!" +} \ No newline at end of file diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 8fdc3e5f30d..afbac9f5bf6 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -127,3 +127,6 @@ jobs: pathToPublish: $(build.artifactstagingdirectory)\SmokeTestBundles artifactType: container artifactName: SmokeTestBundles + + - powershell: .\SmokeTests\SmokeTestAnalysis.ps1 + displayName: Analyze Package Sizes From d412fab7b94ca32fced631189da712921635c787 Mon Sep 17 00:00:00 2001 From: michael-hawker <24302614+michael-hawker@users.noreply.github.com> Date: Fri, 11 Dec 2020 16:18:27 -0800 Subject: [PATCH 6/7] Separate out the own package in a different color --- SmokeTests/SmokeTestAnalysis.ps1 | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/SmokeTests/SmokeTestAnalysis.ps1 b/SmokeTests/SmokeTestAnalysis.ps1 index f65f505f38d..66f5f838123 100644 --- a/SmokeTests/SmokeTestAnalysis.ps1 +++ b/SmokeTests/SmokeTestAnalysis.ps1 @@ -82,7 +82,8 @@ if (Test-Path $PackagePath) $PackageSmokeTestFiles = Get-ChildItem $TempFolder -Recurse -Attributes !Directory -Include "SmokeTest*" # TODO: Make function or regex better to extra package name more easily based on a template string at the top or something... - Write-Host ("{0} Additional Footprint: {1:n0} bytes" -f $Package.Name.substring(10, $Package.Name.Length - 32), (($PackageFiles | Measure-Object -Property Length -sum).Sum + ($PackageSmokeTestFiles | Measure-Object -Property Length -sum).Sum - $BaselineFootprint)) + $PackageShortName = $Package.Name.substring(10, $Package.Name.Length - 32) + Write-Host ("{0} Additional Footprint: {1:n0} bytes" -f $PackageShortName, (($PackageFiles | Measure-Object -Property Length -sum).Sum + ($PackageSmokeTestFiles | Measure-Object -Property Length -sum).Sum - $BaselineFootprint)) # Quick check on the base exe file/symbols differences foreach ($file in $SmokeTestFiles) @@ -118,7 +119,14 @@ if (Test-Path $PackagePath) # List remaining (new) files to this package foreach ($file in $PackageFiles) { - Write-Host (" Additional: {0} = {1:n0}" -f $file.Name, $file.Length) -ForegroundColor Yellow + if ($file.Name -match $PackageShortName) + { + Write-Host (" Lib (self): {0} = {1:n0}" -f $file.Name, $file.Length) -ForegroundColor White + } + else + { + Write-Host (" Additional: {0} = {1:n0}" -f $file.Name, $file.Length) -ForegroundColor Yellow + } } # TODO: Especially if we add comparison to the main branch, we should format as an actual table and colorize via VT: https://stackoverflow.com/a/49038815/8798708 From 05941d94b724a6a7e544ae52d565eddbe3a3992f Mon Sep 17 00:00:00 2001 From: "Michael Hawker MSFT (XAML Llama)" <24302614+michael-hawker@users.noreply.github.com> Date: Tue, 15 Dec 2020 09:14:46 -0800 Subject: [PATCH 7/7] Apply suggestions from code review Co-authored-by: Rosario Pulella --- azure-pipelines.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index afbac9f5bf6..43502fd40b2 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -49,7 +49,7 @@ jobs: ### Unit Tests ### - powershell: .\build\build.ps1 -target=Test - displayName: Test + displayName: Test - task: PublishTestResults@2 inputs: @@ -97,7 +97,6 @@ jobs: inputs: targetPath: .\bin\nupkg artifactName: Packages - ### Smoke Tests ###