-
Notifications
You must be signed in to change notification settings - Fork 1.4k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Split out Smoke Tests and Unit Tests into a separate Jobs (#3617)
## Fixes #3560 This PR does the following: - [x] Split our tests out of the single Job into two separates Jobs, these can hopefully run in parallel? We should also get separate status checks for each job? (Will see when we run this here.) - [x] Adds a UWP Baseline Blank App, this will be useful for #3600 to compare what the minimum app size is currently with no extra dependencies. - [x] Split out UI Tests as a separate step from the Unit Tests - [x] Add Application Metrics Report to Smoke Tests #3600 (could be a separate PR I suppose) ## PR Type - Build or CI related changes ## What is the current behavior? Everything is done in one big job. ## What is the new behavior? We have more granularity on our pipeline ## PR Checklist Please check if your PR fulfills the following requirements: - [ ] Tested code with current [supported SDKs](../readme.md#supported) - [ ] Pull Request has been submitted to the documentation repository [instructions](..\contributing.md#docs). Link: <!-- docs PR link --> - [ ] Sample in sample app has been added / updated (for bug fixes / features) - [ ] Icon has been created (if new sample) following the [Thumbnail Style Guide and templates](https://github.com/windows-toolkit/WindowsCommunityToolkit-design-assets) - [ ] Tests for the changes have been added (for bug fixes / features) (if applicable) - [ ] Header has been added to all new source files (run *build/UpdateHeaders.bat*) - [ ] Contains **NO** breaking changes <!-- If this PR contains a breaking change, please describe the impact and migration path for existing applications below. Please note that breaking changes are likely to be rejected within minor release cycles or held until major versions. --> ## Other information
- Loading branch information
Showing
7 changed files
with
230 additions
and
26 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,149 @@ | ||
| $ErrorActionPreference = "Stop" | ||
| $ProgressPreference="SilentlyContinue" | ||
|
|
||
| Write-Host "Running Smoke Test Package Analyis" | ||
| Write-Host "----------------------------------" | ||
|
|
||
| # Our script is at our SmokeTest root, we want to look for the AppPackages folder | ||
| $PackagePath = $PSScriptRoot + "\AppPackages\" | ||
| $FilePattern = "SmokeTest_{0}_x86_bundle.msixupload" | ||
| $BaselineName = $FilePattern -f "UWPBaseline" | ||
| $TempFolder = "ExplodedArchive" | ||
|
|
||
| function Expand-MsixUploadPackage { | ||
| param ( | ||
| [string]$PackageFile, | ||
| [string]$Destination | ||
| ) | ||
|
|
||
| $ZipUpload = $PackageFile.Replace("msixupload", "zip") | ||
|
|
||
| Move-Item $PackageFile -Destination $ZipUpload | ||
|
|
||
| Expand-Archive $ZipUpload -DestinationPath $Destination | ||
|
|
||
| Move-Item $ZipUpload -Destination $PackageFile | ||
|
|
||
| Push-Location $Destination | ||
|
|
||
| $Bundle = (Get-ChildItem "*.msixbundle").Name | ||
| $ZipBundle = $Bundle.Replace("msixbundle", "zip") | ||
|
|
||
| Move-Item $Bundle -Destination $ZipBundle | ||
|
|
||
| Expand-Archive $ZipBundle -DestinationPath . | ||
|
|
||
| Remove-Item $ZipBundle | ||
|
|
||
| $msix = (Get-ChildItem "*.msix").Name | ||
| $ZipMSIX = $msix.Replace("msix", "zip") | ||
|
|
||
| Move-Item $msix -Destination $ZipMSIX | ||
|
|
||
| Expand-Archive $ZipMSIX -DestinationPath . -Force # Force here as we have some duplicate file names we don't really care about from parent archives | ||
|
|
||
| Remove-Item $ZipMSIX | ||
|
|
||
| Pop-Location | ||
| } | ||
|
|
||
| if (Test-Path $PackagePath) | ||
| { | ||
| Push-Location $PackagePath | ||
|
|
||
| Write-Host "Extracting Baseline..." | ||
|
|
||
| # TODO: Theoretically we could grab bits from the bin directory instead of having to expand each package, not sure about what we ignore though | ||
| Expand-MsixUploadPackage $BaselineName -Destination $TempFolder | ||
|
|
||
| # Get all the base file info only (grab stuff in directories but not the directories themselves) | ||
| $BaselineFiles = Get-ChildItem $TempFolder -Recurse -Attributes !Directory -Exclude "SmokeTest*" | ||
| $SmokeTestFiles = Get-ChildItem $TempFolder -Recurse -Attributes !Directory -Include "SmokeTest*" | ||
|
|
||
| $BaselineFootprint = ($BaselineFiles | Measure-Object -Property Length -sum).Sum + ($SmokeTestFiles | Measure-Object -Property Length -sum).Sum | ||
| Write-Host ("Baseline Footprint: {0:n0} bytes" -f $BaselineFootprint) | ||
| Write-Host "-----------------------------------------" | ||
|
|
||
| $PackageList = Get-ChildItem "$PackagePath*.msixupload" -Exclude $BaselineName | ||
|
|
||
| #$i = 0 | ||
| foreach ($Package in $PackageList) | ||
| { | ||
| #Write-Progress -Id 0 -Activity "Comparing Against Baseline..." -Status "Prepping Package" -PercentComplete (($i++ / $PackageList.count)*100) -CurrentOperation $Package.Name | ||
|
|
||
| # Make sure we've cleaned-up the last archive | ||
| Remove-Item $TempFolder -Recurse -Force | ||
|
|
||
| #$ProgressPreference="SilentlyContinue" | ||
| Expand-MsixUploadPackage $Package.Name -Destination $TempFolder | ||
| #$ProgressPreference="Continue" | ||
|
|
||
| [System.Collections.ArrayList]$PackageFiles = Get-ChildItem $TempFolder -Recurse -Attributes !Directory -Exclude "SmokeTest*" | ||
| $PackageSmokeTestFiles = Get-ChildItem $TempFolder -Recurse -Attributes !Directory -Include "SmokeTest*" | ||
|
|
||
| # TODO: Make function or regex better to extra package name more easily based on a template string at the top or something... | ||
| $PackageShortName = $Package.Name.substring(10, $Package.Name.Length - 32) | ||
| Write-Host ("{0} Additional Footprint: {1:n0} bytes" -f $PackageShortName, (($PackageFiles | Measure-Object -Property Length -sum).Sum + ($PackageSmokeTestFiles | Measure-Object -Property Length -sum).Sum - $BaselineFootprint)) | ||
|
|
||
| # Quick check on the base exe file/symbols differences | ||
| foreach ($file in $SmokeTestFiles) | ||
| { | ||
| $match = $null | ||
| $match = $PackageSmokeTestFiles | Where-Object {$_.Extension -eq $file.Extension} | ||
| if ($null -ne $match) | ||
| { | ||
| Write-Host (" App Diff: ({0}) = {1:n0}" -f $file.Extension, ($match.Length - $file.Length)) -ForegroundColor DarkCyan | ||
| } | ||
| } | ||
|
|
||
| #$j = 0 | ||
| foreach ($file in $BaselineFiles) | ||
| { | ||
| #Write-Progress -Id 1 -ParentId 0 -Activity "Comparing Against Baseline..." -Status "Comparing Package" -PercentComplete (($j++ / $BaselineFiles.count)*100) -CurrentOperation $file.Name | ||
|
|
||
| $match = $null | ||
| $match = $PackageFiles | Where-Object {$_.Name -eq $file.Name} | ||
| if ($null -ne $match) | ||
| { | ||
| # File was in baseline, but has a different size | ||
| if ($match.Length -ne $file.Length) | ||
| { | ||
| Write-Host (" Size Diff: {0} = {1:n0}" -f $file.Name, ($match.Length - $file.Length)) -ForegroundColor Magenta | ||
| } | ||
|
|
||
| # Remove checked files (performance) and also remaining are new | ||
| $PackageFiles.Remove($match) | ||
| } | ||
| } | ||
|
|
||
| # List remaining (new) files to this package | ||
| foreach ($file in $PackageFiles) | ||
| { | ||
| if ($file.Name -match $PackageShortName) | ||
| { | ||
| Write-Host (" Lib (self): {0} = {1:n0}" -f $file.Name, $file.Length) -ForegroundColor White | ||
| } | ||
| else | ||
| { | ||
| Write-Host (" Additional: {0} = {1:n0}" -f $file.Name, $file.Length) -ForegroundColor Yellow | ||
| } | ||
| } | ||
|
|
||
| # TODO: Especially if we add comparison to the main branch, we should format as an actual table and colorize via VT: https://stackoverflow.com/a/49038815/8798708 | ||
|
|
||
| #Write-Progress -Id 1 -ParentId 0 -Activity "Comparing Against Baseline..." -Completed | ||
| Write-Host "-----------------------------------------" | ||
| Write-Host | ||
| } | ||
|
|
||
| #Write-Progress -Id 0 -Activity "Comparing Against Baseline..." -Completed | ||
|
|
||
| # Clean-up | ||
| Remove-Item $TempFolder -Recurse -Force | ||
|
|
||
| Pop-Location | ||
| } | ||
| else | ||
| { | ||
| Write-Error "Path $PackagePath not found for analysis!" | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,12 @@ | ||
| <Page | ||
| x:Class="SmokeTest.MainPage" | ||
| xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation" | ||
| xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" | ||
| xmlns:local="using:SmokeTest" | ||
| xmlns:d="http://schemas.microsoft.com/expression/blend/2008" | ||
| xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006" | ||
| mc:Ignorable="d" | ||
| Background="{ThemeResource ApplicationPageBackgroundThemeBrush}"> | ||
|
|
||
| <Grid/> | ||
| </Page> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,14 @@ | ||
| // Licensed to the .NET Foundation under one or more agreements. | ||
| // The .NET Foundation licenses this file to you under the MIT license. | ||
| // See the LICENSE file in the project root for more information. | ||
|
|
||
| namespace SmokeTest | ||
| { | ||
| public sealed partial class MainPage | ||
| { | ||
| public MainPage() | ||
| { | ||
| InitializeComponent(); | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters