diff --git a/SmokeTests/SmokeTest.csproj b/SmokeTests/SmokeTest.csproj index 1f4c5855eed..ea7c29341d6 100644 --- a/SmokeTests/SmokeTest.csproj +++ b/SmokeTests/SmokeTest.csproj @@ -112,7 +112,7 @@ 2.4.3 - + @@ -120,10 +120,10 @@ - + - + \ No newline at end of file diff --git a/SmokeTests/SmokeTestAnalysis.ps1 b/SmokeTests/SmokeTestAnalysis.ps1 new file mode 100644 index 00000000000..66f5f838123 --- /dev/null +++ b/SmokeTests/SmokeTestAnalysis.ps1 @@ -0,0 +1,149 @@ +$ErrorActionPreference = "Stop" +$ProgressPreference="SilentlyContinue" + +Write-Host "Running Smoke Test Package Analyis" +Write-Host "----------------------------------" + +# Our script is at our SmokeTest root, we want to look for the AppPackages folder +$PackagePath = $PSScriptRoot + "\AppPackages\" +$FilePattern = "SmokeTest_{0}_x86_bundle.msixupload" +$BaselineName = $FilePattern -f "UWPBaseline" +$TempFolder = "ExplodedArchive" + +function Expand-MsixUploadPackage { + param ( + [string]$PackageFile, + [string]$Destination + ) + + $ZipUpload = $PackageFile.Replace("msixupload", "zip") + + Move-Item $PackageFile -Destination $ZipUpload + + Expand-Archive $ZipUpload -DestinationPath $Destination + + Move-Item $ZipUpload -Destination $PackageFile + + Push-Location $Destination + + $Bundle = (Get-ChildItem "*.msixbundle").Name + $ZipBundle = $Bundle.Replace("msixbundle", "zip") + + Move-Item $Bundle -Destination $ZipBundle + + Expand-Archive $ZipBundle -DestinationPath . + + Remove-Item $ZipBundle + + $msix = (Get-ChildItem "*.msix").Name + $ZipMSIX = $msix.Replace("msix", "zip") + + Move-Item $msix -Destination $ZipMSIX + + Expand-Archive $ZipMSIX -DestinationPath . -Force # Force here as we have some duplicate file names we don't really care about from parent archives + + Remove-Item $ZipMSIX + + Pop-Location +} + +if (Test-Path $PackagePath) +{ + Push-Location $PackagePath + + Write-Host "Extracting Baseline..." + + # TODO: Theoretically we could grab bits from the bin directory instead of having to expand each package, not sure about what we ignore though + Expand-MsixUploadPackage $BaselineName -Destination $TempFolder + + # Get all the base file info only (grab stuff in directories but not the directories themselves) + $BaselineFiles = Get-ChildItem $TempFolder -Recurse -Attributes !Directory -Exclude "SmokeTest*" + $SmokeTestFiles = Get-ChildItem $TempFolder -Recurse -Attributes !Directory -Include "SmokeTest*" + + $BaselineFootprint = ($BaselineFiles | Measure-Object -Property Length -sum).Sum + ($SmokeTestFiles | Measure-Object -Property Length -sum).Sum + Write-Host ("Baseline Footprint: {0:n0} bytes" -f $BaselineFootprint) + Write-Host "-----------------------------------------" + + $PackageList = Get-ChildItem "$PackagePath*.msixupload" -Exclude $BaselineName + + #$i = 0 + foreach ($Package in $PackageList) + { + #Write-Progress -Id 0 -Activity "Comparing Against Baseline..." -Status "Prepping Package" -PercentComplete (($i++ / $PackageList.count)*100) -CurrentOperation $Package.Name + + # Make sure we've cleaned-up the last archive + Remove-Item $TempFolder -Recurse -Force + + #$ProgressPreference="SilentlyContinue" + Expand-MsixUploadPackage $Package.Name -Destination $TempFolder + #$ProgressPreference="Continue" + + [System.Collections.ArrayList]$PackageFiles = Get-ChildItem $TempFolder -Recurse -Attributes !Directory -Exclude "SmokeTest*" + $PackageSmokeTestFiles = Get-ChildItem $TempFolder -Recurse -Attributes !Directory -Include "SmokeTest*" + + # TODO: Make function or regex better to extra package name more easily based on a template string at the top or something... + $PackageShortName = $Package.Name.substring(10, $Package.Name.Length - 32) + Write-Host ("{0} Additional Footprint: {1:n0} bytes" -f $PackageShortName, (($PackageFiles | Measure-Object -Property Length -sum).Sum + ($PackageSmokeTestFiles | Measure-Object -Property Length -sum).Sum - $BaselineFootprint)) + + # Quick check on the base exe file/symbols differences + foreach ($file in $SmokeTestFiles) + { + $match = $null + $match = $PackageSmokeTestFiles | Where-Object {$_.Extension -eq $file.Extension} + if ($null -ne $match) + { + Write-Host (" App Diff: ({0}) = {1:n0}" -f $file.Extension, ($match.Length - $file.Length)) -ForegroundColor DarkCyan + } + } + + #$j = 0 + foreach ($file in $BaselineFiles) + { + #Write-Progress -Id 1 -ParentId 0 -Activity "Comparing Against Baseline..." -Status "Comparing Package" -PercentComplete (($j++ / $BaselineFiles.count)*100) -CurrentOperation $file.Name + + $match = $null + $match = $PackageFiles | Where-Object {$_.Name -eq $file.Name} + if ($null -ne $match) + { + # File was in baseline, but has a different size + if ($match.Length -ne $file.Length) + { + Write-Host (" Size Diff: {0} = {1:n0}" -f $file.Name, ($match.Length - $file.Length)) -ForegroundColor Magenta + } + + # Remove checked files (performance) and also remaining are new + $PackageFiles.Remove($match) + } + } + + # List remaining (new) files to this package + foreach ($file in $PackageFiles) + { + if ($file.Name -match $PackageShortName) + { + Write-Host (" Lib (self): {0} = {1:n0}" -f $file.Name, $file.Length) -ForegroundColor White + } + else + { + Write-Host (" Additional: {0} = {1:n0}" -f $file.Name, $file.Length) -ForegroundColor Yellow + } + } + + # TODO: Especially if we add comparison to the main branch, we should format as an actual table and colorize via VT: https://stackoverflow.com/a/49038815/8798708 + + #Write-Progress -Id 1 -ParentId 0 -Activity "Comparing Against Baseline..." -Completed + Write-Host "-----------------------------------------" + Write-Host + } + + #Write-Progress -Id 0 -Activity "Comparing Against Baseline..." -Completed + + # Clean-up + Remove-Item $TempFolder -Recurse -Force + + Pop-Location +} +else +{ + Write-Error "Path $PackagePath not found for analysis!" +} \ No newline at end of file diff --git a/SmokeTests/SmokeTests.proj b/SmokeTests/SmokeTests.proj index 4ae1f8184b2..18b9aee3a8b 100644 --- a/SmokeTests/SmokeTests.proj +++ b/SmokeTests/SmokeTests.proj @@ -4,7 +4,7 @@ x86 Release - Microsoft.Toolkit;Microsoft.Toolkit.HighPerformance;Microsoft.Toolkit.Parsers;Microsoft.Toolkit.Mvvm;Microsoft.Toolkit.Services;Microsoft.Toolkit.Uwp;Microsoft.Toolkit.Uwp.Connectivity;Microsoft.Toolkit.Uwp.DeveloperTools;Microsoft.Toolkit.Uwp.Input.GazeInteraction;Microsoft.Toolkit.Uwp.Notifications;Microsoft.Toolkit.Uwp.UI;Microsoft.Toolkit.Uwp.UI.Animations;Microsoft.Toolkit.Uwp.UI.Controls;Microsoft.Toolkit.Uwp.UI.Controls.DataGrid;Microsoft.Toolkit.Uwp.UI.Controls.Layout;Microsoft.Toolkit.Uwp.UI.Media;Microsoft.Toolkit.Uwp.UI.Controls.Markdown + UWPBaseline;Microsoft.Toolkit;Microsoft.Toolkit.HighPerformance;Microsoft.Toolkit.Parsers;Microsoft.Toolkit.Mvvm;Microsoft.Toolkit.Services;Microsoft.Toolkit.Uwp;Microsoft.Toolkit.Uwp.Connectivity;Microsoft.Toolkit.Uwp.DeveloperTools;Microsoft.Toolkit.Uwp.Input.GazeInteraction;Microsoft.Toolkit.Uwp.Notifications;Microsoft.Toolkit.Uwp.UI;Microsoft.Toolkit.Uwp.UI.Animations;Microsoft.Toolkit.Uwp.UI.Controls;Microsoft.Toolkit.Uwp.UI.Controls.DataGrid;Microsoft.Toolkit.Uwp.UI.Controls.Layout;Microsoft.Toolkit.Uwp.UI.Media;Microsoft.Toolkit.Uwp.UI.Controls.Markdown - + + + + diff --git a/SmokeTests/UWPBaseline/MainPage.xaml.cs b/SmokeTests/UWPBaseline/MainPage.xaml.cs new file mode 100644 index 00000000000..65e09322217 --- /dev/null +++ b/SmokeTests/UWPBaseline/MainPage.xaml.cs @@ -0,0 +1,14 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace SmokeTest +{ + public sealed partial class MainPage + { + public MainPage() + { + InitializeComponent(); + } + } +} diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 16784b4146b..43502fd40b2 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -13,8 +13,9 @@ variables: BuildConfiguration: Release jobs: -- job: ToolkitBuild - timeoutInMinutes: 120 +### BUILD ### +- job: BuildBits + timeoutInMinutes: 60 steps: - task: BatchScript@1 @@ -45,15 +46,10 @@ jobs: - powershell: .\build\build.ps1 -target=Build displayName: Build + ### Unit Tests ### + - powershell: .\build\build.ps1 -target=Test displayName: Test - timeoutInMinutes: 15 - - - powershell: .\build\build.ps1 -target=Package - displayName: Package - - - powershell: .\build\build.ps1 -target=SmokeTest - displayName: SmokeTest - task: PublishTestResults@2 inputs: @@ -62,19 +58,29 @@ jobs: displayName: Publish Test Results condition: always() + - task: PublishPipelineArtifact@1 + displayName: Publish Test WexLogFileOutput + inputs: + targetPath: .\build\WexLogFileOutput + artifactName: WexUnitTestErrorLogFileOutput + condition: failed() + + ### UI Integration Tests ### + + - powershell: .\build\build.ps1 -target=UITest + displayName: UI Integration Tests + - task: PublishPipelineArtifact@1 displayName: Publish UI Test Results inputs: targetPath: .\build\UITestResults.wtl - artifactName: WexLogFileOutput + artifactName: WexUITestLogFileOutput condition: always() - - task: PublishPipelineArtifact@1 - displayName: Publish Test WexLogFileOutput - inputs: - targetPath: .\build\WexLogFileOutput - artifactName: WexErrorLogFileOutput - condition: failed() + ### Package ### + + - powershell: .\build\build.ps1 -target=Package + displayName: Package - task: PowerShell@2 displayName: Authenticode Sign Packages @@ -86,13 +92,28 @@ jobs: ArtifactDirectory: bin\nupkg condition: and(succeeded(), not(eq(variables['build.reason'], 'PullRequest')), not(eq(variables['SignClientSecret'], '')), not(eq(variables['SignClientUser'], ''))) - - task: PublishBuildArtifacts@1 + - task: PublishPipelineArtifact@1 displayName: Publish Package Artifacts inputs: - pathToPublish: .\bin\nupkg - artifactType: container + targetPath: .\bin\nupkg artifactName: Packages +### Smoke Tests ### + +- job: SmokeTests + dependsOn: BuildBits + timeoutInMinutes: 40 + + steps: + - task: DownloadPipelineArtifact@2 + displayName: Download NuGet Packages Artifact + inputs: + artifact: Packages + path: .\bin\nupkg + + - powershell: .\build\build.ps1 -target=SmokeTest + displayName: SmokeTest + - task: CopyFiles@2 inputs: sourceFolder: .\SmokeTests\AppPackages @@ -105,3 +126,6 @@ jobs: pathToPublish: $(build.artifactstagingdirectory)\SmokeTestBundles artifactType: container artifactName: SmokeTestBundles + + - powershell: .\SmokeTests\SmokeTestAnalysis.ps1 + displayName: Analyze Package Sizes diff --git a/build/build.cake b/build/build.cake index edefaa1eea7..4369b25d3d3 100644 --- a/build/build.cake +++ b/build/build.cake @@ -243,7 +243,7 @@ public string getMSTestAdapterPath(){ } Task("Test") - .Description("Runs all Tests") + .Description("Runs all Unit Tests") .Does(() => { Information("\nRunning Unit Tests"); @@ -272,7 +272,11 @@ Task("Test") ArgumentCustomization = arg => arg.Append($"-s {baseDir}/.runsettings"), }; DotNetCoreTest(file.FullPath, testSettings); -}).DoesForEach(GetFiles(taefBinDir + "/**/UITests.Tests.TAEF.dll"), (file) => +}).DeferOnError(); + +Task("UITest") + .Description("Runs all UI Tests") + .DoesForEach(GetFiles(taefBinDir + "/**/UITests.Tests.TAEF.dll"), (file) => { Information("\nRunning TAEF Interaction Tests"); @@ -315,6 +319,7 @@ Task("MSTestUITest") Task("Default") .IsDependentOn("Build") .IsDependentOn("Test") + .IsDependentOn("UITest") .IsDependentOn("Package"); Task("UpdateHeaders")