Skip to content

Commit

Permalink
Fixed issues in live test
Browse files Browse the repository at this point in the history
  • Loading branch information
vidai-msft committed Jan 16, 2023
1 parent 856e4d3 commit 59b1e5a
Show file tree
Hide file tree
Showing 9 changed files with 160 additions and 156 deletions.
6 changes: 4 additions & 2 deletions .azure-pipelines/util/live-test-steps.yml
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,8 @@ jobs:
buildType: 'specific'
project: $(ProjectToDownloadArtifacts)
definition: $(BuildPipelineDefinitionId)
buildVersionToDownload: 'latest'
buildVersionToDownload: 'latestFromBranch'
branchName: 'refs/heads/main'
artifactName: 'artifacts'
targetPath: $(Pipeline.Workspace)

Expand Down Expand Up @@ -91,10 +92,11 @@ jobs:
targetType: filePath
filePath: ./tools/TestFx/Live/InvokeLiveTestCITask.ps1
arguments: -UseWindowsPowerShell ('${{ parameters.psVersion }}' -eq '5.1') -ScriptFile './tools/TestFx/Live/InvokeLiveTestScenarios.ps1 -BuildId $(Build.BuildId) -OSVersion ${{ parameters.vmImage }} -PSVersion ${{ parameters.psVersion }} -RepoLocation $(Build.SourcesDirectory)'
failOnStderr: true

- task: PowerShell@2
displayName: Save live test results to Kusto
condition: succeededOrFailed()
retryCountOnTaskFailure: 3
inputs:
pwsh: true
targetType: filePath
Expand Down
2 changes: 0 additions & 2 deletions src/KeyVault/KeyVault.Test/LiveTests/TestLiveScenarios.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -44,8 +44,6 @@ Invoke-LiveTestScenario -Name "Update key vault" -Description "Test updating pro

# Update EnableRbacAuthorization
$vault = New-AzKeyVault -VaultName $vaultName -ResourceGroupName $rgName -Location $vaultLocation
$vault = $vault | Update-AzKeyVault -EnableRbacAuthorization $true
Assert-True { $vault.EnableRbacAuthorization } "EnableRbacAuthorization should be true"

$vault = $vault | Update-AzKeyVault -EnableRbacAuthorization $false
Assert-False { $vault.EnableRbacAuthorization } "EnableRbacAuthorization should be false"
Expand Down
33 changes: 33 additions & 0 deletions tools/TestFx/Live/DebugLocalLiveTestScenarios.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
param(
[Parameter(Mandatory)]
[ValidateScript({ Test-Path -LiteralPath $_ -PathType Container })]
[string] $RepoLocation
)

$debugDirectory = Join-Path -Path $RepoLocation -ChildPath "artifacts" | Join-Path -ChildPath "Debug"
$accountsModuleDirectory = Join-Path -Path $debugDirectory -ChildPath "Az.Accounts"
Write-Host "Start to import Azure PowerShell modules from artifacts/Debug." -ForegroundColor Green
Write-Host "If you see module import issue, please restart the PowerShell host." -ForegroundColor Magenta

Write-Host "Importing Az.Accounts." -ForegroundColor Green
Import-Module (Join-Path -Path $accountsModuleDirectory -ChildPath "Az.Accounts.psd1")
Get-ChildItem -Path $debugDirectory -Directory -Exclude "Az.Accounts" | Get-ChildItem -File -Filter "*.psd1" | ForEach-Object {
Write-Host "Importing $($_.FullName)." -ForegroundColor Green
Import-Module $_.FullName -Force
}
Write-Host "Successfully imported Azure PowerShell modules from artifacts/Debug" -ForegroundColor Green

$dataLocation = (Get-AzConfig -TestCoverageLocation).Value
if ([string]::IsNullOrWhiteSpace($dataLocation) -or !(Test-Path -LiteralPath $dataLocation -PathType Container)) {
$dataLocation = Join-Path -Path $env:USERPROFILE -ChildPath ".Azure"
}
Write-Host "Data location is `"$dataLocation`"" -ForegroundColor Cyan

$srcDir = Join-Path -Path $RepoLocation -ChildPath "src"
$liveScenarios = Get-ChildItem -Path $srcDir -Recurse -Directory -Filter "LiveTests" | Get-ChildItem -Filter "TestLiveScenarios.ps1" -File
$liveScenarios | ForEach-Object {
$moduleName = [regex]::match($_.FullName, "[\\|\/]src[\\|\/](?<ModuleName>[a-zA-Z]+)[\\|\/]").Groups["ModuleName"].Value
Import-Module "./tools/TestFx/Assert.ps1" -Force
Import-Module "./tools/TestFx/Live/LiveTestUtility.psd1" -ArgumentList $moduleName, "LocalDebug", "LocalDebug", "LocalDebug", $dataLocation -Force
. $_.FullName
}
8 changes: 4 additions & 4 deletions tools/TestFx/Live/InitializeLiveTestEnvironment.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -15,14 +15,14 @@ function InstallLiveTestDesiredPowerShell {
[string] $DesiredVersion
)

Write-Host "Validating desired PowerShell version ..."
Write-Host "Validating desired PowerShell version."

if ($isWinPSDesired) {
powershell -NoLogo -NoProfile -NonInteractive -Command "(Get-Variable -Name PSVersionTable).Value"
Write-Host "Desired Windows Powershell version $DesiredVersion has been installed."
Write-Host "##[section]Desired Windows Powershell version $DesiredVersion has been installed."
}
else {
Write-Host "Installing PowerShell version $DesiredVersion ..."
Write-Host "##[section]Installing PowerShell version $DesiredVersion."

dotnet --version
dotnet new tool-manifest --force
Expand All @@ -36,7 +36,7 @@ function InstallLiveTestDesiredPowerShell {

dotnet tool run pwsh -Version

Write-Host "Desired PowerShell version $DesiredVersion has been installed."
Write-Host "##[section]Desired PowerShell version $DesiredVersion has been installed."
}
}

Expand Down
3 changes: 3 additions & 0 deletions tools/TestFx/Live/InvokeLiveTestCITask.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,12 @@ param (

if ($UseWindowsPowerShell) {
$process = "powershell"
Write-Host "##[section]Using Windows PowerShell"
}
else {
$process = "dotnet tool run pwsh"
Write-Host "##[section]Using PowerShell"
dotnet tool run pwsh -NoLogo -NoProfile -NonInteractive -Version
}

switch ($PSCmdlet.ParameterSetName) {
Expand Down
179 changes: 73 additions & 106 deletions tools/TestFx/Live/InvokeLiveTestScenarios.ps1
Original file line number Diff line number Diff line change
@@ -1,125 +1,92 @@
param (
[Parameter(Mandatory)]
[ValidateScript({ Test-Path -LiteralPath $_ -PathType Container })]
[string] $RepoLocation,
[ValidateNotNullOrEmpty()]
[string] $BuildId,

[Parameter()]
[switch] $DebugMode
)
[Parameter(Mandatory)]
[ValidateNotNullOrEmpty()]
[string] $OSVersion,

dynamicparam {
if (!$DebugMode.IsPresent) {
$dynParams = [Management.Automation.RuntimeDefinedParameterDictionary]::new()
$paramBuildId = [Management.Automation.RuntimeDefinedParameter]::new(
"BuildId",
[string],
[Attribute[]]@(
[Parameter]@{ Mandatory = $true }
[ValidateNotNullOrEmpty]::new()
)
)
$paramOSVersion = [Management.Automation.RuntimeDefinedParameter]::new(
"OSVersion",
[string],
[Attribute[]]@(
[Parameter]@{ Mandatory = $true }
[ValidateNotNullOrEmpty]::new()
)
)
$paramPSVersion = [Management.Automation.RuntimeDefinedParameter]::new(
"PSVersion",
[string],
[Attribute[]]@(
[Parameter]@{ Mandatory = $true }
[ValidateNotNullOrEmpty]::new()
)
)
$dynParams.Add($paramBuildId.Name, $paramBuildId)
$dynParams.Add($paramOSVersion.Name, $paramOSVersion)
$dynParams.Add($paramPSVersion.Name, $paramPSVersion)
$dynParams
}
}
[Parameter(Mandatory)]
[ValidateNotNullOrEmpty()]
[string] $PSVersion,

[Parameter(Mandatory)]
[ValidateScript({ Test-Path -LiteralPath $_ -PathType Container })]
[string] $RepoLocation
)

process {
function FillLiveTestCoverageAdditionalInfo {
[CmdletBinding()]
param (
[Parameter(Mandatory)]
[ValidateScript({ Test-Path -LiteralPath $_ -PathType Container })]
[string] $TestCoverageDataLocation,
function FillLiveTestCoverageAdditionalInfo {
[CmdletBinding()]
param (
[Parameter(Mandatory)]
[ValidateScript({ Test-Path -LiteralPath $_ -PathType Container })]
[string] $TestCoverageDataLocation,

[Parameter(Mandatory)]
[ValidateNotNullOrEmpty()]
[string] $BuildId,
[Parameter(Mandatory)]
[ValidateNotNullOrEmpty()]
[string] $BuildId,

[Parameter(Mandatory)]
[ValidateNotNullOrEmpty()]
[string] $OSVersion,
[Parameter(Mandatory)]
[ValidateNotNullOrEmpty()]
[string] $OSVersion,

[Parameter(Mandatory)]
[ValidateNotNullOrEmpty()]
[string] $PSVersion,
[Parameter(Mandatory)]
[ValidateNotNullOrEmpty()]
[string] $PSVersion,

[Parameter(Mandatory)]
[ValidateNotNullOrEmpty()]
[string] $ModuleName
)
[Parameter(Mandatory)]
[ValidateNotNullOrEmpty()]
[string] $ModuleName
)

$testCoverageUtility = $PSScriptRoot | Split-Path | Join-Path -ChildPath "Coverage" | Join-Path -ChildPath "TestCoverageUtility.psd1"
Import-Module $testCoverageUtility
$module = Get-Module -Name "Az.$ModuleName" -ListAvailable
$moduleDetails = Get-TestCoverageModuleDetails -Module $module
$testCoverageUtility = $PSScriptRoot | Split-Path | Join-Path -ChildPath "Coverage" | Join-Path -ChildPath "TestCoverageUtility.psd1"
Import-Module $testCoverageUtility -Force
$module = Get-Module -Name "Az.$ModuleName" -ListAvailable
$moduleDetails = Get-TestCoverageModuleDetails -Module $module

$testCoverageRawCsv = Join-Path -Path $TestCoverageDataLocation -ChildPath "TestCoverageAnalysis" | Join-Path -ChildPath "Raw" | Join-Path -ChildPath "Az.$ModuleName.csv"
$testCoverageRawCsv = Join-Path -Path $TestCoverageDataLocation -ChildPath "TestCoverageAnalysis" | Join-Path -ChildPath "Raw" | Join-Path -ChildPath "Az.$ModuleName.csv"
if (Test-Path -LiteralPath $testCoverageRawCsv -PathType Leaf) {
(Import-Csv -LiteralPath $testCoverageRawCsv) |
Select-Object @{ Name = "BuildId"; Expression = { "$BuildId" } }, `
@{ Name = "OSVersion"; Expression = { "$OSVersion" } }, `
@{ Name = "PSVersion"; Expression = { "$PSVersion" } }, `
@{ Name = "Module"; Expression = { "$ModuleName" } }, `
"CommandName", @{ Name = "TotalCommands"; Expression = { "$($moduleDetails['TotalCommands'])" } }, `
"ParameterSetName", @{ Name = "TotalParameterSets"; Expression = { "$($moduleDetails['TotalParameterSets'])" } }, `
"Parameters", @{ Name = "TotalParameters"; Expression = { "$($moduleDetails['TotalParameters'])" } }, `
"SourceScript", "LineNumber", "StartDateTime", "EndDateTime", "IsSuccess" |
Select-Object `
@{ Name = "Source"; Expression = { "LiveTest" } }, `
@{ Name = "BuildId"; Expression = { "$BuildId" } }, `
@{ Name = "OSVersion"; Expression = { "$OSVersion" } }, `
@{ Name = "PSVersion"; Expression = { "$PSVersion" } }, `
@{ Name = "Module"; Expression = { "$ModuleName" } }, `
@{ Name = "CommandName"; Expression = { $_.CommandName } }, `
@{ Name = "TotalCommands"; Expression = { "$($moduleDetails['TotalCommands'])" } }, `
@{ Name = "ParameterSetName"; Expression = { $_.ParameterSetName } }, `
@{ Name = "TotalParameterSets"; Expression = { "$($moduleDetails['TotalParameterSets'])" } }, `
@{ Name = "Parameters"; Expression = { $_.Parameters } }, `
@{ Name = "TotalParameters"; Expression = { "$($moduleDetails['TotalParameters'])" } }, `
@{ Name = "SourceScript"; Expression = { $_.SourceScript } }, `
@{ Name = "LineNumber"; Expression = { $_.LineNumber } }, `
@{ Name = "StartDateTime"; Expression = { $_.StartDateTime } }, `
@{ Name = "EndDateTime"; Expression = { $_.EndDateTime } }, `
@{ Name = "IsSuccess"; Expression = { $_.IsSuccess } } |
Export-Csv -LiteralPath $testCoverageRawCsv -Encoding utf8 -NoTypeInformation -Force
}

if ($PSVersion -eq "latest") {
$PSVersion = (Get-Variable -Name PSVersionTable).Value.PSVersion.ToString()
else {
Write-Host "##[warning]No test coverage data was found. Either the test coverage is not enabled or all live test commands were failed for the module `"$ModuleName`"."
}
$DataLocation = (Get-AzConfig -TestCoverageLocation).Value

if ($DebugMode.IsPresent) {
$debugDirectory = Join-Path -Path $RepoLocation -ChildPath "artifacts" | Join-Path -ChildPath "Debug"
$accountsModuleDirectory = Join-Path -Path $debugDirectory -ChildPath "Az.Accounts"
Write-Host "Start to import Azure PowerShell modules from artifacts/Debug." -ForegroundColor Green
Write-Host "If you see module import issue, please restart the PowerShell host." -ForegroundColor Magenta
}

Write-Host "Importing Az.Accounts." -ForegroundColor Green
Import-Module (Join-Path -Path $accountsModuleDirectory -ChildPath "Az.Accounts.psd1")
Get-ChildItem -LiteralPath $debugDirectory -Directory -Exclude "Az.Accounts" | Get-ChildItem -File -Include "*.psd1" | ForEach-Object {
Write-Host "Importing $_.FullName." -ForegroundColor Green
Import-Module $_.FullName -Force
}
Write-Host "Successfully imported Azure PowerShell modules from artifacts/Debug" -ForegroundColor Green
if ($PSVersion -eq "latest") {
$PSVersion = (Get-Variable -Name PSVersionTable).Value.PSVersion.ToString()
}

$BuildId = "LocalDebug"
$OSVersion = "LocalDebug"
$PSVersion = "LocalDebug"
}
else {
$BuildId = $PSBoundParameters["BuildId"]
$OSVersion = $PSBoundParameters["OSVersion"]
$PSVersion = $PSBoundParameters["PSVersion"]
}
$dataLocation = (Get-AzConfig -TestCoverageLocation).Value
if ([string]::IsNullOrWhiteSpace($dataLocation) -or !(Test-Path -LiteralPath $dataLocation -PathType Container)) {
$dataLocation = Join-Path -Path $env:USERPROFILE -ChildPath ".Azure"
}

$srcDir = Join-Path -Path $RepoLocation -ChildPath "src"
$liveScenarios = Get-ChildItem -LiteralPath $srcDir -Recurse -Directory -Filter "LiveTests" | Get-ChildItem -Filter "TestLiveScenarios.ps1" -File
$liveScenarios | ForEach-Object {
$moduleName = [regex]::match($_.FullName, "[\\|\/]src[\\|\/](?<ModuleName>[a-zA-Z]+)[\\|\/]").Groups["ModuleName"].Value
Import-Module "./tools/TestFx/Assert.ps1" -Force
Import-Module "./tools/TestFx/Live/LiveTestUtility.psd1" -ArgumentList $moduleName, $BuildId, $OSVersion, $PSVersion, $DataLocation -Force
. $_
FillLiveTestCoverageAdditionalInfo -TestCoverageDataLocation $DataLocation -BuildId $BuildId -OSVersion $OSVersion -PSVersion $PSVersion -ModuleName $moduleName
}
$srcDir = Join-Path -Path $RepoLocation -ChildPath "src"
$liveScenarios = Get-ChildItem -Path $srcDir -Recurse -Directory -Filter "LiveTests" | Get-ChildItem -Filter "TestLiveScenarios.ps1" -File
$liveScenarios | ForEach-Object {
$moduleName = [regex]::match($_.FullName, "[\\|\/]src[\\|\/](?<ModuleName>[a-zA-Z]+)[\\|\/]").Groups["ModuleName"].Value
Import-Module "./tools/TestFx/Assert.ps1" -Force
Import-Module "./tools/TestFx/Live/LiveTestUtility.psd1" -ArgumentList $moduleName, $BuildId, $OSVersion, $PSVersion, $dataLocation -Force
. $_.FullName
FillLiveTestCoverageAdditionalInfo -TestCoverageDataLocation $dataLocation -BuildId $BuildId -OSVersion $OSVersion -PSVersion $PSVersion -ModuleName $moduleName
}
Loading

0 comments on commit 59b1e5a

Please sign in to comment.