Skip to content

Commit

Permalink
Merge branch 'main' of https://github.com/Azure/azure-sdk-tools into …
Browse files Browse the repository at this point in the history
…toc_mgmt
  • Loading branch information
sima-zhu committed Jul 5, 2022
2 parents 2343881 + 160c203 commit d17fa9d
Show file tree
Hide file tree
Showing 178 changed files with 18,147 additions and 1,289 deletions.
2 changes: 1 addition & 1 deletion .github/CODEOWNERS
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
################

# Git Hub integration and bot rules
/.github/ @AlexGhiondea @jsquire
/.github/ @jsquire @ronniegeraghty

###########
# Eng Sys
Expand Down
20 changes: 12 additions & 8 deletions eng/common/TestResources/New-TestResources.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,10 @@ param (
[ValidateSet('AzureCloud', 'AzureUSGovernment', 'AzureChinaCloud', 'Dogfood')]
[string] $Environment = 'AzureCloud',

[Parameter()]
[ValidateSet('test', 'perf')]
[string] $ResourceType = 'test',

[Parameter()]
[hashtable] $ArmTemplateParameters,

Expand Down Expand Up @@ -223,7 +227,7 @@ function BuildBicepFile([System.IO.FileSystemInfo] $file)
}

$tmp = $env:TEMP ? $env:TEMP : [System.IO.Path]::GetTempPath()
$templateFilePath = Join-Path $tmp "test-resources.$(New-Guid).compiled.json"
$templateFilePath = Join-Path $tmp "$ResourceType-resources.$(New-Guid).compiled.json"

# Az can deploy bicep files natively, but by compiling here it becomes easier to parse the
# outputted json for mismatched parameter declarations.
Expand Down Expand Up @@ -349,7 +353,7 @@ try {
$root = [System.IO.Path]::Combine($repositoryRoot, "sdk", $ServiceDirectory) | Resolve-Path
$templateFiles = @()

'test-resources.json', 'test-resources.bicep' | ForEach-Object {
"$ResourceType-resources.json", "$ResourceType-resources.bicep" | ForEach-Object {
Write-Verbose "Checking for '$_' files under '$root'"
Get-ChildItem -Path $root -Filter "$_" -Recurse | ForEach-Object {
Write-Verbose "Found template '$($_.FullName)'"
Expand Down Expand Up @@ -397,7 +401,7 @@ try {
# string.
if (!$Location) {
$Location = @{
'AzureCloud' = 'westus2';
'AzureCloud' = 'westus';
'AzureUSGovernment' = 'usgovvirginia';
'AzureChinaCloud' = 'chinaeast2';
'Dogfood' = 'westus'
Expand Down Expand Up @@ -586,9 +590,9 @@ try {
# Service principals in the Microsoft AAD tenant must end with an @microsoft.com domain; those in other tenants
# are not permitted to do so. Format the non-MS name so there's an assocation with the Azure SDK.
if ($TenantId -eq '72f988bf-86f1-41af-91ab-2d7cd011db47') {
$displayName = "test-resources-$($baseName)$suffix.microsoft.com"
$displayName = "$ResourceType-resources-$($baseName)$suffix.microsoft.com"
} else {
$displayName = "$($baseName)$suffix.test-resources.azure.sdk"
$displayName = "$($baseName)$suffix.$ResourceType-resources.azure.sdk"
}

$servicePrincipalWrapper = NewServicePrincipalWrapper -subscription $SubscriptionId -resourceGroup $ResourceGroupName -displayName $DisplayName
Expand Down Expand Up @@ -705,7 +709,7 @@ try {
}
}

$preDeploymentScript = $templateFile.originalFilePath | Split-Path | Join-Path -ChildPath 'test-resources-pre.ps1'
$preDeploymentScript = $templateFile.originalFilePath | Split-Path | Join-Path -ChildPath "$ResourceType-resources-pre.ps1"
if (Test-Path $preDeploymentScript) {
Log "Invoking pre-deployment script '$preDeploymentScript'"
&$preDeploymentScript -ResourceGroupName $ResourceGroupName @PSBoundParameters
Expand Down Expand Up @@ -745,7 +749,7 @@ try {

$deploymentOutputs = SetDeploymentOutputs $serviceName $context $deployment $templateFile

$postDeploymentScript = $templateFile.originalFilePath | Split-Path | Join-Path -ChildPath 'test-resources-post.ps1'
$postDeploymentScript = $templateFile.originalFilePath | Split-Path | Join-Path -ChildPath "$ResourceType-resources-post.ps1"
if (Test-Path $postDeploymentScript) {
Log "Invoking post-deployment script '$postDeploymentScript'"
&$postDeploymentScript -ResourceGroupName $ResourceGroupName -DeploymentOutputs $deploymentOutputs @PSBoundParameters
Expand Down Expand Up @@ -903,7 +907,7 @@ This is used for CI automation.
Optional location where resources should be created. If left empty, the default
is based on the cloud to which the template is being deployed:
* AzureCloud -> 'westus2'
* AzureCloud -> 'westus'
* AzureUSGovernment -> 'usgovvirginia'
* AzureChinaCloud -> 'chinaeast2'
* Dogfood -> 'westus'
Expand Down
2 changes: 1 addition & 1 deletion eng/common/TestResources/New-TestResources.ps1.md
Original file line number Diff line number Diff line change
Expand Up @@ -422,7 +422,7 @@ Optional location where resources should be created.
If left empty, the default
is based on the cloud to which the template is being deployed:
* AzureCloud -\> 'westus2'
* AzureCloud -\> 'westus'
* AzureUSGovernment -\> 'usgovvirginia'
* AzureChinaCloud -\> 'chinaeast2'
* Dogfood -\> 'westus'
Expand Down
2 changes: 1 addition & 1 deletion eng/common/TestResources/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ ${env:KEYVAULT_CLIENT_ID} = '<<secret>>'
${env:KEYVAULT_CLIENT_SECRET} = '<<secret>>'
${env:KEYVAULT_SUBSCRIPTION_ID} = 'YOUR SUBSCRIPTION ID'
${env:KEYVAULT_RESOURCE_GROUP} = 'rg-myusername'
${env:KEYVAULT_LOCATION} = 'westus2'
${env:KEYVAULT_LOCATION} = 'westus'
${env:KEYVAULT_SKU} = 'premium'
${env:AZURE_KEYVAULT_URL} = '<<url>>'
```
Expand Down
8 changes: 6 additions & 2 deletions eng/common/TestResources/Remove-TestResources.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,10 @@ param (
[Parameter(ParameterSetName = 'ResourceGroup+Provisioner')]
[switch] $CI,

[Parameter()]
[ValidateSet('test', 'perf')]
[string] $ResourceType = 'test',

[Parameter()]
[switch] $Force,

Expand Down Expand Up @@ -198,7 +202,7 @@ Log "Selected subscription '$subscriptionName'"

if ($ServiceDirectory) {
$root = [System.IO.Path]::Combine("$PSScriptRoot/../../../sdk", $ServiceDirectory) | Resolve-Path
$preRemovalScript = Join-Path -Path $root -ChildPath 'remove-test-resources-pre.ps1'
$preRemovalScript = Join-Path -Path $root -ChildPath "remove-$ResourceType-resources-pre.ps1"
if (Test-Path $preRemovalScript) {
Log "Invoking pre resource removal script '$preRemovalScript'"

Expand All @@ -210,7 +214,7 @@ if ($ServiceDirectory) {
}

# Make sure environment files from New-TestResources -OutFile are removed.
Get-ChildItem -Path $root -Filter test-resources.json.env -Recurse | Remove-Item -Force:$Force
Get-ChildItem -Path $root -Filter "$ResourceType-resources.json.env" -Recurse | Remove-Item -Force:$Force
}

$verifyDeleteScript = {
Expand Down
2 changes: 2 additions & 0 deletions eng/common/TestResources/deploy-test-resources.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ parameters:
DeleteAfterHours: 8
Location: ''
SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources)
ResourceType: test

# SubscriptionConfiguration will be splatted into the parameters of the test
# resources script. It should be JSON in the form:
Expand Down Expand Up @@ -48,6 +49,7 @@ steps:
# pass those in via the ArmTemplateParameters flag, and handle any
# additional parameters from the pipelines via AdditionalParameters
eng/common/TestResources/New-TestResources.ps1 `
-ResourceType '${{ parameters.ResourceType }}' `
-ServiceDirectory '${{ parameters.ServiceDirectory }}' `
-Location '${{ parameters.Location }}' `
-DeleteAfterHours '${{ parameters.DeleteAfterHours }}' `
Expand Down
2 changes: 2 additions & 0 deletions eng/common/TestResources/remove-test-resources.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
parameters:
ServiceDirectory: ''
SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources)
ResourceType: test

# SubscriptionConfiguration will be splat into the parameters of the test
# resources script. It should be JSON in the form:
Expand All @@ -29,6 +30,7 @@ steps:
eng/common/TestResources/Remove-TestResources.ps1 `
@subscriptionConfiguration `
-ResourceType '${{ parameters.ResourceType }}' `
-ServiceDirectory "${{ parameters.ServiceDirectory }}" `
-CI `
-Force `
Expand Down
12 changes: 0 additions & 12 deletions eng/common/pipelines/templates/steps/cosmos-emulator.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,15 +22,3 @@ steps:
-Stage "Launch"
pwsh: true
displayName: Launch Public Cosmos DB Emulator
continueOnError: true

- task: Powershell@2
inputs:
filePath: $(Build.SourcesDirectory)/eng/common/scripts/Cosmos-Emulator.ps1
arguments: >
-EmulatorMsiUrl "${{ parameters.EmulatorMsiUrl }}"
-StartParameters "${{ parameters.StartParameters }}"
-Stage "Launch"
pwsh: true
displayName: Retry Launch of Public Cosmos DB Emulator
condition: failed()
24 changes: 18 additions & 6 deletions eng/common/pipelines/templates/steps/verify-readme.yml
Original file line number Diff line number Diff line change
@@ -1,17 +1,29 @@
parameters:
ScanPath: $(Build.SourcesDirectory)
RepoRoot: $(Build.SourcesDirectory)
SettingsPath: '$(Build.SourcesDirectory)/eng/.docsettings.yml'
DocWardenVersion : '0.7.2'
- name: ScanPath
type: string
default: ''
# Where ScanPath takes a single path, ScanPaths takes a comma separated list of paths to scan
- name: ScanPaths
type: string
default: ''
- name: RepoRoot
type: string
default: $(Build.SourcesDirectory)
- name: SettingsPath
type: string
default: '$(Build.SourcesDirectory)/eng/.docsettings.yml'
- name: DocWardenVersion
type: string
default: ''

steps:
- task: PowerShell@2
displayName: "Verify Readmes"
inputs:
filePath: "eng/common/scripts/Verify-Readme.ps1"
arguments: >
-DocWardenVersion ${{ parameters.DocWardenVersion }}
-ScanPath ${{ parameters.ScanPath }}
-DocWardenVersion '${{ parameters.DocWardenVersion }}'
-ScanPaths '${{ coalesce(parameters.ScanPath, parameters.ScanPaths) }}'
-RepoRoot ${{ parameters.RepoRoot }}
-SettingsPath ${{ parameters.SettingsPath }}
pwsh: true
18 changes: 14 additions & 4 deletions eng/common/pipelines/templates/steps/verify-samples.yml
Original file line number Diff line number Diff line change
@@ -1,15 +1,25 @@
parameters:
- name: ServiceDirectory
type: string
default: not-specified
default: ''
- name: ServiceDirectories
type: string
default: ''
- name: ScriptDirectory
type: string
default: eng/common/scripts

steps:
- pwsh: |
# If the last path segment is an absolute path it will be used entirely.
$root = [System.IO.Path]::Combine('$(Build.SourcesDirectory)', 'sdk', '${{ parameters.ServiceDirectory }}')
Get-ChildItem $root -Filter *.md -Recurse | ${{ parameters.ScriptDirectory }}/Test-SampleMetadata.ps1 -AllowParentProducts
# Take whichever parameter has been set. If set, ServiceDirectory will be a single path or
# ServiceDirectories will be a comma separated list.
$ServiceDirectories = '${{ coalesce(parameters.ServiceDirectory, parameters.ServiceDirectories) }}'
$ScanPaths = @()
foreach ($ServiceDirectory in $ServiceDirectories.Split(',')) {
$ScanPath = [System.IO.Path]::Combine('$(Build.SourcesDirectory)', 'sdk', $ServiceDirectory)
Write-Host "Adding $ScanPath to the scanned paths"
$ScanPaths += $ScanPath
}
Get-ChildItem $ScanPaths -Filter *.md -Recurse | ${{ parameters.ScriptDirectory }}/Test-SampleMetadata.ps1 -AllowParentProducts
displayName: Verify sample metadata
workingDirectory: $(Build.SourcesDirectory)
11 changes: 9 additions & 2 deletions eng/common/scripts/Helpers/PSModule-Helpers.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,15 @@ function Update-PSModulePath()
}

# If we want to use another default repository other then PSGallery we can update the default parameters
function Install-ModuleIfNotInstalled($moduleName, $version, $repositoryUrl = $DefaultPSRepositoryUrl)
function Install-ModuleIfNotInstalled()
{
[CmdletBinding(SupportsShouldProcess = $true)]
param(
[string]$moduleName,
[string]$version,
[string]$repositoryUrl = $DefaultPSRepositoryUrl
)

# Check installed modules
$modules = (Get-Module -ListAvailable $moduleName)
if ($version -as [Version]) {
Expand Down Expand Up @@ -94,4 +101,4 @@ function Install-ModuleIfNotInstalled($moduleName, $version, $repositoryUrl = $D
return $modules[0]
}

Update-PSModulePath
Update-PSModulePath
101 changes: 91 additions & 10 deletions eng/common/scripts/Verify-Readme.ps1
Original file line number Diff line number Diff line change
@@ -1,24 +1,105 @@
# Wrapper Script for Readme Verification
[CmdletBinding()]
param (
[Parameter(Mandatory = $true)]
[Parameter(Mandatory = $false)]
[string]$DocWardenVersion,
[Parameter(Mandatory = $true)]
[string]$ScanPath,
[string]$RepoRoot,
[string]$ScanPaths,
[Parameter(Mandatory = $true)]
[string]$SettingsPath
)
. (Join-Path $PSScriptRoot common.ps1)
$DefaultDocWardenVersion = "0.7.2"
$script:FoundError = $false

function Test-Readme-Files {
param(
[string]$SettingsPath,
[string]$ScanPath,
[string]$RepoRoot)

Write-Host "Scanning..."

if ($RepoRoot)
{
Write-Host "ward scan -d $ScanPath -u $RepoRoot -c $SettingsPath"
ward scan -d $ScanPath -u $RepoRoot -c $SettingsPath
}
else
{
Write-Host "ward scan -d $ScanPath -c $SettingsPath"
ward scan -d $ScanPath -c $SettingsPath
}
# ward scan is what returns the non-zero exit code on failure.
# Since it's being called from a function, that error needs to
# be propagated back so the script can exit appropriately
if ($LASTEXITCODE -ne 0) {
$script:FoundError = $true
}
}

# Verify all of the inputs before running anything
if ([String]::IsNullOrWhiteSpace($DocWardenVersion)) {
$DocWardenVersion = $DefaultDocWardenVersion
}

# verify the doc settings file exists
if (!(Test-Path -Path $SettingsPath -PathType leaf)) {
LogError "Setting file, $SettingsPath, does not exist"
$script:FoundError = $true
}

$scanPathsArray = @()

# Verify that either ScanPath or ScanPaths were set but not both or neither
if ([String]::IsNullOrWhiteSpace($ScanPaths)) {
LogError "ScanPaths cannot be empty."
} else {
$scanPathsArray = $ScanPaths.Split(',')
foreach ($path in $scanPathsArray) {
if (!(Test-Path -Path $path -PathType Container)) {
LogError "path, $path, doesn't exist or isn't a directory"
$script:FoundError = $true
}
}
}

# Exit out now if there were any argument issues
if ($script:FoundError) {
LogError "There were argument failures, please see above for specifics"
exit 1
}

# Echo back the settings
Write-Host "DocWardenVersion=$DocWardenVersion"
Write-Host "SettingsPath=$SettingsPath"

if ($RepoRoot) {
Write-Host "RepoRoot=$RepoRoot"
}

Write-Host "ScanPath=$ScanPaths"

Write-Host "Installing setup tools and DocWarden"
Write-Host "pip install setuptools wheel --quiet"
pip install setuptools wheel --quiet
if ($LASTEXITCODE -ne 0) {
LogError "pip install setuptools wheel --quiet failed with exit code $LASTEXITCODE"
exit 1
}
Write-Host "pip install doc-warden==$DocWardenVersion --quiet"
pip install doc-warden==$DocWardenVersion --quiet

if ($RepoRoot)
{
ward scan -d $ScanPath -u $RepoRoot -c $SettingsPath
if ($LASTEXITCODE -ne 0) {
LogError "pip install doc-warden==$DocWardenVersion --quiet failed with exit code $LASTEXITCODE"
exit 1
}
else
{
ward scan -d $ScanPath -c $SettingsPath

# Finally, do the scanning
foreach ($path in $scanPathsArray) {
Test-Readme-Files $SettingsPath $path $RepoRoot
}

if ($script:FoundError) {
LogError "There were README verification failures, scroll up to see the issue(s)"
exit 1
}
2 changes: 1 addition & 1 deletion eng/common/scripts/Verify-Resource-Ref.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ foreach ($file in $ymlfiles)
$ymlContent = Get-Content $file.FullName -Raw
$ymlObject = ConvertFrom-Yaml $ymlContent -Ordered

if ($ymlObject.Contains("resources"))
if ($ymlObject -and ($ymlObject.Contains("resources")))
{
if ($ymlObject["resources"]["repositories"])
{
Expand Down
Loading

0 comments on commit d17fa9d

Please sign in to comment.