Merged main

doc/analyze_check_versions.md

@@ -6,7 +6,7 @@ This table is to clarify the currently pinned version of tools we run in CI and
 | Tool | Current Version | Next Version | Next Version Merge Date |
 |------|-----------------|--------------|-------------------------|
 Pylint | 3.2.5 | 3.2.7 | 2025-01-13 |
-MyPy | 1.10.1 | TBD | 2025-01-13 |
-Pyright | 1.1.371 | TBD | 2025-01-13 |
+MyPy | 1.10.1 | 1.13.0 | 2025-01-13 |
+Pyright | 1.1.371 | 1.1.389 | 2025-01-13 |
 Sphinx | 7.3.7 | N/A | N/A |
 Black | 24.4.0 | N/A | N/A |

doc/eng_sys_checks.md

@@ -112,6 +112,8 @@ This is the most useful skip, but the following skip variables are also supporte
   - Omit checking that a package's dependencies are on PyPI before releasing.
 - `Skip.KeywordCheck`
   - Omit checking that a package's keywords are correctly formulated before releasing.
+- `Skip.Black`
+  - Omit checking `black` in the `analyze` job.
 
 ## The pyproject.toml
 
@@ -172,7 +174,7 @@ You can enable test logging in a pipeline by setting the queue time variable `PY
 
 `PYTEST_LOG_LEVEL=INFO`
 
-This also works locally with tox by setting the `PYTEST_LOG_LEVEL` environment variable. 
+This also works locally with tox by setting the `PYTEST_LOG_LEVEL` environment variable.
 
 Note that if you want DEBUG level logging with sensitive information unredacted in the test logs, then you still must pass `logging_enable=True` into the client(s) being used in tests.
 
@@ -237,17 +239,17 @@ fail if docstring are invalid, helping to ensure the resulting documentation wil
 
 #### Opt-in to formatting validation
 
-Make the following change to your projects `ci.yml`:
+Ensure that `black = true` is present within your `pyproject.toml`:
 
 ```yml
-extends:
-    template: ../../eng/pipelines/templates/stages/archetype-sdk-client.yml
-    parameters:
-        ...
-        ValidateFormatting: true
-        ...
+[tool.azure-sdk-build]
+...other checks enabled/disabled
+black = true
+...other checks enabled/disabled
 ```
 
+to opt into the black invocation.
+
 #### Running locally
 
 1. Go to package root directory.

eng/common/scripts/Cadl-Project-Generate.ps1

@@ -12,7 +12,7 @@ param (
 $ErrorActionPreference = "Stop"
 . $PSScriptRoot/Helpers/PSModule-Helpers.ps1
 . $PSScriptRoot/common.ps1
-Install-ModuleIfNotInstalled "powershell-yaml" "0.4.1" | Import-Module
+Install-ModuleIfNotInstalled "powershell-yaml" "0.4.7" | Import-Module
 
 function NpmInstallForProject([string]$workingDirectory) {
     Push-Location $workingDirectory

eng/common/scripts/Cadl-Project-Sync.ps1

@@ -9,7 +9,7 @@ param (
 
 $ErrorActionPreference = "Stop"
 . $PSScriptRoot/Helpers/PSModule-Helpers.ps1
-Install-ModuleIfNotInstalled "powershell-yaml" "0.4.1" | Import-Module
+Install-ModuleIfNotInstalled "powershell-yaml" "0.4.7" | Import-Module
 $sparseCheckoutFile = ".git/info/sparse-checkout"
 
 function AddSparseCheckoutPath([string]$subDirectory) {

eng/common/scripts/Helpers/ApiView-Helpers.ps1

@@ -74,6 +74,7 @@ function Process-ReviewStatusCode($statusCode, $packageName, $apiApprovalStatus,
 {
   $apiApproved = $false
   $apiApprovalDetails = "API Review is not approved for package $($packageName). Release pipeline will fail if API review is not approved for a GA version release. You can check http://aka.ms/azsdk/engsys/apireview/faq for more details on API Approval."
+  $apiApprovalDetails += " Once your API is approved, re-trigger the release pipeline again."
 
   $packageNameApproved = $false
   $packageNameApprovalDetails = ""
@@ -115,4 +116,4 @@ function Process-ReviewStatusCode($statusCode, $packageName, $apiApprovalStatus,
 
   $packageNameStatus.IsApproved = $packageNameApproved
   $packageNameStatus.Details = $packageNameApprovalDetails
-}
+}

eng/common/scripts/Helpers/Package-Helpers.ps1

@@ -80,7 +80,7 @@ function CompatibleConvertFrom-Yaml {
   $yqPresent = Get-Command 'yq' -ErrorAction SilentlyContinue
   if (-not $yqPresent) {
     . (Join-Path $PSScriptRoot PSModule-Helpers.ps1)
-    Install-ModuleIfNotInstalled -WhatIf:$false "powershell-yaml" "0.4.1" | Import-Module
+    Install-ModuleIfNotInstalled -WhatIf:$false "powershell-yaml" "0.4.7" | Import-Module
   }
 
   # Process the content (for example, you could convert from YAML here)

eng/common/scripts/TypeSpec-Project-Generate.ps1

@@ -13,7 +13,7 @@ $ErrorActionPreference = "Stop"
 . $PSScriptRoot/Helpers/PSModule-Helpers.ps1
 . $PSScriptRoot/Helpers/CommandInvocation-Helpers.ps1
 . $PSScriptRoot/common.ps1
-Install-ModuleIfNotInstalled "powershell-yaml" "0.4.1" | Import-Module
+Install-ModuleIfNotInstalled "powershell-yaml" "0.4.7" | Import-Module
 
 function NpmInstallForProject([string]$workingDirectory) {
     Push-Location $workingDirectory

eng/common/scripts/TypeSpec-Project-Process.ps1

@@ -14,7 +14,7 @@ param (
 
 . $PSScriptRoot/common.ps1
 . $PSScriptRoot/Helpers/PSModule-Helpers.ps1
-Install-ModuleIfNotInstalled "powershell-yaml" "0.4.1" | Import-Module
+Install-ModuleIfNotInstalled "powershell-yaml" "0.4.7" | Import-Module
 
 function CreateUpdate-TspLocation([System.Object]$tspConfig, [string]$TypeSpecProjectDirectory, [string]$CommitHash, [string]$repo, [string]$repoRoot, [ref]$isNewSdkProject) {
   $additionalDirs = @()
@@ -225,4 +225,4 @@ if ($SkipSyncAndGenerate -and !$isNewSdkProject) {
   }
 }
 
-return $sdkProjectFolder
+return $sdkProjectFolder

eng/common/scripts/TypeSpec-Project-Sync.ps1

@@ -11,7 +11,7 @@ param (
 
 $ErrorActionPreference = "Stop"
 . $PSScriptRoot/Helpers/PSModule-Helpers.ps1
-Install-ModuleIfNotInstalled "powershell-yaml" "0.4.1" | Import-Module
+Install-ModuleIfNotInstalled "powershell-yaml" "0.4.7" | Import-Module
 $sparseCheckoutFile = ".git/info/sparse-checkout"
 
 function AddSparseCheckoutPath([string]$subDirectory) {

eng/common/scripts/Update-DocsMsToc.ps1

@@ -55,7 +55,7 @@ param(
 . $PSScriptRoot/common.ps1
 . $PSScriptRoot/Helpers/PSModule-Helpers.ps1
 
-Install-ModuleIfNotInstalled "powershell-yaml" "0.4.1" | Import-Module
+Install-ModuleIfNotInstalled "powershell-yaml" "0.4.7" | Import-Module
 
 Set-StrictMode -Version 3
 

eng/common/scripts/Verify-ChangeLogs.ps1

@@ -7,30 +7,17 @@ Set-StrictMode -Version 3
 
 . (Join-Path $PSScriptRoot common.ps1)
 
-function ShouldVerifyChangeLog ($ServiceDirectory, $PackageName) {
-    $jsonCiYmlPath = Join-Path $ServiceDirectory "ci.yml"
-
-    if (Test-Path $jsonCiYmlPath)
-    {
-        $ciYml = Get-Content $jsonCiYmlPath -Raw | yq -o=json | ConvertFrom-Json -AsHashTable
-
-        if ($ciYml.extends -and $ciYml.extends.parameters -and $ciYml.extends.parameters.Artifacts) {
-            $packagesCheckingChangeLog = $ciYml.extends.parameters.Artifacts `
-                | Where-Object { -not ($_["skipVerifyChangelog"] -eq $true) } `
-                | Select-Object -ExpandProperty name
-            if ($packagesCheckingChangeLog -contains $PackageName)
-            {
-                return $true
-            } else {
-                return $false
-            }
-        }
+
+function ShouldVerifyChangeLog ($PkgArtifactDetails) {
+  if ($PkgArtifactDetails) {
+    if ($PkgArtifactDetails.PSObject.Properties["skipVerifyChangeLog"] -eq $true) {
+      return $false
     }
-}
 
-if (-not (Get-Command 'yq' -ErrorAction SilentlyContinue)) {
-    Write-Host "Error: 'yq' is not installed or not found in PATH. Please remedy this before running this script."
-    exit 1
+    return $true
+  }
+
+  return $false
 }
 
 # find which packages we need to confirm the changelog for
@@ -41,7 +28,7 @@ $allPassing = $true
 foreach($propertiesFile in $packageProperties) {
   $PackageProp = Get-Content -Path $propertiesFile | ConvertFrom-Json
 
-  if (-not (ShouldVerifyChangeLog -ServiceDirectory (Join-Path $RepoRoot "sdk" $PackageProp.ServiceDirectory) -PackageName $PackageProp.Name)) {
+  if (-not (ShouldVerifyChangeLog $PackageProp.ArtifactDetails)) {
         Write-Host "Skipping changelog verification for $($PackageProp.Name)"
         continue
   }

eng/common/scripts/Verify-Resource-Ref.ps1

@@ -1,5 +1,5 @@
 . (Join-Path $PSScriptRoot common.ps1)
-Install-Module -Name powershell-yaml -RequiredVersion 0.4.1 -Force -Scope CurrentUser
+Install-Module -Name powershell-yaml -RequiredVersion 0.4.7 -Force -Scope CurrentUser
 $ymlfiles = Get-ChildItem $RepoRoot -recurse | Where-Object {$_ -like '*.yml'}
 $affectedRepos = [System.Collections.ArrayList]::new()
 
@@ -48,4 +48,4 @@ if ($affectedRepos.Count -gt 0)
     exit 1
 }
 
-Write-Output "All repository resources in yaml files reference a valid tag"
+Write-Output "All repository resources in yaml files reference a valid tag"

eng/common/scripts/Verify-RestApiSpecLocation.ps1

@@ -50,7 +50,7 @@ if (-not $GitHubPat) {
 
 Write-Host "The spec used to release SDK should be from the main branch of Azure/azure-rest-api-specs repository."
 Write-Host "ServiceDir:$ServiceDirectory, PackageName:$PackageName, ArtifactLocation:$ArtifactLocation, PackageInfoDirectory:$PackageInfoDirectory."
-Install-ModuleIfNotInstalled "powershell-yaml" "0.4.1" | Import-Module
+Install-ModuleIfNotInstalled "powershell-yaml" "0.4.7" | Import-Module
 
 # This function is used to verify the 'require' and 'input-file' settings in autorest.md point to the main branch of Azure/azure-rest-api-specs repository
 # input-file may be: 

eng/common/scripts/job-matrix/job-matrix-functions.ps1

@@ -213,7 +213,7 @@ function GetMatrixConfigFromFile([String] $config) {
 }
 
 function GetMatrixConfigFromYaml([String] $yamlConfig) {
-    Install-ModuleIfNotInstalled "powershell-yaml" "0.4.1" | Import-Module
+    Install-ModuleIfNotInstalled "powershell-yaml" "0.4.7" | Import-Module
     # ConvertTo then from json is to make sure the nested values are in PSCustomObject
     [MatrixConfig]$config = ConvertFrom-Yaml $yamlConfig -Ordered | ConvertTo-Json -Depth 100 | ConvertFrom-Json
     return GetMatrixConfig $config

eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1

@@ -414,7 +414,7 @@ function CheckDependencies()
         }
     )
 
-    Install-ModuleIfNotInstalled "powershell-yaml" "0.4.1" | Import-Module
+    Install-ModuleIfNotInstalled "powershell-yaml" "0.4.7" | Import-Module
 
     $shouldError = $false
     foreach ($dep in $deps) {

eng/emitter-package.json

@@ -1,7 +1,7 @@
 {
   "main": "dist/src/index.js",
   "dependencies": {
-    "@azure-tools/typespec-python": "0.36.4"
+    "@azure-tools/typespec-python": "0.36.5"
   },
   "devDependencies": {
     "@azure-tools/typespec-autorest": "~0.48.0",

eng/pipelines/templates/jobs/ci.yml

@@ -49,9 +49,6 @@ parameters:
   - name: VerifyAutorest
     type: boolean
     default: false
-  - name: ValidateFormatting
-    type: boolean
-    default: false
   - name: UnsupportedToxEnvironments
     type: string
     default: ''
@@ -229,7 +226,6 @@ jobs:
         TestPipeline: ${{ parameters.TestPipeline }}
         Artifacts: ${{ parameters.Artifacts }}
         VerifyAutorest: ${{ parameters.VerifyAutorest }}
-        ValidateFormatting: ${{ parameters.ValidateFormatting }}
         GenerateApiReviewForManualOnly: ${{ parameters.GenerateApiReviewForManualOnly }}
 
   - template: /eng/common/pipelines/templates/jobs/generate-job-matrix.yml
@@ -244,20 +240,26 @@ jobs:
       MatrixConfigs: ${{ parameters.MatrixConfigs }}
       MatrixFilters: ${{ parameters.MatrixFilters }}
       MatrixReplace: ${{ parameters.MatrixReplace }}
-      PreGenerationSteps:
-
-        - template: /eng/pipelines/templates/steps/download-package-artifacts.yml
-
-        - task: Powershell@2
-          inputs:
-            pwsh: true
-            filePath: eng/scripts/distribute-packages-to-matrix.ps1
-            arguments: >-
-              -PackageInfoFolder "$(Build.ArtifactStagingDirectory)/PackageInfo"
-              -PlatformMatrix "${{ parameters.MatrixConfigs[0].Path }}"
-          displayName: 'Distribute Packages to Matrix'
-          condition: and(eq(variables['Build.Reason'], 'PullRequest'), eq('${{ parameters.ServiceDirectory }}','auto'))
-
+      ${{ if eq(parameters.ServiceDirectory, 'auto') }}:
+        SparseCheckoutPaths:
+          - "**/setup.py"
+          - "**/pyproject.toml"
+          - "tools/"
+          - "**/ci*.yml"
+          - "**/_version.py"
+          - "**/version.py"
+          - "**/README.md"
+          - "**/CHANGELOG.md"
+          - "**/*matrix*.json"
+        EnablePRGeneration: true
+        PRMatrixSetting: "TargetingString"
+        PreGenerationSteps:
+          - task: UsePythonVersion@0
+            inputs:
+              versionSpec: '3.12'
+          - template: /eng/common/pipelines/templates/steps/save-package-properties.yml
+            parameters:
+              ServiceDirectory: ${{parameters.ServiceDirectory}}
       CloudConfig:
         Cloud: Public
       AdditionalParameters:

eng/pipelines/templates/stages/archetype-sdk-client.yml

@@ -63,9 +63,6 @@ parameters:
   - name: VerifyAutorest
     type: boolean
     default: false
-  - name: ValidateFormatting
-    type: boolean
-    default: false
   - name: TestProxy
     type: boolean
     default: true
@@ -107,7 +104,6 @@ extends:
               MatrixFilters: ${{ parameters.MatrixFilters }}
               MatrixReplace: ${{ parameters.MatrixReplace }}
               VerifyAutorest: ${{ parameters.VerifyAutorest }}
-              ValidateFormatting: ${{ parameters.ValidateFormatting }}
               TestProxy: ${{ parameters.TestProxy }}
               GenerateApiReviewForManualOnly: ${{ parameters.GenerateApiReviewForManualOnly }}
 

eng/pipelines/templates/stages/platform-matrix.json

@@ -17,8 +17,8 @@
     {
       "MacTestConfig": {
         "macos311": {
-          "OSVmImage": "env:LINUXVMIMAGE",
-          "Pool": "env:LINUXPOOL",
+          "OSVmImage": "env:MACVMIMAGE",
+          "Pool": "env:MACPOOL",
           "PythonVersion": "3.11",
           "CoverageArg": "--disablecov",
           "TestSamples": "false"

eng/pipelines/templates/steps/analyze.yml

@@ -5,7 +5,6 @@ parameters:
   Artifacts: []
   TestPipeline: false
   VerifyAutorest: false
-  ValidateFormatting: false
   GenerateApiReviewForManualOnly: false
 
 # Please use `$(TargetingString)` to refer to the python packages glob string. This variable is set from resolve-package-targeting.yml.
@@ -110,11 +109,9 @@ steps:
       ServiceDirectory: ${{ parameters.ServiceDirectory }}
       AdditionalTestArgs: ${{ parameters.AdditionalTestArgs }}
 
-  - ${{ if parameters.ValidateFormatting }}:
-    - template: run_black.yml
-      parameters:
-        ServiceDirectory: ${{ parameters.ServiceDirectory }}
-        ValidateFormatting: ${{ parameters.ValidateFormatting }}
+  - template: run_black.yml
+    parameters:
+      ServiceDirectory: ${{ parameters.ServiceDirectory }}
 
   - task: PythonScript@0
     displayName: 'Run Keyword Validation Check'