Merge branch 'master' of https://github.com/Azure/azure-sdk-for-python …

…into enum-meta

* 'master' of https://github.com/Azure/azure-sdk-for-python:
  bump six dependencies in some libraries (#16496)
  call on_error if timeout in flush (#16485)
  Sync eng/common directory with azure-sdk-tools for PR 1365 (#16505)
  Fix min dependency tests - update azure core (#16504)
  Sync eng/common directory with azure-sdk-tools for PR 1364 (#16503)
  Ma arch feedback (#16502)
  Adding a new limitation to the README file. (#16475)
  [Blob][Datalake] STG76 Preview (#16349)
  append code coverage over each other (#16202)
  Arch preview feedback (#16441)
  Support CAE in azure-identity (#16323)
  [EventHubs] Support for Custom endpoint adddress and custom certificate  (#16295)
  [Communication] - Phone Number Management - Added support for AAD auth (#16075)
  fix live tests (#16495)

.coveragerc

@@ -6,6 +6,11 @@ omit =
 	*/test*
 	env*
 
+[paths]
+source =
+	sdk/
+	**/sdk
+
 [report]
 exclude_lines =
 	pragma: no cover

eng/ci_tools.txt

@@ -3,7 +3,7 @@ cryptography==3.1
 setuptools==44.1.0; python_version == '2.7'
 setuptools==46.4.0; python_version >= '3.5'
 virtualenv==20.0.23
-wheel==0.34.2 
+wheel==0.34.2
 Jinja2==2.11.2
 packaging==20.4
 tox==3.15.0
@@ -18,6 +18,7 @@ coverage==4.5.4
 codecov==2.1.0
 beautifulsoup4==4.9.1
 pkginfo==1.5.0.1
+pip==20.2
 
 # locking packages defined as deps from azure-sdk-tools or azure-devtools
 pytoml==0.1.21

eng/common/pipelines/templates/steps/cosmos-emulator.yml

@@ -9,8 +9,87 @@ steps:
       Write-Host "Target Dir: $targetDir"
       msiexec /a ${{ parameters.EmulatorMsiUrl }} TARGETDIR=$targetDir /qn | wait-process
     displayName: Download and Extract Public Cosmos DB Emulator
+  - powershell: |
+      Write-Host "Deleting Cosmos DB Emulator data"
+      if (Test-Path $Env:LOCALAPPDATA\CosmosDbEmulator) { Remove-Item -Recurse -Force $Env:LOCALAPPDATA\CosmosDbEmulator }
+    displayName: Delete Cosmos DB Emulator data
+  - powershell: |
+      Write-Host "Getting Cosmos DB Emulator Version"
+      $ProductName = "Azure Cosmos DB Emulator"
+      $Emulator = (Join-Path $env:temp (Join-Path $ProductName "Microsoft.Azure.Cosmos.Emulator.exe"))
+      $fileVersion = Get-ChildItem $Emulator
+      Write-Host $Emulator $fileVersion.VersionInfo
+    displayName: Get Cosmos DB Emulator Version
   - powershell: |
       Write-Host "Launching Cosmos DB Emulator"
-      Import-Module "$env:temp\Azure Cosmos DB Emulator\PSModules\Microsoft.Azure.CosmosDB.Emulator"
-      Start-CosmosDbEmulator -NoUI ${{ parameters.StartParameters }}
+      $ProductName = "Azure Cosmos DB Emulator"
+      $Emulator = (Join-Path $env:temp (Join-Path $ProductName "Microsoft.Azure.Cosmos.Emulator.exe"))
+      if (!(Test-Path $Emulator)) {
+        Write-Error "The emulator is not installed where expected at '$Emulator'"
+        return
+      }
+      $process = Start-Process $Emulator -ArgumentList "/getstatus" -PassThru -Wait
+      switch ($process.ExitCode) {
+        1 {
+          Write-Host "The emulator is already starting"
+          return
+        }
+        2 {
+          Write-Host "The emulator is already running"
+          return
+        }
+        3 {
+          Write-Host "The emulator is stopped"
+        }
+        default {
+          Write-Host "Unrecognized exit code $process.ExitCode"
+          return
+        }
+      }
+      $argumentList = ""
+      if (-not [string]::IsNullOrEmpty("${{ parameters.StartParameters }}")) {
+          $argumentList += , "${{ parameters.StartParameters }}"
+      } else {
+        # Use the default params if none provided
+        $argumentList = "/noexplorer /noui /enablepreview /disableratelimiting /enableaadauthentication"
+      }
+      Write-Host "Starting emulator process: $Emulator $argumentList"
+      $process=Start-Process $Emulator -ArgumentList $argumentList -ErrorAction Stop -PassThru
+      Write-Host "Emulator process started: $($process.Name), $($process.FileVersion)"
+      $Timeout = 600
+      $result="NotYetStarted"
+      $complete = if ($Timeout -gt 0) {
+        $start = [DateTimeOffset]::Now
+        $stop = $start.AddSeconds($Timeout)
+        {
+          $result -eq "Running" -or [DateTimeOffset]::Now -ge $stop
+        }
+      }
+      else {
+        {
+          $result -eq "Running"
+        }
+      }
+      do {
+        $process = Start-Process $Emulator -ArgumentList "/getstatus" -PassThru -Wait
+        switch ($process.ExitCode) {
+          1 {
+            Write-Host "The emulator is starting"
+          }
+          2 {
+            Write-Host "The emulator is running"
+            $result="Running"
+            return
+          }
+          3 {
+            Write-Host "The emulator is stopped"
+          }
+          default {
+            Write-Host "Unrecognized exit code $process.ExitCode"
+          }
+        }
+        Start-Sleep -Seconds 5
+      }
+      until ($complete.Invoke())
+      Write-Error "The emulator failed to reach Running status within ${Timeout} seconds"
     displayName: Start Cosmos DB Emulator

eng/common/scripts/Create-APIReview.ps1

@@ -8,7 +8,8 @@ Param (
   [string] $APIKey,
   [Parameter(Mandatory=$True)]
   [string] $APILabel,
-  [string] $PackageName = ""
+  [string] $PackageName,
+  [string] $ConfigFileDir = ""
 )
 
 
@@ -82,17 +83,19 @@ else
 }
 
 $FoundFailure = $False
-$pkgInfoPath = Join-Path -Path $ArtifactPath "PackageInfo"
+if (-not $ConfigFileDir)
+{
+    $ConfigFileDir = Join-Path -Path $ArtifactPath "PackageInfo"
+}
 foreach ($pkgName in $responses.Keys)
 {    
     $respCode = $responses[$pkgName]
     if ($respCode -ne '200')
     {
-        $pkgPropPath = Join-Path -Path $pkgInfoPath ($PackageName + ".json")
+        $pkgPropPath = Join-Path -Path $ConfigFileDir "$PackageName.json"
         if (-Not (Test-Path $pkgPropPath))
         {
             Write-Host " Package property file path $($pkgPropPath) is invalid."
-            $FoundFailure = $True
         }
         else
         {

eng/common/scripts/Save-Package-Properties.ps1

@@ -13,7 +13,7 @@ if ($allPackageProperties)
     New-Item -ItemType Directory -Force -Path $outDirectory
     foreach($pkg in $allPackageProperties)
     {
-        if ($pkg.IsNewSDK)
+        if ($pkg.IsNewSdk)
         {
             Write-Host "Package Name: $($pkg.Name)"
             Write-Host "Package Version: $($pkg.Version)"

eng/pipelines/templates/jobs/archetype-sdk-client.yml

@@ -39,31 +39,31 @@ parameters:
         Pool: $(LinuxPool)
         OSVmImage:
         PythonVersion: '2.7'
-        CoverageArg: ''
+        CoverageArg: '--disablecov'
         RunForPR: true
       Linux_Python35:
         Pool: $(LinuxPool)
         OSVmImage:
         PythonVersion: '3.5'
-        CoverageArg: ''
+        CoverageArg: '--disablecov'
         RunForPR: false
       Linux_Python38:
         Pool: $(LinuxPool)
         OSVmImage:
         PythonVersion: '3.8'
-        CoverageArg: ''
+        CoverageArg: '--disablecov'
         RunForPR: true
       Windows_Python35:
         Pool: $(WindowsPool)
         OSVmImage:
         PythonVersion: '3.5'
-        CoverageArg: ''
+        CoverageArg: '--disablecov'
         RunForPR: true
       MacOS_Python27:
         Pool:
         OSVmImage: 'macOS-10.15'
         PythonVersion: '2.7'
-        CoverageArg: ''
+        CoverageArg: '--disablecov'
         RunForPR: false
       Linux_pypy3:
         Pool: $(LinuxPool)
@@ -100,7 +100,7 @@ jobs:
 
     steps:
     - template: ../steps/build-artifacts.yml
-      parameters: 
+      parameters:
         ServiceDirectory: ${{ parameters.ServiceDirectory }}
         BuildTargetingString: ${{ parameters.BuildTargetingString }}
         BeforePublishSteps: ${{ parameters.BeforePublishSteps }}
@@ -133,7 +133,7 @@ jobs:
         CheckLinkGuidance: $true
 
     - template: ../steps/analyze.yml
-      parameters: 
+      parameters:
         ServiceDirectory: ${{ parameters.ServiceDirectory }}
         BuildTargetingString: ${{ parameters.BuildTargetingString }}
         TestMarkArgument: ${{ parameters.TestMarkArgument }}
@@ -168,7 +168,7 @@ jobs:
           ${{ if or(eq(matrixEntry.value.RunForPR, 'true'), ne(variables['Build.Reason'], 'PullRequest')) }}:
             ${{ matrixEntry.key }}:
               ${{ insert }}: ${{ matrixEntry.value }}
-        
+
     pool:
       name: $[coalesce(variables['Pool'], '')]
       vmImage: $[coalesce(variables['OSVmImage'], '')]
@@ -195,7 +195,7 @@ jobs:
         {
           $toxenvvar = '$(Run.ToxCustomEnvs)'
         }
-        
+
         echo "##vso[task.setvariable variable=toxenv]$toxenvvar"
       displayName: "Set Tox Environment"
 
@@ -211,10 +211,10 @@ jobs:
         ToxTestEnv: $(toxenv)
         ToxEnvParallel: ${{ parameters.ToxEnvParallel }}
         InjectedPackages: $(InjectedPackages)
-        BeforeTestSteps: 
+        BeforeTestSteps:
           - task: DownloadPipelineArtifact@0
             inputs:
-              artifactName: 'artifacts' 
+              artifactName: 'artifacts'
               targetPath: $(Build.ArtifactStagingDirectory)
 
           - template: ../steps/set-dev-build.yml
@@ -237,6 +237,6 @@ jobs:
 
     steps:
     - template: ../steps/test_regression.yml
-      parameters: 
+      parameters:
         ServiceDirectory: ${{ parameters.ServiceDirectory }}
         BuildTargetingString: ${{ parameters.BuildTargetingString }}

eng/pipelines/templates/jobs/archetype-sdk-tests.yml

@@ -1,49 +1,85 @@
 parameters:
-  ServiceDirectory: ''
-  EnvVars: {}
-  MaxParallel: 0
-  BeforeTestSteps: []
-  AfterTestSteps: []
-  BuildTargetingString: 'azure-*'
-  AdditionalTestArgs: ''
-  TestMarkArgument: ''
-  InjectedPackages: ''
-  BuildDocs: true
-  JobName: Test
-  AllocateResourceGroup: true
-  DeployArmTemplate: false
-  TestTimeoutInMinutes: 120
-  TestSamples: false
-  Location: ''
-  Matrix:
-    Linux_Python35:
-      Pool: azsdk-pool-mms-ubuntu-1804-general
-      OSVmImage: MMSUbuntu18.04
-      PythonVersion: '3.5'
-      CoverageArg: '--disablecov'
-    MacOs_Python37:
-      Pool: Azure Pipelines
-      OSVmImage: 'macOS-10.15'
-      PythonVersion: '3.7'
-      CoverageArg: '--disablecov'
-    Windows_Python27:
-      Pool: azsdk-pool-mms-win-2019-general
-      OSVmImage: MMS2019
-      PythonVersion: '2.7'
-      CoverageArg: '--disablecov'
-    Linux_PyPy3:
-      Pool: azsdk-pool-mms-ubuntu-1804-general
-      OSVmImage: MMSUbuntu18.04
-      PythonVersion: 'pypy3'
-      CoverageArg: '--disablecov'
-    Linux_Python39:
-      Pool: azsdk-pool-mms-ubuntu-1804-general
-      OSVmImage: MMSUbuntu18.04
-      PythonVersion: '3.9'
-      CoverageArg: ''
-  CloudConfigurations:
-    AzureCloud:
-      SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources)
+  - name: ServiceDirectory
+    type: string
+    default: ''
+  - name: EnvVars
+    type: object
+    default: {}
+  - name: MaxParallel
+    type: number
+    default: 0
+  - name: BeforeTestSteps
+    type: object
+    default: []
+  - name: AfterTestSteps
+    type: object
+    default: []
+  - name: BuildTargetingString
+    type: string
+    default: 'azure-*'
+  - name: AdditionalTestArgs
+    type: string
+    default: ''
+  - name: TestMarkArgument
+    type: string
+    default: ''
+  - name: InjectedPackages
+    type: string
+    default: ''
+  - name: BuildDocs
+    type: boolean
+    default: true
+  - name: JobName
+    type: string
+    default: 'Test'
+  - name: AllocateResourceGroup
+    type: boolean
+    default: true
+  - name: DeployArmTemplate
+    type: boolean
+    default: false
+  - name: TestTimeoutInMinutes
+    type: number
+    default: 120
+  - name: TestSamples
+    type: boolean
+    default: false
+  - name: Location
+    type: string
+    default: ''
+  - name: Matrix
+    type: object
+    default:
+      Linux_Python35:
+        Pool: azsdk-pool-mms-ubuntu-1804-general
+        OSVmImage: MMSUbuntu18.04
+        PythonVersion: '3.5'
+        CoverageArg: '--disablecov'
+      MacOs_Python37:
+        Pool: Azure Pipelines
+        OSVmImage: 'macOS-10.15'
+        PythonVersion: '3.7'
+        CoverageArg: '--disablecov'
+      Windows_Python27:
+        Pool: azsdk-pool-mms-win-2019-general
+        OSVmImage: MMS2019
+        PythonVersion: '2.7'
+        CoverageArg: '--disablecov'
+      Linux_PyPy3:
+        Pool: azsdk-pool-mms-ubuntu-1804-general
+        OSVmImage: MMSUbuntu18.04
+        PythonVersion: 'pypy3'
+        CoverageArg: '--disablecov'
+      Linux_Python39:
+        Pool: azsdk-pool-mms-ubuntu-1804-general
+        OSVmImage: MMSUbuntu18.04
+        PythonVersion: '3.9'
+        CoverageArg: ''
+  - name: CloudConfigurations
+    type: object
+    default:
+      AzureCloud:
+        SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources)
 
 jobs:
   - ${{ each cloudConfig in parameters.CloudConfigurations }}: