[DataFactory] Add new features in version 4.14.0

.config/dotnet-tools.json

@@ -0,0 +1,12 @@
+{
+  "version": 1,
+  "isRoot": true,
+  "tools": {
+    "dotnet-reportgenerator-globaltool": {
+      "version": "4.8.0",
+      "commands": [
+        "reportgenerator"
+      ]
+    }
+  }
+}

.github/CODEOWNERS

@@ -489,7 +489,7 @@ sdk/trafficmanager/Microsoft.Azure.Management.TrafficManager/            @tmsupp
 #/<NotInRepo>/            @chlahav
 
 # PRLabel: %Search
-/sdk/search/                                    @brjohnstmsft @arv100kri @bleroy @tg-msft @heaths
+/sdk/search/                                    @brjohnstmsft @arv100kri @bleroy @Mohit-Chakraborty @tg-msft @heaths
 
 # ServiceLabel: %Search %Service Attention
 /sdk/search/Microsoft.*/                        @brjohnstmsft @arv100kri @bleroy @tjacobhi @markheff @miwelsh
@@ -562,7 +562,7 @@ sdk/trafficmanager/Microsoft.Azure.Management.TrafficManager/            @tmsupp
 /sdk/support/Microsoft.Azure.Management.Support/           @shahbj79 @mit2nil @aygoya @ganganarayanan
 
 # ServiceLabel: %Synapse %Service Attention
-/sdk/synapse/            @aim-for-better @idear1203
+/sdk/synapse/            @aim-for-better @idear1203 @chamons
 
 # PRLabel: %Tables
 /sdk/tables/                                    @christothes

CONTRIBUTING.md

@@ -9,7 +9,7 @@
 - Install VS 2019 (Community or higher) and make sure you have the latest updates (https://www.visualstudio.com/).
   - Need at least .NET Framework 4.6.1 and 4.7 development tools
 - Install the **.NET Core cross-platform development** workloads in VisualStudio
-- Install **.NET Core 3.1.301 SDK** for your specific platform. (or a higher version within the 3.1.*** band)  (https://dotnet.microsoft.com/download/dotnet-core/3.1)
+- Install **.NET Core 5.0.100 SDK** for your specific platform. (or a higher version within the 5.0.*** band)  (https://dotnet.microsoft.com/download/dotnet-core/5.0)
 - Install the latest version of git (https://git-scm.com/downloads)
 - Install [NodeJS](https://nodejs.org/en/) (14.x.x) if you plan to use [C# code generation](https://github.com/Azure/autorest.csharp).
 
@@ -55,8 +55,11 @@ Nuget package will be created in root directory under \artifacts\packages\Debug
 ### Using the command line:
 
 Run e.g. `msbuild eng\mgmt.proj /t:"Runtests" /p:Scope=Compute`
-In the above example _RunTests_ will build and run tests for Compute only or you can use command line CLI
-`dotnet test Compute\Microsoft.Azure.Management.Compute\tests\Microsoft.Azure.Management.Tests.csproj`
+In the above example _RunTests_ will build and run tests for Compute only or you can use command line CLI:
+
+```bash
+dotnet test Compute\Microsoft.Azure.Management.Compute\tests\Microsoft.Azure.Management.Tests.csproj
+```
 
 ### Non-Windows command line build
 
@@ -67,6 +70,22 @@ Now you can use the same command on non-windows as above for e.g. on Ubuntu you
 - `dotnet msbuild eng\mgmt.proj /t:CreateNugetPackage /p:scope=Compute`
 - `dotnet msbuild build.proj /t:Util /p:UtilityName=InstallPsModules`
 
+### Code Coverage
+
+If you want to enable code coverage reporting, on the command line pass `/p:CollectCoverage=true` like so:
+
+```bash
+dotnet tool restore
+dotnet test /p:CollectCoverage=true
+```
+
+On developers' machines, you can open `index.html` from within the `TestResults` directory in each of your test projects.
+Coverage reports can also be found in Azure Pipelines on the "Code Coverage" tab after a pull request validation build completes.
+All covered projects should have 70% or better test coverage.
+
+By default, all _Azure.*_ libraries are covered, and any project that sets the `IsClientLibrary=true` MSBuild property.
+To exclude a project, set `ExcludeFromCodeCoverage=true` in the project's MSBuild properties before other targets are imported.
+
 ### Update build tools
 
 Build tools are now downloaded as part of a nuget package under `root\restoredPackages\microsoft.internal.netsdkbuild.mgmt.tools`

common/Perf/Azure.Test.Perf/PerfProgram.cs

@@ -96,7 +96,6 @@ private static async Task Run(Type testType, PerfOptions options)
 
             try
             {
-
                 try
                 {
                     await tests[0].GlobalSetupAsync();

common/SmokeTests/smoke-tests.yml

@@ -76,6 +76,8 @@ jobs:
       - pwsh: Get-Content ./common/SmokeTests/SmokeTest/SmokeTest.csproj
         displayName: Show SmokeTest.csproj
 
+      - template: /eng/pipelines/templates/steps/install-dotnet.yml
+
       - pwsh: dotnet restore ./common/SmokeTests/SmokeTest/SmokeTest.csproj
         displayName: dotnet restore
 

common/Stress/Azure.Test.Stress/StressMetrics.cs

@@ -225,10 +225,10 @@ private static string FormatTable(IEnumerable<(string Key, object Value)> data)
             foreach (var kvp in data)
             {
                 sb.Append(kvp.Key);
-                sb.Append(":");
+                sb.Append(':');
                 for (var i = kvp.Key.Length + 1; i < longestKeyLength + padding + 1; i++)
                 {
-                    sb.Append(" ");
+                    sb.Append(' ');
                 }
                 sb.Append(kvp.Value);
                 sb.AppendLine();

common/Stress/Azure.Test.Stress/StressTest.cs

@@ -79,7 +79,6 @@ protected static string GetEnvironmentVariable(string name)
                 throw new InvalidOperationException($"Undefined environment variable {name}");
             }
             return value;
-
         }
     }
 }

doc/ApiDocGeneration/Generate-Api-Docs.ps1

@@ -62,6 +62,7 @@ $DocOutApiDir = "${DocOutDir}/api"
 $DocOutHtmlDir = "${DocOutDir}/_site"
 $MDocTool = "${BinDirectory}/mdoc/mdoc.exe"
 $DocFxTool = "${BinDirectory}/docfx/docfx.exe"
+$DocCommonGenDir = "$RepoRoot/eng/common/docgeneration"
 
 if ($LibType -eq 'management') {
     $ArtifactName = $ArtifactName.Substring($ArtifactName.LastIndexOf('.Management') + 1)
@@ -128,7 +129,7 @@ Write-Verbose "Copy over generated yml and other assets"
 Copy-Item "${YamlOutDir}/*"-Destination "${DocOutApiDir}" -Recurse -Force
 Copy-Item "${DocGenDir}/assets/docfx.json" -Destination "${DocOutDir}" -Recurse -Force
 New-Item -Path "${DocOutDir}" -Name templates -ItemType directory
-Copy-Item "${DocGenDir}/templates/**" -Destination "${DocOutDir}/templates" -Recurse -Force
+Copy-Item "${DocCommonGenDir}/templates/**" -Destination "${DocOutDir}/templates" -Recurse -Force
 
 Write-Verbose "Create Toc for Site Navigation"
 New-Item "${DocOutDir}/toc.yml" -Force
@@ -138,7 +139,7 @@ Write-Verbose "Build Doc Content"
 & "${DocFxTool}" build "${DocOutDir}/docfx.json"
 
 Write-Verbose "Copy over site Logo"
-Copy-Item "${DocGenDir}/assets/logo.svg" -Destination "${DocOutHtmlDir}" -Recurse -Force
+Copy-Item "${DocCommonGenDir}/assets/logo.svg" -Destination "${DocOutHtmlDir}" -Recurse -Force
 
 Write-Verbose "Compress and copy HTML into the staging Area"
 Compress-Archive -Path "${DocOutHtmlDir}/*" -DestinationPath "${ArtifactStagingDirectory}/${ArtifactName}/${ArtifactName}.docs.zip" -CompressionLevel Fastest

eng/.docsettings.yml

@@ -148,6 +148,8 @@ known_content_issues:
   - ['sdk/extensions/Azure.Extensions.AspNetCore.Configuration.Secrets/README.md', 'azure-sdk-tools/issues/404']
   - ['sdk/extensions/Microsoft.Azure.WebJobs.Extensions.Clients/README.md','azure-sdk-tools/issues/404']
   - ['sdk/extensions/Microsoft.Extensions.Azure/README.md','#5499']
+  - ['sdk/eventgrid/Microsoft.Azure.Messaging.EventGrid.CloudNativeCloudEvents/README.md', '#15423']
+  - ['sdk/eventhub/README.md','azure-sdk-tools/issues/42']
   - ['sdk/search/README.md','azure-sdk-tools/issues/42']
   - ['sdk/formrecognizer/Azure.AI.FormRecognizer/README.md','#5499']
   - ['sdk/formrecognizer/Azure.AI.FormRecognizer/README.md','#11492']
@@ -156,6 +158,7 @@ known_content_issues:
   - ['sdk/storage/Azure.Storage.Blobs.Cryptography/README.md','#11492']
   - ['sdk/storage/Azure.Storage.Common/README.md','#11492']
   - ['sdk/storage/Azure.Storage.Files.DataLake/README.md','#11492']
+  - ['sdk/storage/Azure.Storage.Files.DataLake/perf/README.md','#11492']
   - ['sdk/storage/Azure.Storage.Files.Shares/README.md','#11492']
   - ['sdk/storage/Azure.Storage.Queues/README.md','#11492']
   - ['sdk/synapse/Microsoft.Azure.Synapse/README.md','#11492']

eng/CodeAnalysis.ruleset

@@ -142,11 +142,11 @@
         <Rule Id="SA1502"  Action="None" />          <!-- Element should not be on a single line -->
         <Rule Id="SA1503"  Action="None" />          <!-- Braces should not be omitted -->
         <Rule Id="SA1504"  Action="None" />          <!-- All accessors should be single-line or multi-line -->
-        <Rule Id="SA1505"  Action="None" />          <!-- Opening braces should not be followed by blank line -->
+        <Rule Id="SA1505"  Action="Warning" />       <!-- Opening braces should not be followed by blank line -->
         <Rule Id="SA1506"  Action="None" />          <!-- Element documentation headers should not be followed by blank line -->
-        <Rule Id="SA1507"  Action="None" />          <!-- Code should not contain multiple blank lines in a row -->
-        <Rule Id="SA1508"  Action="None" />          <!-- Closing braces should not be preceded by blank line -->
-        <Rule Id="SA1509"  Action="None" />          <!-- Opening braces should not be preceded by blank line -->
+        <Rule Id="SA1507"  Action="Warning" />       <!-- Code should not contain multiple blank lines in a row -->
+        <Rule Id="SA1508"  Action="Warning" />       <!-- Closing braces should not be preceded by blank line -->
+        <Rule Id="SA1509"  Action="Warning" />       <!-- Opening braces should not be preceded by blank line -->
         <Rule Id="SA1510"  Action="None" />          <!-- Chained statement blocks should not be preceded by blank line -->
         <Rule Id="SA1511"  Action="None" />          <!-- While-do footer should not be preceded by blank line -->
         <Rule Id="SA1512"  Action="None" />          <!-- Single-line comments should not be followed by blank line -->

eng/CodeCoverage.runsettings

@@ -0,0 +1,18 @@
+<?xml version="1.0"?>
+<RunSettings>
+  <DataCollectionRunSettings>
+    <DataCollectors>
+      <DataCollector friendlyName="XPlat Code Coverage" enabled="true">
+        <Configuration>
+          <Format>cobertura</Format>
+          <ExcludeByAttribute>ExcludeFromCodeCoverageAttribute,GeneratedCodeAttribute,Obsolete</ExcludeByAttribute>
+          <IncludeTestAssembly>false</IncludeTestAssembly>
+          <SingleHit>false</SingleHit>
+          <SkipAutoProps>true</SkipAutoProps>
+        </Configuration>
+      </DataCollector>
+      <!-- Enable logging to diagnose test host failures -->
+      <DataCollector friendlyName="blame" enabled="true" />
+    </DataCollectors>
+  </DataCollectionRunSettings>
+</RunSettings>

eng/CodeCoverage.targets

@@ -1,13 +1,89 @@
 <Project>
-  <ItemGroup Condition="'$(IsTestProject)' == 'true'">
-    <PackageReference Include="coverlet.msbuild">
+  <PropertyGroup>
+    <!-- Conditions for which projects should be covered by default. -->
+    <_IsCodeCoverable Condition="'$(IsClientLibrary)' == 'true'">true</_IsCodeCoverable>
+  </PropertyGroup>
+
+  <PropertyGroup Condition="'$(CollectCoverage)' == 'true' and '$(_IsCodeCoverable)' == 'true' and '$(IsTestProject)' == 'true' and '$(ExcludeFromCodeCoverage)' != 'true'">
+    <CodeCoverageDirectory Condition="'$(CodeCoverageDirectory)' == ''">$([System.IO.Path]::GetFullPath("$(MSBuildProjectDirectory)\.."))</CodeCoverageDirectory>
+    <SkipCoverageReport Condition="'$(SkipCoverageReport)' == '' and ('$(ContinuousIntegrationBuild)' == 'true' or '$(TF_BUILD)' == 'true')">true</SkipCoverageReport>
+    <VSTestCollect Condition="'$(VSTestCollect)' == ''">XPlat Code Coverage</VSTestCollect>
+    <VSTestSetting Condition="'$(VSTestSetting)' == ''">$(MSBuildThisFileDirectory)CodeCoverage.runsettings</VSTestSetting>
+    <_CollectCoverage>true</_CollectCoverage>
+    <_TestResultsDirectory>$(MSBuildProjectDirectory)\TestResults</_TestResultsDirectory>
+  </PropertyGroup>
+
+  <ItemGroup Condition="'$(_CollectCoverage)' == 'true'">
+    <!--
+      Use VSTest integration to work around test host crashes on larger collections:
+      https://github.com/coverlet-coverage/coverlet/blob/master/Documentation/VSTestIntegration.md
+    -->
+    <PackageReference Include="coverlet.collector">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
     </PackageReference>
   </ItemGroup>
 
-<!-- Allows Collection of Code Coverage for Deterministic Builds 
-    https://github.com/coverlet-coverage/coverlet/blob/master/Documentation/DeterministicBuild.md -->
+  <!-- Clean up previous TestResults so reports are recent. -->
+  <Target Name="_CleanPreviousCodeCoverage"
+          BeforeTargets="VSTest"
+          Condition="'$(_CollectCoverage)' == 'true'">
+    <RemoveDir Directories="$(_TestResultsDirectory)" />
+  </Target>
+
+  <!-- Set a variable to indicate that a report can be generated. -->
+  <Target Name="_SetCoverageCollected"
+          AfterTargets="VSTest"
+          Condition="'$(_CollectCoverage)' == 'true'">
+    <ItemGroup>
+      <_CoverageFiles Include="$(_TestResultsDirectory)\**\coverage.cobertura.xml" />
+    </ItemGroup>
+    <PropertyGroup>
+      <_CoverageCollected Condition="'@(_CoverageFiles)' != ''">true</_CoverageCollected>
+    </PropertyGroup>
+
+    <!-- Indicate to the pipeline a report can be generated. -->
+    <Message Importance="high"
+             Condition="'$(_CoverageCollected)' == 'true' and '$(TF_BUILD)' == 'true'"
+             Text="##vso[task.setvariable variable=coverage.collected]true" />
+  </Target>
+
+  <!-- Should be similar to what's in the pipelines, though generate a full HTML report. -->
+  <Target Name="GenerateCodeCoverageReport"
+          AfterTargets="VSTest"
+          DependsOnTargets="_SetCoverageCollected"
+          Condition="'$(_CollectCoverage)' == 'true' and '$(SkipCoverageReport)' != 'true' and '$(_CoverageCollected)' == 'true'">
+    <PropertyGroup>
+      <CoverageReportCommandLine>dotnet tool run reportgenerator --</CoverageReportCommandLine>
+      <CoverageReportCommandLine>$(CoverageReportCommandLine) "-reports:$(_TestResultsDirectory)\**\coverage.cobertura.xml"</CoverageReportCommandLine>
+      <CoverageReportCommandLine>$(CoverageReportCommandLine) -reporttypes:Html</CoverageReportCommandLine>
+      <CoverageReportCommandLine>$(CoverageReportCommandLine) "-targetdir:$(_TestResultsDirectory)"</CoverageReportCommandLine>
+      <CoverageReportCommandLine>$(CoverageReportCommandLine) "-filefilters:+$(CodeCoverageDirectory)\**"</CoverageReportCommandLine>
+    </PropertyGroup>
+    <Exec Command="$(CoverageReportCommandLine)"
+          IgnoreExitCode="true"
+          StandardErrorImportance="high"
+          StandardOutputImportance="low" />
+  </Target>
+
+  <Target Name="_ValidateSourceFileNames"
+          BeforeTargets="CoreBuild"
+          Condition="'$(_IsCodeCoverable)' == 'true'">
+    <ItemGroup>
+      <!-- Prevent https://github.com/Azure/azure-sdk-for-net/issues/17164 from becoming an issue further in the build process -->
+      <_ContainsCurlyBraces Include="@(Compile)" Condition="$([MSBuild]::ValueOrDefault('%(Directory)%(Filename)', '').Contains('{')) or $([MSBuild]::ValueOrDefault('%(Directory)%(Filename)', '').Contains('}'))" />
+    </ItemGroup>
+    <Error
+      Text="File name '%(_ContainsCurlyBraces.FullPath)' cannot contain { or }; remove type parameters from the file name, or change {T} to OfT and disable SA1649 if the class has a non-generic counterpart (https://github.com/Azure/azure-sdk-for-net/issues/17164)."
+      Condition="'@(_ContainsCurlyBraces)' != ''" />
+  </Target>
+
+  <!--
+    Allows Collection of Code Coverage for Deterministic Builds:
+    https://github.com/coverlet-coverage/coverlet/blob/master/Documentation/DeterministicBuild.md
+
+    This needs to be available in all projects.
+  -->
   <ItemGroup>
     <SourceRoot Include="$(NuGetPackageRoot)" />
   </ItemGroup>
@@ -17,7 +93,8 @@
           Returns="@(_LocalTopLevelSourceRoot)"
           Condition="'$(DeterministicSourcePaths)' == 'true'">
     <ItemGroup>
-      <_LocalTopLevelSourceRoot Include="@(SourceRoot)" Condition="'%(SourceRoot.NestedRoot)' == ''"/>
+      <_LocalTopLevelSourceRoot Include="@(SourceRoot)"
+                                Condition="'%(SourceRoot.NestedRoot)' == ''"/>
     </ItemGroup>
   </Target>
-</Project>
+</Project>

eng/Directory.Build.Data.props

@@ -25,6 +25,10 @@
       CA2231; <!-- Override Equality operators -->
       CA2225; <!-- Provide alternative to implicit operators -->
       CA1714; <!-- Flags should have plural names -->
+      CA1062; <!-- Public parameter should be checked for null -->
+      CA1031; <!-- Don't catch generic exceptions -->
+      CA2000; <!-- Call dispose on IDisposable objects -->
+      CA2012; <!-- ValueTask should only be awaited once - conflicts with EnsureCompleted check -->
     </NoWarn>
 
     <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
@@ -51,7 +55,7 @@
     <GenerateAPIListing Condition="'$(IsShippingClientLibrary)' == 'true'">true</GenerateAPIListing>
     <UpdateSourceOnBuild Condition="'$(UpdateSourceOnBuild)' == ''">$(AZURE_DEV_UPDATESOURCESONBUILD)</UpdateSourceOnBuild>
     <PowerShellExe Condition="'$(PowerShellExe)' == ''">pwsh</PowerShellExe>
-    <CoverletOutputFormat Condition="'$(CoverletOutputFormat)' == '' and '$(CollectCoverage)' == 'true'">cobertura</CoverletOutputFormat>
+    <InheritDocEnabled>true</InheritDocEnabled>
   </PropertyGroup>
 
   <PropertyGroup Condition="'$(IsShippingClientLibrary)' == 'true' and '$(TF_BUILD)' == 'true'">
@@ -68,6 +72,12 @@
     </NoWarn>
   </PropertyGroup>
 
+  <PropertyGroup Condition="'$(InheritDocEnabled)' != 'false' and '$(TargetFramework)'=='netstandard2.0'">
+    <NoWarn>
+      $(NoWarn);IDT001<!-- InheritDoc related to malformed XML in netstandard.xml -->
+    </NoWarn>
+  </PropertyGroup>
+
   <!-- CodeAnalysis RuleSet -->
   <PropertyGroup Condition="'$(IsClientLibrary)' == 'true'">
     <CodeAnalysisRuleSet>$(RepoEngPath)\CodeAnalysis.ruleset</CodeAnalysisRuleSet>
@@ -85,8 +95,9 @@
 
   <PropertyGroup Condition="'$(IsTestProject)' == 'true' or '$(IsTestSupportProject)' == 'true' or '$(IsSamplesProject)' == 'true' or '$(IsPerfProject)' == 'true' or '$(IsStressProject)' == 'true'">
     <IsPackable>false</IsPackable>
-    <RequiredTargetFrameworks>netcoreapp2.1</RequiredTargetFrameworks>
-    <RequiredTargetFrameworks Condition="'$(OS)' == 'Windows_NT'">netcoreapp2.1;net461</RequiredTargetFrameworks>
+    <RequiredTargetFrameworks>netcoreapp2.1;net5.0</RequiredTargetFrameworks>
+    <!-- Also test net461 on Windows; it's listed first so that coverage reports are for netcoreapp2.1 (the "primary"). -->
+    <RequiredTargetFrameworks Condition="'$(OS)' == 'Windows_NT'">net461;netcoreapp2.1;net5.0</RequiredTargetFrameworks>
   </PropertyGroup>
 
   <Import Project="$(RepoRoot)/sdk/core/Azure.Core/src/Azure.Core.props" Condition="'$(IsMgmtClientLibrary)' == 'true'"/>