Skip to content

Commit

Permalink
Merge pull request Azure#2 from Azure/main
Browse files Browse the repository at this point in the history
pr
  • Loading branch information
seilorjunior authored Aug 16, 2022
2 parents 183f33a + a0cd194 commit 37babd3
Show file tree
Hide file tree
Showing 164 changed files with 6,401 additions and 4,729 deletions.
32 changes: 30 additions & 2 deletions eng/Directory.Build.Common.targets
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
<Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">

<Import Project="Versioning.targets" />

Expand Down Expand Up @@ -330,5 +330,33 @@
<Message Condition="'$(IsShippingLibrary)' == 'true'" Text="'$(PackageRootDirectory)' '$(ServiceDirectory)' '$(PackageId)' '$(VersionForProperties)' '$(PackageSdkType)' '$(PackageIsNewSdk)'" Importance="High" />
</Target>

<Target Name="_SetProjectDependsOnInnerTarget">
<PropertyGroup>
<InnerTargets>ProjectDependsOnInner</InnerTargets>
</PropertyGroup>
</Target>

<Target
Name="ProjectDependsOn"
DependsOnTargets="_SetProjectDependsOnInnerTarget;DispatchToInnerBuilds"
Condition="'$(IsCrossTargetingBuild)' == 'true'"
Returns="@(ProjectsToInclude)">

<RemoveDuplicates Inputs="@(InnerOutput)">
<Output TaskParameter="Filtered" ItemName="ProjectsToInclude"/>
</RemoveDuplicates>
</Target>

<Target Name="ProjectDependsOnInner" DependsOnTargets="ResolveReferences" Returns="@(ProjectsToInclude)">
<ItemGroup>
<_LibrariesReferenced Include="@(ReferencePath -> '%(Filename)')"/>
<_DependsOnGiven Include="@(_LibrariesReferenced)" Condition="'%(Identity)' == '$(TestDependsOnDependency)'" />
</ItemGroup>

<ItemGroup Condition="'@(_DependsOnGiven)' != '' and '$(IsClientLibrary)' == 'true'" >
<ProjectsToInclude Include="$(MSBuildProjectFullPath)" />
</ItemGroup>
</Target>

<Import Project="$(CentralPackageVersionPackagePath)\Sdk.targets" />
</Project>
</Project>
2 changes: 1 addition & 1 deletion eng/Packages.Data.props
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,7 @@
All should have PrivateAssets="All" set so they don't become package dependencies
-->
<ItemGroup>
<PackageReference Update="Microsoft.Azure.AutoRest.CSharp" Version="3.0.0-beta.20220810.2" PrivateAssets="All" />
<PackageReference Update="Microsoft.Azure.AutoRest.CSharp" Version="3.0.0-beta.20220816.1" PrivateAssets="All" />
<PackageReference Update="Azure.ClientSdk.Analyzers" Version="0.1.1-dev.20220111.2" PrivateAssets="All" />
<PackageReference Update="coverlet.collector" Version="1.3.0" PrivateAssets="All" />
<PackageReference Update="Microsoft.CodeAnalysis.FxCopAnalyzers" Version="3.3.1" PrivateAssets="All" />
Expand Down
2 changes: 1 addition & 1 deletion eng/common/testproxy/target_version.txt
Original file line number Diff line number Diff line change
@@ -1 +1 @@
1.0.0-dev.20220805.3
1.0.0-dev.20220810.2
47 changes: 36 additions & 11 deletions eng/pipelines/templates/jobs/ci.tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,6 @@ parameters:
- name: SDKType
type: string
default: all
- name: ServiceDirectory
type: string
default: not-specified
- name: ServiceToTest
type: string
default: ''
- name: TestSetupSteps
type: stepList
default: []
Expand All @@ -24,6 +18,12 @@ parameters:
- name: CloudConfig
type: object
default: {}
- name: ProjectListOverrideFilePropertyName
type: string
default: ''
- name: ServiceDirectory
type: string
default: ''

jobs:
- job:
Expand All @@ -43,10 +43,23 @@ jobs:
- ${{ if not(contains(variables['Build.DefinitionName'], '-pr - ')) }}:
- template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
parameters:
${{ if eq(parameters.ProjectListOverrideFilePropertyName, '') }}:
Paths:
- "/*"
- "!SessionRecords"
- "/sdk/${{ parameters.ServiceDirectory }}/**/SessionRecords"
${{ else }}:
Paths:
- "/*"
- "!SessionRecords"
- "/sdk/${{ parameters.ServiceToTest }}/**/SessionRecords"
- "/sdk/*/**/SessionRecords"
- ${{ if ne(parameters.ProjectListOverrideFilePropertyName, '') }}:
- task: DownloadPipelineArtifact@2
inputs:
artifact: DependencyTestProjectReferences
patterns: "**/$(${{ parameters.ProjectListOverrideFilePropertyName }})"
path: $(Build.SourcesDirectory)

- pwsh: |
$(Build.SourcesDirectory)/eng/common/scripts/trust-proxy-certificate.ps1
displayName: 'Language Specific Certificate Trust'
Expand All @@ -63,19 +76,31 @@ jobs:
- template: /eng/pipelines/templates/steps/install-dotnet.yml
parameters:
NuGetCacheKey: Test
- pwsh: |
if ("${{ parameters.ProjectListOverrideFilePropertyName }}" -ne '') {
Write-Host "##vso[task.setvariable variable=ProjectListOverrideFileName]$(${{ parameters.ProjectListOverrideFilePropertyName }})"
Write-Host "##vso[task.setvariable variable=CodeCoverageFilePattern]$(Build.SourcesDirectory)\sdk\**\coverage.cobertura.xml"
}
else
{
Write-Host "##vso[task.setvariable variable=ProjectListOverrideFileName]"
Write-Host "##vso[task.setvariable variable=CodeCoverageFilePattern]$(Build.SourcesDirectory)\sdk\${{parameters.ServiceDirectory}}\**\coverage.cobertura.xml"
}
displayName: Set variable for the project list file and coverage directory
- pwsh: >-
dotnet test eng/service.proj
--filter '(TestCategory!=Manually) & (TestCategory!=Live) & ($(AdditionalTestFilters))'
--framework $(TestTargetFramework)
--logger 'trx;LogFileName=$(TestTargetFramework).trx' --logger:'console;verbosity=normal'
--blame-crash-dump-type full --blame-hang-dump-type full --blame-hang-timeout ${{parameters.TestTimeoutInMinutes}}minutes
/p:SDKType=${{ parameters.SDKType }}
/p:ServiceDirectory=${{ parameters.ServiceToTest }}
/p:ServiceDirectory=${{ parameters.ServiceDirectory }}
/p:IncludeSrc=false /p:IncludeSamples=false /p:IncludePerf=false /p:IncludeStress=false
/p:RunApiCompat=false /p:InheritDocEnabled=false
/p:Configuration=$(BuildConfiguration)
/p:CollectCoverage=$(CollectCoverage) /p:CodeCoverageDirectory=$(Build.SourcesDirectory)\sdk\${{parameters.ServiceDirectory}}
/p:EnableSourceLink=false
/p:CollectCoverage=$(CollectCoverage)
/p:EnableSourceLink=false
/p:ProjectListOverrideFile=$(ProjectListOverrideFileName)
$(AdditionalTestArguments)
displayName: "Build & Test ($(TestTargetFramework))"
- task: PublishTestResults@2
Expand All @@ -91,7 +116,7 @@ jobs:
condition: and(succeededOrFailed(), eq(variables['CollectCoverage'], 'true'), eq(variables['coverage.collected'], 'true'))
displayName: Generate Code Coverage Reports
inputs:
reports: $(Build.SourcesDirectory)\sdk\${{parameters.ServiceDirectory}}\**\coverage.cobertura.xml
reports: $(CodeCoverageFilePattern)
targetdir: $(Build.ArtifactStagingDirectory)\coverage
reporttypes: Cobertura
filefilters: +$(Build.SourcesDirectory)\sdk\${{parameters.ServiceDirectory}}\**
Expand Down
42 changes: 37 additions & 5 deletions eng/pipelines/templates/jobs/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,9 @@ parameters:
- name: MatrixReplace
type: object
default: []

- name: TestDependsOnDependency
type: string
default: not-specified
jobs:
- job: Build
pool:
Expand Down Expand Up @@ -226,18 +228,48 @@ jobs:
parameters:
BaselineFilePath: $(Build.sourcesdirectory)/eng/dotnet.gdnbaselines
ServiceDirectory: ${{ parameters.ServiceDirectory }}

- template: /eng/common/pipelines/templates/jobs/archetype-sdk-tests-generate.yml
parameters:
GenerateJobName: generate_target_service_test_matrix
JobTemplatePath: /eng/pipelines/templates/jobs/ci.tests.yml
MatrixConfigs: ${{ parameters.MatrixConfigs }}
MatrixFilters: ${{ parameters.MatrixFilters }}
MatrixReplace: ${{ parameters.MatrixReplace }}
CloudConfig:
Cloud: Public
Cloud: public
AdditionalParameters:
SDKType: ${{ parameters.SDKType }}
ServiceToTest: ${{ coalesce(parameters.ServiceToTest, parameters.ServiceDirectory) }}
ServiceDirectory: ${{ parameters.ServiceDirectory }}
TestSetupSteps: ${{ parameters.TestSetupSteps }}
TestTimeoutInMinutes: ${{ parameters.TestTimeoutInMinutes }}
TestTimeoutInMinutes: ${{ parameters.TestTimeoutInMinutes }}
- ${{ if ne(parameters.TestDependsOnDependency, 'not-specified') }}:
- template: /eng/common/pipelines/templates/jobs/archetype-sdk-tests-generate.yml
parameters:
GenerateJobName: generate_target_dependencies_test_matrix
JobTemplatePath: /eng/pipelines/templates/jobs/ci.tests.yml
MatrixConfigs: ${{ parameters.MatrixConfigs }}
MatrixFilters: ${{ parameters.MatrixFilters }}
MatrixReplace: ${{ parameters.MatrixReplace }}
CloudConfig:
Cloud: public
SparseCheckoutPaths:
- "/*"
- "!/sdk/*/**/SessionRecords"
PreGenerationSteps:
- ${{ each config in parameters.MatrixConfigs }}:
- template: /eng/pipelines/templates/steps/dependency.tests.yml
parameters:
TestDependsOnDependency: ${{ parameters.TestDependsOnDependency }}
# The value for ProjectListOverrideFilePropertyName should be the same as AdditionalParameters ProjectListOverrideFilePropertyName below
ProjectListOverrideFilePropertyName: 'ProjectListOverrideFile'
ProjectFilesOutputFolder: $(Build.ArtifactStagingDirectory)
ExcludeTargetTestProjects: true
ServiceDirectory: ${{ parameters.ServiceDirectory }}
MatrixConfigsFile: ${{ config.Path }}
AdditionalParameters:
SDKType: ${{ parameters.SDKType }}
ServiceDirectory: ${{ parameters.ServiceDirectory }}
TestSetupSteps: ${{ parameters.TestSetupSteps }}
TestTimeoutInMinutes: ${{ parameters.TestTimeoutInMinutes }}
# The value for ProjectListOverrideFilePropertyName should be the same as dependency.tests.yml parameter ProjectListOverrideFilePropertyName
ProjectListOverrideFilePropertyName: 'ProjectListOverrideFile'
4 changes: 4 additions & 0 deletions eng/pipelines/templates/stages/archetype-sdk-client.yml
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,9 @@ parameters:
- name: MatrixReplace
type: object
default: []
- name: TestDependsOnDependency
type: string
default: not-specified

variables:
- template: ../variables/globals.yml
Expand Down Expand Up @@ -85,6 +88,7 @@ stages:
- ${{ if eq(variables['System.TeamProject'], 'internal') }}:
- BuildConfiguration=Debug/Release
- ${{ parameters.MatrixReplace }}
TestDependsOnDependency: ${{ parameters.TestDependsOnDependency }}

# The Prerelease and Release stages are conditioned on whether we are building a pull request and the branch.
- ${{if and(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['System.TeamProject'], 'internal'))}}:
Expand Down
56 changes: 56 additions & 0 deletions eng/pipelines/templates/steps/dependency.tests.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
parameters:
- name: ProjectFilesOutputFolder
type: string
default: "$(Build.ArtifactStagingDirectory)"
- name: NumOfTestProjectsPerJob
type: number
default: 20
- name: OutputProjectFilePath
type: string
default: "$(System.DefaultWorkingDirectory)/projects.txt"
- name: TestDependsOnDependency
type: string
default: "not-specific"
- name: ProjectListOverrideFilePropertyName
type: string
default: ''
- name: MatrixConfigsFile
type: string
default: './eng/pipelines/templates/stages/platform-matrix.json'
- name: ExcludeTargetTestProjects
type: boolean
default: false
- name: ServiceDirectory
type: string
default: 'not-specific'
steps:
- template: /eng/pipelines/templates/steps/install-dotnet.yml
- pwsh: |
dotnet build /t:ProjectDependsOn ./eng/service.proj `
/p:TestDependsOnDependency="${{parameters.TestDependsOnDependency}}" `
/p:IncludeSrc=false /p:IncludeStress=false /p:IncludeSamples=false `
/p:IncludePerf=false /p:RunApiCompat=false `
/p:InheritDocEnabled=false /p:BuildProjectReferences=false `
/p:OutputProjectFilePath="${{parameters.OutputProjectFilePath}}"
displayName: Find all test dependencies
- task: PowerShell@2
inputs:
targetType: filePath
filePath: eng/scripts/splittestdependencies/Generate-Dependency-Test-References.ps1
arguments: >-
-ProjectListFilePath ${{ parameters.OutputProjectFilePath }}
-ProjectFilesOutputFolder ${{ parameters.ProjectFilesOutputFolder }}
-NumOfTestProjectsPerJob ${{ parameters.NumOfTestProjectsPerJob }}
-MatrixConfigsFile ${{parameters.MatrixConfigsFile}}
-ProjectFileConfigName ${{parameters.ProjectListOverrideFilePropertyName}}
-ExcludeTargetTestProjects: $${{parameters.ExcludeTargetTestProjects}}
-ServiceDirectoryToExclude ${{parameters.ServiceDirectory}}
ignoreLASTEXITCODE: true
pwsh: true
displayName: Split testing packages into different files

- task: PublishBuildArtifacts@1
inputs:
pathToPublish: $(Build.ArtifactStagingDirectory)
artifactName: DependencyTestProjectReferences
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
<#
.SYNOPSIS
Split the test dependencies into different project files.
.DESCRIPTION
.NET pipelines will run centain number of test dependencies in each entry of job matrix.
The Script is to split all test dependencies into difference sets for later use.
The script is able to test locally using Pester. Please run tests as following every time you made changes to the script:
```
Invoke-Pester -Output Detailed $PSScriptRoot\tests\Generate-Dependency-Test-References-Tests.ps1
```
.PARAMETER ProjectListFilePath
The file path contains all test dependencies. E.g. The file generated by 'eng/service.proj' in dependecny.tests.yml `Find all test dependencies` step
.PARAMETER ProjectFilesOutputFolder
The folder for all test dependency split files e.g. `Project_1.props` which are ready to publish. E.g. $(Build.ArtifactStagingDirectory)
.PARAMETER NumOfTestProjectsPerJob
The number of test run on each job.
.PARAMETER MatrixConfigsFile
The file path of the matrix job configuration. E.g. "/eng/pipelines/templates/stages/platform-matrix.json"
.PARAMETER ProjectFileConfigName
The config name of the project file name. Hardcoded in "\eng\pipelines\templates\jobs\ci.yml" `PreGenerationSteps` for matix generation. E.g. 'ProjectListOverrideFile'
.PARAMETER ExcludeTargetTestProjects
The flag to indicate whether exclude target test package or not.
.PARAMETER ServiceDirectoryToExclude
The service directory which needs to exclude. E.g. "core"
.EXAMPLE
Run script with default parameters.
Generate-Dependency-Test-References.ps1 -ProjectFilesOutputFolder "$(Build.ArtifactStagingDirectory)/projects" -ProjectListFilePath "eng/projects.txt" `
-MatrixConfigsFile "eng/pipelines/templates/stages/platform-matrix.json" -ProjectFileConfigName "ProjectListOverrideFile" -ExcludeTargetTestProjects $true `
-ServiceDirectoryToExclude "core"
#>
Param (
[ValidateNotNullOrEmpty()]
[string] $ProjectListFilePath,
[string] $ProjectFilesOutputFolder="./",
[int] $NumOfTestProjectsPerJob = 20,
[ValidateNotNullOrEmpty()]
[string] $MatrixConfigsFile,
[ValidateNotNullOrEmpty()]
[string] $ProjectFileConfigName,
[boolean] $ExcludeTargetTestProjects=$false,
[string] $ServiceDirectoryToExclude
)

. (Join-Path $PSScriptRoot generate-dependency-functions.ps1)
Set-StrictMode -Version 3

$projGroups = Split-Project-File-To-Groups `
-ProjectFile $ProjectListFilePath `
-NumberOfTestsPerJob $NumOfTestProjectsPerJob `
-ExcludeService $ExcludeTargetTestProjects `
-ServiceDirToExclude $ServiceDirectoryToExclude

$projFiles = Write-Test-Dependency-Group-To-Files `
-ProjectGroups $projGroups `
-MatrixOutputFolder $ProjectFilesOutputFolder

Write-Project-Files-To-Matrix `
-ProjectFiles $projFiles `
-MatrixJsonPath $MatrixConfigsFile `
-MatrixOutputFolder $ProjectFilesOutputFolder `
-ProjectFileConfigName $ProjectFileConfigName
Loading

0 comments on commit 37babd3

Please sign in to comment.