Skip to content

Commit

Permalink
Use sparse checkout in release, ci and live tests (#20901)
Browse files Browse the repository at this point in the history
* Use sparse checkout in release, ci and live tests

* Optimize sparse checkout multiple directories

* Sparse checkout, recording improvements

* Add xml glob for sparse-checkout in release step

* Update sparse checkout for Build and Analyze CI jobs

* Add CODEOWNERS to sparse checkout for docs.ms publishing.

* Check out test-resources.json files. Clean up sparse checkout paths

* Use path definition for docs metadata release

* Simplify recording checkout inclusion/exclusion

* Sparse checkout cspell config

* Use named checkout directories variable for generate_from_source_pom.py script

* Re-order ci steps
  • Loading branch information
benbp authored May 26, 2021
1 parent d575c6d commit 9712dd0
Show file tree
Hide file tree
Showing 6 changed files with 169 additions and 77 deletions.
32 changes: 5 additions & 27 deletions eng/pipelines/templates/jobs/ci.tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -55,9 +55,12 @@ jobs:
- template: ../variables/globals.yml

steps:
- template: /eng/common/pipelines/templates/steps/verify-agent-os.yml
- template: /eng/pipelines/templates/steps/initialize-test-environment.yml
parameters:
AgentImage: $(OSVmImage)
Artifacts: ${{ parameters.Artifacts }}
AdditionalModules: ${{ parameters.AdditionalModules }}
ServiceDirectory: ${{ parameters.ServiceDirectory }}
CheckoutRecordings: true

- template: ../steps/cache-maven-repository.yml

Expand All @@ -68,31 +71,6 @@ jobs:

- ${{ parameters.PreTestSteps }}

- task: UsePythonVersion@0
displayName: 'Use Python 3.6'
inputs:
versionSpec: '3.6'
condition: and(succeeded(), eq(variables['TestFromSource'],'true'))

- task: PythonScript@0
displayName: 'Set versions for source build'
inputs:
scriptPath: 'eng/versioning/set_versions.py'
arguments: '--build-type client --pst'
condition: and(succeeded(), eq(variables['TestFromSource'],'true'))

- task: PythonScript@0
displayName: 'Update versions for source build'
inputs:
scriptPath: 'eng/versioning/update_versions.py'
arguments: '--update-type library --build-type client --sr'
condition: and(succeeded(), eq(variables['ShouldRunSourceTests'],'true'))

- template: ../steps/generate-project-list.yml
parameters:
Artifacts: ${{parameters.Artifacts}}
AdditionalModules: ${{parameters.AdditionalModules}}

- task: PythonScript@0
# The root pom.xml contains both track 1, data, and track 2, client libraries.
# This task calls a Python script that will go through all of the pom files
Expand Down
71 changes: 58 additions & 13 deletions eng/pipelines/templates/jobs/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,15 @@ jobs:
ArtifactName: 'packages'

steps:
- template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
parameters:
Paths:
- 'sdk/${{ parameters.ServiceDirectory }}'
- '**/*.xml'
- '**/*.md'
- '!sdk/**/test-recordings'
- '!sdk/**/session-records'

- script: |
echo "##vso[build.addbuildtag]Scheduled"
displayName: 'Tag scheduled builds'
Expand Down Expand Up @@ -83,17 +92,29 @@ jobs:
condition: and(succeeded(), eq(variables['SetDevVersion'],'true'))
displayName: Apply version settings to repository
- template: ../steps/generate-project-list.yml
parameters:
Artifacts: ${{parameters.Artifacts}}
AdditionalModules: ${{parameters.AdditionalModules}}

- task: PythonScript@0
displayName: 'Generate directories variable for sparse checkout'
inputs:
scriptPath: 'eng/scripts/generate_from_source_pom.py'
arguments: '--set-pipeline-variable CheckoutDirectories --project-list $(ProjectList)'
workingDirectory: '$(System.DefaultWorkingDirectory)'

- template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
parameters:
Paths: $(CheckoutDirectories)
SkipDefaultCheckout: true

- script: |
pip install markdown2 BeautifulSoup4
displayName: 'pip install markdown2 and BeautifulSoup4'
- template: ../steps/cache-maven-repository.yml

- template: ../steps/generate-project-list.yml
parameters:
Artifacts: ${{parameters.Artifacts}}
AdditionalModules: ${{parameters.AdditionalModules}}

- task: Maven@3
displayName: 'Build and Package, JDK Version: $(JavaBuildVersion)'
inputs:
Expand Down Expand Up @@ -139,7 +160,24 @@ jobs:
vmImage: MMSUbuntu18.04

steps:
- task: UsePythonVersion@0
displayName: 'Use Python 3.6'
inputs:
versionSpec: '3.6'

- template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
parameters:
Paths:
- 'sdk/${{ parameters.ServiceDirectory }}'
- '**/*.xml'
- '**/*.md'
- '.vscode/cspell.json'
- '!sdk/**/test-recordings'
- '!sdk/**/session-records'

- template: /eng/common/pipelines/templates/steps/check-spelling.yml
parameters:
CspellConfigPath: .vscode/cspell.json

- task: PowerShell@2
displayName: 'Verify versions in POM files'
Expand All @@ -149,10 +187,22 @@ jobs:
filePath: eng/versioning/pom_file_version_scanner.ps1
arguments: -Debug:$$(IsDebug)

- task: UsePythonVersion@0
displayName: 'Use Python 3.6'
- template: ../steps/generate-project-list.yml
parameters:
Artifacts: ${{ parameters.Artifacts }}
AdditionalModules: ${{ parameters.AdditionalModules }}

- task: PythonScript@0
displayName: 'Generate directories variable for sparse checkout'
inputs:
versionSpec: '3.6'
scriptPath: 'eng/scripts/generate_from_source_pom.py'
arguments: '--set-pipeline-variable CheckoutDirectories --project-list $(ProjectList)'
workingDirectory: '$(System.DefaultWorkingDirectory)'

- template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
parameters:
Paths: $(CheckoutDirectories)
SkipDefaultCheckout: true

- template: /eng/common/pipelines/templates/steps/set-test-pipeline-version.yml
parameters:
Expand Down Expand Up @@ -200,11 +250,6 @@ jobs:
Options: --batch-mode
MavenOptions: $(MemoryOptions) $(LoggingOptions)

- template: ../steps/generate-project-list.yml
parameters:
Artifacts: ${{parameters.Artifacts}}
AdditionalModules: ${{parameters.AdditionalModules}}

# maven dependency:tree needs to be able to resolve dependencies, so these should be installed.
- task: Maven@3
displayName: 'Build and Install'
Expand Down
31 changes: 4 additions & 27 deletions eng/pipelines/templates/jobs/live.tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -43,9 +43,11 @@ jobs:
container: $[ variables['Container'] ]

steps:
- template: /eng/common/pipelines/templates/steps/verify-agent-os.yml
- template: /eng/pipelines/templates/steps/initialize-test-environment.yml
parameters:
AgentImage: $(OSVmImage)
Artifacts: ${{ parameters.Artifacts }}
AdditionalModules: ${{ parameters.AdditionalModules }}
ServiceDirectory: ${{ parameters.ServiceDirectory }}

- ${{ if ne(parameters.DisableAzureResourceCreation, 'true') }}:
- template: /eng/common/TestResources/build-test-resource-config.yml
Expand Down Expand Up @@ -73,31 +75,6 @@ jobs:

- ${{ parameters.PreSteps }}

- template: ../steps/generate-project-list.yml
parameters:
Artifacts: ${{parameters.Artifacts}}
AdditionalModules: ${{parameters.AdditionalModules}}

- task: UsePythonVersion@0
displayName: 'Use Python 3.6'
inputs:
versionSpec: '3.6'
condition: and(succeeded(), eq(variables['TestFromSource'],'true'))

- task: PythonScript@0
displayName: 'Set versions for source build'
inputs:
scriptPath: 'eng/versioning/set_versions.py'
arguments: '--build-type client --pst'
condition: and(succeeded(), eq(variables['TestFromSource'],'true'))

- task: PythonScript@0
displayName: 'Update versions for source build'
inputs:
scriptPath: 'eng/versioning/update_versions.py'
arguments: '--update-type library --build-type client --sr'
condition: and(succeeded(), eq(variables['ShouldRunSourceTests'],'true'))

- task: Maven@3
displayName: 'Build and Install, JDK Version: $(JavaTestVersion)'
inputs:
Expand Down
41 changes: 33 additions & 8 deletions eng/pipelines/templates/stages/archetype-java-release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,12 @@ stages:
vmImage: MMSUbuntu18.04

steps:
- template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
parameters:
Paths:
- '${{ parameters.ServiceDirectory }}'
- '**/*.xml'

- task: UsePythonVersion@0
displayName: 'Use Python 3.6'
inputs:
Expand Down Expand Up @@ -124,13 +130,14 @@ stages:
runOnce:
deploy:
steps:
- checkout: self
- template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
- template: /eng/common/pipelines/templates/steps/retain-run.yml
- template: /eng/common/pipelines/templates/steps/create-tags-and-git-release.yml
parameters:
ArtifactLocation: $(Pipeline.Workspace)/${{parameters.ArtifactName}}-signed/${{artifact.groupId}}/${{artifact.name}}
PackageRepository: Maven
ReleaseSha: $(Build.SourceVersion)

- ${{if ne(artifact.skipPublishPackage, 'true')}}:
- deployment: PublishPackage
displayName: "Publish to Maven Central"
Expand All @@ -146,10 +153,15 @@ stages:
runOnce:
deploy:
steps:
- checkout: self
path: azure-sdk-for-java
- checkout: azure-sdk-build-tools
path: azure-sdk-build-tools
- template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
parameters:
SkipDefaultCheckout: true
Repositories:
- Name: Azure/azure-sdk-for-java
Commitish: $(Build.SourceVersion)
WorkingDirectory: $(Pipeline.Workspace)/azure-sdk-for-java
- template: tools/gpg/gpg.yml@azure-sdk-build-tools
- template: /eng/pipelines/templates/steps/java-publishing.yml
parameters:
Expand Down Expand Up @@ -188,7 +200,10 @@ stages:
runOnce:
deploy:
steps:
- checkout: self
- template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
parameters:
paths:
- .github/CODEOWNERS
- template: /eng/common/pipelines/templates/steps/get-pr-owners.yml
parameters:
TargetVariable: "OwningGHUser"
Expand All @@ -209,6 +224,8 @@ stages:
CIConfigs: $(CIConfigs)
CloseAfterOpenForTesting: '${{ parameters.TestPipeline }}'
SkipPackageJson: ${{artifact.skipUpdatePackageJson}}
SparseCheckoutPaths:
- docs-ref-services

- ${{if ne(artifact.skipPublishDocGithubIo, 'true')}}:
- deployment: PublishDocs
Expand All @@ -225,7 +242,7 @@ stages:
runOnce:
deploy:
steps:
- checkout: self
- template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
- pwsh: |
Get-ChildItem -Recurse $(Pipeline.Workspace)/${{parameters.ArtifactName}}-signed/${{artifact.groupId}}/${{artifact.name}}
workingDirectory: $(Pipeline.Workspace)
Expand Down Expand Up @@ -255,7 +272,10 @@ stages:
runOnce:
deploy:
steps:
- checkout: self
- template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
parameters:
Paths:
- '**/*.xml'

- task: UsePythonVersion@0
displayName: 'Use Python 3.6'
Expand Down Expand Up @@ -295,10 +315,15 @@ stages:
name: Azure Pipelines
vmImage: vs2017-win2016
steps:
- checkout: self
path: azure-sdk-for-java
- checkout: azure-sdk-build-tools
path: azure-sdk-build-tools
- template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
parameters:
SkipDefaultCheckout: true
Repositories:
- Name: Azure/azure-sdk-for-java
Commitish: $(Build.SourceVersion)
WorkingDirectory: $(Pipeline.Workspace)/azure-sdk-for-java

- download: current
artifact: ${{parameters.ArtifactName}}-signed
Expand Down
62 changes: 62 additions & 0 deletions eng/pipelines/templates/steps/initialize-test-environment.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
parameters:
- name: ServiceDirectory
type: string
- name: Artifacts
type: object
default: []
- name: AdditionalModules
type: object
default: []
- name: CheckoutRecordings
type: boolean
default: false

steps:
- template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
parameters:
Paths:
- '**/*.xml'
- 'sdk/${{ parameters.ServiceDirectory }}'
- ${{ if not(parameters.CheckoutRecordings) }}:
- '!sdk/**/test-recordings'
- '!sdk/**/session-records'

- task: UsePythonVersion@0
displayName: 'Use Python 3.6'
inputs:
versionSpec: '3.6'

- template: /eng/common/pipelines/templates/steps/verify-agent-os.yml
parameters:
AgentImage: $(OSVmImage)

- task: PythonScript@0
displayName: 'Set versions for source build'
inputs:
scriptPath: 'eng/versioning/set_versions.py'
arguments: '--build-type client --pst'
condition: and(succeeded(), eq(variables['TestFromSource'], 'true'))

- task: PythonScript@0
displayName: 'Update versions for source build'
inputs:
scriptPath: 'eng/versioning/update_versions.py'
arguments: '--update-type library --build-type client --sr'
condition: and(succeeded(), eq(variables['ShouldRunSourceTests'],'true'))

- template: generate-project-list.yml
parameters:
Artifacts: ${{ parameters.Artifacts }}
AdditionalModules: ${{ parameters.AdditionalModules }}

- task: PythonScript@0
displayName: 'Generate directories variable for sparse checkout'
inputs:
scriptPath: 'eng/scripts/generate_from_source_pom.py'
arguments: '--set-pipeline-variable CheckoutDirectories --project-list $(ProjectList)'
workingDirectory: '$(System.DefaultWorkingDirectory)'

- template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
parameters:
SkipDefaultCheckout: true
Paths: $(CheckoutDirectories)
Loading

0 comments on commit 9712dd0

Please sign in to comment.