Add 1es changes for job/matrix generation and publish

eng/common/pipelines/templates/jobs/generate-job-matrix.yml

@@ -0,0 +1,140 @@
+parameters:
+- name: AdditionalParameters
+  type: object
+- name: DependsOn
+  type: object
+  default: null
+- name: CloudConfig
+  type: object
+  default: {}
+- name: MatrixConfigs
+  type: object
+  default: []
+- name: MatrixFilters
+  type: object
+  default: []
+- name: MatrixReplace
+  type: object
+  default: {}
+- name: JobTemplatePath
+  type: string
+# Set this to false to do a full checkout for private repositories with the azure pipelines service connection
+- name: SparseCheckout
+  type: boolean
+  default: true
+- name: SparseCheckoutPaths
+  type: object
+  default: []
+- name: Pool
+  type: string
+  default: $(LINUXPOOL)
+- name: OsVmImage
+  type: string
+  default: $(LINUXVMIMAGE)
+- name: Os
+  type: string
+  default: linux
+# This parameter is only necessary if there are multiple invocations of this template within the SAME STAGE.
+# When that occurs, provide a name other than the default value.
+- name: GenerateJobName
+  type: string
+  default: 'generate_job_matrix'
+- name: PreGenerationSteps
+  type: stepList
+  default: []
+# Mappings to OS name required at template compile time by 1es pipeline templates
+- name: Pools
+  type: object
+  default:
+    - name: Linux
+      filter: .*Linux.*Pool$
+      os: linux
+    - name: Windows
+      filter: .*Windows.*Pool$
+      os: windows
+    - name: Mac
+      filter: .*MacPool$
+      os: macOS
+
+jobs:
+- job: ${{ parameters.GenerateJobName }}
+  variables:
+    - template: /eng/pipelines/templates/variables/image.yml
+    - name: skipComponentGovernanceDetection
+      value: true
+    - name: displayNameFilter
+      value: $[ coalesce(variables.jobMatrixFilter, '.*') ]
+  pool:
+    name: ${{ parameters.Pool }}
+    vmImage: ${{ parameters.OsVmImage }}
+    os: ${{ parameters.Os }}
+  ${{ if parameters.DependsOn }}:
+    dependsOn: ${{ parameters.DependsOn }}
+  steps:
+    # Skip sparse checkout for the `azure-sdk-for-<lang>-pr` private mirrored repositories
+    # as we require the github service connection to be loaded.
+    - ${{ if and(parameters.SparseCheckout, not(contains(variables['Build.DefinitionName'], '-pr - '))) }}:
+      - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
+        parameters:
+          ${{ if ne(length(parameters.SparseCheckoutPaths), 0) }}:
+            Paths: ${{ parameters.SparseCheckoutPaths }}
+          ${{ if and(eq(length(parameters.SparseCheckoutPaths), 0), ne(parameters.AdditionalParameters.ServiceDirectory, '')) }}:
+            Paths:
+              - "sdk/${{ parameters.AdditionalParameters.ServiceDirectory }}"
+
+    - ${{ parameters.PreGenerationSteps }}
+
+    - ${{ each config in parameters.MatrixConfigs }}:
+      - ${{ each pool in parameters.Pools }}:
+        - ${{ if eq(config.GenerateVMJobs, 'true') }}:
+          - task: Powershell@2
+            inputs:
+              pwsh: true
+              filePath: eng/common/scripts/job-matrix/Create-JobMatrix.ps1
+              arguments: >
+                -ConfigPath ${{ config.Path }}
+                -Selection ${{ config.Selection }}
+                -DisplayNameFilter '$(displayNameFilter)'
+                -Filters '${{ join(''',''', parameters.MatrixFilters) }}', 'container=^$', 'SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}', 'Pool=${{ pool.filter }}'
+                -Replace '${{ join(''',''', parameters.MatrixReplace) }}'
+                -NonSparseParameters '${{ join(''',''', config.NonSparseParameters) }}'
+            displayName: Create ${{ pool.name }} Matrix ${{ config.Name }}
+            name: vm_job_matrix_${{ config.Name }}_${{ pool.name }}
+
+        - ${{ if eq(config.GenerateContainerJobs, 'true') }}:
+          - task: Powershell@2
+            inputs:
+              pwsh: true
+              filePath: eng/common/scripts/job-matrix/Create-JobMatrix.ps1
+              arguments: >
+                -ConfigPath ${{ config.Path }}
+                -Selection ${{ config.Selection }}
+                -DisplayNameFilter '$(displayNameFilter)'
+                -Filters '${{ join(''',''', parameters.MatrixFilters) }}', 'container=^$', 'SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}', 'Pool=${{ pool.filter }}'
+                -NonSparseParameters '${{ join(''',''', config.NonSparseParameters) }}'
+            displayName: Create ${{ pool.name }} Container Matrix ${{ config.Name }}
+            name: container_job_matrix_${{ config.Name }}_${{ pool.name }}
+
+- ${{ each config in parameters.MatrixConfigs }}:
+  - ${{ each pool in parameters.Pools }}:
+    - ${{ if eq(config.GenerateVMJobs, 'true') }}:
+      - template: ${{ parameters.JobTemplatePath }}
+        parameters:
+          UsePlatformContainer: false
+          OSName: ${{ pool.os }}
+          Matrix: dependencies.${{ parameters.GenerateJobName }}.outputs['vm_job_matrix_${{ config.Name }}_${{ pool.name }}.matrix']
+          DependsOn: ${{ parameters.GenerateJobName }}
+          CloudConfig: ${{ parameters.CloudConfig }}
+          ${{ each param in parameters.AdditionalParameters }}:
+            ${{ param.key }}: ${{ param.value }}
+
+    - ${{ if eq(config.GenerateContainerJobs, 'true') }}:
+      - template: ${{ parameters.JobTemplatePath }}
+        parameters:
+          UsePlatformContainer: true
+          OSName: ${{ pool.os }}
+          Matrix: dependencies.${{ parameters.GenerateJobName }}.outputs['vm_job_matrix_${{ config.Name }}_${{ pool.name }}.matrix']
+          DependsOn: ${{ parameters.GenerateJobName }}
+          CloudConfig: ${{ parameters.CloudConfig }}
+          ${{ each param in parameters.AdditionalParameters }}:
+            ${{ param.key }}: ${{ param.value }}

eng/common/pipelines/templates/steps/publish-1es-artifact.yml

@@ -0,0 +1,30 @@
+# This step is used to prevent duplication of artifact publishes when there is an issue that would prevent the overall success of the job.
+# Ensuring that we only publish when successful (and two a differently named artifact otherwise) will allow easy retry on a build pipeline
+# without running into the "cannot override artifact" failure when we finally do get a passing run.
+
+# ArtifactName - The name of the artifact in the "successful" case.
+# ArtifactPath - The path we will be publishing.
+# CustomCondition - Used if there is additional logic necessary to prevent attempt of publish.
+
+parameters:
+  ArtifactName: ''
+  ArtifactPath: ''
+  CustomCondition: true
+
+steps:
+  - pwsh: |
+      Write-Host "##vso[task.setvariable variable=PublishArtifactName;]${{ parameters.ArtifactName }}"
+    condition: and(succeeded(), ${{ parameters.CustomCondition }})
+    displayName: Set Artifact Name
+
+  - pwsh: |
+      Write-Host "##vso[task.setvariable variable=PublishArtifactName;]${{ parameters.ArtifactName }}-FailedAttempt$(System.JobAttempt)"
+    condition: and(failed(), ${{ parameters.CustomCondition }})
+    displayName: Set Failed Artifact Name
+
+  - task: 1ES.PublishPipelineArtifact@1
+    condition: and(succeeded(), ${{ parameters.CustomCondition }})
+    displayName: 'Publish ${{ parameters.ArtifactName }} Artifacts'
+    inputs:
+      artifact: '$(PublishArtifactName)'
+      path: '${{ parameters.ArtifactPath }}'