diff --git a/.azure/pipelines/build.yaml b/.azure/pipelines/build.yaml index 5c2ba8e905..d33810ff2a 100644 --- a/.azure/pipelines/build.yaml +++ b/.azure/pipelines/build.yaml @@ -8,7 +8,7 @@ trigger: schedules: - cron: "0 0 * * *" - displayName: Daily midnight build (including CodeQL) + displayName: 'Daily midnight build (including CodeQL)' branches: include: - main @@ -26,7 +26,7 @@ parameters: - name: version_prefix displayName: Version prefix type: string - default: 7.0.0 + default: 8.0.0 - name: include_suffix displayName: Append version suffix type: boolean @@ -61,217 +61,18 @@ parameters: type: boolean variables: - build_flags: ' /m /v:m' - solution: 'Orleans.sln' - codesign_runtime: '2.1.x' - # Auto-injection is not necessary because the tasks are explicitly included where they're enabled. - Codeql.SkipTaskAutoInjection: true - ${{ if eq(variables['System.TeamProject'], 'GitHub - PR Builds') }}: - pool_name: 'orleans-pr-hosted-pool' - official_build: false - ${{ else }}: - pool_name: 'orleans-build-hosted-pool' - official_build: true - # Do not let CodeQL3000 Extension gate scan frequency. - Codeql.Cadence: 0 - # Enable CodeQL3000 unconditionally so it may be run on any branch. - Codeql.Enabled: true - # Ignore test and infrastructure code. - Codeql.SourceRoot: src - # CodeQL3000 needs this plumbed along as a variable to enable TSA. Don't use TSA in manual builds. - Codeql.TSAEnabled: ${{ eq(variables['Build.Reason'], 'Schedule') }} - # Default expects tsaoptions.json under SourceRoot. - Codeql.TSAOptionsPath: '$(Build.SourcesDirectory)/.config/tsaoptions.json' - # Do not slow builds down w/ the CodeQL3000 tasks unless this is a nightly build or it's requested. - runCodeQL3000: ${{ or(eq(variables['Build.Reason'], 'Schedule'), and(eq(variables['Build.Reason'], 'Manual'), eq(parameters.runCodeQL3000, 'true'))) }} +- template: templates/vars.yaml jobs: - -# Build, sign dlls, build nuget pkgs, then sign them -- job: Build - pool: $(pool_name) - displayName: Build and create NuGet packages - steps: - - checkout: self - - task: UseDotNet@2 - displayName: 'Use .NET Core sdk' - inputs: - useGlobalJson: true - - ${{ if eq(variables.runCodeQL3000, 'true') }}: - - task: CodeQL3000Init@0 - displayName: CodeQL Initialize - # This task only tags a build if it actually does CodeQL3000 work. - # Those tasks no-op while the analysis is considered up to date i.e. for runs w/in a few days of each other. - - script: "echo ##vso[build.addbuildtag]CodeQL3000" - displayName: 'Set CI CodeQL3000 tag' - condition: ne(variables.CODEQL_DIST,'') - - task: DotNetCoreCLI@2 - displayName: Build - inputs: - command: build - arguments: '$(build_flags) /bl:${{parameters.build_configuration}}-Build.binlog /p:Configuration=${{parameters.build_configuration}} $(solution)' - env: - VersionPrefix: ${{parameters.version_prefix}} - ${{ if eq(parameters.include_suffix, true) }}: - VersionSuffix: ${{parameters.version_suffix}} - OfficialBuild: $(official_build) - - ${{ if eq(variables.runCodeQL3000, 'true') }}: - - task: CodeQL3000Finalize@0 - displayName: CodeQL Finalize - # DLL code signing - - ${{ if eq(parameters.codesign, true) }}: - - task: UseDotNet@2 - displayName: 'Codesign: Use .NET Core' - inputs: - packageType: runtime - version: $(codesign_runtime) - - task: CopyFiles@2 - displayName: 'Codesign: Copy Files for signing' - inputs: - SourceFolder: '$(build.sourcesdirectory)' - Contents: | - src/**/bin/${{parameters.build_configuration}}/**/Orleans*.dll - src/**/bin/${{parameters.build_configuration}}/**/Microsoft.Orleans.*.dll - !src/BootstrapBuild/** - TargetFolder: '$(build.artifactstagingdirectory)\codesign' - CleanTargetFolder: true - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - displayName: 'Codesign: ESRP CodeSigning' - inputs: - ConnectedServiceName: 'CodeSign Service (NuGet)' - FolderPath: '$(build.artifactstagingdirectory)\codesign' - Pattern: '*' - signConfigType: inlineSignParams - inlineOperation: | - [ - { - "keyCode": "CP-230012", - "operationSetCode": "SigntoolSign", - "parameters": [ - { - "parameterName": "OpusName", - "parameterValue": "Microsoft" - }, - { - "parameterName": "OpusInfo", - "parameterValue": "http://www.microsoft.com" - }, - { - "parameterName": "FileDigest", - "parameterValue": "/fd \"SHA256\"" - }, - { - "parameterName": "PageHash", - "parameterValue": "/NPH" - }, - { - "parameterName": "TimeStamp", - "parameterValue": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" - } - ], - "toolName": "sign", - "toolVersion": "1.0" - }, - { - "keyCode": "CP-230012", - "operationSetCode": "SigntoolVerify", - "parameters": [ ], - "toolName": "sign", - "toolVersion": "1.0" - } - ] - SessionTimeout: 180 - VerboseLogin: true - - task: CopyFiles@2 - displayName: 'Codesign: Copy Signed Files Back' - inputs: - SourceFolder: '$(build.artifactstagingdirectory)\codesign' - Contents: '**\*' - TargetFolder: '$(build.sourcesdirectory)' - OverWrite: true - # End DLL code signing - - task: CmdLine@2 - displayName: Pack - inputs: - script: 'dotnet pack --no-build --no-restore $(build_flags) /bl:${{parameters.build_configuration}}-Pack.binlog /p:Configuration=${{parameters.build_configuration}} $(solution)' - env: - VersionPrefix: ${{parameters.version_prefix}} - ${{ if eq(parameters.include_suffix, true) }}: - VersionSuffix: ${{parameters.version_suffix}} - OfficialBuild: $(official_build) - # NuGet code signing - - ${{ if eq(parameters.codesign, true) }}: - - task: UseDotNet@2 - displayName: 'Codesign: Use .NET Core' - inputs: - packageType: runtime - version: $(codesign_runtime) - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 - displayName: 'Codesign: ESRP CodeSigning (nuget)' - inputs: - ConnectedServiceName: 'CodeSign Service (NuGet)' - FolderPath: '$(build.sourcesdirectory)/Artifacts/${{parameters.build_configuration}}' - Pattern: '*.nupkg' - signConfigType: inlineSignParams - inlineOperation: | - [ - { - "keyCode": "CP-401405", - "operationSetCode": "NuGetSign", - "parameters": [], - "toolName": "sign", - "toolVersion": "1.0" - }, - { - "keyCode": "CP-401405", - "operationSetCode": "NuGetVerify", - "parameters": [ ], - "toolName": "sign", - "toolVersion": "1.0" - } - ] - SessionTimeout: 180 - VerboseLogin: true - # End NuGet code signing - - ${{ if ne(variables['System.TeamProject'], 'GitHub - PR Builds') }}: - - task: PublishBuildArtifacts@1 - displayName: 'Publish Artifact: Nuget Packages' - inputs: - PathtoPublish: '$(build.sourcesdirectory)/Artifacts/${{parameters.build_configuration}}' - ArtifactName: nuget - -# Tests -- ${{ if and(eq(parameters.skip_test, false), ne(variables.runCodeQL3000, 'true')) }}: - - ${{ each category in parameters.tests_categories }}: - - ${{ each framework in parameters.frameworks }}: - - job: - displayName: ${{category}} on ${{framework}} - timeoutInMinutes: 120 - pool: $(pool_name) - dependsOn: Build - steps: - - checkout: self - - task: UseDotNet@2 - inputs: - useGlobalJson: true - displayName: 'Use .NET Core sdk' - - task: DotNetCoreCLI@2 - displayName: Build - inputs: - command: build - arguments: '$(build_flags) /bl:${{parameters.build_configuration}}-Build.binlog /p:Configuration=${{parameters.build_configuration}} $(solution)' - - ${{ if ne(variables['System.TeamProject'], 'GitHub - PR Builds') }}: - - task: DownloadSecureFile@1 - name: secretFile - displayName: 'Download secrets' - inputs: - secureFile: 'OrleansTestSecrets.json' - - task: DotNetCoreCLI@2 - displayName: Test - inputs: - command: 'test' - testRunTitle: ${{category}} on ${{framework}} - arguments: '--no-build --framework ${{framework}} --configuration "${{parameters.build_configuration}}" --filter Category=${{category}} -- -parallel none -noshadow' - ${{ if ne(variables['System.TeamProject'], 'GitHub - PR Builds') }}: - env: - ORLEANS_SECRETFILE: $(secretFile.secureFilePath) +- template: templates/build.yaml + parameters: + build_configuration: ${{ parameters.build_configuration }} + version_prefix: ${{ parameters.version_prefix }} + include_suffix: ${{ parameters.include_suffix }} + version_suffix: ${{ parameters.version_suffix }} + codesign: ${{ parameters.codesign }} + skip_test: ${{ parameters.skip_test }} + publish_nightly: false + frameworks: ${{ parameters.frameworks }} + tests_categories: ${{ parameters.tests_categories }} + runCodeQL3000: ${{ parameters.runCodeQL3000 }} \ No newline at end of file diff --git a/.azure/pipelines/nightly-main.yaml b/.azure/pipelines/nightly-main.yaml new file mode 100644 index 0000000000..d794f3f504 --- /dev/null +++ b/.azure/pipelines/nightly-main.yaml @@ -0,0 +1,24 @@ +trigger: none +pr: none + +schedules: +- cron: "0 0 * * *" + displayName: Publish nightly packages + branches: + include: + - main + always: false + +variables: +- template: templates/vars.yaml + +jobs: +- template: templates/build.yaml + parameters: + build_configuration: Release + version_prefix: '8.1.0' + include_suffix: true + version_suffix: nightly.$(Build.BuildNumber) + codesign: true + publish_nightly: true + skip_test: true diff --git a/.azure/pipelines/templates/build.yaml b/.azure/pipelines/templates/build.yaml new file mode 100644 index 0000000000..d008b8b261 --- /dev/null +++ b/.azure/pipelines/templates/build.yaml @@ -0,0 +1,256 @@ +parameters: + - name: build_configuration + displayName: Build configuration + type: string + default: Release + values: + - Release + - Debug + - name: version_prefix + displayName: Version prefix + type: string + default: 8.0.0 + - name: include_suffix + displayName: Append version suffix + type: boolean + default: true + - name: version_suffix + displayName: Version suffix + type: string + default: ci.$(Build.BuildNumber) + - name: codesign + displayName: Enable code signing + type: boolean + default: false + - name: skip_test + displayName: Skip tests + type: boolean + default: false + - name: publish_nightly + displayName: Publish to orleans-nightly + type: boolean + default: false + - name: frameworks + displayName: Frameworks + type: object + default: + - net8.0 + - name: tests_categories + displayName: Test categories + type: object + default: + - BVT + - SlowBVT + - Functional + - name: runCodeQL3000 + default: false + displayName: Run CodeQL3000 tasks + type: boolean + +jobs: + +# Build, sign dlls, build nuget pkgs, then sign them +- job: Build + pool: $(pool_name) + displayName: Build and create NuGet packages + steps: + - checkout: self + - task: UseDotNet@2 + displayName: 'Use .NET Core sdk' + inputs: + useGlobalJson: true + - ${{ if eq(variables.runCodeQL3000, 'true') }}: + - task: CodeQL3000Init@0 + displayName: CodeQL Initialize + # This task only tags a build if it actually does CodeQL3000 work. + # Those tasks no-op while the analysis is considered up to date i.e. for runs w/in a few days of each other. + - script: "echo ##vso[build.addbuildtag]CodeQL3000" + displayName: 'Set CI CodeQL3000 tag' + condition: ne(variables.CODEQL_DIST,'') + - task: DotNetCoreCLI@2 + displayName: Build + inputs: + command: build + arguments: '$(build_flags) /bl:${{parameters.build_configuration}}-Build.binlog /p:Configuration=${{parameters.build_configuration}} $(solution)' + env: + VersionPrefix: ${{parameters.version_prefix}} + ${{ if eq(parameters.include_suffix, true) }}: + VersionSuffix: ${{parameters.version_suffix}} + OfficialBuild: $(official_build) + - ${{ if eq(variables.runCodeQL3000, 'true') }}: + - task: CodeQL3000Finalize@0 + displayName: CodeQL Finalize + # DLL code signing + - ${{ if eq(parameters.codesign, true) }}: + - task: UseDotNet@2 + displayName: 'Codesign: Use .NET Core' + inputs: + packageType: runtime + version: $(codesign_runtime) + - task: CopyFiles@2 + displayName: 'Codesign: Copy Files for signing' + inputs: + SourceFolder: '$(build.sourcesdirectory)' + Contents: | + src/**/bin/${{parameters.build_configuration}}/**/Orleans*.dll + src/**/bin/${{parameters.build_configuration}}/**/Microsoft.Orleans.*.dll + !src/BootstrapBuild/** + TargetFolder: '$(build.artifactstagingdirectory)\codesign' + CleanTargetFolder: true + - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + displayName: 'Codesign: ESRP CodeSigning' + inputs: + ConnectedServiceName: 'CodeSign Service (NuGet)' + FolderPath: '$(build.artifactstagingdirectory)\codesign' + Pattern: '*' + signConfigType: inlineSignParams + inlineOperation: | + [ + { + "keyCode": "CP-230012", + "operationSetCode": "SigntoolSign", + "parameters": [ + { + "parameterName": "OpusName", + "parameterValue": "Microsoft" + }, + { + "parameterName": "OpusInfo", + "parameterValue": "http://www.microsoft.com" + }, + { + "parameterName": "FileDigest", + "parameterValue": "/fd \"SHA256\"" + }, + { + "parameterName": "PageHash", + "parameterValue": "/NPH" + }, + { + "parameterName": "TimeStamp", + "parameterValue": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" + } + ], + "toolName": "sign", + "toolVersion": "1.0" + }, + { + "keyCode": "CP-230012", + "operationSetCode": "SigntoolVerify", + "parameters": [ ], + "toolName": "sign", + "toolVersion": "1.0" + } + ] + SessionTimeout: 180 + VerboseLogin: true + - task: CopyFiles@2 + displayName: 'Codesign: Copy Signed Files Back' + inputs: + SourceFolder: '$(build.artifactstagingdirectory)\codesign' + Contents: '**\*' + TargetFolder: '$(build.sourcesdirectory)' + OverWrite: true + # End DLL code signing + - task: CmdLine@2 + displayName: Pack + inputs: + script: 'dotnet pack --no-build --no-restore $(build_flags) /bl:${{parameters.build_configuration}}-Pack.binlog /p:Configuration=${{parameters.build_configuration}} $(solution)' + env: + VersionPrefix: ${{parameters.version_prefix}} + ${{ if eq(parameters.include_suffix, true) }}: + VersionSuffix: ${{parameters.version_suffix}} + OfficialBuild: $(official_build) + # NuGet code signing + - ${{ if eq(parameters.codesign, true) }}: + - task: UseDotNet@2 + displayName: 'Codesign: Use .NET Core' + inputs: + packageType: runtime + version: $(codesign_runtime) + - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + displayName: 'Codesign: ESRP CodeSigning (nuget)' + inputs: + ConnectedServiceName: 'CodeSign Service (NuGet)' + FolderPath: '$(build.sourcesdirectory)/Artifacts/${{parameters.build_configuration}}' + Pattern: '*.nupkg' + signConfigType: inlineSignParams + inlineOperation: | + [ + { + "keyCode": "CP-401405", + "operationSetCode": "NuGetSign", + "parameters": [], + "toolName": "sign", + "toolVersion": "1.0" + }, + { + "keyCode": "CP-401405", + "operationSetCode": "NuGetVerify", + "parameters": [ ], + "toolName": "sign", + "toolVersion": "1.0" + } + ] + SessionTimeout: 180 + VerboseLogin: true + # End NuGet code signing + - ${{ if ne(variables['System.TeamProject'], 'GitHub - PR Builds') }}: + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: Nuget Packages' + inputs: + PathtoPublish: '$(build.sourcesdirectory)/Artifacts/${{parameters.build_configuration}}' + ArtifactName: nuget + +# Tests +- ${{ if and(eq(parameters.skip_test, false), ne(variables.runCodeQL3000, 'true')) }}: + - ${{ each category in parameters.tests_categories }}: + - ${{ each framework in parameters.frameworks }}: + - job: + displayName: ${{category}} on ${{framework}} + timeoutInMinutes: 120 + pool: $(pool_name) + dependsOn: Build + steps: + - checkout: self + - task: UseDotNet@2 + inputs: + useGlobalJson: true + displayName: 'Use .NET Core sdk' + - task: DotNetCoreCLI@2 + displayName: Build + inputs: + command: build + arguments: '$(build_flags) /bl:${{parameters.build_configuration}}-Build.binlog /p:Configuration=${{parameters.build_configuration}} $(solution)' + - ${{ if ne(variables['System.TeamProject'], 'GitHub - PR Builds') }}: + - task: DownloadSecureFile@1 + name: secretFile + displayName: 'Download secrets' + inputs: + secureFile: 'OrleansTestSecrets.json' + - task: DotNetCoreCLI@2 + displayName: Test + inputs: + command: 'test' + testRunTitle: ${{category}} on ${{framework}} + arguments: '--no-build --framework ${{framework}} --configuration "${{parameters.build_configuration}}" --filter Category=${{category}} -- -parallel none -noshadow' + ${{ if ne(variables['System.TeamProject'], 'GitHub - PR Builds') }}: + env: + ORLEANS_SECRETFILE: $(secretFile.secureFilePath) + +# Publish packages to nightly feed +- ${{ if and(eq(parameters.codesign, true), eq(parameters.publish_nightly, true)) }}: + - job: Publish + dependsOn: Build + steps: + - download: current + artifact: nuget + - task: NuGetAuthenticate@1 + displayName: 'NuGet Authenticate' + - task: NuGetCommand@2 + displayName: 'NuGet push' + inputs: + command: push + publishVstsFeed: 'orleans-public/orleans-nightly' + packagesToPush: $(Pipeline.Workspace)/nuget/**/*.nupkg + allowPackageConflicts: true \ No newline at end of file diff --git a/.azure/pipelines/templates/vars.yaml b/.azure/pipelines/templates/vars.yaml new file mode 100644 index 0000000000..6d25fbd64e --- /dev/null +++ b/.azure/pipelines/templates/vars.yaml @@ -0,0 +1,26 @@ +# It seems that variables must be defined in their own file when using templates + +variables: + build_flags: ' /m /v:m' + solution: 'Orleans.sln' + codesign_runtime: '2.1.x' + # Auto-injection is not necessary because the tasks are explicitly included where they're enabled. + Codeql.SkipTaskAutoInjection: true + ${{ if eq(variables['System.TeamProject'], 'GitHub - PR Builds') }}: + pool_name: 'orleans-pr-hosted-pool' + official_build: false + ${{ else }}: + pool_name: 'orleans-build-hosted-pool' + official_build: true + # Do not let CodeQL3000 Extension gate scan frequency. + Codeql.Cadence: 0 + # Enable CodeQL3000 unconditionally so it may be run on any branch. + Codeql.Enabled: true + # Ignore test and infrastructure code. + Codeql.SourceRoot: src + # CodeQL3000 needs this plumbed along as a variable to enable TSA. Don't use TSA in manual builds. + Codeql.TSAEnabled: ${{ eq(variables['Build.Reason'], 'Schedule') }} + # Default expects tsaoptions.json under SourceRoot. + Codeql.TSAOptionsPath: '$(Build.SourcesDirectory)/.config/tsaoptions.json' + # Do not slow builds down w/ the CodeQL3000 tasks unless this is a nightly build or it's requested. + runCodeQL3000: ${{ or(eq(variables['Build.Reason'], 'Schedule'), and(eq(variables['Build.Reason'], 'Manual'), eq(parameters.runCodeQL3000, 'true'))) }}