diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 5fefcb3b7e266..068e578887892 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -100,6 +100,13 @@ if(NOT WIN32) message(WARNING "Instrument is only supported on Windows now") set(onnxruntime_ENABLE_INSTRUMENT OFF) endif() +else() + check_cxx_compiler_flag(/d2FH4- HAS_D2FH4) + if (HAS_D2FH4) + message("Enabling /d2FH4-") + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /d2FH4-") + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /d2FH4-") + endif() endif() if(onnxruntime_USE_OPENMP) diff --git a/tools/ci_build/github/azure-pipelines/azure-pipelines-py-packaging.yml b/tools/ci_build/github/azure-pipelines/azure-pipelines-py-packaging.yml index 914f930ad2cea..7063df1dd52de 100644 --- a/tools/ci_build/github/azure-pipelines/azure-pipelines-py-packaging.yml +++ b/tools/ci_build/github/azure-pipelines/azure-pipelines-py-packaging.yml @@ -122,10 +122,7 @@ jobs: - template: templates/clean-agent-build-directory-step.yml - job: Windows_py_Wheels - workspace: - clean: all - pool: Win-CPU - timeoutInMinutes: 120 + pool: 'Win-CPU-2019' strategy: matrix: Python35: @@ -135,38 +132,68 @@ jobs: Python37: python.version: '3.7' variables: - buildDirectory: '$(Build.SourcesDirectory)\build' - steps: - - task: CondaEnvironment@1 - inputs: - createCustomEnvironment: true - environmentName: 'py$(python.version)' - packageSpecs: 'python=$(python.version)' - cleanEnvironment: true + OrtPackageId: 'Microsoft.ML.OnnxRuntime' + MsbuildArguments: '-maxcpucount' + OnnxRuntimeBuildDirectory: '$(Build.BinariesDirectory)' + EnvSetupScript: setup_env.bat + buildArch: x64 + setVcvars: true + BuildConfig: 'Release' + timeoutInMinutes: 60 + workspace: + clean: all + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: $(python.version) + addToPath: true + architecture: 'x64' + - template: templates/set-test-data-variables-step.yml - - task: BatchScript@1 - displayName: 'Run build script' - inputs: - filename: 'build.bat' - arguments: ' --build_dir $(buildDirectory) --config Release --use_openmp --build_wheel' - workingFolder: "$(Build.SourcesDirectory)" + - task: BatchScript@1 + displayName: 'setup env' + inputs: + filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\$(EnvSetupScript)' + modifyEnvironment: true + workingFolder: '$(Build.BinariesDirectory)' - - task: CopyFiles@2 - displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)' - inputs: - SourceFolder: '$(buildDirectory)' - Contents: '**\dist\*.whl' - TargetFolder: '$(Build.ArtifactStagingDirectory)' + - script: | + python -m pip install -q pyopenssl setuptools wheel numpy + + workingDirectory: '$(Build.BinariesDirectory)' + displayName: 'Install python modules' - - task: PublishBuildArtifacts@1 - displayName: 'Publish Artifact: ONNXRuntime python wheel' - inputs: - ArtifactName: onnxruntime + - task: PythonScript@0 + displayName: 'Download test data' + inputs: + scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\github\download_test_data.py' + arguments: --test_data_url $(TestDataUrl) --build_dir $(Build.BinariesDirectory) + workingDirectory: $(Build.BinariesDirectory) - - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 - displayName: 'Component Detection' + - task: PythonScript@0 + displayName: 'BUILD' + inputs: + scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' + arguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_generator "Visual Studio 16 2019" --build_wheel --use_automl --enable_onnx_tests --parallel' + workingDirectory: '$(Build.BinariesDirectory)' + + - task: CopyFiles@2 + displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)' + inputs: + SourceFolder: '$(Build.BinariesDirectory)\$(BuildConfig)' + Contents: '**\dist\*.whl' + TargetFolder: '$(Build.ArtifactStagingDirectory)' - - template: templates/clean-agent-build-directory-step.yml + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: ONNXRuntime python wheel' + inputs: + ArtifactName: onnxruntime + + - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 + displayName: 'Component Detection' + condition: succeeded() + + - template: templates/clean-agent-build-directory-step.yml - job: Windows_py_GPU_Wheels workspace: @@ -185,13 +212,11 @@ jobs: Python37: python.version: '3.7' steps: - - task: CondaEnvironment@1 - inputs: - createCustomEnvironment: true - environmentName: 'py$(python.version)' - packageSpecs: 'python=$(python.version)' - cleanEnvironment: true - + - task: UsePythonVersion@0 + inputs: + versionSpec: $(python.version) + addToPath: true + architecture: 'x64' - task: BatchScript@1 displayName: 'Setup VS2017 env vars' inputs: diff --git a/tools/ci_build/github/azure-pipelines/nuget/cpu-esrp-pipeline.yml b/tools/ci_build/github/azure-pipelines/nuget/cpu-esrp-pipeline.yml index 2bd66515dc072..062ff28790fba 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/cpu-esrp-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/cpu-esrp-pipeline.yml @@ -15,6 +15,6 @@ schedules: jobs: - template: templates/cpu.yml parameters: - AgentPool : $(AgentPoolWin) + AgentPool : 'Win-CPU-2019' DoEsrp: 'true' DoCompliance: 'true' diff --git a/tools/ci_build/github/azure-pipelines/nuget/cpu-pipeline.yml b/tools/ci_build/github/azure-pipelines/nuget/cpu-pipeline.yml index a1baef74398d8..0a84e1daef6de 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/cpu-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/cpu-pipeline.yml @@ -1,11 +1,10 @@ # Defined as pipeline variables # variables: -# AgentPoolWin : 'Win-CPU' # AgentPoolLinux : 'Linux-CPU' # AgentPoolMacOS : 'macOS-10.13' jobs: - template: templates/cpu.yml parameters: - AgentPool : $(AgentPoolWin) + AgentPool : 'Win-CPU-2019' DoEsrp: 'false' diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml index 7a58bc99947b2..c43d9e674a9ad 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml @@ -1,6 +1,5 @@ # Defined as pipeline variables # variables: -# AgentPoolWin : 'Win-CPU' # AgentPoolLinux : 'Linux-CPU' # AgentPoolMacOS : 'macOS-10.13' @@ -9,11 +8,11 @@ parameters: DoCompliance: 'false' jobs: -- template: ../../templates/win-ci.yml +- template: ../../templates/win-ci-2019.yml parameters: - AgentPool : $(AgentPoolWin) + AgentPool : 'Win-CPU-2019' JobName: 'Windows_CI_Dev' - BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --use_telemetry' + BuildCommand: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --use_openmp --build_shared_lib --enable_onnx_tests --use_telemetry --cmake_generator "Visual Studio 16 2019"' DoDebugBuild: 'false' DoNugetPack : 'true' DoCompliance: ${{ parameters.DoCompliance }} @@ -24,11 +23,11 @@ jobs: mkdir $(Build.ArtifactStagingDirectory)\testdata copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\custom_op_library.* $(Build.ArtifactStagingDirectory)\testdata -- template: ../../templates/win-x86-ci.yml +- template: ../../templates/win-x86-ci-2019.yml parameters: - AgentPool : $(AgentPoolWin) + AgentPool : 'Win-CPU-2019' JobName: 'Windows_CI_Dev_x86' - BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --x86 --use_telemetry' + BuildCommand: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --use_openmp --build_shared_lib --enable_onnx_tests --x86 --use_telemetry --cmake_generator "Visual Studio 16 2019"' DoDebugBuild: 'false' DoNugetPack : 'true' DoCompliance: 'false' @@ -103,7 +102,7 @@ jobs: - job: NuGet_Packaging workspace: clean: all - pool: $(AgentPoolWin) + pool: 'Win-CPU-2019' dependsOn: - Windows_CI_Dev - Windows_CI_Dev_x86 diff --git a/tools/ci_build/github/azure-pipelines/templates/compliance.yml b/tools/ci_build/github/azure-pipelines/templates/compliance.yml index 467dfe53e205a..f96857856bc4a 100644 --- a/tools/ci_build/github/azure-pipelines/templates/compliance.yml +++ b/tools/ci_build/github/azure-pipelines/templates/compliance.yml @@ -1,10 +1,4 @@ steps: -- task: SDLNativeRules@2 - displayName: 'Run the PREfast SDL Native Rules for MSBuild' - inputs: - userProvideBuildInfo: 'auto' - continueOnError: true - - task: CredScan@2 displayName: 'Run CredScan' inputs: @@ -17,7 +11,19 @@ steps: arguments: 'analyze $(Build.BinariesDirectory)\RelWithDebInfo\*.dll --recurse --verbose' continueOnError: true -- task: PublishSecurityAnalysisLogs@2 +- task: securedevelopmentteam.vss-secure-development-tools.build-task-prefast.SDLNativeRules@2 + displayName: 'Run the PREfast SDL Native Rules for MSBuild' + continueOnError: true + +- task: securedevelopmentteam.vss-secure-development-tools.build-task-report.SdtReport@1 + displayName: 'Create Security Analysis Report' + inputs: + BinSkim: true + BinSkimBreakOn: WarningAbove + CredScan: true + SDLNativeRules: true + +- task: securedevelopmentteam.vss-secure-development-tools.build-task-publishsecurityanalysislogs.PublishSecurityAnalysisLogs@2 displayName: 'Publish Security Analysis Logs' continueOnError: true diff --git a/tools/ci_build/github/azure-pipelines/templates/win-ci-2019.yml b/tools/ci_build/github/azure-pipelines/templates/win-ci-2019.yml new file mode 100644 index 0000000000000..c018d6f02089e --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/templates/win-ci-2019.yml @@ -0,0 +1,188 @@ +parameters: + DoDebugBuild: 'true' + DoCompliance: 'false' + BuildCommand: '' + JobName: 'Windows_CI_Dev' + DoNugetPack: 'false' + NuPackScript : '' + ArtifactName: 'drop-nuget' + DoEsrp: 'false' + DoTestCoverage: 'false' + BuildArch: 'x64' + SetVcvars: 'false' + msbuildArchitecture: 'amd64' + EnvSetupScript: 'setup_env.bat' + CudaVersion: '' + AgentPool: 'Win-CPU' + AgentDemands: [] + OrtPackageId: Microsoft.ML.OnnxRuntime +jobs: +- job: ${{ parameters.JobName }} + timeoutInMinutes: 120 + workspace: + clean: all + pool: + name: ${{ parameters.AgentPool }} + demands: ${{ parameters.AgentDemands }} + variables: + buildDirectory: '$(Build.BinariesDirectory)' + BuildCommand: ${{ parameters.BuildCommand }} + OnnxRuntimeBuildDirectory: '$(Build.BinariesDirectory)' + DotNetExe: 'dotnet.exe' + CUDA_VERSION: ${{ parameters.CudaVersion }} + + steps: + - powershell: | + if($env:TELEMETRYGUID) + { + $length = $env:TELEMETRYGUID.length + $fileContent = "#define ENABLE_TELEMETRY`n#define TraceLoggingOptionMicrosoftTelemetry() \ + TraceLoggingOptionGroup("+$env:TELEMETRYGUID.substring(1, $length-2)+")" + New-Item -Path "$(Build.SourcesDirectory)\include\onnxruntime\core\platform\windows\TraceLoggingConfigPrivate.h" -ItemType "file" -Value "$fileContent" -Force + } + displayName: 'Create TraceLoggingConfigPrivate.h For WinML Telemetry' + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.7' + addToPath: true + architecture: ${{ parameters.BuildArch }} + + - task: BatchScript@1 + displayName: 'setup env' + inputs: + filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\${{ parameters.EnvSetupScript }}' + modifyEnvironment: true + workingFolder: '$(Build.BinariesDirectory)' + + - script: | + python -m pip install -q pyopenssl setuptools wheel numpy + workingDirectory: '$(Build.BinariesDirectory)' + displayName: 'Install python modules' + + + - task: PythonScript@0 + displayName: 'Generate cmake config' + inputs: + scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' + arguments: '$(BuildCommand) --update --config RelWithDebInfo' + workingDirectory: '$(Build.BinariesDirectory)' + + - task: VSBuild@1 + displayName: 'Build' + inputs: + solution: '$(Build.BinariesDirectory)\RelWithDebInfo\onnxruntime.sln' + platform: 'x64' + configuration: RelWithDebInfo + msbuildArchitecture: ${{ parameters.BuildArch }} + maximumCpuCount: true + logProjectEvents: true + workingFolder: '$(Build.BinariesDirectory)\RelWithDebInfo' + createLogFile: true + clean: true + + # Build RelWithDebInfo -- this variable required to build C# + - script: | + @echo ##vso[task.setvariable variable=Configuration]RelWithDebInfo + + - template: set-test-data-variables-step.yml + + - task: NuGetToolInstaller@0 + displayName: Use Nuget 4.9 + inputs: + versionSpec: 4.9.4 + + - task: PythonScript@0 + displayName: 'Download test data' + inputs: + scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\github\download_test_data.py' + arguments: --test_data_url $(TestDataUrl) --build_dir $(Build.BinariesDirectory) + workingDirectory: $(Build.BinariesDirectory) + + - task: DotNetCoreCLI@2 + displayName: 'Restore nuget packages' + inputs: + command: restore + projects: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln' + configuration: 'RelWithDebInfo' + arguments: '--configuration RelWithDebInfo -p:Platform="Any CPU" -p:OrtPackageId=${{ parameters.OrtPackageId }}' + workingDirectory: '$(Build.SourcesDirectory)\csharp' + + - task: DotNetCoreCLI@2 + displayName: 'Build C#' + inputs: + command: build + projects: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln' + configuration: 'RelWithDebInfo' + arguments: '--configuration RelWithDebInfo -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=${{ parameters.OrtPackageId }}' + workingDirectory: '$(Build.SourcesDirectory)\csharp' + + - task: DotNetCoreCLI@2 + displayName: 'Test C#' + inputs: + command: test + projects: '$(Build.SourcesDirectory)\csharp\test\Microsoft.ML.OnnxRuntime.Tests\Microsoft.ML.OnnxRuntime.Tests.csproj' + configuration: 'RelWithDebInfo' + arguments: '--configuration RelWithDebInfo -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=${{ parameters.OrtPackageId }}' + workingDirectory: '$(Build.SourcesDirectory)\csharp' + + - script: | + mklink /D /J $(Build.BinariesDirectory)\RelWithDebInfo\models $(Build.BinariesDirectory)\models + DIR dist\ /S /B > wheel_filename_file + set /p WHEEL_FILENAME= wheel_filename_file + set /p WHEEL_FILENAME=