Windows CI changes(#2650)

microsoft · Dec 13, 2019 · a46a28b · a46a28b
1 parent 18bdde3
commit a46a28b
Show file tree

Hide file tree

Showing 15 changed files with 546 additions and 120 deletions.
diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt
@@ -100,6 +100,13 @@ if(NOT WIN32)
     message(WARNING "Instrument is only supported on Windows now")
     set(onnxruntime_ENABLE_INSTRUMENT OFF)
   endif()
+else()
+  check_cxx_compiler_flag(/d2FH4- HAS_D2FH4)
+  if (HAS_D2FH4)
+    message("Enabling /d2FH4-")
+    set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /d2FH4-")
+    set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /d2FH4-")
+  endif()
 endif()
 
 if(onnxruntime_USE_OPENMP)

diff --git a/tools/ci_build/github/azure-pipelines/azure-pipelines-py-packaging.yml b/tools/ci_build/github/azure-pipelines/azure-pipelines-py-packaging.yml
@@ -122,10 +122,7 @@ jobs:
     - template: templates/clean-agent-build-directory-step.yml
 
 - job: Windows_py_Wheels
-  workspace:
-    clean: all
-  pool: Win-CPU
-  timeoutInMinutes:  120  
+  pool: 'Win-CPU-2019'
   strategy:
     matrix:
       Python35:
@@ -135,38 +132,68 @@ jobs:
       Python37:
         python.version: '3.7'
   variables:
-    buildDirectory: '$(Build.SourcesDirectory)\build'
-  steps:
-    - task: CondaEnvironment@1
-      inputs:
-        createCustomEnvironment: true
-        environmentName: 'py$(python.version)'
-        packageSpecs: 'python=$(python.version)'
-        cleanEnvironment: true
+    OrtPackageId: 'Microsoft.ML.OnnxRuntime'
+    MsbuildArguments: '-maxcpucount'
+    OnnxRuntimeBuildDirectory: '$(Build.BinariesDirectory)'
+    EnvSetupScript: setup_env.bat
+    buildArch: x64
+    setVcvars: true
+    BuildConfig: 'Release'
+  timeoutInMinutes: 60
+  workspace:
+    clean: all
+  steps:    
+  - task: UsePythonVersion@0
+    inputs: 
+      versionSpec: $(python.version) 
+      addToPath: true 
+      architecture: 'x64'
+  - template: templates/set-test-data-variables-step.yml
 
-    - task: BatchScript@1
-      displayName: 'Run build script'
-      inputs:
-        filename: 'build.bat'
-        arguments: ' --build_dir $(buildDirectory) --config Release --use_openmp --build_wheel'
-        workingFolder: "$(Build.SourcesDirectory)"
+  - task: BatchScript@1
+    displayName: 'setup env'
+    inputs:
+      filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\$(EnvSetupScript)'
+      modifyEnvironment: true
+      workingFolder: '$(Build.BinariesDirectory)'
 
-    - task: CopyFiles@2
-      displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)'
-      inputs:
-        SourceFolder: '$(buildDirectory)'
-        Contents: '**\dist\*.whl'
-        TargetFolder: '$(Build.ArtifactStagingDirectory)'
+  - script: |
+     python -m pip install -q pyopenssl setuptools wheel numpy     
+   
+    workingDirectory: '$(Build.BinariesDirectory)'
+    displayName: 'Install python modules'        
 
-    - task: PublishBuildArtifacts@1
-      displayName: 'Publish Artifact: ONNXRuntime python wheel'
-      inputs:
-        ArtifactName: onnxruntime
+  - task: PythonScript@0
+    displayName: 'Download test data'
+    inputs:
+      scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\github\download_test_data.py'
+      arguments: --test_data_url $(TestDataUrl) --build_dir $(Build.BinariesDirectory)
+      workingDirectory: $(Build.BinariesDirectory)
 
-    - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
-      displayName: 'Component Detection'
+  - task: PythonScript@0
+    displayName: 'BUILD'
+    inputs:
+      scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
+      arguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_generator "Visual Studio 16 2019" --build_wheel --use_automl --enable_onnx_tests --parallel'
+      workingDirectory: '$(Build.BinariesDirectory)'     
+
+  - task: CopyFiles@2
+    displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)'
+    inputs:
+      SourceFolder: '$(Build.BinariesDirectory)\$(BuildConfig)'
+      Contents: '**\dist\*.whl'
+      TargetFolder: '$(Build.ArtifactStagingDirectory)'
 
-    - template: templates/clean-agent-build-directory-step.yml
+  - task: PublishBuildArtifacts@1
+    displayName: 'Publish Artifact: ONNXRuntime python wheel'
+    inputs:
+      ArtifactName: onnxruntime
+
+  - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
+    displayName: 'Component Detection'
+    condition: succeeded()
+
+  - template: templates/clean-agent-build-directory-step.yml
 
 - job:  Windows_py_GPU_Wheels
   workspace:
@@ -185,13 +212,11 @@ jobs:
       Python37:
         python.version: '3.7'
   steps:
-    - task: CondaEnvironment@1
-      inputs:
-        createCustomEnvironment: true
-        environmentName: 'py$(python.version)'
-        packageSpecs: 'python=$(python.version)'
-        cleanEnvironment: true
-
+    - task: UsePythonVersion@0
+      inputs: 
+        versionSpec: $(python.version) 
+        addToPath: true 
+        architecture: 'x64'
     - task: BatchScript@1
       displayName: 'Setup VS2017 env vars'
       inputs:

diff --git a/tools/ci_build/github/azure-pipelines/nuget/cpu-esrp-pipeline.yml b/tools/ci_build/github/azure-pipelines/nuget/cpu-esrp-pipeline.yml
@@ -15,6 +15,6 @@ schedules:
 jobs: 
 - template: templates/cpu.yml
   parameters:
-    AgentPool : $(AgentPoolWin)
+    AgentPool : 'Win-CPU-2019'
     DoEsrp: 'true'
     DoCompliance: 'true'
diff --git a/tools/ci_build/github/azure-pipelines/nuget/cpu-pipeline.yml b/tools/ci_build/github/azure-pipelines/nuget/cpu-pipeline.yml
@@ -1,11 +1,10 @@
 # Defined as pipeline variables
 # variables:
-#   AgentPoolWin : 'Win-CPU'
 #   AgentPoolLinux : 'Linux-CPU'
 #   AgentPoolMacOS : 'macOS-10.13'
 
 jobs: 
 - template: templates/cpu.yml
   parameters:
-    AgentPool : $(AgentPoolWin)
+    AgentPool : 'Win-CPU-2019'
     DoEsrp: 'false'
diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml
@@ -1,6 +1,5 @@
 # Defined as pipeline variables
 # variables:
-#   AgentPoolWin : 'Win-CPU'
 #   AgentPoolLinux : 'Linux-CPU'
 #   AgentPoolMacOS : 'macOS-10.13'
 
@@ -9,11 +8,11 @@ parameters:
   DoCompliance: 'false'
 
 jobs: 
-- template: ../../templates/win-ci.yml
+- template: ../../templates/win-ci-2019.yml
   parameters:
-    AgentPool : $(AgentPoolWin)
+    AgentPool : 'Win-CPU-2019'
     JobName: 'Windows_CI_Dev'
-    BuildCommand:  '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --use_telemetry'
+    BuildCommand:  '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --use_openmp --build_shared_lib --enable_onnx_tests --use_telemetry --cmake_generator "Visual Studio 16 2019"'
     DoDebugBuild: 'false'
     DoNugetPack : 'true'
     DoCompliance: ${{ parameters.DoCompliance }}
@@ -24,11 +23,11 @@ jobs:
      mkdir $(Build.ArtifactStagingDirectory)\testdata
      copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\custom_op_library.* $(Build.ArtifactStagingDirectory)\testdata
 
-- template: ../../templates/win-x86-ci.yml
+- template: ../../templates/win-x86-ci-2019.yml
   parameters:
-    AgentPool : $(AgentPoolWin)
+    AgentPool : 'Win-CPU-2019'
     JobName: 'Windows_CI_Dev_x86'
-    BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_openmp --build_shared_lib  --build_csharp --enable_onnx_tests --x86 --use_telemetry'
+    BuildCommand: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --use_openmp --build_shared_lib --enable_onnx_tests --x86 --use_telemetry --cmake_generator "Visual Studio 16 2019"'
     DoDebugBuild: 'false'
     DoNugetPack : 'true'
     DoCompliance: 'false'
@@ -103,7 +102,7 @@ jobs:
 - job: NuGet_Packaging
   workspace:
     clean: all
-  pool: $(AgentPoolWin)
+  pool: 'Win-CPU-2019'
   dependsOn:
   - Windows_CI_Dev
   - Windows_CI_Dev_x86

diff --git a/tools/ci_build/github/azure-pipelines/templates/compliance.yml b/tools/ci_build/github/azure-pipelines/templates/compliance.yml
@@ -1,10 +1,4 @@
 steps:
-- task: SDLNativeRules@2
-  displayName: 'Run the PREfast SDL Native Rules for MSBuild'
-  inputs:
-    userProvideBuildInfo: 'auto'
-  continueOnError: true
-
 - task: CredScan@2
   displayName: 'Run CredScan'
   inputs:
@@ -17,7 +11,19 @@ steps:
     arguments: 'analyze $(Build.BinariesDirectory)\RelWithDebInfo\*.dll  --recurse --verbose'
   continueOnError: true
 
-- task: PublishSecurityAnalysisLogs@2
+- task: securedevelopmentteam.vss-secure-development-tools.build-task-prefast.SDLNativeRules@2
+  displayName: 'Run the PREfast SDL Native Rules for MSBuild'
+  continueOnError: true
+
+- task: securedevelopmentteam.vss-secure-development-tools.build-task-report.SdtReport@1
+  displayName: 'Create Security Analysis Report'
+  inputs:
+    BinSkim: true
+    BinSkimBreakOn: WarningAbove
+    CredScan: true
+    SDLNativeRules: true
+
+- task: securedevelopmentteam.vss-secure-development-tools.build-task-publishsecurityanalysislogs.PublishSecurityAnalysisLogs@2
   displayName: 'Publish Security Analysis Logs'
   continueOnError: true