Merge branch 'main' into natke-patch-2

microsoft · May 30, 2023 · ef2beaa · ef2beaa
2 parents f0f42d6 + 30eb7af
commit ef2beaa
Show file tree

Hide file tree

Showing 49 changed files with 549 additions and 232 deletions.
diff --git a/.pipelines/OneBranch.Official.yml b/.pipelines/OneBranch.Official.yml
@@ -47,8 +47,9 @@ extends:
         break: true # always break the build on binskim issues in addition to TSA upload
         analyzeTargetGlob: '**\RelWithDebInfo\ortextensions.dll'  # avoid scanning the 3rd party DLLs.
       codeql:
-        python:
+        compiled:
           enabled: true
+        cadence: 10
       policheck:
         break: true # always break the build on policheck issues. You can disable it by setting to 'false'
         exclusionsFile: '$(REPOROOT)\.config\policheck_exclusions.xml'

diff --git a/.pipelines/android_packaging.yml b/.pipelines/android_packaging.yml
@@ -44,6 +44,8 @@ jobs:
               VERSION=$(cat ./version.txt)
               AAR_PATH="$(Build.BinariesDirectory)/android_aar/aar_out/$(buildConfig)/com/microsoft/onnxruntime/onnxruntime-extensions-android/${VERSION}/onnxruntime-extensions-android-${VERSION}.aar"
 
+              # Do not output ##vso[] commands with `set -x` or they may be parsed again and include a trailing quote.
+              set +x
               echo "##vso[task.setvariable variable=ORT_EXTENSIONS_AAR_PATH]${AAR_PATH}"
             displayName: Build onnxruntime-extensions AAR package
 

diff --git a/.pipelines/ci.yml b/.pipelines/ci.yml
@@ -385,6 +385,8 @@ jobs:
               VERSION=$(cat ./version.txt)
               AAR_PATH="$(Build.BinariesDirectory)/android_aar/aar_out/com/microsoft/onnxruntime/onnxruntime-extensions-android/${VERSION}/onnxruntime-extensions-android-${VERSION}.aar"
 
+              # Do not output ##vso[] commands with `set -x` or they may be parsed again and include a trailing quote.
+              set +x
               echo "##vso[task.setvariable variable=ORT_EXTENSIONS_AAR_PATH]${AAR_PATH}"
             displayName: Build onnxruntime-extensions AAR package
 

diff --git a/.pipelines/nuget.yml b/.pipelines/nuget.yml
@@ -14,6 +14,11 @@ parameters:
   type: boolean
   default: false
 
+- name: NugetVersionSuffix
+  displayName: Update nuget version suffix (e.g. alpha/beta/rc, only if publishing to nuget.org, otherwise leave as "none").
+  type: string
+  default: none
+
 trigger:
   branches:
     exclude:
@@ -30,4 +35,5 @@ stages:
     DoCompliance: ${{ parameters.DoCompliance }}
     DoEsrp: ${{ parameters.DoEsrp }}
     IsReleaseBuild: ${{ parameters.IsReleaseBuild }}
+    NugetVersionSuffix: ${{ parameters.NugetVersionSuffix }}
     OrtNugetPackageId: 'Microsoft.ML.OnnxRuntime.Extensions'
diff --git a/.pipelines/templates/build-package-for-android-aar.yml b/.pipelines/templates/build-package-for-android-aar.yml
@@ -87,6 +87,8 @@ stages:
           VERSION=$(cat ./version.txt)
           AAR_PATH="$(Build.BinariesDirectory)/android_aar/aar_out/$(buildConfig)/com/microsoft/onnxruntime/onnxruntime-extensions-android/${VERSION}/onnxruntime-extensions-android-${VERSION}.aar"
 
+          # Do not output ##vso[] commands with `set -x` or they may be parsed again and include a trailing quote.
+          set +x
           echo "##vso[task.setvariable variable=ORT_EXTENSIONS_AAR_PATH]${AAR_PATH}"
         workingDirectory: '$(Build.SourcesDirectory)'
     - bash: |

diff --git a/.pipelines/templates/build-package-for-linux.yml b/.pipelines/templates/build-package-for-linux.yml
@@ -36,6 +36,7 @@ jobs:
 
     # Currently we can only run tests on x64 as the arm64 tests have a failure
     # https://github.com/microsoft/onnxruntime-extensions/issues/417
+    # NOTE: on arm64 machine, CMake version needs to be updated since we now require CMake 3.25 or newer.
     - ${{ if eq(parameters.OrtExtensionsArch, 'x64') }}:
       - bash: |
           export CFLAGS="${{parameters.OrtExtensionsCFlags}}"
@@ -45,6 +46,10 @@ jobs:
         displayName: 'build onnxruntime-extensions and run tests'
     - ${{ else }}:
       - bash: |
+          sudo apt remove cmake
+          pip install cmake --upgrade
+          export PATH=~/.local/bin:$PATH
+          cmake --version
           export CFLAGS="${{parameters.OrtExtensionsCFlags}}"
           export CXXFLAGS="${{parameters.OrtExtensionsCXXFlags}}"
           ./build_lib.sh --build_dir $(Build.BinariesDirectory)/out/ --config RelWithDebInfo --parallel ${{parameters.AdditionalBuildFlags}}

diff --git a/.pipelines/templates/build-package-for-nuget.yml b/.pipelines/templates/build-package-for-nuget.yml
@@ -14,6 +14,10 @@ parameters:
   type: boolean
   default: false
 
+- name: NugetVersionSuffix
+  displayName: Nuget version suffix
+  type: string
+
 - name: OrtNugetPackageId
   displayName: Package name for nuget
   type: string
@@ -219,7 +223,8 @@ stages:
           python $(Build.SourcesDirectory)\tools\ci_build\update_nuspec_for_native_nuget.py `
             --package_version $OrtExtVersion `
             --commit_id $(Build.SourceVersion) `
-            --is_release_build ${{ parameters.IsReleaseBuild }} 
+            --is_release_build ${{ parameters.IsReleaseBuild }} `
+            --nuget_version_suffix ${{ parameters.NugetVersionSuffix }}
 
           cat $(Build.SourcesDirectory)\nuget\NativeNuget.nuspec
         workingDirectory: '$(Build.SourcesDirectory)'

diff --git a/.pipelines/templates/run-with-android-emulator-steps.yml b/.pipelines/templates/run-with-android-emulator-steps.yml
@@ -14,6 +14,8 @@ steps:
       --start --emulator-extra-args="-partition-size 4096" \
       --emulator-pid-file "${ORT_EXTENSIONS_BUILD_ANDROID_EMULATOR_PID_FILE}"
 
+    # Do not output ##vso[] commands with `set -x` or they may be parsed again and include a trailing quote.
+    set +x
     echo "##vso[task.setvariable variable=ORT_EXTENSIONS_BUILD_ANDROID_EMULATOR_PID_FILE]${ORT_EXTENSIONS_BUILD_ANDROID_EMULATOR_PID_FILE}"
   displayName: "Create and start Android emulator"
 

diff --git a/.pipelines/templates/run-with-ios-simulator-steps.yml b/.pipelines/templates/run-with-ios-simulator-steps.yml
@@ -8,6 +8,8 @@ steps:
 
     ORT_EXTENSIONS_BUILD_SIMULATOR_ID=$(xcrun simctl create iPhoneSimulatorForPipeline com.apple.CoreSimulator.SimDeviceType.iPhone-8)
 
+    # Do not output ##vso[] commands with `set -x` or they may be parsed again and include a trailing quote.
+    set +x
     echo "##vso[task.setvariable variable=ORT_EXTENSIONS_BUILD_SIMULATOR_ID]${ORT_EXTENSIONS_BUILD_SIMULATOR_ID}"
   displayName: "Create iPhone simulator"
 

diff --git a/.pipelines/templates/set-package-version-variable-step.yml b/.pipelines/templates/set-package-version-variable-step.yml
@@ -25,5 +25,7 @@ steps:
       VERSION="${BASE_VERSION}-dev+$(Build.BuildId).${SHORT_COMMIT_HASH}"
     fi
 
+    # Do not output ##vso[] commands with `set -x` or they may be parsed again and include a trailing quote.
+    set +x
     echo "##vso[task.setvariable variable=${{ parameters.PackageVersionVariableName }}]${VERSION}"
   displayName: "Set \"${{ parameters.PackageVersionVariableName }}\" variable to package version"
diff --git a/.pipelines/wheels_win32.yml b/.pipelines/wheels_win32.yml
@@ -1,7 +1,7 @@
 jobs:
 - job: windows
   timeoutInMinutes: 120
-  pool: {vmImage: 'windows-latest', name: 'Win-CPU-2021'}
+  pool: {name: 'onnxruntime-Win-CPU-2022'}
   variables:
     CIBW_BUILD: "cp3{7,8,9,10}-*amd64"
 

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.20)
+cmake_minimum_required(VERSION 3.25)
 project(onnxruntime_extensions LANGUAGES C CXX)
 
 # set(CMAKE_VERBOSE_MAKEFILE ON)
@@ -145,10 +145,15 @@ set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 
 set_property(GLOBAL PROPERTY USE_FOLDERS ON)
 
-set(CMAKE_FIND_FRAMEWORK NEVER CACHE STRING "...")
+# set both regular and cache variables to NEVER. the regular variable has a default of FIRST defined by cmake,
+# but due to CMP0126 that will exist in parallel to the cached variable if the CMake minimum version is >= 3.25.
+# if we don't set this to NEVER (or possibly LAST) the builds of the wheel for different python versions will fail
+# as it will find the system python version first and not the correct python version for the wheel.
+set(CMAKE_FIND_FRAMEWORK "NEVER")
+set(CMAKE_FIND_FRAMEWORK "NEVER" CACHE STRING "...")
 
 if(NOT "${CMAKE_FIND_FRAMEWORK}" STREQUAL "NEVER")
-  message(FATAL_ERROR "CMAKE_FIND_FRAMEWORK is not NEVER")
+  message(STATUS "CMAKE_FIND_FRAMEWORK is ${CMAKE_FIND_FRAMEWORK} not NEVER.")
 endif()
 
 # External dependencies

diff --git a/ThirdPartyNotices.txt b/ThirdPartyNotices.txt
@@ -45,7 +45,7 @@ blingfire 0831265c1aca95ca02eca5bf1155e4251e545328
 
 _____
 
-dlib v19.22
+dlib a12824d42584e292ecb3bad05c4b32c2015a7b89
 
 Boost Software License - Version 1.0 - August 17th, 2003
 
@@ -72,7 +72,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 DEALINGS IN THE SOFTWARE.
 _____
 
-google/re2 2020-11-01
+google/re2 2021-06-01
 
 Copyright (c) 2009 The RE2 Authors. All rights reserved.
 
@@ -122,7 +122,7 @@ Viatcheslav Ostapenko <[email protected]>
 
 _____
 
-nlohmann/json
+nlohmann/json v3.10.5
 
 MIT License
 
@@ -571,3 +571,52 @@ sentencepiece 0.1.96
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
+
+_____
+
+dr_libs dd762b861ecadf5ddd5fb03e9ca1db6707b54fbb
+
+    MIT No Attribution
+
+    Copyright 2020 David Reid
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy of
+    this software and associated documentation files (the "Software"), to deal in
+    the Software without restriction, including without limitation the rights to
+    use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+    of the Software, and to permit persons to whom the Software is furnished to do
+    so.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE.
+
+_____
+
+gsl 3.24.0
+
+    Copyright (c) 2015 Microsoft Corporation. All rights reserved. 
+
+    This code is licensed under the MIT License (MIT). 
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy 
+    of this software and associated documentation files (the "Software"), to deal 
+    in the Software without restriction, including without limitation the rights 
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies 
+    of the Software, and to permit persons to whom the Software is furnished to do 
+    so, subject to the following conditions: 
+
+    The above copyright notice and this permission notice shall be included in all 
+    copies or substantial portions of the Software. 
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 
+    THE SOFTWARE.
diff --git a/cgmanifest.json b/cgmanifest.json
@@ -5,7 +5,7 @@
       "component": {
         "type": "git",
         "git": {
-          "commitHash": "v1.6.0",
+          "commitHash": "v1.14.1",
           "repositoryUrl": "https://github.com/microsoft/onnxruntime.git"
         }
       }
@@ -14,7 +14,7 @@
       "component": {
         "type": "git",
         "git": {
-          "commitHash": "0dab03ba7bc438d7ba3eac2b2c1eb39ed520f928",
+          "commitHash": "6e511679de8ab0feefc1cdac1505b2fac5548e42",
           "repositoryUrl": "https://github.com/protocolbuffers/protobuf.git"
         }
       }
@@ -32,7 +32,7 @@
       "component": {
         "type": "git",
         "git": {
-          "commitHash": "v19.22",
+          "commitHash": "a12824d42584e292ecb3bad05c4b32c2015a7b89",
           "repositoryUrl": "https://github.com/davisking/dlib.git"
         }
       }
@@ -41,7 +41,16 @@
       "component": {
         "type": "git",
         "git": {
-          "commitHash": "2020-11-01",
+          "commitHash": "dd762b861ecadf5ddd5fb03e9ca1db6707b54fbb",
+          "repositoryUrl": "https://github.com/mackron/dr_libs.git"
+        }
+      }
+    },
+    {
+      "component": {
+        "type": "git",
+        "git": {
+          "commitHash": "2021-06-01",
           "repositoryUrl": "https://github.com/google/re2.git"
         }
       }
@@ -59,7 +68,16 @@
       "component": {
         "type": "git",
         "git": {
-          "commitHash": "v3.7.3",
+          "commitHash": "3.24.0",
+          "repositoryUrl": "https://github.com/microsoft/GSL.git"
+        }
+      }
+    },
+    {
+      "component": {
+        "type": "git",
+        "git": {
+          "commitHash": "v3.10.5",
           "repositoryUrl": "https://github.com/nlohmann/json.git"
         }
       }
@@ -77,8 +95,8 @@
       "component": {
         "type": "git",
         "git": {
-          "commitHash": "9d  12-Jan-2020",
-          "repositoryUrl": "https://github.com/opencv/3rdparty/libjpeg"
+          "commitHash": "364702b1c98943e4e306e745389d3f464010f069",
+          "repositoryUrl": "https://github.com/opencv/opencv/tree/4.x/3rdparty/libjpeg"
         },
         "comments": "Used by OpenCV"
       }
@@ -87,8 +105,8 @@
       "component": {
         "type": "git",
         "git": {
-          "commitHash": "2.4.0",
-          "repositoryUrl": "https://github.com/opencv/3rdparty/openjpeg"
+          "commitHash": "a2fc479c0b36d1786a9570ddb76f2ab72626994b",
+          "repositoryUrl": "https://github.com/opencv/opencv/tree/4.x/3rdparty/openjpeg"
         },
         "comments": "Used by OpenCV"
       }
@@ -97,8 +115,8 @@
       "component": {
         "type": "git",
         "git": {
-          "commitHash": "1.6.37",
-          "repositoryUrl": "https://github.com/opencv/3rdparty/libpng"
+          "commitHash": "d9bf522b271ed026813cbe35399b5aead3c9b670",
+          "repositoryUrl": "https://github.com/opencv/opencv/tree/4.x/3rdparty/libpng"
         },
         "comments": "Used by OpenCV"
       }
@@ -107,7 +125,7 @@
       "component": {
         "type": "git",
         "git": {
-          "commitHash": "v2.6.0",
+          "commitHash": "v2.6.2",
           "repositoryUrl": "https://github.com/pybind/pybind11.git"
         }
       }
@@ -125,7 +143,7 @@
       "component": {
         "type": "git",
         "git": {
-          "commitHash": "v1.2.11",
+          "commitHash": "v1.2.13",
           "repositoryUrl": "https://github.com/madler/zlib.git"
         },
         "comments": "Used by OpenCV"

diff --git a/cmake/ext_ortlib.cmake b/cmake/ext_ortlib.cmake
@@ -11,24 +11,26 @@ else()
   # default to 1.11.1 if not specified
   set(ONNXRUNTIME_VER "1.11.1" CACHE STRING "ONNX Runtime version")
 
-  if(CMAKE_HOST_APPLE)
+  if(APPLE)
     set(ONNXRUNTIME_URL "v${ONNXRUNTIME_VER}/onnxruntime-osx-universal2-${ONNXRUNTIME_VER}.tgz")
-  elseif(CMAKE_HOST_WIN32)
-    if(CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64")
+  elseif(WIN32)
+    set(ONNXRUNTIME_BINARY_PLATFORM "x64")
+
+    # override if generator platform is set
+    if (CMAKE_GENERATOR_PLATFORM)
       if (CMAKE_GENERATOR_PLATFORM STREQUAL "Win32")
-        set(ONNXRUNTIME_URL "v${ONNXRUNTIME_VER}/onnxruntime-win-x86-${ONNXRUNTIME_VER}.zip")
-      else()
-        set(ONNXRUNTIME_URL "v${ONNXRUNTIME_VER}/onnxruntime-win-x64-${ONNXRUNTIME_VER}.zip")
-      endif()
-    elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL "ARM64")
-      if (CMAKE_GENERATOR_PLATFORM STREQUAL "ARM")
-        set(ONNXRUNTIME_URL "v${ONNXRUNTIME_VER}/onnxruntime-win-arm-${ONNXRUNTIME_VER}.zip")
-      else()
-        set(ONNXRUNTIME_URL "v${ONNXRUNTIME_VER}/onnxruntime-win-arm64-${ONNXRUNTIME_VER}.zip")
+        set(ONNXRUNTIME_BINARY_PLATFORM "x86")
+      elseif (CMAKE_GENERATOR_PLATFORM STREQUAL "ARM64" OR CMAKE_GENERATOR_PLATFORM STREQUAL "ARM64EC")
+        set(ONNXRUNTIME_BINARY_PLATFORM "arm64")
+      elseif (CMAKE_GENERATOR_PLATFORM STREQUAL "ARM")
+        set(ONNXRUNTIME_BINARY_PLATFORM "arm")
       endif()
-    else()
-      message(FATAL_ERROR "Unexpected CMAKE_SYSTEM_PROCESSOR of ${CMAKE_SYSTEM_PROCESSOR}.")
+    elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "ARM64")
+      # or if building on arm64 machine
+      set(ONNXRUNTIME_BINARY_PLATFORM "arm64")
     endif()
+
+    set(ONNXRUNTIME_URL "v${ONNXRUNTIME_VER}/onnxruntime-win-${ONNXRUNTIME_BINARY_PLATFORM}-${ONNXRUNTIME_VER}.zip")
   else()
     # Linux or other, using Linux package to retrieve the headers
     set(ONNXRUNTIME_URL "v${ONNXRUNTIME_VER}/onnxruntime-linux-x64-${ONNXRUNTIME_VER}.tgz")