upgrade protobuf to 3.20.2 and onnx to 1.13 (#14279)

### Description upgrade protobuf to 3.20.2, same as onnx 1.13.0 ### Motivation and Context Per component governance requirement and Fixes #14060 unused-parameter error occurs in 2 conditions. 1. compile protolbuf `onnxruntime_src/cmake/external/protobuf/src/google/protobuf/repeated_ptr_field.h:752:66: error: unused parameter ‘prototype’ [-Werror=unused-parameter]` 2. include onnx_pb.h ``` 2023-01-28T10:20:15.0410853Z FAILED: CMakeFiles/onnxruntime_pybind11_state.dir/onnxruntime_src/onnxruntime/python/onnxruntime_pybind_iobinding.cc.o ...... 2023-01-28T10:20:15.0466024Z from /build/Debug/_deps/onnx-src/onnx/onnx_pb.h:51, 2023-01-28T10:20:15.0466958Z from /onnxruntime_src/include/onnxruntime/core/framework/to_tensor_proto_element_type.h:10, .... 2023-01-28T10:20:15.0609678Z /build/Debug/_deps/onnx-build/onnx/onnx-operators-ml.pb.h:1178:25: required from here 2023-01-28T10:20:15.0610895Z /onnxruntime_src/cmake/external/protobuf/src/google/protobuf/repeated_ptr_field.h:752:66: error: unused parameter ‘prototype’ [-Werror=unused-parameter] 2023-01-28T10:20:15.0611707Z cc1plus: all warnings being treated as errors ``` https://dev.azure.com/onnxruntime/2a773b67-e88b-4c7f-9fc0-87d31fea8ef2/_apis/build/builds/874605/logs/22
microsoft · Feb 3, 2023 · 34d8bcb · 34d8bcb
1 parent a004c96
commit 34d8bcb
Show file tree

Hide file tree

Showing 17 changed files with 157 additions and 26 deletions.
diff --git a/cgmanifests/generated/cgmanifest.json b/cgmanifests/generated/cgmanifest.json
@@ -112,7 +112,7 @@
       "component": {
         "type": "git",
         "git": {
-          "commitHash": "5a5f8a5935762397aa68429b5493084ff970f774",
+          "commitHash": "9b7bca2a723ff94edcd007d93b5d0cf1838591dc",
           "repositoryUrl": "https://github.com/onnx/onnx.git"
         },
         "comments": "git submodule at cmake/external/onnx"
@@ -318,6 +318,16 @@
         "comments": "onnx_tensorrt"
       }
     },
+    {
+      "component": {
+        "type": "git",
+        "git": {
+          "commitHash": "a20c65f2cd549445fda907f7b83894c8eb7427d6",
+          "repositoryUrl": "https://github.com/protocolbuffers/protobuf.git"
+        },
+        "comments": "protobuf"
+      }
+    },
     {
       "component": {
         "type": "git",

diff --git a/cmake/deps.txt b/cmake/deps.txt
@@ -23,10 +23,10 @@ microsoft_gsl;https://github.com/microsoft/GSL/archive/refs/tags/v4.0.0.zip;cf36
 microsoft_wil;https://github.com/microsoft/wil/archive/5f4caba4e7a9017816e47becdd918fcc872039ba.zip;fd119887d0d17c37adf1fc227b054befa28158ad
 mimalloc;https://github.com/microsoft/mimalloc/archive/refs/tags/v2.0.3.zip;e4f37b93b2da78a5816c2495603a4188d316214b
 mp11;https://github.com/boostorg/mp11/archive/refs/tags/boost-1.79.0.zip;c8f04e378535ededbe5af52c8f969d2dedbe73d5
-onnx;https://github.com/onnx/onnx/archive/1ba785612a79fe749aa1e478336e534743372639.zip;6c22c32fabafac81107186ea6bedb31741a3142d
+onnx;https://github.com/onnx/onnx/archive/refs/tags/v1.13.0.zip;8dda5079cdb5a134b08b0c73f4592a6404fc2dc6
 #use the commit where it's several commits after 8.5-GA branch (https://github.com/onnx/onnx-tensorrt/commit/369d6676423c2a6dbf4a5665c4b5010240d99d3c)
 onnx_tensorrt;https://github.com/onnx/onnx-tensorrt/archive/369d6676423c2a6dbf4a5665c4b5010240d99d3c.zip;62119892edfb78689061790140c439b111491275
-protobuf;https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.18.3.zip;b95bf7e9de9c2249b6c1f2ca556ace49999e90bd
+protobuf;https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.20.2.zip;9f71dad95fb83438e88822a9969fc93773fd8c48
 psimd;https://github.com/Maratyszcza/psimd/archive/072586a71b55b7f8c584153d223e95687148a900.zip;1f5454b01f06f9656b77e4a5e2e31d7422487013
 pthreadpool;https://github.com/Maratyszcza/pthreadpool/archive/1787867f6183f056420e532eec640cba25efafea.zip;e43e80781560c5ab404a4da20f34d846f5f5d101
 pybind11;https://github.com/pybind/pybind11/archive/refs/tags/v2.10.1.zip;769b6aa67a77f17a770960f604b727645b6f6a13

diff --git a/cmake/external/onnxruntime_external_deps.cmake b/cmake/external/onnxruntime_external_deps.cmake
@@ -115,7 +115,7 @@ FetchContent_Declare(
   URL_HASH SHA1=${DEP_SHA1_protobuf}
   SOURCE_SUBDIR  cmake
   PATCH_COMMAND ${ONNXRUNTIME_PROTOBUF_PATCH_COMMAND}
-  FIND_PACKAGE_ARGS 3.18.0 NAMES Protobuf
+  FIND_PACKAGE_ARGS 3.20.2 NAMES Protobuf
 )
 set(protobuf_BUILD_TESTS OFF CACHE BOOL "Build protobuf tests" FORCE)
 if (CMAKE_SYSTEM_NAME STREQUAL "Android")

diff --git a/cmake/external/protobuf b/cmake/external/protobuf
diff --git a/cmake/onnxruntime_unittests.cmake b/cmake/onnxruntime_unittests.cmake
@@ -763,7 +763,7 @@ if (MSVC)
   # The warning means the type of two integral values around a binary operator is narrow than their result.
   # If we promote the two input values first, it could be more tolerant to integer overflow.
   # However, this is test code. We are less concerned.
-  target_compile_options(onnxruntime_test_all PRIVATE "/wd26451")
+  target_compile_options(onnxruntime_test_all PRIVATE "/wd26451" "/wd4244")
 else()
   target_compile_options(onnxruntime_test_all PRIVATE "-Wno-parentheses")
 endif()

diff --git a/cmake/patches/onnx/onnx.patch b/cmake/patches/onnx/onnx.patch
@@ -1,8 +1,16 @@
 diff --git a/CMakeLists.txt b/CMakeLists.txt
-index 8e595855..789ec80a 100644
+index 4dd56b6e..018da488 100644
 --- a/CMakeLists.txt
 +++ b/CMakeLists.txt
-@@ -545,20 +545,9 @@ endif()
+@@ -397,6 +397,7 @@ if (MSVC)
+   endif()
+ else()
+   # On non-Windows, hide all symbols we don't need
++  set(EXTRA_FLAGS "-Wno-unused-parameter")
+   set(ONNX_API_DEFINE "-DONNX_API=__attribute__\(\(__visibility__\(\"default\"\)\)\)")
+   set_target_properties(onnx_proto PROPERTIES CXX_VISIBILITY_PRESET hidden)
+   set_target_properties(onnx_proto PROPERTIES VISIBILITY_INLINES_HIDDEN 1)
+@@ -548,20 +549,9 @@ endif()
  if(MSVC)
    target_compile_options(onnx_proto
                           PRIVATE /MP
@@ -23,3 +31,29 @@ index 8e595855..789ec80a 100644
                                   ${EXTRA_FLAGS})
    if(ONNX_USE_PROTOBUF_SHARED_LIBS)
        target_compile_options(onnx_proto
+diff --git a/onnx/onnx_pb.h b/onnx/onnx_pb.h
+index 0aab3e26..0f859267 100644
+--- a/onnx/onnx_pb.h
++++ b/onnx/onnx_pb.h
+@@ -47,10 +47,21 @@
+ #define ONNX_API ONNX_IMPORT
+ #endif
+
++//  onnx/onnx-operators-ml.pb.h:1178:25:   required from here
++//  protobuf/src/google/protobuf/repeated_ptr_field.h:752:66: error: unused parameter ‘prototype’ [-Werror=unused-parameter]
++#if defined(__GNUC__)
++#pragma GCC diagnostic push
++#pragma GCC diagnostic ignored "-Wunused-parameter"
++#endif
++
+ #ifdef ONNX_ML
+ #include "onnx/onnx-ml.pb.h"
+ #else
+ #include "onnx/onnx.pb.h"
+ #endif
+
++#if defined(__GNUC__)
++#pragma GCC diagnostic pop
++#endif
++
+ #endif // ! ONNX_ONNX_PB_H
diff --git a/cmake/patches/protobuf/protobuf_cmake.patch b/cmake/patches/protobuf/protobuf_cmake.patch
@@ -1,25 +1,31 @@
 diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt
-index 61a5c3ded..68d7c5598 100644
+index ac92442a1..e930cbd2e 100644
 --- a/cmake/CMakeLists.txt
 +++ b/cmake/CMakeLists.txt
-@@ -199,12 +199,8 @@ if (MSVC)
-   endif()
+@@ -240,9 +240,7 @@ if (MSVC)
    # MSVC warning suppressions
    add_definitions(
--    /wd4018 # 'expression' : signed/unsigned mismatch
      /wd4065 # switch statement contains 'default' but no 'case' labels
--    /wd4146 # unary minus operator applied to unsigned type, result still unsigned
 -    /wd4244 # 'conversion' conversion from 'type1' to 'type2', possible loss of data
      /wd4251 # 'identifier' : class 'type' needs to have dll-interface to be used by clients of class 'type2'
 -    /wd4267 # 'var' : conversion from 'size_t' to 'type', possible loss of data
      /wd4305 # 'identifier' : truncation from 'type1' to 'type2'
      /wd4307 # 'operator' : integral constant overflow
      /wd4309 # 'conversion' : truncation of constant value
-@@ -212,7 +208,6 @@ if (MSVC)
+@@ -250,7 +248,6 @@ if (MSVC)
      /wd4355 # 'this' : used in base member initializer list
      /wd4506 # no definition for inline function 'function'
      /wd4800 # 'type' : forcing value to bool 'true' or 'false' (performance warning)
 -    /wd4996 # The compiler encountered a deprecated declaration.
    )
    # Allow big object
    add_definitions(/bigobj)
+@@ -272,6 +269,8 @@ if (MSVC)
+   endif()
+
+   configure_file(version.rc.in ${CMAKE_CURRENT_BINARY_DIR}/version.rc @ONLY)
++else (MSVC)
++  set(CMAKE_CXX_FLAGS  "${CMAKE_CXX_FLAGS} -Wno-unused-parameter")
+ endif (MSVC)
+
+
diff --git a/onnxruntime/test/python/onnx_backend_test_series.py b/onnxruntime/test/python/onnx_backend_test_series.py
@@ -143,6 +143,9 @@ def create_backend_test(test_name=None):
         if backend.supports_device("OPENVINO_CPU_FP16"):
             current_failing_tests += apply_filters(filters, "current_failing_tests_OPENVINO_CPU_FP16")
 
+        if backend.supports_device("OPENVINO"):
+            current_failing_tests += apply_filters(filters, "current_failing_tests_OPENVINO_opset18")
+
         if backend.supports_device("MIGRAPHX"):
             current_failing_tests += apply_filters(filters, "current_failing_tests_MIGRAPHX")
 

diff --git a/onnxruntime/test/testdata/onnx_backend_test_series_filters.jsonc b/onnxruntime/test/testdata/onnx_backend_test_series_filters.jsonc
@@ -345,6 +345,83 @@
         "^test_training_dropout_mask", // Runs but there's accuracy mismatch
         "^test_training_dropout_default" // Runs but there's accuracy mismatch
     ],
+    "current_failing_tests_OPENVINO_opset18": [
+        // pending opset 18 support, RUNTIME_EXCEPTION : Encountered unknown exception in Initialize()
+        "^test_center_crop_pad_crop_axes_chw",
+        "^test_center_crop_pad_crop_axes_chw_expanded",
+        "^test_center_crop_pad_crop_axes_hwc",
+        "^test_center_crop_pad_crop_axes_hwc_expanded",
+        "^test_group_normalization_epsilon",
+        "^test_group_normalization_example",
+        "^test_layer_normalization_2d_axis1_expanded_ver18",
+        "^test_layer_normalization_2d_axis_negative_1_expanded_ver18",
+        "^test_layer_normalization_3d_axis1_epsilon_expanded_ver18",
+        "^test_layer_normalization_3d_axis2_epsilon_expanded_ver18",
+        "^test_layer_normalization_3d_axis_negative_1_epsilon_expanded_ver18",
+        "^test_layer_normalization_3d_axis_negative_2_epsilon_expanded_ver18",
+        "^test_layer_normalization_4d_axis1_expanded_ver18",
+        "^test_layer_normalization_4d_axis2_expanded_ver18",
+        "^test_layer_normalization_4d_axis3_expanded_ver18",
+        "^test_layer_normalization_4d_axis_negative_1_expanded_ver18",
+        "^test_layer_normalization_4d_axis_negative_2_expanded_ver18",
+        "^test_layer_normalization_4d_axis_negative_3_expanded_ver18",
+        "^test_layer_normalization_default_axis_expanded_ver18",
+        "^test_reduce_l1_do_not_keepdims_example_expanded",
+        "^test_reduce_l1_do_not_keepdims_random_expanded",
+        "^test_reduce_log_sum_asc_axes_expanded",
+        "^test_reduce_log_sum_desc_axes_expanded",
+        "^test_reduce_sum_square_do_not_keepdims_example_expanded",
+        "^test_reduce_sum_square_do_not_keepdims_random_expanded",
+        "^test_scatter_elements_with_reduction_max",
+        "^test_scatter_elements_with_reduction_min",
+        "^test_scatternd_max",
+        "^test_scatternd_min",
+        // pending opset 18 support, test failures
+        "^test_group_normalization_epsilon_expanded",
+        "^test_group_normalization_example_expanded",
+        "^test_logsoftmax_large_number_expanded_ver18",
+        "^test_mvn_expanded_ver18_cpu",
+        "^test_reduce_l1_do_not_keepdims_example",
+        "^test_reduce_l1_do_not_keepdims_random",
+        "^test_reduce_l1_keep_dims_example",
+        "^test_reduce_l1_keep_dims_random",
+        "^test_reduce_l1_negative_axes_keep_dims_example",
+        "^test_reduce_l1_negative_axes_keep_dims_random",
+        "^test_reduce_l2_do_not_keepdims_example",
+        "^test_reduce_l2_do_not_keepdims_random",
+        "^test_reduce_l2_keep_dims_example",
+        "^test_reduce_l2_keep_dims_random",
+        "^test_reduce_l2_negative_axes_keep_dims_example",
+        "^test_reduce_l2_negative_axes_keep_dims_random",
+        "^test_reduce_log_sum_asc_axes",
+        "^test_reduce_log_sum_desc_axes",
+        "^test_reduce_log_sum_negative_axes",
+        "^test_reduce_max_do_not_keepdims_example",
+        "^test_reduce_max_do_not_keepdims_random",
+        "^test_reduce_max_keepdims_example",
+        "^test_reduce_max_keepdims_random",
+        "^test_reduce_max_negative_axes_keepdims_example",
+        "^test_reduce_max_negative_axes_keepdims_random",
+        "^test_reduce_mean_do_not_keepdims_example",
+        "^test_reduce_mean_do_not_keepdims_random",
+        "^test_reduce_mean_keepdims_example",
+        "^test_reduce_mean_keepdims_random",
+        "^test_reduce_mean_negative_axes_keepdims_example",
+        "^test_reduce_mean_negative_axes_keepdims_random",
+        "^test_reduce_prod_do_not_keepdims_example",
+        "^test_reduce_prod_do_not_keepdims_random",
+        "^test_reduce_prod_keepdims_example",
+        "^test_reduce_prod_keepdims_random",
+        "^test_reduce_prod_negative_axes_keepdims_example",
+        "^test_reduce_prod_negative_axes_keepdims_random",
+        "^test_reduce_sum_square_do_not_keepdims_example",
+        "^test_reduce_sum_square_do_not_keepdims_random",
+        "^test_reduce_sum_square_keepdims_example",
+        "^test_reduce_sum_square_keepdims_random",
+        "^test_reduce_sum_square_negative_axes_keepdims_example",
+        "^test_reduce_sum_square_negative_axes_keepdims_random",
+        "^test_softmax_large_number_expanded_ver18"
+    ],
     "current_failing_tests_MIGRAPHX": [
         "^test_constant_pad_cpu",
         "^test_round_cpu",

diff --git a/tools/ci_build/github/azure-pipelines/templates/download-deps.yml b/tools/ci_build/github/azure-pipelines/templates/download-deps.yml
@@ -11,7 +11,7 @@ steps:
       packageType: upack
       feed: '/7424c8e4-5c62-490e-95c4-79446f31017c'
       definition: '517c4f6f-5437-4392-a70d-4f15ec5be2f0'
-      version: 1.0.24
+      version: 1.0.25
       downloadPath: $(Build.BinariesDirectory)/deps
 
 # The private ADO project
@@ -22,7 +22,7 @@ steps:
       packageType: upack
       feed: '/4c7631f5-24c0-4307-8822-1aa8f180c325'
       definition: 'fd9dd5ad-b73e-4678-890e-edcf680dbc1a'
-      version: 1.0.24
+      version: 1.0.25
       downloadPath: $(Build.BinariesDirectory)/deps
 
 # You can add more ADO accounts at here.
diff --git a/tools/ci_build/github/linux/docker/inference/x64/python/cpu/scripts/requirements.txt b/tools/ci_build/github/linux/docker/inference/x64/python/cpu/scripts/requirements.txt
@@ -4,6 +4,6 @@ pytest
 setuptools>=41.4.0
 wheel
 git+http://github.com/onnx/onnx.git@5a5f8a5935762397aa68429b5493084ff970f774#egg=onnx
-protobuf==3.18.3
+protobuf==3.20.2
 sympy==1.10.1
 flatbuffers
diff --git a/tools/ci_build/github/linux/docker/scripts/manylinux/requirements.txt b/tools/ci_build/github/linux/docker/scripts/manylinux/requirements.txt
@@ -3,7 +3,7 @@ mypy
 pytest
 setuptools>=41.4.0
 wheel
-onnx==1.12.0
-protobuf==3.18.3
+onnx==1.13.0
+protobuf==3.20.2
 sympy==1.10.1
 flatbuffers
diff --git a/tools/ci_build/github/linux/docker/scripts/requirements.txt b/tools/ci_build/github/linux/docker/scripts/requirements.txt
@@ -4,9 +4,9 @@ mypy
 pytest
 setuptools>=41.4.0
 wheel>=0.35.1
-onnx==1.12.0
+onnx==1.13.0
 argparse
 sympy==1.10.1
 flatbuffers
-protobuf==3.18.1
-packaging
+protobuf==3.20.2
+packaging
diff --git a/tools/ci_build/github/linux/tvm/requirements.txt b/tools/ci_build/github/linux/tvm/requirements.txt
@@ -4,10 +4,10 @@ scipy
 wheel
 setuptools
 onnx==1.11.0
-protobuf==3.18.3
+protobuf==3.20.2
 flatbuffers
 tornado
 psutil
 xgboost
 cloudpickle
-pytest
+pytest
diff --git a/tools/ci_build/github/windows/helpers.ps1 b/tools/ci_build/github/windows/helpers.ps1
@@ -372,6 +372,7 @@ function Install-Protobuf {
 
     pushd .
     $url=Get-DownloadURL -name protobuf -src_root $src_root
+    Write-Host "Downloading protobuf from $url"
     $temp_dir = Get-TempDirectory
     $protobuf_src_dir = Join-Path $temp_dir "protobuf"
     $download_finished = DownloadAndExtract -Uri $url -InstallDirectory $protobuf_src_dir -Force

diff --git a/tools/ci_build/github/windows/install_third_party_deps.ps1 b/tools/ci_build/github/windows/install_third_party_deps.ps1
@@ -76,7 +76,7 @@ Install-Pybind -cmake_path $cmake_path -src_root $ort_src_root -build_config $bu
 Install-Protobuf -cmake_path $cmake_path -src_root $ort_src_root -build_config $build_config -cmake_extra_args $cmake_extra_args
 
 # TODO: parse it from deps.txt
-$protobuf_version="3.18.3"
+$protobuf_version="3.20.2"
 
 # ONNX doesn't allow us to specify CMake's path
 Install-ONNX -build_config $build_config -src_root $ort_src_root -protobuf_version $protobuf_version
diff --git a/tools/ci_build/requirements.txt b/tools/ci_build/requirements.txt
@@ -1,6 +1,6 @@
 # packages used by transformers tool test
 packaging
-protobuf==3.20.1
+protobuf==3.20.2
 numpy==1.24.0
 coloredlogs==15.0
 transformers==4.24.0