Move QAttention/QEmbedLayerNormalization op defs to quantization_defs…

….cc (#10507)

.gitmodules

@@ -72,3 +72,12 @@
 [submodule "cmake/external/onnx-tensorrt"]
 	path = cmake/external/onnx-tensorrt
 	url = https://github.com/onnx/onnx-tensorrt.git
+[submodule "cmake/external/pthreadpool"]
+	path = cmake/external/pthreadpool
+	url = https://github.com/Maratyszcza/pthreadpool.git
+[submodule "cmake/external/FP16"]
+	path = cmake/external/FP16
+	url = https://github.com/Maratyszcza/FP16.git
+[submodule "cmake/external/XNNPACK"]
+	path = cmake/external/XNNPACK
+	url = https://github.com/google/XNNPACK.git

cmake/CMakeLists.txt

@@ -192,9 +192,19 @@ elseif (NOT WIN32 AND NOT APPLE)
     message("Building ONNX Runtime for ${CMAKE_SYSTEM_PROCESSOR}")
 endif()
 
+# option to enable XNNPACK
+option(onnxruntime_USE_XNNPACK "Enable XNNPACK as an alternative math library on ARM" OFF)
+
 # Single output director for all binaries
 set(RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin CACHE PATH "Single output directory for all binaries.")
 
+function(onnxruntime_add_include_to_target dst_target)
+    foreach(src_target ${ARGN})
+        target_include_directories(${dst_target} PRIVATE $<TARGET_PROPERTY:${src_target},INTERFACE_INCLUDE_DIRECTORIES>)
+        target_compile_definitions(${dst_target} PRIVATE $<TARGET_PROPERTY:${src_target},INTERFACE_COMPILE_DEFINITIONS>)
+    endforeach()
+endfunction()
+
 function(set_msvc_c_cpp_compiler_warning_level warning_level)
   if (NOT "${warning_level}" MATCHES "^[0-4]$")
     message(FATAL_ERROR "Expected warning_level value of 0-4, got '${warning_level}'.")
@@ -784,6 +794,9 @@ else()
   else()
     set(PROTOBUF_LIB libprotobuf-lite)
   endif()
+  if (TARGET ${PROTOBUF_LIB})
+    set_target_properties(${PROTOBUF_LIB} PROPERTIES FOLDER "External/Protobuf")
+  endif()
 endif()
 
 if (UNIX AND onnxruntime_ENABLE_LTO AND NOT onnxruntime_PREFER_SYSTEM_LIB)
@@ -818,7 +831,6 @@ if (onnxruntime_USE_CUDA AND "${onnxruntime_CUDNN_HOME}" STREQUAL "")
   message(FATAL_ERROR "onnxruntime_CUDNN_HOME required for onnxruntime_USE_CUDA")
 endif()
 
-
 get_filename_component(ONNXRUNTIME_ROOT "${ONNXRUNTIME_ROOT}" ABSOLUTE)
 get_filename_component(ORTTRAINING_ROOT "${ORTTRAINING_ROOT}" ABSOLUTE)
 get_filename_component(REPO_ROOT "${REPO_ROOT}" ABSOLUTE)
@@ -1140,6 +1152,26 @@ if (onnxruntime_USE_COREML)
     list(APPEND ORT_PROVIDER_CMAKE_FLAGS -Donnxruntime_USE_COREML=1)
     list(APPEND ONNXRUNTIME_PROVIDER_NAMES coreml)
 endif()
+if (onnxruntime_USE_XNNPACK)
+  set(ONNXRUNTIME_XNNPACK_KERNELS_LIBRARY onnxruntime_xnnpack)
+  list(APPEND ORT_PROVIDER_FLAGS  -DUSE_XNNPACK=1)
+  list(APPEND ORT_PROVIDER_CMAKE_FLAGS -Donnxruntime_USE_XNNPACK=1)
+  set(XNNPACK_DIR external/xnnpack)
+  set(XNNPACK_INCLUDE_DIR ${XNNPACK_DIR}/include)
+  set(XNNPACK_BUILD_TESTS OFF CACHE INTERNAL "")
+  set(XNNPACK_USE_SYSTEM_LIBS ON CACHE INTERNAL "")  
+  set(XNNPACK_BUILD_BENCHMARKS OFF CACHE INTERNAL "")
+  set(FP16_BUILD_TESTS OFF CACHE INTERNAL "")
+  set(FP16_BUILD_BENCHMARKS OFF CACHE INTERNAL "")
+  set(CLOG_SOURCE_DIR "${PYTORCH_CPUINFO_DIR}/deps/clog")
+  set(CPUINFO_SOURCE_DIR ${PYTORCH_CPUINFO_DIR})
+  add_subdirectory(external/FP16)  
+  add_subdirectory(external/pthreadpool)
+  #add_subdirectory(external/xnnpack)
+  add_subdirectory(xnnpack)
+  set_target_properties(XNNPACK PROPERTIES FOLDER "External")
+  set_target_properties(pthreadpool PROPERTIES FOLDER "External")
+endif()
 function(onnxruntime_set_compile_flags target_name)
     target_compile_definitions(${target_name} PUBLIC EIGEN_USE_THREADS)
     if (MSVC)
@@ -1250,13 +1282,6 @@ function(onnxruntime_add_executable target_name)
   endif()
 endfunction()
 
-function(onnxruntime_add_include_to_target dst_target)
-    foreach(src_target ${ARGN})
-        target_include_directories(${dst_target} PRIVATE $<TARGET_PROPERTY:${src_target},INTERFACE_INCLUDE_DIRECTORIES>)
-        target_compile_definitions(${dst_target} PRIVATE $<TARGET_PROPERTY:${src_target},INTERFACE_COMPILE_DEFINITIONS>)
-    endforeach()
-endfunction()
-
 set(onnxruntime_EXTERNAL_DEPENDENCIES onnx_proto)
 
 # ACL
@@ -1871,6 +1896,10 @@ if (CMAKE_SYSTEM_NAME STREQUAL "Android")
   list(APPEND onnxruntime_EXTERNAL_LIBRARIES log)
 endif()
 
+if (onnxruntime_USE_XNNPACK)
+  list(APPEND onnxruntime_EXTERNAL_LIBRARIES XNNPACK pthreadpool)
+endif()
+
 if (WIN32)
   list(APPEND onnxruntime_EXTERNAL_LIBRARIES ${SYS_PATH_LIB} Shlwapi)
   list(APPEND onnxruntime_EXTERNAL_LIBRARIES debug Dbghelp)

cmake/onnxruntime.cmake

@@ -162,6 +162,7 @@ endif()
 set(onnxruntime_INTERNAL_LIBRARIES
   onnxruntime_session
   ${onnxruntime_libs}
+  # CUDA, ROCM, TENSORRT, MIGRAPHX, DNNL, and OpenVINO are dynamically loaded at runtime
   ${PROVIDERS_ACL}
   ${PROVIDERS_ARMNN}
   ${PROVIDERS_COREML}
@@ -175,11 +176,13 @@ set(onnxruntime_INTERNAL_LIBRARIES
   ${PROVIDERS_INTERNAL_TESTING}
   ${onnxruntime_winml}
   onnxruntime_optimizer
+  onnxruntime_xnnpack
   onnxruntime_providers
   ${onnxruntime_tvm_libs}
   onnxruntime_framework
-  onnxruntime_graph
+  onnxruntime_graph  
   onnxruntime_util
+  onnxruntime_xnnpack_schemas
   ${ONNXRUNTIME_MLAS_LIBS}
   onnxruntime_common
   onnxruntime_flatbuffers

cmake/onnxruntime_graph.cmake

@@ -10,6 +10,12 @@ file(GLOB_RECURSE onnxruntime_graph_src CONFIGURE_DEPENDS
 # create empty list for any excludes
 set(onnxruntime_graph_src_exclude_patterns)
 
+if (onnxruntime_USE_XNNPACK)
+  onnxruntime_add_static_library(onnxruntime_xnnpack_schemas ${ONNXRUNTIME_ROOT}/core/xnnpack/schema/xnnpack_onnx_defs.cc)
+  onnxruntime_add_include_to_target(onnxruntime_xnnpack_schemas onnxruntime_common onnx onnx_proto ${PROTOBUF_LIB})
+  set(ONNXRUNTIME_XNNPACK_SCHEMAS_LIBRARY onnxruntime_xnnpack_schemas)
+  set_target_properties(onnxruntime_xnnpack_schemas PROPERTIES FOLDER "ONNXRuntime")
+endif()
 if (onnxruntime_MINIMAL_BUILD)
   # remove schema registration support
   list(APPEND onnxruntime_graph_src_exclude_patterns

cmake/onnxruntime_mlas.cmake

@@ -106,7 +106,6 @@ function(setup_mlas_source_for_windows)
       ${MLAS_SRC_DIR}/arm/sgemmc.cpp
     )
   elseif(onnxruntime_target_platform STREQUAL "x64")
-
     file(GLOB_RECURSE mlas_platform_srcs_avx CONFIGURE_DEPENDS
       "${MLAS_SRC_DIR}/intrinsics/avx/*.cpp"
     )

cmake/onnxruntime_providers.cmake

@@ -64,6 +64,14 @@ file(GLOB_RECURSE onnxruntime_providers_srcs CONFIGURE_DEPENDS
   "${ONNXRUNTIME_ROOT}/core/providers/cpu/*.cc"
 )
 
+if(onnxruntime_USE_XNNPACK)
+  onnxruntime_add_static_library(onnxruntime_xnnpack ${ONNXRUNTIME_ROOT}/core/xnnpack/conv.h ${ONNXRUNTIME_ROOT}/core/xnnpack/conv.cc ${ONNXRUNTIME_ROOT}/core/xnnpack/xnnpack_kernels.cc)
+  target_include_directories(onnxruntime_xnnpack PRIVATE ${ONNXRUNTIME_ROOT} ${eigen_INCLUDE_DIRS})
+  set_target_properties(onnxruntime_xnnpack PROPERTIES FOLDER "ONNXRuntime")
+  onnxruntime_add_include_to_target(onnxruntime_xnnpack onnxruntime_common onnxruntime_framework onnx onnx_proto ${PROTOBUF_LIB} flatbuffers XNNPACK pthreadpool)
+  #list(REMOVE_ITEM  onnxruntime_providers_srcs ${ONNXRUNTIME_ROOT}/core/providers/cpu/nn/conv.h ${ONNXRUNTIME_ROOT}/core/providers/cpu/nn/conv.cc)
+endif()
+
 if(onnxruntime_DISABLE_ML_OPS)
   list(FILTER onnxruntime_providers_srcs EXCLUDE REGEX ".*/ml/.*")
 endif()

cmake/onnxruntime_python.cmake

@@ -137,22 +137,27 @@ endif()
 target_link_libraries(onnxruntime_pybind11_state PRIVATE
     onnxruntime_session
     ${onnxruntime_libs}
+    # CUDA, ROCM, TENSORRT, MIGRAPHX, DNNL, and OpenVINO are dynamically loaded at runtime
+    ${PROVIDERS_ACL}
+    ${PROVIDERS_ARMNN}
+    ${PROVIDERS_COREML}
+    ${PROVIDERS_DML}
+    ${PROVIDERS_NNAPI}
     ${PROVIDERS_NUPHAR}
     ${PROVIDERS_STVM}
-    ${PROVIDERS_VITISAI}
-    ${PROVIDERS_NNAPI}
-    ${PROVIDERS_COREML}
     ${PROVIDERS_RKNPU}
-    ${PROVIDERS_DML}
-    ${PROVIDERS_ACL}
-    ${PROVIDERS_ARMNN}
+    ${PROVIDERS_ROCM}
+    ${PROVIDERS_VITISAI}
+    ${PROVIDERS_INTERNAL_TESTING}
+    ${onnxruntime_winml}
     onnxruntime_optimizer
+    onnxruntime_xnnpack
     onnxruntime_providers
-    onnxruntime_util
     ${onnxruntime_tvm_libs}
     onnxruntime_framework
+    onnxruntime_graph  
     onnxruntime_util
-    onnxruntime_graph
+    onnxruntime_xnnpack_schemas
     ${ONNXRUNTIME_MLAS_LIBS}
     onnxruntime_common
     onnxruntime_flatbuffers

cmake/onnxruntime_session.cmake

@@ -20,6 +20,9 @@ source_group(TREE ${REPO_ROOT} FILES ${onnxruntime_session_srcs})
 onnxruntime_add_static_library(onnxruntime_session ${onnxruntime_session_srcs})
 install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/session  DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core)
 onnxruntime_add_include_to_target(onnxruntime_session onnxruntime_common onnxruntime_framework onnx onnx_proto ${PROTOBUF_LIB} flatbuffers)
+if(onnxruntime_USE_XNNPACK)
+  onnxruntime_add_include_to_target(onnxruntime_session XNNPACK pthreadpool)
+endif()
 if(onnxruntime_ENABLE_INSTRUMENT)
   target_compile_definitions(onnxruntime_session PUBLIC ONNXRUNTIME_ENABLE_INSTRUMENT)
 endif()

cmake/onnxruntime_unittests.cmake

@@ -485,24 +485,28 @@ endif()
 
 set(ONNXRUNTIME_TEST_LIBS
     onnxruntime_session
-    ${ONNXRUNTIME_INTEROP_TEST_LIBS}
     ${onnxruntime_libs}
     # CUDA, ROCM, TENSORRT, MIGRAPHX, DNNL, and OpenVINO are dynamically loaded at runtime
-    ${PROVIDERS_NUPHAR}
-    ${PROVIDERS_NNAPI}
-    ${PROVIDERS_RKNPU}
-    ${PROVIDERS_DML}
     ${PROVIDERS_ACL}
     ${PROVIDERS_ARMNN}
     ${PROVIDERS_COREML}
-    # ${PROVIDERS_STVM}
+    ${PROVIDERS_DML}
+    ${PROVIDERS_NNAPI}
+    ${PROVIDERS_NUPHAR}
+    ${PROVIDERS_STVM}
+    ${PROVIDERS_RKNPU}
+    ${PROVIDERS_ROCM}
+    ${PROVIDERS_VITISAI}
+    ${PROVIDERS_INTERNAL_TESTING}
+    ${onnxruntime_winml}
     onnxruntime_optimizer
+    onnxruntime_xnnpack
     onnxruntime_providers
-    onnxruntime_util
     ${onnxruntime_tvm_libs}
     onnxruntime_framework
+    onnxruntime_graph  
     onnxruntime_util
-    onnxruntime_graph
+    onnxruntime_xnnpack_schemas
     ${ONNXRUNTIME_MLAS_LIBS}
     onnxruntime_common
     onnxruntime_flatbuffers
@@ -1253,7 +1257,10 @@ if (NOT onnxruntime_ENABLE_TRAINING_TORCH_INTEROP)
       set_property(TEST onnxruntime4j_test APPEND PROPERTY DEPENDS onnxruntime4j_jni)
   endif()
 endif()
-
+if (NOT onnxruntime_MINIMAL_BUILD)
+  onnxruntime_add_executable(layout_transformer ${REPO_ROOT}/onnxruntime/tool/layout_transformer.cc ${REPO_ROOT}/onnxruntime/core/providers/cpu/tensor/transpose.cc)
+  target_link_libraries(layout_transformer PRIVATE onnxruntime_optimizer onnxruntime_util onnxruntime_framework onnxruntime_graph onnxruntime_mlas onnxruntime_common onnxruntime_flatbuffers ${onnxruntime_EXTERNAL_LIBRARIES})
+endif()
 # limit to only test on windows first, due to a runtime path issue on linux
 if (NOT onnxruntime_MINIMAL_BUILD AND NOT onnxruntime_EXTENDED_MINIMAL_BUILD
                                   AND NOT ${CMAKE_SYSTEM_NAME} MATCHES "Darwin|iOS"
@@ -1296,3 +1303,25 @@ if (onnxruntime_USE_STVM)
 endif()
 
 include(onnxruntime_fuzz_test.cmake)
+
+#if(onnxruntime_USE_XNNPACK)
+if(FALSE)
+    onnxruntime_add_executable(onnxruntime_xnnpack_test   "${TEST_SRC_DIR}/xnnpack/main.cpp" "${TEST_SRC_DIR}/xnnpack/model.c" "${TEST_SRC_DIR}/xnnpack/input.c")
+    if(${CMAKE_SYSTEM_NAME} STREQUAL "iOS")
+      set_target_properties(onnxruntime_xnnpack_test PROPERTIES
+        XCODE_ATTRIBUTE_CODE_SIGNING_ALLOWED "NO"
+    )
+    endif()
+    target_include_directories(onnxruntime_xnnpack_test PRIVATE ${ONNXRUNTIME_ROOT}
+            ${CMAKE_CURRENT_BINARY_DIR})
+    target_link_libraries(onnxruntime_xnnpack_test PRIVATE  onnxruntime_session
+  onnxruntime_optimizer
+  onnxruntime_providers
+  onnxruntime_util
+  onnxruntime_framework
+  onnxruntime_graph
+  ${ONNXRUNTIME_MLAS_LIBS}
+  onnxruntime_common
+  onnxruntime_flatbuffers ${onnxruntime_EXTERNAL_LIBRARIES})
+    set_target_properties(onnxruntime_xnnpack_test PROPERTIES FOLDER "ONNXRuntimeTest")
+endif()