diff --git a/cgmanifests/cgmanifest.json b/cgmanifests/cgmanifest.json index d95bfbb65e5b7..74097043abdb5 100644 --- a/cgmanifests/cgmanifest.json +++ b/cgmanifests/cgmanifest.json @@ -39,17 +39,17 @@ "commitHash": "638d7d2407de27f98f542f61a37a33c90a2e75a9", "repositoryUrl": "https://github.com/microsoft/onnxruntime-tvm.git" }, - "comments": "needed for nuphar" + "comments": "needed for Nuphar EP" } }, { "component": { "type": "git", "git": { - "commitHash": "7b3a22e465dd6aca4729504a19beb4bc23312755", + "commitHash": "36b48a5707321adba8a70e14da443566a9391e5a", "repositoryUrl": "https://github.com/apache/tvm.git" }, - "comments": "needed for EP STVM" + "comments": "needed for TVM EP" } }, { diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 650caa6652386..5abf37a6d581a 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -60,7 +60,7 @@ option(onnxruntime_BUILD_OBJC "Build Objective-C library" OFF) option(onnxruntime_USE_PREINSTALLED_EIGEN "Use pre-installed EIGEN. Need to provide eigen_SOURCE_PATH if turn this on." OFF) option(onnxruntime_BUILD_BENCHMARKS "Build ONNXRuntime micro-benchmarks" OFF) -option(onnxruntime_USE_TVM "Build TVM for code generation." OFF) +option(onnxruntime_USE_NUPHAR_TVM "Build TVM for code generation for NUPHAR EP" OFF) option(onnxruntime_USE_LLVM "Build TVM with LLVM" OFF) option(onnxruntime_BUILD_FOR_NATIVE_MACHINE "Enable this option for turning on optimization specific to this machine" OFF) @@ -104,7 +104,8 @@ option(onnxruntime_USE_MIMALLOC "Override new/delete and arena allocator with mi #It's default OFF because it's experimental now. option(onnxruntime_PREFER_SYSTEM_LIB "Experimental: Build with the preinstalled libraries in your system" OFF) option(onnxruntime_USE_ROCM "Build with AMD GPU support" OFF) -option(onnxruntime_USE_STVM "Build with standalone TVM support" OFF) +option(onnxruntime_USE_TVM "Build with TVM support" OFF) +option(onnxruntime_TVM_CUDA_RUNTIME "Build TVM with CUDA support" OFF) # Options related to reducing the binary size produced by the build option(onnxruntime_DISABLE_CONTRIB_OPS "Disable contrib ops" OFF) @@ -818,7 +819,6 @@ if (onnxruntime_USE_CUDA AND "${onnxruntime_CUDNN_HOME}" STREQUAL "") message(FATAL_ERROR "onnxruntime_CUDNN_HOME required for onnxruntime_USE_CUDA") endif() - get_filename_component(ONNXRUNTIME_ROOT "${ONNXRUNTIME_ROOT}" ABSOLUTE) get_filename_component(ORTTRAINING_ROOT "${ORTTRAINING_ROOT}" ABSOLUTE) get_filename_component(REPO_ROOT "${REPO_ROOT}" ABSOLUTE) @@ -1100,10 +1100,10 @@ if (onnxruntime_USE_NUPHAR) list(APPEND ORT_PROVIDER_CMAKE_FLAGS -Donnxruntime_USE_NUPHAR=1) list(APPEND ONNXRUNTIME_PROVIDER_NAMES nuphar) endif() -if (onnxruntime_USE_STVM) - list(APPEND ORT_PROVIDER_FLAGS -DUSE_STVM=1) - list(APPEND ORT_PROVIDER_CMAKE_FLAGS -Donnxruntime_USE_STVM=1) - list(APPEND ONNXRUNTIME_PROVIDER_NAMES stvm) +if (onnxruntime_USE_TVM) + list(APPEND ORT_PROVIDER_FLAGS -DUSE_TVM=1) + list(APPEND ORT_PROVIDER_CMAKE_FLAGS -Donnxruntime_USE_TVM=1) + list(APPEND ONNXRUNTIME_PROVIDER_NAMES tvm) endif() if (onnxruntime_USE_WINML) list(APPEND ORT_PROVIDER_FLAGS -DUSE_WINML=1) @@ -1337,67 +1337,97 @@ if (onnxruntime_USE_DNNL) include(dnnl) endif() -# TVM -if (onnxruntime_USE_TVM) +# NUPHAR EP +if (onnxruntime_USE_NUPHAR_TVM) if (NOT TARGET tvm) message(STATUS "Include TVM(*).") include(tvm) endif() if (onnxruntime_USE_CUDA) - if (onnxruntime_USE_STVM) - set(USE_CUDA ${onnxruntime_CUDA_HOME} CACHE BOOL "Only defined for TVM") - set(USE_MKLDNN ON CACHE BOOL "Only defined for TVM") - set(USE_CUDNN ON CACHE BOOL "Only defined for TVM") - endif() - if (onnxruntime_USE_NUPHAR) - set(USE_CUDA ON CACHE BOOL "Only defined for TVM") - endif() + set(USE_CUDA ON CACHE BOOL "Only defined for TVM") endif() if (onnxruntime_USE_LLVM) set(USE_LLVM ON CACHE BOOL "Only defined for TVM") - add_definitions(-DUSE_TVM_WITH_LLVM) + add_definitions(-DUSE_NUPHAR_TVM_WITH_LLVM) endif() - if (onnxruntime_USE_STVM) - set(USE_OPENMP gnu CACHE STRING "Only defined for TVM") - set(USE_MICRO OFF CACHE BOOL "Only defined for TVM") - endif() - message(STATUS "TVM BEFORE USE_LLVM=${USE_LLVM} USE_OPENMP=${USE_OPENMP} USE_MICRO=${USE_MICRO} CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} USE_CUDA=${USE_CUDA} USE_GTEST=${USE_GTEST} CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}") + message(STATUS "TVM BEFORE USE_LLVM=${USE_LLVM} USE_OPENMP=${USE_OPENMP} CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} USE_CUDA=${USE_CUDA} USE_GTEST=${USE_GTEST} CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}") message(STATUS "tvm_SOURCE_DIR=${tvm_SOURCE_DIR}") message(STATUS "tvm_BINARY_DIR=${tvm_BINARY_DIR}") add_subdirectory(${tvm_SOURCE_DIR} ${tvm_BINARY_DIR} EXCLUDE_FROM_ALL) - message(STATUS "TVM AFTER USE_LLVM=${USE_LLVM} USE_OPENMP=${USE_OPENMP} USE_MICRO=${USE_MICRO} CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} USE_CUDA=${USE_CUDA} USE_GTEST=${USE_GTEST} CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}") + message(STATUS "TVM AFTER USE_LLVM=${USE_LLVM} USE_OPENMP=${USE_OPENMP} CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} USE_CUDA=${USE_CUDA} USE_GTEST=${USE_GTEST} CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}") - if (onnxruntime_USE_NUPHAR) - set_target_properties(tvm PROPERTIES FOLDER ${tvm_SOURCE_DIR}) - set_target_properties(tvm_topi PROPERTIES FOLDER ${tvm_SOURCE_DIR}) - set_target_properties(tvm_runtime PROPERTIES FOLDER ${tvm_SOURCE_DIR}) + set_target_properties(tvm PROPERTIES FOLDER ${tvm_SOURCE_DIR}) + set_target_properties(tvm_topi PROPERTIES FOLDER ${tvm_SOURCE_DIR}) + set_target_properties(tvm_runtime PROPERTIES FOLDER ${tvm_SOURCE_DIR}) - set(TVM_INCLUDES ${tvm_SOURCE_DIR}/include - ${tvm_SOURCE_DIR}/3rdparty/dmlc-core/include - ${tvm_SOURCE_DIR}/3rdparty/dlpack/include - $ - $) - endif() + set(TVM_INCLUDES ${tvm_SOURCE_DIR}/include + ${tvm_SOURCE_DIR}/3rdparty/dmlc-core/include + ${tvm_SOURCE_DIR}/3rdparty/dlpack/include + $ + $) + + add_definitions(-DUSE_NUPHAR_TVM) + + set(onnxruntime_tvm_libs onnxruntime_codegen_tvm) - if (onnxruntime_USE_STVM) - set_target_properties(tvm PROPERTIES FOLDER ${tvm_SOURCE_DIR}) + # needs to link with stdc++fs in Linux + if (UNIX) + if (NOT APPLE) + set(FS_STDLIB stdc++fs) + endif() + endif() + list(APPEND onnxruntime_EXTERNAL_LIBRARIES tvm ${FS_STDLIB}) + list(APPEND onnxruntime_EXTERNAL_DEPENDENCIES tvm) - set(TVM_INCLUDES ${tvm_SOURCE_DIR}/include - ${tvm_SOURCE_DIR}/3rdparty/dmlc-core/include - ${tvm_SOURCE_DIR}/3rdparty/dlpack/include - $) + if (WIN32 AND MSVC) + # wd4100: identifier' : unreferenced formal parameter + # wd4244: conversion from 'int' to 'char', possible loss of data + # wd4251: class X needs to have dll-interface to be used by clients of class Y + # wd4267: 'initializing': conversion from 'size_t' to 'int', possible loss of data + # wd4275: non dll-interface class X used as base for dll-interface class Y + # wd4389: signed/unsigned mismatch + # wd4456: declaration of X hides previous local declaration + set(DISABLED_WARNINGS_FOR_TVM "/wd4100" "/wd4244" "/wd4251" "/wd4267" "/wd4275" "/wd4389" "/wd4456") + else() + set(DISABLED_WARNINGS_FOR_TVM "-Wno-error=ignored-qualifiers" "-Wno-unused-parameter" "-Wno-ignored-qualifiers") + list(APPEND DISABLED_WARNINGS_FOR_TVM "-Wno-error=unused-parameter") + if (HAS_CATCH_VALUE) + #TODO: send a PR to TVM and fix it + list(APPEND DISABLED_WARNINGS_FOR_TVM "-Wno-error=catch-value") + endif() endif() - add_definitions(-DUSE_TVM) + include(onnxruntime_codegen.cmake) +endif(onnxruntime_USE_NUPHAR_TVM) + +# TVM EP +if (onnxruntime_USE_TVM) + if (NOT TARGET tvm) + message(STATUS "Include TVM(*).") + include(tvm) + endif() - if (onnxruntime_USE_NUPHAR) - set(onnxruntime_tvm_libs onnxruntime_codegen_tvm) + if (onnxruntime_USE_LLVM) + set(USE_LLVM ON CACHE BOOL "Only defined for TVM") endif() - if (onnxruntime_USE_STVM) - set(onnxruntime_tvm_libs onnxruntime_providers_stvm) + + if (onnxruntime_TVM_CUDA_RUNTIME) + set(USE_CUDA ON CACHE BOOL "Only defined for TVM" FORCE) endif() + set(USE_OPENMP gnu CACHE STRING "Only defined for TVM") + add_subdirectory(${tvm_SOURCE_DIR} ${tvm_BINARY_DIR} EXCLUDE_FROM_ALL) + + set_target_properties(tvm PROPERTIES FOLDER ${tvm_SOURCE_DIR}) + + set(TVM_INCLUDES ${tvm_SOURCE_DIR}/include + ${tvm_SOURCE_DIR}/3rdparty/dmlc-core/include + ${tvm_SOURCE_DIR}/3rdparty/dlpack/include + $) + + set(onnxruntime_tvm_libs onnxruntime_providers_tvm) + # needs to link with stdc++fs in Linux if (UNIX) if (NOT APPLE) @@ -1712,29 +1742,6 @@ if (onnxruntime_USE_ROCM) add_definitions(-DROCM_VERSION=${ROCM_VERSION_DEV_INT}) endif() -if (onnxruntime_USE_TVM) - if (WIN32 AND MSVC) - # wd4100: identifier' : unreferenced formal parameter - # wd4244: conversion from 'int' to 'char', possible loss of data - # wd4251: class X needs to have dll-interface to be used by clients of class Y - # wd4267: 'initializing': conversion from 'size_t' to 'int', possible loss of data - # wd4275: non dll-interface class X used as base for dll-interface class Y - # wd4389: signed/unsigned mismatch - # wd4456: declaration of X hides previous local declaration - set(DISABLED_WARNINGS_FOR_TVM "/wd4100" "/wd4244" "/wd4251" "/wd4267" "/wd4275" "/wd4389" "/wd4456") - else() - set(DISABLED_WARNINGS_FOR_TVM "-Wno-error=ignored-qualifiers" "-Wno-unused-parameter" "-Wno-ignored-qualifiers") - list(APPEND DISABLED_WARNINGS_FOR_TVM "-Wno-error=unused-parameter") - if (HAS_CATCH_VALUE) - #TODO: send a PR to TVM and fix it - list(APPEND DISABLED_WARNINGS_FOR_TVM "-Wno-error=catch-value") - endif() - endif() - if (onnxruntime_USE_NUPHAR) - include(onnxruntime_codegen.cmake) - endif() -endif() - if (onnxruntime_ENABLE_MICROSOFT_INTERNAL) add_definitions(-DMICROSOFT_INTERNAL) endif() diff --git a/cmake/external/tvm.cmake b/cmake/external/tvm.cmake index dc78f13483d6f..d3c1a1900ef60 100644 --- a/cmake/external/tvm.cmake +++ b/cmake/external/tvm.cmake @@ -1,10 +1,10 @@ -if (onnxruntime_USE_STVM) - message(STATUS "onnxruntime_USE_STVM: Fetch tvm for STVM.") +if (onnxruntime_USE_TVM) + message(STATUS "onnxruntime_USE_TVM: Fetch tvm for TVM EP") FetchContent_Declare( tvm GIT_REPOSITORY https://github.com/apache/tvm.git - GIT_TAG v0.8.0 + GIT_TAG 36b48a5707321adba8a70e14da443566a9391e5a ) FetchContent_GetProperties(tvm) @@ -13,14 +13,11 @@ if (onnxruntime_USE_STVM) endif() set(tvm_INCLUDE_DIRS ${tvm_SOURCE_DIR}/include) - set(onnxruntime_STVM_HOME ${tvm_SOURCE_DIR}) - message(STATUS "Define onnxruntime_STVM_HOME.") - message(STATUS ${onnxruntime_STVM_HOME}) endif() if (onnxruntime_USE_NUPHAR) - message(STATUS "onnxruntime_USE_NUPHAR: Fetch onnxruntime-tvm for NUPHAR.") + message(STATUS "onnxruntime_USE_NUPHAR: Fetch onnxruntime-tvm for NUPHAR EP") FetchContent_Declare( tvm diff --git a/cmake/onnxruntime.cmake b/cmake/onnxruntime.cmake index 6fe4c6497a9ca..64068a03a09f1 100644 --- a/cmake/onnxruntime.cmake +++ b/cmake/onnxruntime.cmake @@ -168,7 +168,7 @@ set(onnxruntime_INTERNAL_LIBRARIES ${PROVIDERS_DML} ${PROVIDERS_NNAPI} ${PROVIDERS_NUPHAR} - ${PROVIDERS_STVM} + ${PROVIDERS_TVM} ${PROVIDERS_RKNPU} ${PROVIDERS_ROCM} ${PROVIDERS_VITISAI} diff --git a/cmake/onnxruntime_csharp.cmake b/cmake/onnxruntime_csharp.cmake index aad34009974a1..f0eecab652d36 100644 --- a/cmake/onnxruntime_csharp.cmake +++ b/cmake/onnxruntime_csharp.cmake @@ -34,8 +34,8 @@ if (onnxruntime_USE_NUPHAR) STRING(APPEND CSHARP_PREPROCESSOR_DEFINES "USE_NUPHAR;") endif() -if (onnxruntime_USE_STVM) - STRING(APPEND CSHARP_PREPROCESSOR_DEFINES "USE_STVM,") +if (onnxruntime_USE_TVM) + STRING(APPEND CSHARP_PREPROCESSOR_DEFINES "USE_TVM,") endif() if (onnxruntime_USE_OPENVINO) diff --git a/cmake/onnxruntime_providers.cmake b/cmake/onnxruntime_providers.cmake index eab1918de701d..02f890f33d4a4 100644 --- a/cmake/onnxruntime_providers.cmake +++ b/cmake/onnxruntime_providers.cmake @@ -149,8 +149,8 @@ endif() if(onnxruntime_USE_ROCM) set(PROVIDERS_ROCM onnxruntime_providers_rocm) endif() -if (onnxruntime_USE_STVM) - set(PROVIDERS_STVM onnxruntime_providers_stvm) +if (onnxruntime_USE_TVM) + set(PROVIDERS_TVM onnxruntime_providers_tvm) endif() @@ -482,7 +482,7 @@ if (onnxruntime_USE_CUDA) ) # disable a warning from the CUDA headers about unreferenced local functions #target_compile_options(onnxruntime_providers_cuda PRIVATE /wd4505) - if (onnxruntime_USE_TVM) + if (onnxruntime_USE_NUPHAR_TVM) target_compile_options(onnxruntime_providers_cuda PRIVATE ${DISABLED_WARNINGS_FOR_TVM}) endif() set(onnxruntime_providers_cuda_static_library_flags @@ -656,8 +656,8 @@ endif() if (onnxruntime_USE_NUPHAR) add_definitions(-DUSE_NUPHAR=1) - if (NOT onnxruntime_USE_TVM) - message(FATAL_ERROR "onnxruntime_USE_TVM required for onnxruntime_USE_NUPHAR") + if (NOT onnxruntime_USE_NUPHAR_TVM) + message(FATAL_ERROR "onnxruntime_USE_NUPHAR_TVM required for onnxruntime_USE_NUPHAR") endif() if (NOT onnxruntime_USE_LLVM) @@ -1274,45 +1274,39 @@ if (onnxruntime_USE_ROCM) endif() -if (onnxruntime_USE_STVM) - if (NOT TARGET tvm) - message(STATUS "Include TVM.") - include(tvm) - endif() - add_definitions(-DUSE_STVM=1) +if (onnxruntime_USE_TVM) + add_definitions(-DUSE_TVM=1) - file (GLOB_RECURSE onnxruntime_providers_stvm_cc_srcs CONFIGURE_DEPENDS - "${ONNXRUNTIME_ROOT}/core/providers/stvm/*.h" - "${ONNXRUNTIME_ROOT}/core/providers/stvm/*.cc" + file (GLOB_RECURSE onnxruntime_providers_tvm_cc_srcs CONFIGURE_DEPENDS + "${ONNXRUNTIME_ROOT}/core/providers/tvm/*.h" + "${ONNXRUNTIME_ROOT}/core/providers/tvm/*.cc" ) - source_group(TREE ${ONNXRUNTIME_ROOT}/core FILES ${onnxruntime_providers_stvm_cc_srcs}) - onnxruntime_add_static_library(onnxruntime_providers_stvm ${onnxruntime_providers_stvm_cc_srcs}) + source_group(TREE ${ONNXRUNTIME_ROOT}/core FILES ${onnxruntime_providers_tvm_cc_srcs}) + onnxruntime_add_static_library(onnxruntime_providers_tvm ${onnxruntime_providers_tvm_cc_srcs}) if ( CMAKE_COMPILER_IS_GNUCC ) - target_compile_options(onnxruntime_providers_stvm PRIVATE -Wno-unused-parameter -Wno-missing-field-initializers) + target_compile_options(onnxruntime_providers_tvm PRIVATE -Wno-unused-parameter -Wno-missing-field-initializers) endif() - target_include_directories(onnxruntime_providers_stvm PRIVATE - ${onnxruntime_STVM_HOME}/include - ${onnxruntime_STVM_HOME}/3rdparty/dlpack/include - ${onnxruntime_STVM_HOME}/3rdparty/dmlc-core/include + target_include_directories(onnxruntime_providers_tvm PRIVATE + ${TVM_INCLUDES} ${PYTHON_INLCUDE_DIRS}) - onnxruntime_add_include_to_target(onnxruntime_providers_stvm onnxruntime_common onnx tvm) + onnxruntime_add_include_to_target(onnxruntime_providers_tvm onnxruntime_common onnx tvm) - add_dependencies(onnxruntime_providers_stvm ${onnxruntime_EXTERNAL_DEPENDENCIES}) + add_dependencies(onnxruntime_providers_tvm ${onnxruntime_EXTERNAL_DEPENDENCIES}) - target_link_libraries(onnxruntime_providers_stvm PRIVATE + target_link_libraries(onnxruntime_providers_tvm PRIVATE onnx tvm onnxruntime_common onnxruntime_framework ) - set_target_properties(onnxruntime_providers_stvm PROPERTIES FOLDER "ONNXRuntime") - set_target_properties(onnxruntime_providers_stvm PROPERTIES LINKER_LANGUAGE CXX) + set_target_properties(onnxruntime_providers_tvm PROPERTIES FOLDER "ONNXRuntime") + set_target_properties(onnxruntime_providers_tvm PROPERTIES LINKER_LANGUAGE CXX) - target_compile_options(onnxruntime_providers_stvm PRIVATE -Wno-error=type-limits) - target_compile_definitions(onnxruntime_providers_stvm PUBLIC DMLC_USE_LOGGING_LIBRARY=) + target_compile_options(onnxruntime_providers_tvm PRIVATE -Wno-error=type-limits) + target_compile_definitions(onnxruntime_providers_tvm PUBLIC DMLC_USE_LOGGING_LIBRARY=) - install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/providers/stvm DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core/providers) + install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/providers/tvm DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core/providers) endif() diff --git a/cmake/onnxruntime_python.cmake b/cmake/onnxruntime_python.cmake index d8760123b18ca..51daf0663018b 100644 --- a/cmake/onnxruntime_python.cmake +++ b/cmake/onnxruntime_python.cmake @@ -138,7 +138,7 @@ target_link_libraries(onnxruntime_pybind11_state PRIVATE onnxruntime_session ${onnxruntime_libs} ${PROVIDERS_NUPHAR} - ${PROVIDERS_STVM} + ${PROVIDERS_TVM} ${PROVIDERS_VITISAI} ${PROVIDERS_NNAPI} ${PROVIDERS_COREML} @@ -669,25 +669,27 @@ if (onnxruntime_USE_NUPHAR) ) endif() -if (onnxruntime_USE_STVM) - file(GLOB onnxruntime_python_providers_stvm_srcs CONFIGURE_DEPENDS - "${ONNXRUNTIME_ROOT}/python/providers/stvm/*.py" +if (onnxruntime_USE_TVM) + file(GLOB onnxruntime_python_providers_tvm_srcs CONFIGURE_DEPENDS + "${ONNXRUNTIME_ROOT}/python/providers/tvm/*.py" ) add_custom_command( TARGET onnxruntime_pybind11_state POST_BUILD COMMAND ${CMAKE_COMMAND} -E make_directory $/onnxruntime/providers - COMMAND ${CMAKE_COMMAND} -E make_directory $/onnxruntime/providers/stvm + COMMAND ${CMAKE_COMMAND} -E make_directory $/onnxruntime/providers/tvm COMMAND ${CMAKE_COMMAND} -E copy - ${onnxruntime_python_providers_stvm_srcs} - $/onnxruntime/providers/stvm + ${onnxruntime_python_providers_tvm_srcs} + $/onnxruntime/providers/tvm COMMAND ${CMAKE_COMMAND} -E copy - $ + $ $/onnxruntime/capi/ + # TODO(vvchernov): why? COMMAND ${CMAKE_COMMAND} -E copy ${tvm_BINARY_DIR}/libtvm* ${tvm_SOURCE_DIR}/python/tvm ) + # TODO(vvchernov): repeat? add_custom_command( TARGET onnxruntime_pybind11_state POST_BUILD WORKING_DIRECTORY ${tvm_SOURCE_DIR}/python diff --git a/cmake/onnxruntime_unittests.cmake b/cmake/onnxruntime_unittests.cmake index 17647c91941dc..e181acc76516f 100644 --- a/cmake/onnxruntime_unittests.cmake +++ b/cmake/onnxruntime_unittests.cmake @@ -9,7 +9,7 @@ set(TEST_INC_DIR ${ONNXRUNTIME_ROOT}) if (onnxruntime_ENABLE_TRAINING) list(APPEND TEST_INC_DIR ${ORTTRAINING_ROOT}) endif() -if (onnxruntime_USE_TVM) +if (onnxruntime_USE_NUPHAR_TVM) list(APPEND TEST_INC_DIR ${TVM_INCLUDES}) endif() @@ -460,10 +460,10 @@ if(onnxruntime_USE_COREML) endif() if(onnxruntime_USE_NUPHAR) - file(GLOB_RECURSE onnxruntime_test_tvm_src CONFIGURE_DEPENDS - "${TEST_SRC_DIR}/tvm/*.h" - "${TEST_SRC_DIR}/tvm/*.cc" - ) + file(GLOB_RECURSE onnxruntime_test_nuphar_src CONFIGURE_DEPENDS + "${TEST_SRC_DIR}/nuphar_tvm/*.h" + "${TEST_SRC_DIR}/nuphar_tvm/*.cc" + ) list(APPEND onnxruntime_test_framework_src_patterns ${TEST_SRC_DIR}/framework/nuphar/*) list(APPEND onnxruntime_test_framework_libs onnxruntime_providers_nuphar) @@ -495,7 +495,7 @@ set(ONNXRUNTIME_TEST_LIBS ${PROVIDERS_ACL} ${PROVIDERS_ARMNN} ${PROVIDERS_COREML} - # ${PROVIDERS_STVM} + # ${PROVIDERS_TVM} onnxruntime_optimizer onnxruntime_providers onnxruntime_util @@ -550,17 +550,17 @@ if(onnxruntime_USE_COREML) endif() endif() -if (onnxruntime_USE_STVM) - file (GLOB_RECURSE onnxruntime_test_stvm_src CONFIGURE_DEPENDS - "${ONNXRUNTIME_ROOT}/test/stvm/*.h" - "${ONNXRUNTIME_ROOT}/test/stvm/*.cc" +if (onnxruntime_USE_TVM) + file (GLOB_RECURSE onnxruntime_test_tvm_src CONFIGURE_DEPENDS + "${TEST_SRC_DIR}/tvm/*.h" + "${TEST_SRC_DIR}/tvm/*.cc" ) - list(APPEND onnxruntime_test_providers_dependencies onnxruntime_providers_stvm) + list(APPEND onnxruntime_test_providers_dependencies onnxruntime_providers_tvm) endif() if(WIN32) - if (onnxruntime_USE_TVM) + if (onnxruntime_USE_NUPHAR_TVM) list(APPEND disabled_warnings ${DISABLED_WARNINGS_FOR_TVM}) endif() endif() @@ -645,11 +645,11 @@ if (onnxruntime_ENABLE_TRAINING) endif() if (onnxruntime_USE_NUPHAR) - list(APPEND all_tests ${onnxruntime_test_tvm_src}) + list(APPEND all_tests ${onnxruntime_test_nuphar_src}) endif() -if (onnxruntime_USE_STVM) - list(APPEND all_tests ${onnxruntime_test_stvm_src}) +if (onnxruntime_USE_TVM) + list(APPEND all_tests ${onnxruntime_test_tvm_src}) endif() if (onnxruntime_USE_OPENVINO) @@ -794,7 +794,7 @@ if (NOT onnxruntime_ENABLE_TRAINING_TORCH_INTEROP) ) endif() if(WIN32) - if (onnxruntime_USE_TVM) + if (onnxruntime_USE_NUPHAR_TVM) add_custom_command( TARGET ${test_data_target} POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy $ $ @@ -851,7 +851,7 @@ if (onnxruntime_ENABLE_TRAINING_TORCH_INTEROP) endif() set_target_properties(onnx_test_runner PROPERTIES FOLDER "ONNXRuntimeTest") -if (onnxruntime_USE_TVM) +if (onnxruntime_USE_NUPHAR_TVM) if (WIN32) target_link_options(onnx_test_runner PRIVATE "/STACK:4000000") endif() @@ -1050,7 +1050,7 @@ if (NOT onnxruntime_ENABLE_TRAINING_TORCH_INTEROP) target_link_libraries(onnxruntime_perf_test PRIVATE onnxruntime_language_interop onnxruntime_pyop) endif() - if (onnxruntime_USE_TVM) + if (onnxruntime_USE_NUPHAR_TVM) if (WIN32) target_link_options(onnxruntime_perf_test PRIVATE "/STACK:4000000") endif() @@ -1287,12 +1287,12 @@ if (NOT onnxruntime_MINIMAL_BUILD AND NOT onnxruntime_EXTENDED_MINIMAL_BUILD endif() endif() -if (onnxruntime_USE_STVM) - # find_library(STVM_LIBS NAMES libtvm.so PATHS ${onnxruntime_STVM_HOME}/lib) - # link_directories(onnxruntime_test_all ${STVM_LIBS}) +if (onnxruntime_USE_TVM) + # find_library(TVM_LIBS NAMES libtvm.so PATHS ${tvm_SOURCE_DIR}/lib) + # link_directories(onnxruntime_test_all ${TVM_LIBS}) find_library(PYTHON_LIBS NAMES libpython PATHS /usr/local/lib) #target_link_libraries(onnxruntime_test_all PRIVATE ${PYTHON_LIBRARIES} -lutil) - # set(CMAKE_SHARED_LINKER_FLAGS "-Wl,-rpath,${STVM_LIBS}") + # set(CMAKE_SHARED_LINKER_FLAGS "-Wl,-rpath,${TVM_LIBS}") endif() include(onnxruntime_fuzz_test.cmake) diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.shared.cs b/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.shared.cs index 5b3c05e9c9527..fea145b018ac8 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.shared.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.shared.cs @@ -851,7 +851,7 @@ IntPtr[] outputValues /* An array of output value pointers. Array must be alloca public static extern IntPtr /*(OrtStatus*)*/ OrtSessionOptionsAppendExecutionProvider_Nuphar(IntPtr /*(OrtSessionOptions*) */ options, int allow_unaligned_buffers, IntPtr /*(char char*)*/ settings); [DllImport(NativeLib.DllName, CharSet = CharSet.Ansi)] - public static extern IntPtr /*(OrtStatus*)*/ OrtSessionOptionsAppendExecutionProvider_Stvm(IntPtr /*(OrtSessionOptions*) */ options, IntPtr /*(char char*)*/ settings); + public static extern IntPtr /*(OrtStatus*)*/ OrtSessionOptionsAppendExecutionProvider_Tvm(IntPtr /*(OrtSessionOptions*) */ options, IntPtr /*(char char*)*/ settings); #endif /// /// Append a TensorRT EP instance (configured based on given provider options) to the native OrtSessionOptions instance diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.shared.cs b/csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.shared.cs index 34db0bd12f314..1a076dc6864c4 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.shared.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.shared.cs @@ -150,15 +150,15 @@ public static SessionOptions MakeSessionOptionWithNupharProvider(String settings } /// - /// A helper method to construct a SessionOptions object for Stvm execution. + /// A helper method to construct a SessionOptions object for TVM execution. /// Use only if you have the onnxruntime package specific to this Execution Provider. /// /// settings string, comprises of comma separated key:value pairs. default is empty - /// A SessionsOptions() object configured for execution with Stvm - public static SessionOptions MakeSessionOptionWithStvmProvider(String settings = "") + /// A SessionsOptions() object configured for execution with TVM + public static SessionOptions MakeSessionOptionWithTvmProvider(String settings = "") { SessionOptions options = new SessionOptions(); - options.AppendExecutionProvider_Stvm(settings); + options.AppendExecutionProvider_Tvm(settings); return options; } @@ -360,16 +360,16 @@ public void AppendExecutionProvider_Nuphar(string settings = "") /// /// Use only if you have the onnxruntime package specific to this Execution Provider. /// - /// string with Stvm specific settings - public void AppendExecutionProvider_Stvm(string settings = "") + /// string with TVM specific settings + public void AppendExecutionProvider_Tvm(string settings = "") { #if __MOBILE__ - throw new NotSupportedException("The Stvm Execution Provider is not supported in this build"); + throw new NotSupportedException("The TVM Execution Provider is not supported in this build"); #else var settingsPinned = GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(settings), GCHandleType.Pinned); using (var pinnedSettingsName = new PinnedGCHandle(settingsPinned)) { - NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionOptionsAppendExecutionProvider_Stvm(handle, pinnedSettingsName.Pointer)); + NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionOptionsAppendExecutionProvider_Tvm(handle, pinnedSettingsName.Pointer)); } #endif } diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Common/InferenceTest.cs b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Common/InferenceTest.cs index c41be5443c8fe..202025df7bb11 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Common/InferenceTest.cs +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Common/InferenceTest.cs @@ -118,8 +118,8 @@ public void TestSessionOptions() opt.AppendExecutionProvider_Nnapi(0); #endif -#if USE_STVM - opt.AppendExecutionProvider_Stvm("Vulkan -device=amd_apu"); +#if USE_TVM + opt.AppendExecutionProvider_Tvm("Vulkan -device=amd_apu"); #endif #if USE_NUPHAR diff --git a/docs/STVM_EP.md b/docs/TVM_EP.md similarity index 60% rename from docs/STVM_EP.md rename to docs/TVM_EP.md index 369c96cc37800..a80f8d2c47bca 100644 --- a/docs/STVM_EP.md +++ b/docs/TVM_EP.md @@ -1,10 +1,10 @@ -# Standalone TVM (STVM) Execution Provider +# TVM Execution Provider ## Contents - [Introduction](#introduction) -- [Build](#build) -- [Configuration options](#configuration-option) +- [Build](#build-onnx-runtime-with-the-tvm-execution-provider) +- [Configuration options](#configuration-options) - [Performance Tuning](#performance-tuning) - [Samples](#samples) - [Known issues](#known-issues) @@ -12,22 +12,25 @@ ## Introduction -STVM is an execution provider for ONNX Runtime that is built on top of Apache TVM. It enables ONNX Runtime users to leverage Apache TVM model optimizations. -STVM EP is currently in "Preview". It's been tested to work on a handful of models on Linux, but not on Windows or MacOS. +TVM is an execution provider for ONNX Runtime that is built on top of Apache TVM. It enables ONNX Runtime users to leverage Apache TVM model optimizations. +TVM EP is currently in "Preview". It's been tested to work on a handful of models on Linux, but not on Windows or MacOS. -### Build ONNX Runtime with the STVM Execution Provider +## Build ONNX Runtime with the TVM Execution Provider Install the minimal pre-requisites on Ubuntu/Debian like linux operating systems: -``` +```bash apt-get install -y python3 python3-dev python3-pip python3-setuptools gcc libtinfo-dev zlib1g-dev build-essential cmake libedit-dev libxml2-dev llvm-12 pip3 install numpy decorator attrs ``` -Clone this repo. +Also, the current implementation has `NVidia GPU` support for TVM EP. For now, you can use only `NVidia GPU` with CUDA Toolkit support. +To do this, make sure you have installed the NVidia driver and CUDA Toolkit. +More detailed instructions can be found on the [official page](https://developer.nvidia.com/cuda-toolkit). +Clone this repo. In order to build ONNXRT you will need to have CMake 3.18 or higher. In Ubuntu 20.04 you can use the following commands to install the latest version of CMake: -``` +```bash sudo apt-get update sudo apt-get install gpg wget @@ -42,26 +45,43 @@ sudo apt-get install kitware-archive-keyring sudo apt-get install cmake ``` -Build ONNX Runtime: -``` -./build.sh --config Release --enable_pybind --build_wheel --skip_tests --parallel --use_stvm --skip_onnx_tests +Build ONNX Runtime (TVM x86): +```bash +./build.sh --config Release --enable_pybind --build_wheel --parallel --skip_tests --skip_onnx_tests --use_tvm ``` -This command builds both TVM and onnxruntime-stvm. It creates two wheel, one for each project. -Build the python API for ONNX Runtime instead of using the standard package: +Build ONNX Runtime (TVM with CUDA support): +```bash +./build.sh --config Release --enable_pybind --build_wheel --parallel --skip_tests --skip_onnx_tests --use_tvm --tvm_cuda_runtime ``` + +This command builds both `TVM` and `onnxruntime-tvm`. It creates two wheel, one for each project. +Build the python API for ONNX Runtime instead of using the standard package. Instructions for this are given below. + +Package for TVM: +```bash cd -pip3 uninstall onnxruntime onnxruntime-stvm tvm -y -whl_path=$(find ./build/Linux/Release/dist -name "*.whl") +python3 -m pip uninstall tvm -y +whl_path=$(find ./build//Release/_deps/tvm-src/python/dist -name "*.whl") python3 -m pip install $whl_path ``` -Alternatively, you can set PYTHONPATH to tell python where to find the ONNXRT library and the TVM library. + +Package for TVM EP: +```bash +cd +python3 -m pip3 uninstall onnxruntime onnxruntime-tvm -y +whl_path=$(find ./build//Release/dist -name "*.whl") +python3 -m pip install $whl_path ``` -export PYTHONPATH=$ORT_PYTHON_HOME:$TVM_PYTHON_HOME:${PYTHONPATH} + +Alternatively, you can set `PYTHONPATH` to tell python where to find the ONNXRT library and the TVM library. +```bash +export PYTHONPATH=/build//Release:${PYTHONPATH} +export PYTHONPATH=/build//Release/_deps/tvm-src/python:${PYTHONPATH} ``` ## Configuration options -STVM Executor Provider can be configured with the following provider options: +TVM Executor Provider can be configured with the following provider options: ```python po = [dict(target=client_target, target_host=client_target_host, @@ -70,7 +90,7 @@ po = [dict(target=client_target, tuning_file_path=client_tuning_logfile, input_names = input_names_str, input_shapes = input_shapes_str)] -stvm_session = onnxruntime.InferenceSession(model_path, providers=["StvmExecutionProvider"], provider_options=po) +tvm_session = onnxruntime.InferenceSession(model_path, providers=["TvmExecutionProvider"], provider_options=po) ```
@@ -87,7 +107,7 @@ input_shapes = "[1 3 224 224] [1 2]" ``` ## Performance Tuning -TVM optimizes machine learning models through an automated tuning process that produces model variants specific to targeted hardware architectures. This process also generates 'tuning logs' that the STVM EP relies on to maximize model performance. These logs can be acquired for your model by either using TVM as described here: +TVM optimizes machine learning models through an automated tuning process that produces model variants specific to targeted hardware architectures. This process also generates 'tuning logs' that the TVM EP relies on to maximize model performance. These logs can be acquired for your model by either using TVM as described here: AutoTVM: https://tvm.apache.org/docs/how_to/tune_with_autotvm/index.html @@ -97,19 +117,19 @@ https://tvm.apache.org/docs/how_to/tune_with_autoscheduler/index.html or by using logs generated through the OctoML platform (https://onnx.octoml.ai) using instructions [here](https://help.octoml.ai/en/articles/5814452-using-octoml-platform-logs-with-onnx-rt-tvm-ep) -Using the STVM EP with TVM tuning logs also requires users to turn off ONNX Runtime preprocessing. To do this, the following `SessionOptions()` can be used: +Using the TVM EP with TVM tuning logs also requires users to turn off ONNX Runtime preprocessing. To do this, the following `SessionOptions()` can be used: ``` so = onnxruntime.SessionOptions() so.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_DISABLE_ALL -stvm_session = onnxruntime.InferenceSession(model_path, sess_options=so, providers=["StvmExecutionProvider"], provider_options=po) +tvm_session = onnxruntime.InferenceSession(model_path, sess_options=so, providers=["TvmExecutionProvider"], provider_options=po) ``` ## Samples -- [Sample notebook for ResNet50 inference with STVM EP](https://github.com/octoml/onnxruntime/blob/STVM_EP_PR/docs/python/inference/notebooks/onnxruntime-stvm-tutorial.ipynb) +- [Sample notebook for ResNet50 inference with TVM EP](https://github.com/octoml/onnxruntime/blob/vc/rename/docs/python/inference/notebooks/onnxruntime-tvm-tutorial.ipynb) ## Known issues -- At this moment, the STVM EP has only been verified on UNIX/Linux systems. +- At this moment, the TVM EP has only been verified on UNIX/Linux systems. - CUDA/GPU support is still in pre-alpha mode and results are expected to change. It is recommended that only CPU targets are used. - Some compatibility issues have been found between ONNX and Google protobuf. `AttributeError: module 'google.protobuf.internal.containers' has no attribute 'MutableMapping'`. This usually occurss during `import onnx` in any python scripts for protobuf version >= 3.19.0 and ONNX version <= 1.8.1. To resolve the issue Google protobuf and ONNX can be reinstalled separately or together using: ``` @@ -122,25 +142,3 @@ pip3 install protobuf==3.19.1 The following pair of ONNX and protobuf versions have been found to be compatible: - 3.17.3 and 1.8.0 - 3.19.1 and 1.10.1 - -When use onnxruntime-stvm after it was build from the source, the following error may happen: - -``` -terminate called after throwing an instance of 'tvm::runtime::InternalError' - what(): [12:01:11] ..._deps/tvm-src/src/runtime/registry.cc:69: ---------------------------------------------------------------- -An error occurred during the execution of TVM. -For more information, please see: https://tvm.apache.org/docs/errors.html ---------------------------------------------------------------- - Check failed: (can_override) is false: Global PackedFunc arith.CreateAnalyzer is already registered - -Aborted (core dumped) -``` - -It means both onnxruntime and tvm loaded a different dynamic library ``libtvm.[so|dll]``. -To solve that, `tvm` must be imported first: - -```python -import tvm -import onnxruntime -``` diff --git a/docs/python/inference/notebooks/onnxruntime-stvm-tutorial.ipynb b/docs/python/inference/notebooks/onnxruntime-tvm-tutorial.ipynb similarity index 85% rename from docs/python/inference/notebooks/onnxruntime-stvm-tutorial.ipynb rename to docs/python/inference/notebooks/onnxruntime-tvm-tutorial.ipynb index cba197ad00993..06ee56b505ef9 100644 --- a/docs/python/inference/notebooks/onnxruntime-stvm-tutorial.ipynb +++ b/docs/python/inference/notebooks/onnxruntime-tvm-tutorial.ipynb @@ -5,14 +5,14 @@ "id": "72476497", "metadata": {}, "source": [ - "# ONNX Runtime: Tutorial for STVM execution provider\n", + "# ONNX Runtime: Tutorial for TVM execution provider\n", "\n", - "This notebook shows a simple example for model inference with STVM EP.\n", + "This notebook shows a simple example for model inference with TVM EP.\n", "\n", "\n", "#### Tutorial Roadmap:\n", "1. Prerequistes\n", - "2. Accuracy check for STVM EP\n", + "2. Accuracy check for TVM EP\n", "3. Configuration options" ] }, @@ -25,7 +25,7 @@ "\n", "Make sure that you have installed all the necessary dependencies described in the corresponding paragraph of the documentation.\n", "\n", - "Also, make sure you have the `tvm` and `onnxruntime-stvm` packages in your pip environment. \n", + "Also, make sure you have the `tvm` and `onnxruntime-tvm` packages in your pip environment. \n", "\n", "If you are using `PYTHONPATH` variable expansion, make sure it contains the following paths: `/onnxruntime/cmake/external/tvm_update/python` and `/onnxruntime/build/Linux/Release`." ] @@ -72,7 +72,7 @@ "source": [ "import tvm.testing\n", "from tvm.contrib.download import download_testdata\n", - "import onnxruntime.providers.stvm # nessesary to register tvm_onnx_import_and_compile and others" + "import onnxruntime.providers.tvm # nessesary to register tvm_onnx_import_and_compile and others" ] }, { @@ -163,8 +163,8 @@ " return get_onnxruntime_output(model, inputs, \"CPUExecutionProvider\")\n", "\n", "\n", - "def get_stvm_onnxruntime_output(model: ModelProto, inputs: List) -> np.ndarray:\n", - " return get_onnxruntime_output(model, inputs, \"StvmExecutionProvider\")" + "def get_tvm_onnxruntime_output(model: ModelProto, inputs: List) -> np.ndarray:\n", + " return get_onnxruntime_output(model, inputs, \"TvmExecutionProvider\")" ] }, { @@ -201,10 +201,10 @@ " model.opset_import[0].version = opset\n", "\n", " ort_out = get_cpu_onnxruntime_output(model, inputs)\n", - " stvm_out = get_stvm_onnxruntime_output(model, inputs)\n", - " for stvm_val, ort_val in zip(stvm_out, ort_out):\n", - " tvm.testing.assert_allclose(ort_val, stvm_val, rtol=rtol, atol=atol)\n", - " assert ort_val.dtype == stvm_val.dtype" + " tvm_out = get_tvm_onnxruntime_output(model, inputs)\n", + " for tvm_val, ort_val in zip(tvm_out, ort_out):\n", + " tvm.testing.assert_allclose(ort_val, tvm_val, rtol=rtol, atol=atol)\n", + " assert ort_val.dtype == tvm_val.dtype" ] }, { @@ -254,9 +254,9 @@ "id": "90fb7c5c", "metadata": {}, "source": [ - "## 2. Accuracy check for STVM EP \n", + "## 2. Accuracy check for TVM EP \n", "\n", - "This section will check the accuracy. The check will be to compare the output tensors for `CPUExecutionProvider` and `STVMExecutionProvider`. See the description of `verify_with_ort_with_inputs` function used above.\n", + "This section will check the accuracy. The check will be to compare the output tensors for `CPUExecutionProvider` and `TvmExecutionProvider`. See the description of `verify_with_ort_with_inputs` function used above.\n", "\n", "\n", "### Check for simple architectures" @@ -302,29 +302,28 @@ "metadata": {}, "outputs": [ { - "name": "stdout", + "name": "stderr", "output_type": "stream", "text": [ - "STVM ep options:\n", + "/home/onnxruntime/onnxruntime/core/providers/tvm/tvm_execution_provider.cc:480: TVM EP options:\n", "target: llvm -mcpu=skylake-avx512\n", "target_host: llvm -mcpu=skylake-avx512\n", "opt level: 3\n", "freeze weights: 1\n", "tuning file path: \n", - "tuning type: Ansor\n", + "tuning type: AutoTVM\n", "convert layout to NHWC: 0\n", "input tensor names: \n", "input tensor shapes: \n", - "Build TVM graph executor\n", - "****************** Success! ******************\n" + "/home/tvm/python/tvm/relay/build_module.py:414: UserWarning: target_host parameter is going to be deprecated. Please pass in tvm.target.Target(target, host=target_host) instead.\n", + " warnings.warn(\n" ] }, { - "name": "stderr", + "name": "stdout", "output_type": "stream", "text": [ - "/home/agladyshev/workspace/tvm/python/tvm/relay/build_module.py:414: UserWarning: target_host parameter is going to be deprecated. Please pass in tvm.target.Target(target, host=target_host) instead.\n", - " warnings.warn(\n" + "****************** Success! ******************\n" ] } ], @@ -363,40 +362,21 @@ "metadata": {}, "outputs": [ { - "name": "stdout", + "name": "stderr", "output_type": "stream", "text": [ - "STVM ep options:\n", + "/home/onnxruntime/onnxruntime/core/providers/tvm/tvm_execution_provider.cc:480: TVM EP options:\n", "target: llvm -mcpu=skylake-avx512\n", "target_host: llvm -mcpu=skylake-avx512\n", "opt level: 3\n", "freeze weights: 1\n", "tuning file path: \n", - "tuning type: Ansor\n", + "tuning type: AutoTVM\n", "convert layout to NHWC: 0\n", "input tensor names: \n", - "input tensor shapes: \n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/agladyshev/workspace/tvm/python/tvm/driver/build_module.py:235: UserWarning: Specifying name with IRModule input is useless\n", - " warnings.warn(\"Specifying name with IRModule input is useless\")\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Build TVM graph executor\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ + "input tensor shapes: \n", + "/home/tvm/python/tvm/driver/build_module.py:235: UserWarning: Specifying name with IRModule input is useless\n", + " warnings.warn(\"Specifying name with IRModule input is useless\")\n", "One or more operators have not been tuned. Please tune your model for better performance. Use DEBUG logging level to see more details.\n" ] }, @@ -424,7 +404,7 @@ "source": [ "## 3. Configuration options\n", "\n", - "This section shows how you can configure STVM EP using custom options. For more details on the options used, see the corresponding section of the documentation." + "This section shows how you can configure TVM EP using custom options. For more details on the options used, see the corresponding section of the documentation." ] }, { @@ -434,7 +414,7 @@ "metadata": {}, "outputs": [], "source": [ - "provider_name = \"StvmExecutionProvider\"\n", + "provider_name = \"TvmExecutionProvider\"\n", "provider_options = dict(target=\"llvm -mtriple=x86_64-linux-gnu\",\n", " target_host=\"llvm -mtriple=x86_64-linux-gnu\",\n", " opt_level=3,\n", @@ -465,10 +445,10 @@ "metadata": {}, "outputs": [ { - "name": "stdout", + "name": "stderr", "output_type": "stream", "text": [ - "STVM ep options:\n", + "/home/onnxruntime/onnxruntime/core/providers/tvm/tvm_execution_provider.cc:480: TVM EP options:\n", "target: llvm -mtriple=x86_64-linux-gnu\n", "target_host: llvm -mtriple=x86_64-linux-gnu\n", "opt level: 3\n", @@ -477,18 +457,23 @@ "tuning type: Ansor\n", "convert layout to NHWC: 0\n", "input tensor names: \n", - "input tensor shapes: \n", - "Build TVM graph executor\n", + "input tensor shapes: \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ "****************** Output shape: (1, 1000) ******************\n" ] } ], "source": [ - "stvm_session = onnxruntime.InferenceSession(onnx_model.SerializeToString(),\n", - " providers=[provider_name],\n", - " provider_options=[provider_options]\n", - " )\n", - "output = stvm_session.run(output_names, input_dict)[0]\n", + "tvm_session = onnxruntime.InferenceSession(onnx_model.SerializeToString(),\n", + " providers=[provider_name],\n", + " provider_options=[provider_options]\n", + " )\n", + "output = tvm_session.run(output_names, input_dict)[0]\n", "print(f\"****************** Output shape: {output.shape} ******************\")" ] } diff --git a/include/onnxruntime/core/graph/constants.h b/include/onnxruntime/core/graph/constants.h index 09588e5ddedda..d7cd4a919a047 100644 --- a/include/onnxruntime/core/graph/constants.h +++ b/include/onnxruntime/core/graph/constants.h @@ -35,7 +35,7 @@ constexpr const char* kAclExecutionProvider = "ACLExecutionProvider"; constexpr const char* kArmNNExecutionProvider = "ArmNNExecutionProvider"; constexpr const char* kRocmExecutionProvider = "ROCMExecutionProvider"; constexpr const char* kCoreMLExecutionProvider = "CoreMLExecutionProvider"; -constexpr const char* kStvmExecutionProvider = "StvmExecutionProvider"; +constexpr const char* kTvmExecutionProvider = "TvmExecutionProvider"; constexpr const char* kExecutionProviderSharedLibraryPath = "shared_lib_path"; constexpr const char* kExecutionProviderSharedLibraryEntry = "provider_factory_entry_point"; diff --git a/include/onnxruntime/core/providers/stvm/stvm_provider_factory.h b/include/onnxruntime/core/providers/stvm/stvm_provider_factory.h deleted file mode 100644 index 3e0b52aa6318f..0000000000000 --- a/include/onnxruntime/core/providers/stvm/stvm_provider_factory.h +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#include "onnxruntime_c_api.h" - -#ifdef __cplusplus -extern "C" { -#endif - -ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_Stvm, _In_ OrtSessionOptions* options, _In_ const char* settings); - -#ifdef __cplusplus -} -#endif - - diff --git a/include/onnxruntime/core/providers/tvm/tvm_provider_factory.h b/include/onnxruntime/core/providers/tvm/tvm_provider_factory.h new file mode 100644 index 0000000000000..6d2478a94881d --- /dev/null +++ b/include/onnxruntime/core/providers/tvm/tvm_provider_factory.h @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#ifndef TVM_EXECUTION_PROVIDER_FACTORY_H +#define TVM_EXECUTION_PROVIDER_FACTORY_H + +#include "onnxruntime_c_api.h" + +#ifdef __cplusplus +extern "C" { +#endif + +ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_Tvm, _In_ OrtSessionOptions* options, _In_ const char* settings); + +#ifdef __cplusplus +} +#endif + +#endif // TVM_EXECUTION_PROVIDER_FACTORY_H diff --git a/java/src/main/java/ai/onnxruntime/OrtSession.java b/java/src/main/java/ai/onnxruntime/OrtSession.java index ba957df4f024c..6dcaf8e451abc 100644 --- a/java/src/main/java/ai/onnxruntime/OrtSession.java +++ b/java/src/main/java/ai/onnxruntime/OrtSession.java @@ -889,6 +889,17 @@ public void addNuphar(boolean allowUnalignedBuffers, String settings) throws Ort addNuphar(OnnxRuntime.ortApiHandle, nativeHandle, allowUnalignedBuffers ? 1 : 0, settings); } + /** + * Adds TVM as an execution backend. + * + * @param settings See the documentation for valid settings strings. + * @throws OrtException If there was an error in native code. + */ + public void addTvm(String settings) throws OrtException { + checkClosed(); + addTvm(OnnxRuntime.ortApiHandle, nativeHandle, settings); + } + /** * Adds DirectML as an execution backend. * @@ -1034,6 +1045,9 @@ private native void addNuphar( long apiHandle, long nativeHandle, int allowUnalignedBuffers, String settings) throws OrtException; + private native void addTvm(long apiHandle, long nativeHandle, String settings) + throws OrtException; + private native void addDirectML(long apiHandle, long nativeHandle, int deviceId) throws OrtException; diff --git a/java/src/main/native/ai_onnxruntime_OrtSession_SessionOptions.c b/java/src/main/native/ai_onnxruntime_OrtSession_SessionOptions.c index c91463722db9a..e7f4c15fea712 100644 --- a/java/src/main/native/ai_onnxruntime_OrtSession_SessionOptions.c +++ b/java/src/main/native/ai_onnxruntime_OrtSession_SessionOptions.c @@ -18,7 +18,7 @@ #include "onnxruntime/core/providers/dnnl/dnnl_provider_factory.h" #include "onnxruntime/core/providers/nnapi/nnapi_provider_factory.h" #include "onnxruntime/core/providers/nuphar/nuphar_provider_factory.h" -#include "onnxruntime/core/providers/stvm/stvm_provider_factory.h" +#include "onnxruntime/core/providers/tvm/tvm_provider_factory.h" #include "onnxruntime/core/providers/openvino/openvino_provider_factory.h" #include "onnxruntime/core/providers/tensorrt/tensorrt_provider_factory.h" #include "onnxruntime/core/providers/acl/acl_provider_factory.h" @@ -466,19 +466,19 @@ JNIEXPORT void JNICALL Java_ai_onnxruntime_OrtSession_00024SessionOptions_addNup /* * Class:: ai_onnxruntime_OrtSession_SessionOptions - * Method: addStvm + * Method: addTvm * Signature: (JILjava/lang/String)V */ -JNIEXPORT void JNICALL Java_ai_onnxruntime_OrtSession_00024SessionOptions_addStvm +JNIEXPORT void JNICALL Java_ai_onnxruntime_OrtSession_00024SessionOptions_addTvm (JNIEnv * jniEnv, jobject jobj, jlong apiHandle, jlong handle, jstring settingsString) { (void)jobj; - #ifdef USE_STVM + #ifdef USE_TVM const char* settings = (*jniEnv)->GetStringUTFChars(jniEnv, settingsString, NULL); - checkOrtStatus(jniEnv,(const OrtApi*)apiHandle,OrtSessionOptionsAppendExecutionProvider_Stvm((OrtSessionOptions*) handle, settings)); + checkOrtStatus(jniEnv,(const OrtApi*)apiHandle,OrtSessionOptionsAppendExecutionProvider_Tvm((OrtSessionOptions*) handle, settings)); (*jniEnv)->ReleaseStringUTFChars(jniEnv,settingsString,settings); #else - (void)apiHandle;(void)handle;(void)settingsString; // Parameters used when Stvm is defined. - throwOrtException(jniEnv,convertErrorCode(ORT_INVALID_ARGUMENT),"This binary was not compiled with Stvm support."); + (void)apiHandle;(void)handle;(void)settingsString; // Parameters used when TVM is defined. + throwOrtException(jniEnv,convertErrorCode(ORT_INVALID_ARGUMENT),"This binary was not compiled with TVM support."); #endif } diff --git a/onnxruntime/core/framework/utils.cc b/onnxruntime/core/framework/utils.cc index 96bdb44b65edf..2e6e15451b128 100644 --- a/onnxruntime/core/framework/utils.cc +++ b/onnxruntime/core/framework/utils.cc @@ -96,7 +96,7 @@ bool ProviderIsCpuBased(const std::string& provider_type) { return provider_type == onnxruntime::kCpuExecutionProvider || provider_type == onnxruntime::kDnnlExecutionProvider || provider_type == onnxruntime::kNupharExecutionProvider || - provider_type == onnxruntime::kStvmExecutionProvider || + provider_type == onnxruntime::kTvmExecutionProvider || provider_type == onnxruntime::kVitisAIExecutionProvider || provider_type == onnxruntime::kOpenVINOExecutionProvider || provider_type == onnxruntime::kNnapiExecutionProvider || diff --git a/onnxruntime/core/platform/windows/stacktrace.cc b/onnxruntime/core/platform/windows/stacktrace.cc index 4e159b8eddb48..d52d78a885118 100644 --- a/onnxruntime/core/platform/windows/stacktrace.cc +++ b/onnxruntime/core/platform/windows/stacktrace.cc @@ -33,7 +33,7 @@ class CaptureStackTrace { std::vector GetStackTrace() { #ifndef NDEBUG // TVM need to run with shared CRT, so won't work with debug helper now -#if !(defined USE_TVM) && !(defined _OPSCHEMA_LIB_) +#if !(defined USE_NUPHAR_TVM) && !(defined _OPSCHEMA_LIB_) return detail::CaptureStackTrace().Trace(); #else return {}; @@ -45,7 +45,7 @@ std::vector GetStackTrace() { namespace detail { #ifndef NDEBUG -#if !(defined USE_TVM) && !(defined _OPSCHEMA_LIB_) +#if !(defined USE_NUPHAR_TVM) && !(defined _OPSCHEMA_LIB_) class SymbolHelper { public: SymbolHelper() noexcept { diff --git a/onnxruntime/core/providers/get_execution_providers.cc b/onnxruntime/core/providers/get_execution_providers.cc index 14030ede1065d..ffeab88a3aa20 100644 --- a/onnxruntime/core/providers/get_execution_providers.cc +++ b/onnxruntime/core/providers/get_execution_providers.cc @@ -74,8 +74,8 @@ constexpr ProviderInfo kProvidersInPriorityOrder[] = #endif }, { - kStvmExecutionProvider, -#ifdef USE_STVM + kTvmExecutionProvider, +#ifdef USE_TVM true, #else false, diff --git a/onnxruntime/core/providers/nuphar/nuphar_execution_provider.cc b/onnxruntime/core/providers/nuphar/nuphar_execution_provider.cc index 16f54665409e1..806a593e9d858 100644 --- a/onnxruntime/core/providers/nuphar/nuphar_execution_provider.cc +++ b/onnxruntime/core/providers/nuphar/nuphar_execution_provider.cc @@ -29,7 +29,7 @@ thread_local std::unique_ptr> NupharExe thread_local int NupharExecutionProvider::per_model_fused_count_ = 0; static std::string GetCurrentHostTargetString() { -#if USE_TVM_WITH_LLVM +#if USE_NUPHAR_TVM_WITH_LLVM // auto detect from CPU ID const auto& cpu_id_info = CPUIDInfo::GetCPUIDInfo(); if (cpu_id_info.HasAVX512f()) { @@ -42,7 +42,7 @@ static std::string GetCurrentHostTargetString() { return llvm_target_str; #else return stackvm_target_str; -#endif // USE_TVM_WITH_LLVM +#endif // USE_NUPHAR_TVM_WITH_LLVM } NupharExecutionProvider::NupharExecutionProvider(const NupharExecutionProviderInfo& info) diff --git a/onnxruntime/core/providers/nuphar/nuphar_execution_provider.h b/onnxruntime/core/providers/nuphar/nuphar_execution_provider.h index 1f9dd49bfa326..ede5af0b03e5e 100644 --- a/onnxruntime/core/providers/nuphar/nuphar_execution_provider.h +++ b/onnxruntime/core/providers/nuphar/nuphar_execution_provider.h @@ -23,11 +23,11 @@ class CodeGenTarget; constexpr const char* llvm_target_str = "llvm"; constexpr const char* stackvm_target_str = "stackvm"; -#ifdef USE_TVM_WITH_LLVM +#ifdef USE_NUPHAR_TVM_WITH_LLVM constexpr const char* default_nuphar_target_str = llvm_target_str; #else constexpr const char* default_nuphar_target_str = stackvm_target_str; -#endif // USE_TVM_WITH_LLVM +#endif // USE_NUPHAR_TVM_WITH_LLVM // Information needed to construct Nuphar execution providers. struct NupharExecutionProviderInfo { diff --git a/onnxruntime/core/providers/stvm/stvm_provider_factory.cc b/onnxruntime/core/providers/stvm/stvm_provider_factory.cc deleted file mode 100644 index 79c13a08b115c..0000000000000 --- a/onnxruntime/core/providers/stvm/stvm_provider_factory.cc +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#include -#include - -#include "core/providers/stvm/stvm_provider_factory.h" -#include "stvm_execution_provider.h" - -#include "core/session/abi_session_options_impl.h" - -namespace onnxruntime { - -struct StvmProviderFactory : IExecutionProviderFactory { - StvmProviderFactory(const StvmExecutionProviderInfo& info) : info_{info} {} - ~StvmProviderFactory() = default; - - std::unique_ptr CreateProvider() override { - return std::make_unique(info_); - } - - private: - StvmExecutionProviderInfo info_; -}; - -std::shared_ptr CreateExecutionProviderFactory_Stvm(const char* settings) { - StvmExecutionProviderInfo info = StvmExecutionProviderInfo::FromOptionsString(settings); - return std::make_shared(info); -} - -std::shared_ptr CreateExecutionProviderFactory_Stvm(const StvmExecutionProviderInfo& info) -{ - return std::make_shared(info); -} -} // namespace onnxruntime - -ORT_API_STATUS_IMPL(OrtSessionOptionsAppendExecutionProvider_Stvm, - _In_ OrtSessionOptions* options, - _In_ const char* settings) { - onnxruntime::StvmExecutionProviderInfo info = onnxruntime::StvmExecutionProviderInfo::FromOptionsString(settings); - options->provider_factories.push_back(onnxruntime::CreateExecutionProviderFactory_Stvm(info)); - return nullptr; -} diff --git a/onnxruntime/core/providers/stvm/symbols.txt b/onnxruntime/core/providers/stvm/symbols.txt deleted file mode 100644 index 0fbccbe287573..0000000000000 --- a/onnxruntime/core/providers/stvm/symbols.txt +++ /dev/null @@ -1 +0,0 @@ -OrtSessionOptionsAppendExecutionProvider_Stvm diff --git a/onnxruntime/core/providers/tvm/symbols.txt b/onnxruntime/core/providers/tvm/symbols.txt new file mode 100644 index 0000000000000..8d903acd9ea76 --- /dev/null +++ b/onnxruntime/core/providers/tvm/symbols.txt @@ -0,0 +1 @@ +OrtSessionOptionsAppendExecutionProvider_Tvm diff --git a/onnxruntime/core/providers/stvm/stvm_allocator.cc b/onnxruntime/core/providers/tvm/tvm_allocator.cc similarity index 82% rename from onnxruntime/core/providers/stvm/stvm_allocator.cc rename to onnxruntime/core/providers/tvm/tvm_allocator.cc index 3784e49e23bf4..4fadff5c57b9a 100644 --- a/onnxruntime/core/providers/stvm/stvm_allocator.cc +++ b/onnxruntime/core/providers/tvm/tvm_allocator.cc @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -#include "stvm_allocator.h" +#include "tvm_allocator.h" #include "core/framework/allocatormgr.h" #include "core/framework/session_state.h" #include "xpu_data_transfer.h" @@ -9,8 +9,7 @@ namespace onnxruntime { - -void* STVMAllocator::Alloc(size_t size) { +void* TVMAllocator::Alloc(size_t size) { void* p = nullptr; if (size > 0) { DLDataType dl_type{kDLInt, 8, 1}; @@ -21,7 +20,7 @@ void* STVMAllocator::Alloc(size_t size) { return p; } -void STVMAllocator::Free(void* p) { +void TVMAllocator::Free(void* p) { TVMDeviceFreeDataSpace(ctx, p); } diff --git a/onnxruntime/core/providers/stvm/stvm_allocator.h b/onnxruntime/core/providers/tvm/tvm_allocator.h similarity index 52% rename from onnxruntime/core/providers/stvm/stvm_allocator.h rename to onnxruntime/core/providers/tvm/tvm_allocator.h index 4fc5b941203a6..11854024879c9 100644 --- a/onnxruntime/core/providers/stvm/stvm_allocator.h +++ b/onnxruntime/core/providers/tvm/tvm_allocator.h @@ -1,24 +1,24 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -#ifndef STVM_ALLOCATOR -#define STVM_ALLOCATOR +#ifndef TVM_ALLOCATOR +#define TVM_ALLOCATOR #include "core/framework/allocator.h" -#include "stvm_common.h" +#include "tvm_common.h" namespace onnxruntime { -#define STVM_ALLOC_ALIGN 128 +#define TVM_ALLOC_ALIGN 128 -class STVMAllocator : public IAllocator { +class TVMAllocator : public IAllocator { public: - STVMAllocator() : STVMAllocator(OrtMemoryInfo("TVM", - OrtAllocatorType::OrtDeviceAllocator, - OrtDevice(OrtDevice::CPU, OrtDevice::MemType::DEFAULT, 0), - 0, - OrtMemTypeDefault)) {} - explicit STVMAllocator(const OrtMemoryInfo& info) + TVMAllocator() : TVMAllocator(OrtMemoryInfo("TVM", + OrtAllocatorType::OrtDeviceAllocator, + OrtDevice(OrtDevice::CPU, OrtDevice::MemType::DEFAULT, 0), + 0, + OrtMemTypeDefault)) {} + explicit TVMAllocator(const OrtMemoryInfo& info) : IAllocator(info) { switch (info.device.Type()) { case OrtDevice::CPU: @@ -39,4 +39,4 @@ class STVMAllocator : public IAllocator { }; } // namespace onnxruntime -#endif // STVM_ALLOCATOR +#endif // TVM_ALLOCATOR diff --git a/onnxruntime/core/providers/stvm/stvm_api.cc b/onnxruntime/core/providers/tvm/tvm_api.cc similarity index 69% rename from onnxruntime/core/providers/stvm/stvm_api.cc rename to onnxruntime/core/providers/tvm/tvm_api.cc index a4f09873f5ad3..6afcb8a0142a3 100644 --- a/onnxruntime/core/providers/stvm/stvm_api.cc +++ b/onnxruntime/core/providers/tvm/tvm_api.cc @@ -1,15 +1,21 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -#include "stvm_api.h" #include "core/common/common.h" +#include "tvm_api.h" + #include #include -namespace stvm { -tvm::runtime::Module TVMCompile(const std::string& onnx_txt, +namespace onnxruntime { +namespace tvm { + +using TvmIntArray = ::tvm::Array<::tvm::Integer>; +using TvmPackedFunc = ::tvm::PackedFunc; + +TvmModule TVMCompile(const std::string& onnx_txt, const std::string& model_path, const std::string& target, const std::string& target_host, @@ -21,20 +27,20 @@ tvm::runtime::Module TVMCompile(const std::string& onnx_txt, const std::string& tuning_logfile, const std::string& tuning_type) { - tvm::Array> shapes; + ::tvm::Array shapes; for (size_t i = 0; i < input_shapes.size(); ++i) { - tvm::Array shape; + TvmIntArray shape; for (auto& dim : input_shapes[i]) { - shape.push_back(tvm::Integer(dim)); + shape.push_back(::tvm::Integer(dim)); } shapes.push_back(shape); } - const tvm::PackedFunc* compile = tvm::runtime::Registry::Get("tvm_onnx_import_and_compile"); + const TvmPackedFunc* compile = ::tvm::runtime::Registry::Get("tvm_onnx_import_and_compile"); ORT_ENFORCE(compile != nullptr, "Unable to retrieve 'tvm_onnx_import_and_compile'."); - tvm::runtime::Module mod = (*compile)( + TvmModule mod = (*compile)( TVMByteArray{onnx_txt.data(), onnx_txt.size()}, model_path, target, @@ -49,29 +55,29 @@ tvm::runtime::Module TVMCompile(const std::string& onnx_txt, return mod; } -void TVMSetInputs(tvm::runtime::Module& mod, +void TVMSetInputs(TvmModule& mod, std::vector& inds, std::vector& inputs) { // TODO(vvchernov): set_input_zero_copy is more preferable but it does not satisfy alignment conditions. //tvm::PackedFunc set_input = mod.GetFunction("set_input_zero_copy", false); - tvm::PackedFunc set_input = mod.GetFunction("set_input", false); + TvmPackedFunc set_input = mod.GetFunction("set_input", false); for (auto& i : inds) { set_input(i, &inputs[i]); } } -void TVMGetOutputShapes(tvm::runtime::Module& mod, +void TVMGetOutputShapes(TvmModule& mod, size_t num_outputs, std::vector>& output_shapes) { output_shapes.clear(); - tvm::PackedFunc get_output = mod.GetFunction("get_output", false); + TvmPackedFunc get_output = mod.GetFunction("get_output", false); for (size_t i = 0; i < num_outputs; ++i) { - tvm::runtime::NDArray output_array = get_output(i); - tvm::runtime::ShapeTuple shape_tuple = output_array.Shape(); + ::tvm::runtime::NDArray output_array = get_output(i); + ::tvm::runtime::ShapeTuple shape_tuple = output_array.Shape(); size_t dims_num = shape_tuple.size(); std::vector dims; for (size_t j = 0; j < dims_num; ++j) { @@ -81,19 +87,20 @@ void TVMGetOutputShapes(tvm::runtime::Module& mod, } } -void TVMRun(tvm::runtime::Module& mod, +void TVMRun(TvmModule& mod, std::vector& outputs, - [[maybe_unused]] tvm::runtime::TVMRetValue *ret) + [[maybe_unused]] ::tvm::runtime::TVMRetValue *ret) { - const tvm::PackedFunc* run = tvm::runtime::Registry::Get("tvm_run"); + const TvmPackedFunc* run = ::tvm::runtime::Registry::Get("tvm_run"); ORT_ENFORCE(run != nullptr, "Unable to retrieve 'tvm_run'."); (*run)(mod); - tvm::PackedFunc get_output = mod.GetFunction("get_output", false); + TvmPackedFunc get_output = mod.GetFunction("get_output", false); for (size_t i = 0; i < outputs.size(); ++i) { get_output(i, &outputs[i]); } } -} // namespace stvm +} // namespace tvm +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/stvm/stvm_api.h b/onnxruntime/core/providers/tvm/tvm_api.h similarity index 65% rename from onnxruntime/core/providers/stvm/stvm_api.h rename to onnxruntime/core/providers/tvm/tvm_api.h index 41d4b5f08ba63..af13952ab007f 100644 --- a/onnxruntime/core/providers/stvm/stvm_api.h +++ b/onnxruntime/core/providers/tvm/tvm_api.h @@ -1,13 +1,14 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -#ifndef STVM_API_H -#define STVM_API_H +#ifndef TVM_API_H +#define TVM_API_H -#include "stvm_common.h" +#include "tvm_common.h" -namespace stvm { - tvm::runtime::Module TVMCompile(const std::string& onnx_txt, +namespace onnxruntime { +namespace tvm { + TvmModule TVMCompile(const std::string& onnx_txt, const std::string& model_path, const std::string& target, const std::string& target_host, @@ -18,11 +19,12 @@ namespace stvm { bool nhwc = false, const std::string& tuning_logfile = "", const std::string& tuning_type = "AutoTVM"); - void TVMSetInputs(tvm::runtime::Module& mod, std::vector& inds, std::vector& inputs); - void TVMGetOutputShapes(tvm::runtime::Module& mod, + void TVMSetInputs(TvmModule& mod, std::vector& inds, std::vector& inputs); + void TVMGetOutputShapes(TvmModule& mod, size_t num_outputs, std::vector>& output_shapes); - void TVMRun(tvm::runtime::Module& mod, std::vector& outputs, tvm::runtime::TVMRetValue *ret); -} // namespace stvm + void TVMRun(TvmModule& mod, std::vector& outputs, ::tvm::runtime::TVMRetValue *ret); +} // namespace tvm +} // namespace onnxruntime -#endif // STVM_API_H \ No newline at end of file +#endif // TVM_API_H \ No newline at end of file diff --git a/onnxruntime/core/providers/stvm/stvm_common.h b/onnxruntime/core/providers/tvm/tvm_common.h similarity index 65% rename from onnxruntime/core/providers/stvm/stvm_common.h rename to onnxruntime/core/providers/tvm/tvm_common.h index a926a451cfc8d..54f0f6f00acd4 100644 --- a/onnxruntime/core/providers/stvm/stvm_common.h +++ b/onnxruntime/core/providers/tvm/tvm_common.h @@ -1,11 +1,13 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -#ifndef STVM_COMMON_H -#define STVM_COMMON_H +#ifndef TVM_COMMON_H +#define TVM_COMMON_H #include #include #include -#endif // STVM_COMMON_H +using TvmModule = tvm::runtime::Module; + +#endif // TVM_COMMON_H diff --git a/onnxruntime/core/providers/stvm/stvm_execution_provider.cc b/onnxruntime/core/providers/tvm/tvm_execution_provider.cc similarity index 81% rename from onnxruntime/core/providers/stvm/stvm_execution_provider.cc rename to onnxruntime/core/providers/tvm/tvm_execution_provider.cc index 4df2132a384dc..3b25b3cfbd3e5 100644 --- a/onnxruntime/core/providers/stvm/stvm_execution_provider.cc +++ b/onnxruntime/core/providers/tvm/tvm_execution_provider.cc @@ -12,37 +12,38 @@ #include "core/graph/model.h" #include "core/common/cpuid_info.h" -#include "stvm_execution_provider.h" +#include "tvm_execution_provider.h" #include "xpu_data_transfer.h" -#include "stvm_allocator.h" -#include "stvm_utils.h" -#include "stvm_api.h" +#include "tvm_allocator.h" +#include "tvm_utils.h" +#include "tvm_api.h" + using namespace ONNX_NAMESPACE; namespace onnxruntime { // Information to construct kernel function state. -struct STVMFuncState { +struct TVMFuncState { AllocateFunc allocate_func = nullptr; DestroyFunc release_func = nullptr; AllocatorHandle allocator = nullptr; - tvm::runtime::Module* module = nullptr; - std::function>& input_shapes)> compiler = nullptr; }; -class STVMRunner { +class TVMRunner { public: using TVMTensorShape = std::vector; using TVMTensorShapes = std::vector; using InputsInfoMap = std::map; using ORTGraphNodes = std::vector; - STVMRunner() = delete; - ~STVMRunner() = default; + TVMRunner() = delete; + ~TVMRunner() = default; - STVMRunner(StvmExecutionProvider* ep, + TVMRunner(TvmExecutionProvider* ep, const std::string& name, const Graph& graph) { // Extract input shapes @@ -92,7 +93,7 @@ class STVMRunner { size_t num_outputs = ort_outputs_info.size(); if (update_output_shapes_) { - stvm::TVMGetOutputShapes(*mod_, num_outputs, output_shapes_); + tvm::TVMGetOutputShapes(*mod_, num_outputs, output_shapes_); } else { for (auto i = 0u; i < num_outputs; i++) { TensorShape ort_shape = utils::GetTensorShapeFromTensorShapeProto(*ort_outputs_info[i]->Shape()); @@ -152,7 +153,7 @@ class STVMRunner { dl_tensors_inputs[counter] = t; inds[counter++] = i; } - stvm::TVMSetInputs(*mod_, inds, dl_tensors_inputs); + tvm::TVMSetInputs(*mod_, inds, dl_tensors_inputs); size_t num_outputs = tensors_outputs_.size(); for (auto i = 0u; i < num_outputs; i++) { @@ -173,8 +174,8 @@ class STVMRunner { tensors_outputs_[i].data = ort.GetTensorMutableData(output_tensor); } - tvm::runtime::TVMRetValue rvalue; - stvm::TVMRun(*mod_, tensors_outputs_, &rvalue); + ::tvm::runtime::TVMRetValue rvalue; + tvm::TVMRun(*mod_, tensors_outputs_, &rvalue); return Status::OK(); } @@ -188,14 +189,9 @@ class STVMRunner { ishape.resize(dims); for (int j = 0; j < dims; ++j) { int64_t dim = int64_t(ort_shape[j]); - if (dim > 0) { - ishape[j] = dim; - } else { - LOGS_DEFAULT(WARNING) << "input dimension is not positive value (dim = " << dim << "). " << - "It is replaced by 1. if it needs another value please use provider options to correct it"; - ishape[j] = 1; - update_output_shapes_ = true; - } + ORT_ENFORCE(dim > 0, "Input dimension is not positive value (dim = " + std::to_string(dim) + "). " + + "Please use provider options to setup input_names and input_shapes"); + ishape[j] = dim; } inputs_info_[indx] = ishape; } @@ -216,20 +212,20 @@ class STVMRunner { } private: - tvm::runtime::Module* mod_; + TvmModule* mod_; InputsInfoMap inputs_info_{}; bool update_output_shapes_ = false; TVMTensorShapes output_shapes_; std::vector tensors_outputs_; }; -StvmExecutionProvider::StvmExecutionProvider(const StvmExecutionProviderInfo& info) - : IExecutionProvider{kStvmExecutionProvider}, +TvmExecutionProvider::TvmExecutionProvider(const TvmExecutionProviderInfo& info) + : IExecutionProvider{kTvmExecutionProvider}, info_{info} { ProcessInfo(); AllocatorCreationInfo default_memory_info = {[](int) { - return std::make_unique(); + return std::make_unique(); }, 0, false}; allocator_ = CreateAllocator(default_memory_info); @@ -238,20 +234,20 @@ StvmExecutionProvider::StvmExecutionProvider(const StvmExecutionProviderInfo& in // Get environment variables const Env& env_instance = Env::Default(); - const std::string dump_subgraphs_env = env_instance.GetEnvironmentVar(stvm_env_vars::kDumpSubgraphs); + const std::string dump_subgraphs_env = env_instance.GetEnvironmentVar(tvm::env_vars::kDumpSubgraphs); if (!dump_subgraphs_env.empty()) { dump_subgraphs_ = std::stoi(dump_subgraphs_env) != 0; } } -StvmExecutionProvider::~StvmExecutionProvider() {} +TvmExecutionProvider::~TvmExecutionProvider() {} -AllocatorPtr StvmExecutionProvider::GetAllocator(int id, OrtMemType mem_type) const { +AllocatorPtr TvmExecutionProvider::GetAllocator(int id, OrtMemType mem_type) const { return allocator_; } std::vector> -StvmExecutionProvider::GetCapability(const GraphViewer& graph_viewer, +TvmExecutionProvider::GetCapability(const GraphViewer& graph_viewer, const std::vector& /*kernel_registries*/) const { std::vector> result; if (graph_viewer.IsSubgraph()) { @@ -299,7 +295,7 @@ StvmExecutionProvider::GetCapability(const GraphViewer& graph_viewer, return result; } -common::Status StvmExecutionProvider::Compile(const std::vector& nodes, +common::Status TvmExecutionProvider::Compile(const std::vector& nodes, std::vector& node_compute_funcs) { for (auto* fused_node : nodes) { auto func_body = fused_node->GetFunctionBody(); @@ -330,18 +326,18 @@ common::Status StvmExecutionProvider::Compile(const std::vector& nodes, } NodeComputeInfo compute_info; - compute_info.create_state_func = std::bind(&StvmExecutionProvider::CreateStateFunc, + compute_info.create_state_func = std::bind(&TvmExecutionProvider::CreateStateFunc, this, std::placeholders::_1, std::placeholders::_2); compute_info.release_state_func = [](FunctionState state) { if (state) - delete static_cast(state); + delete static_cast(state); }; // TODO(vvchernov): implement ops checking and mechanism of gracefully passing the responsibility to other EPs // if the checking fails due to unsupported op(s) - runners_[func_name] = std::make_shared(this, func_name, node_graph); + runners_[func_name] = std::make_shared(this, func_name, node_graph); compute_info.compute_func = *runners_[func_name].get(); node_compute_funcs.push_back(compute_info); @@ -349,17 +345,17 @@ common::Status StvmExecutionProvider::Compile(const std::vector& nodes, return Status::OK(); } -std::unique_ptr StvmExecutionProvider::GetDataTransfer() const { +std::unique_ptr TvmExecutionProvider::GetDataTransfer() const { if (GPUTargetCheck()) { return std::make_unique(); } else if (info_.target.find("llvm") != std::string::npos) { - return std::make_unique(); + return std::make_unique(); } else { - ORT_NOT_IMPLEMENTED("STVM GetDataTransfer is not implemented for target ", info_.target); + ORT_NOT_IMPLEMENTED("TVM GetDataTransfer is not implemented for target ", info_.target); } } -bool StvmExecutionProvider::GPUTargetCheck() const { +bool TvmExecutionProvider::GPUTargetCheck() const { //TODO(vvchernov): target or target host? bool check = ( info_.target.find("cuda") != std::string::npos || @@ -370,7 +366,7 @@ bool StvmExecutionProvider::GPUTargetCheck() const { return check; } -size_t StvmExecutionProvider::split(const std::string &txt, std::vector &strs, char ch) const { +size_t TvmExecutionProvider::split(const std::string &txt, std::vector &strs, char ch) const { size_t pos = txt.find( ch ); size_t initialPos = 0; strs.clear(); @@ -387,7 +383,7 @@ size_t StvmExecutionProvider::split(const std::string &txt, std::vector tmp_strs; std::vector names_strs; - std::string names_str = StvmExecutionProviderInfo::whitespace_trimming(info_.input_names_str); - std::string shapes_str = StvmExecutionProviderInfo::whitespace_trimming(info_.input_shapes_str); + std::string names_str = TvmExecutionProviderInfo::whitespace_trimming(info_.input_names_str); + std::string shapes_str = TvmExecutionProviderInfo::whitespace_trimming(info_.input_shapes_str); ORT_ENFORCE(split(names_str, names_strs, ' '), "There is no any input tensor names!"); size_t inp_tensors_num = names_strs.size(); @@ -454,29 +450,29 @@ void StvmExecutionProvider::ProcessInfo() { PrintInfo(); } -void StvmExecutionProvider::ProcessCPUTarget() { +void TvmExecutionProvider::ProcessCPUTarget() { const auto& cpu_id_info = CPUIDInfo::GetCPUIDInfo(); // auto detect from CPU ID if (cpu_id_info.HasAVX512Skylake()) { - info_.target = stvm_cpu_targets::LLVM_TARGET_SKYLAKE_AVX512; + info_.target = tvm::cpu_targets::LLVM_TARGET_SKYLAKE_AVX512; } else if (cpu_id_info.HasAVX512f()) { - info_.target = stvm_cpu_targets::LLVM_TARGET_AVX512; + info_.target = tvm::cpu_targets::LLVM_TARGET_AVX512; } else if (cpu_id_info.HasAVX2()) { - info_.target = stvm_cpu_targets::LLVM_TARGET_AVX2; + info_.target = tvm::cpu_targets::LLVM_TARGET_AVX2; } else if (cpu_id_info.HasAVX()) { - info_.target = stvm_cpu_targets::LLVM_TARGET_AVX; + info_.target = tvm::cpu_targets::LLVM_TARGET_AVX; } else { // TODO(vvchernov): extend mechanism of auto-definition of cpu target info_.target = llvm_target_str; } } -void StvmExecutionProvider::ProcessGPUTarget() { +void TvmExecutionProvider::ProcessGPUTarget() { ORT_NOT_IMPLEMENTED("GPU target auto-defenition is not implemented now!"); } -void StvmExecutionProvider::PrintInfo() const { - LOG(INFO) << "STVM ep options:\n" << +void TvmExecutionProvider::PrintInfo() const { + LOG(INFO) << "TVM EP options:\n" << "target: " << info_.target << "\n" << "target_host: " << info_.target_host << "\n" << "opt level: " << info_.opt_level << "\n" << @@ -488,13 +484,13 @@ void StvmExecutionProvider::PrintInfo() const { "input tensor shapes: " << info_.input_shapes_str; } -int StvmExecutionProvider::CreateStateFunc(ComputeContext* context, FunctionState* state) { - auto* state_ptr = new STVMFuncState(); +int TvmExecutionProvider::CreateStateFunc(ComputeContext* context, FunctionState* state) { + auto* state_ptr = new TVMFuncState(); *state_ptr = {context->allocate_func, context->release_func, context->allocator_handle, nullptr, - std::bind(&StvmExecutionProvider::CompileFunc, + std::bind(&TvmExecutionProvider::CompileFunc, this, std::placeholders::_1, std::placeholders::_2)}; @@ -502,24 +498,24 @@ int StvmExecutionProvider::CreateStateFunc(ComputeContext* context, FunctionStat return 0; } -tvm::runtime::Module* StvmExecutionProvider::CompileFunc(std::string func_name, +TvmModule* TvmExecutionProvider::CompileFunc(std::string func_name, const TVMTensorShapes& input_shapes) { if (modules_.count(func_name)) { return modules_[func_name].get(); } - tvm::runtime::Module mod_f = stvm::TVMCompile(buffers_[func_name], - model_paths_[func_name], - info_.target, - info_.target_host, - info_.opt_level, - opsets_[func_name], - info_.freeze_weights, - input_shapes, - info_.to_nhwc, - info_.tuning_file_path, - info_.tuning_type); - auto module_ptr = std::make_shared(); + TvmModule mod_f = tvm::TVMCompile(buffers_[func_name], + model_paths_[func_name], + info_.target, + info_.target_host, + info_.opt_level, + opsets_[func_name], + info_.freeze_weights, + input_shapes, + info_.to_nhwc, + info_.tuning_file_path, + info_.tuning_type); + auto module_ptr = std::make_shared(); *module_ptr = mod_f; modules_[func_name] = module_ptr; // Release memory after module generation diff --git a/onnxruntime/core/providers/stvm/stvm_execution_provider.h b/onnxruntime/core/providers/tvm/tvm_execution_provider.h similarity index 63% rename from onnxruntime/core/providers/stvm/stvm_execution_provider.h rename to onnxruntime/core/providers/tvm/tvm_execution_provider.h index 9e1f568edf04a..633a872e4154f 100644 --- a/onnxruntime/core/providers/stvm/stvm_execution_provider.h +++ b/onnxruntime/core/providers/tvm/tvm_execution_provider.h @@ -1,8 +1,8 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -#ifndef STVM_EXECUTION_PROVIDER_H -#define STVM_EXECUTION_PROVIDER_H +#ifndef TVM_EXECUTION_PROVIDER_H +#define TVM_EXECUTION_PROVIDER_H #include #include @@ -11,29 +11,31 @@ #include "core/common/logging/logging.h" #include "core/framework/execution_provider.h" -#include "core/providers/stvm/stvm_execution_provider_info.h" #include "core/platform/ort_mutex.h" -#include "stvm_common.h" +#include "tvm_common.h" +#include "tvm_execution_provider_info.h" namespace onnxruntime { -namespace stvm_env_vars { - static const std::string kDumpSubgraphs = "ORT_STVM_DUMP_SUBGRAPHS"; -} // namespace stvm_env_vars +namespace tvm { +namespace env_vars { + static const std::string kDumpSubgraphs = "ORT_TVM_DUMP_SUBGRAPHS"; +} // namespace env_vars +} // namespace tvm -class STVMRunner; +class TVMRunner; -class StvmExecutionProvider : public IExecutionProvider { - friend STVMRunner; +class TvmExecutionProvider : public IExecutionProvider { + friend TVMRunner; using TVMTensorShape = std::vector; using TVMTensorShapes = std::vector; - using STVMRunners = std::unordered_map>; - using STVMModules = std::unordered_map>; + using TVMRunners = std::unordered_map>; + using TVMModules = std::unordered_map>; public: - explicit StvmExecutionProvider(const StvmExecutionProviderInfo& info); - virtual ~StvmExecutionProvider(); + explicit TvmExecutionProvider(const TvmExecutionProviderInfo& info); + virtual ~TvmExecutionProvider(); std::vector> GetCapability(const onnxruntime::GraphViewer& graph, @@ -53,19 +55,19 @@ class StvmExecutionProvider : public IExecutionProvider { void PrintInfo() const; // Bindings for compute info int CreateStateFunc(ComputeContext*, FunctionState*); - tvm::runtime::Module* CompileFunc(std::string func_name, const TVMTensorShapes& input_shapes); + TvmModule* CompileFunc(std::string func_name, const TVMTensorShapes& input_shapes); private: - STVMRunners runners_; + TVMRunners runners_; std::unordered_map buffers_; std::unordered_map opsets_; std::unordered_map model_paths_; bool dump_subgraphs_ = false; - OrtMutex stvm_mu_; + OrtMutex tvm_mu_; AllocatorPtr allocator_; - StvmExecutionProviderInfo info_; - STVMModules modules_; + TvmExecutionProviderInfo info_; + TVMModules modules_; }; } // namespace onnxruntime -#endif // STVM_EXECUTION_PROVIDER_H +#endif // TVM_EXECUTION_PROVIDER_H diff --git a/onnxruntime/core/providers/stvm/stvm_execution_provider_info.cc b/onnxruntime/core/providers/tvm/tvm_execution_provider_info.cc similarity index 62% rename from onnxruntime/core/providers/stvm/stvm_execution_provider_info.cc rename to onnxruntime/core/providers/tvm/tvm_execution_provider_info.cc index 079b2742faed8..8170c829c4524 100644 --- a/onnxruntime/core/providers/stvm/stvm_execution_provider_info.cc +++ b/onnxruntime/core/providers/tvm/tvm_execution_provider_info.cc @@ -4,13 +4,14 @@ #include #include -#include "core/providers/stvm/stvm_execution_provider_info.h" - #include "core/common/common.h" #include "core/framework/provider_options_utils.h" +#include "tvm_execution_provider_info.h" + + namespace onnxruntime { -namespace stvm { +namespace tvm { namespace provider_option_names { constexpr const char* kTarget = "target"; constexpr const char* kTargetHost = "target_host"; @@ -35,9 +36,9 @@ static const std::unordered_set valid_keys { }; } // namespace provider_option_names -} // namespace stvm +} // namespace tvm -std::string StvmExecutionProviderInfo::whitespace_trimming(const std::string& str) { +std::string TvmExecutionProviderInfo::whitespace_trimming(const std::string& str) { const std::string WHITESPACE = " \n\r\t\f\v"; size_t start = str.find_first_not_of(WHITESPACE); if (start == std::string::npos) { @@ -49,26 +50,26 @@ std::string StvmExecutionProviderInfo::whitespace_trimming(const std::string& st } } -StvmExecutionProviderInfo StvmExecutionProviderInfo::FromProviderOptions(const ProviderOptions& options) { - StvmExecutionProviderInfo info{}; +TvmExecutionProviderInfo TvmExecutionProviderInfo::FromProviderOptions(const ProviderOptions& options) { + TvmExecutionProviderInfo info{}; ORT_THROW_IF_ERROR( ProviderOptionsParser{} - .AddAssignmentToReference(stvm::provider_option_names::kTarget, info.target) - .AddAssignmentToReference(stvm::provider_option_names::kTargetHost, info.target_host) - .AddAssignmentToReference(stvm::provider_option_names::kOptLevel, info.opt_level) - .AddAssignmentToReference(stvm::provider_option_names::kFreezeWeights, info.freeze_weights) - .AddAssignmentToReference(stvm::provider_option_names::kToNHWC, info.to_nhwc) - .AddAssignmentToReference(stvm::provider_option_names::kTuningFilePath, info.tuning_file_path) - .AddAssignmentToReference(stvm::provider_option_names::kTuningType, info.tuning_type) - .AddAssignmentToReference(stvm::provider_option_names::kInputNames, info.input_names_str) - .AddAssignmentToReference(stvm::provider_option_names::kInputShapes, info.input_shapes_str) + .AddAssignmentToReference(tvm::provider_option_names::kTarget, info.target) + .AddAssignmentToReference(tvm::provider_option_names::kTargetHost, info.target_host) + .AddAssignmentToReference(tvm::provider_option_names::kOptLevel, info.opt_level) + .AddAssignmentToReference(tvm::provider_option_names::kFreezeWeights, info.freeze_weights) + .AddAssignmentToReference(tvm::provider_option_names::kToNHWC, info.to_nhwc) + .AddAssignmentToReference(tvm::provider_option_names::kTuningFilePath, info.tuning_file_path) + .AddAssignmentToReference(tvm::provider_option_names::kTuningType, info.tuning_type) + .AddAssignmentToReference(tvm::provider_option_names::kInputNames, info.input_names_str) + .AddAssignmentToReference(tvm::provider_option_names::kInputShapes, info.input_shapes_str) .Parse(options)); return info; } -StvmExecutionProviderInfo StvmExecutionProviderInfo::FromOptionsString(const char* opt_str) { +TvmExecutionProviderInfo TvmExecutionProviderInfo::FromOptionsString(const char* opt_str) { std::string settings{opt_str}; ProviderOptions options; if (!settings.empty()) { @@ -93,8 +94,8 @@ StvmExecutionProviderInfo StvmExecutionProviderInfo::FromOptionsString(const cha value = whitespace_trimming(value); // Check keys of obtained options - if (stvm::provider_option_names::valid_keys.count(key) == 0) { - ORT_NOT_IMPLEMENTED("StvmOptions: unknown option (", key, ")"); + if (tvm::provider_option_names::valid_keys.count(key) == 0) { + ORT_NOT_IMPLEMENTED("TvmOptions: unknown option (", key, ")"); } options[key] = value; diff --git a/onnxruntime/core/providers/stvm/stvm_execution_provider_info.h b/onnxruntime/core/providers/tvm/tvm_execution_provider_info.h similarity index 72% rename from onnxruntime/core/providers/stvm/stvm_execution_provider_info.h rename to onnxruntime/core/providers/tvm/tvm_execution_provider_info.h index d3adc5d3f73c9..5f91f2d91b531 100644 --- a/onnxruntime/core/providers/stvm/stvm_execution_provider_info.h +++ b/onnxruntime/core/providers/tvm/tvm_execution_provider_info.h @@ -1,8 +1,8 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -#ifndef STVM_EXECUTION_PROVIDER_INFO_H -#define STVM_EXECUTION_PROVIDER_INFO_H +#ifndef TVM_EXECUTION_PROVIDER_INFO_H +#define TVM_EXECUTION_PROVIDER_INFO_H #include #include @@ -18,20 +18,22 @@ constexpr const char* llvm_target_str = "llvm"; constexpr const char* cpu_target_str = "cpu"; constexpr const char* gpu_target_str = "gpu"; -namespace stvm_cpu_targets { +namespace tvm { +namespace cpu_targets { // TODO(vvchernov): avx and avx512 need more careful differentiation for target const std::string LLVM_TARGET_AVX = "llvm -mcpu=corei7-avx"; const std::string LLVM_TARGET_AVX2 = "llvm -mcpu=core-avx2"; const std::string LLVM_TARGET_SKYLAKE_AVX512 = "llvm -mcpu=skylake-avx512"; const std::string LLVM_TARGET_AVX512 = "llvm -mcpu=skylake-avx512"; -} +} // namespace cpu_targets +} // namespace tvm constexpr const unsigned int default_opt_level = 3; -using STVMInputShapes = std::unordered_map>; +using TVMInputShapes = std::unordered_map>; // Information needed to construct an TVM execution provider. -struct StvmExecutionProviderInfo { +struct TvmExecutionProviderInfo { std::string target{default_target_str}; std::string target_host{default_target_str}; unsigned int opt_level{default_opt_level}; @@ -41,13 +43,13 @@ struct StvmExecutionProviderInfo { std::string tuning_type{"AutoTVM"}; std::string input_names_str{""}; std::string input_shapes_str{""}; - STVMInputShapes input_shapes{}; + TVMInputShapes input_shapes{}; static std::string whitespace_trimming(const std::string& str); - static StvmExecutionProviderInfo FromProviderOptions(const ProviderOptions& options); - static StvmExecutionProviderInfo FromOptionsString(const char* options); + static TvmExecutionProviderInfo FromProviderOptions(const ProviderOptions& options); + static TvmExecutionProviderInfo FromOptionsString(const char* options); }; } // namespace onnxruntime -#endif // STVM_EXECUTION_PROVIDER_INFO_H +#endif // TVM_EXECUTION_PROVIDER_INFO_H diff --git a/onnxruntime/core/providers/tvm/tvm_provider_factory.cc b/onnxruntime/core/providers/tvm/tvm_provider_factory.cc new file mode 100644 index 0000000000000..b63077e3b311c --- /dev/null +++ b/onnxruntime/core/providers/tvm/tvm_provider_factory.cc @@ -0,0 +1,44 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include +#include + +#include "core/providers/tvm/tvm_provider_factory.h" +#include "core/session/abi_session_options_impl.h" + +#include "tvm_execution_provider.h" + + +namespace onnxruntime { + +struct TvmProviderFactory : IExecutionProviderFactory { + TvmProviderFactory(const TvmExecutionProviderInfo& info) : info_{info} {} + ~TvmProviderFactory() = default; + + std::unique_ptr CreateProvider() override { + return std::make_unique(info_); + } + + private: + TvmExecutionProviderInfo info_; +}; + +std::shared_ptr CreateExecutionProviderFactory_Tvm(const char* settings) { + TvmExecutionProviderInfo info = TvmExecutionProviderInfo::FromOptionsString(settings); + return std::make_shared(info); +} + +std::shared_ptr CreateExecutionProviderFactory_Tvm(const TvmExecutionProviderInfo& info) +{ + return std::make_shared(info); +} +} // namespace onnxruntime + +ORT_API_STATUS_IMPL(OrtSessionOptionsAppendExecutionProvider_Tvm, + _In_ OrtSessionOptions* options, + _In_ const char* settings) { + onnxruntime::TvmExecutionProviderInfo info = onnxruntime::TvmExecutionProviderInfo::FromOptionsString(settings); + options->provider_factories.push_back(onnxruntime::CreateExecutionProviderFactory_Tvm(info)); + return nullptr; +} diff --git a/onnxruntime/core/providers/stvm/stvm_utils.h b/onnxruntime/core/providers/tvm/tvm_utils.h similarity index 93% rename from onnxruntime/core/providers/stvm/stvm_utils.h rename to onnxruntime/core/providers/tvm/tvm_utils.h index 95c6a66e2169e..15f51c3fd43ab 100644 --- a/onnxruntime/core/providers/stvm/stvm_utils.h +++ b/onnxruntime/core/providers/tvm/tvm_utils.h @@ -1,10 +1,10 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -#ifndef STVM_UTILS_H -#define STVM_UTILS_H +#ifndef TVM_UTILS_H +#define TVM_UTILS_H -#include "stvm_common.h" +#include "tvm_common.h" #include "core/session/onnxruntime_cxx_api.h" #include "core/framework/ortdevice.h" @@ -50,4 +50,4 @@ inline DLDevice GetDLDevice(const OrtDevice& device) { } // namespace onnxruntime -#endif // STVM_UTILS_H +#endif // TVM_UTILS_H diff --git a/onnxruntime/core/providers/stvm/xpu_data_transfer.cc b/onnxruntime/core/providers/tvm/xpu_data_transfer.cc similarity index 91% rename from onnxruntime/core/providers/stvm/xpu_data_transfer.cc rename to onnxruntime/core/providers/tvm/xpu_data_transfer.cc index 3bbdcfb5c3b33..4efb171dda849 100644 --- a/onnxruntime/core/providers/stvm/xpu_data_transfer.cc +++ b/onnxruntime/core/providers/tvm/xpu_data_transfer.cc @@ -4,7 +4,7 @@ #include "core/framework/tensor.h" #include "xpu_data_transfer.h" -#include "stvm_utils.h" +#include "tvm_utils.h" namespace onnxruntime { XPUDataTransfer::XPUDataTransfer() { @@ -63,11 +63,11 @@ DLDevice XPUDataTransfer::get_context(const OrtDevice& device) const return GetDLDevice(device); } -bool StvmCPUDataTransfer::CanCopy(const OrtDevice& src_device, const OrtDevice& dst_device) const { +bool TvmCPUDataTransfer::CanCopy(const OrtDevice& src_device, const OrtDevice& dst_device) const { return src_device.Type() == OrtDevice::CPU && dst_device.Type() == OrtDevice::CPU; } -common::Status StvmCPUDataTransfer::CopyTensor(const Tensor& src, Tensor& dst, int /*exec_queue_id*/) const { +common::Status TvmCPUDataTransfer::CopyTensor(const Tensor& src, Tensor& dst, int /*exec_queue_id*/) const { const void* src_data = src.DataRaw(); void* dst_data = dst.MutableDataRaw(); if (src_data == dst_data) { diff --git a/onnxruntime/core/providers/stvm/xpu_data_transfer.h b/onnxruntime/core/providers/tvm/xpu_data_transfer.h similarity index 90% rename from onnxruntime/core/providers/stvm/xpu_data_transfer.h rename to onnxruntime/core/providers/tvm/xpu_data_transfer.h index 1d5d655fa4e8c..f07c11794390a 100644 --- a/onnxruntime/core/providers/stvm/xpu_data_transfer.h +++ b/onnxruntime/core/providers/tvm/xpu_data_transfer.h @@ -5,7 +5,7 @@ #define XPU_DATA_TRANSFER #include "core/framework/data_transfer.h" -#include "stvm_common.h" +#include "tvm_common.h" namespace onnxruntime { @@ -22,9 +22,9 @@ class XPUDataTransfer : public IDataTransfer { DLDevice get_context(const OrtDevice& device) const; }; -class StvmCPUDataTransfer : public IDataTransfer { +class TvmCPUDataTransfer : public IDataTransfer { public: - StvmCPUDataTransfer() = default; + TvmCPUDataTransfer() = default; // Dampen MSVC warning about not fully overriding CopyTensor using IDataTransfer::CopyTensor; bool CanCopy(const OrtDevice& src_device, const OrtDevice& dst_device) const override; diff --git a/onnxruntime/core/session/provider_stubs.cc b/onnxruntime/core/session/provider_stubs.cc index d320ad3c0f237..067f8a600e2aa 100644 --- a/onnxruntime/core/session/provider_stubs.cc +++ b/onnxruntime/core/session/provider_stubs.cc @@ -58,12 +58,12 @@ ORT_API_STATUS_IMPL(OrtSessionOptionsAppendExecutionProvider_Nuphar, } #endif -#ifndef USE_STVM -ORT_API_STATUS_IMPL(OrtSessionOptionsAppendExecutionProvider_Stvm, +#ifndef USE_TVM +ORT_API_STATUS_IMPL(OrtSessionOptionsAppendExecutionProvider_Tvm, _In_ OrtSessionOptions* options, _In_ const char* settings) { ORT_UNUSED_PARAMETER(options); ORT_UNUSED_PARAMETER(settings); - return CreateNotEnabledStatus("Stvm"); + return CreateNotEnabledStatus("Tvm"); } #endif diff --git a/onnxruntime/python/onnxruntime_pybind_state.cc b/onnxruntime/python/onnxruntime_pybind_state.cc index 56c59406bcd56..4fdd183e35905 100644 --- a/onnxruntime/python/onnxruntime_pybind_state.cc +++ b/onnxruntime/python/onnxruntime_pybind_state.cc @@ -647,15 +647,15 @@ std::unique_ptr CreateExecutionProviderInstance( nuphar_settings.clear(); return p; #endif - } else if (type == kStvmExecutionProvider) { -#if USE_STVM - onnxruntime::StvmExecutionProviderInfo info{}; + } else if (type == kTvmExecutionProvider) { +#if USE_TVM + onnxruntime::TvmExecutionProviderInfo info{}; const auto it = provider_options_map.find(type); if (it != provider_options_map.end()) { - info = onnxruntime::StvmExecutionProviderInfo::FromProviderOptions(it->second); + info = onnxruntime::TvmExecutionProviderInfo::FromProviderOptions(it->second); } - return onnxruntime::CreateExecutionProviderFactory_Stvm(info)->CreateProvider(); + return onnxruntime::CreateExecutionProviderFactory_Tvm(info)->CreateProvider(); #endif } else if (type == kVitisAIExecutionProvider) { #if USE_VITISAI diff --git a/onnxruntime/python/onnxruntime_pybind_state_common.h b/onnxruntime/python/onnxruntime_pybind_state_common.h index 5477f3c91ad43..093fced91c3ba 100644 --- a/onnxruntime/python/onnxruntime_pybind_state_common.h +++ b/onnxruntime/python/onnxruntime_pybind_state_common.h @@ -23,7 +23,7 @@ struct OrtStatus { char msg[1]; // a null-terminated string }; -#define BACKEND_DEVICE BACKEND_PROC BACKEND_DNNL BACKEND_OPENVINO BACKEND_NUPHAR BACKEND_STVM BACKEND_OPENBLAS BACKEND_MIGRAPHX BACKEND_ACL BACKEND_ARMNN BACKEND_DML +#define BACKEND_DEVICE BACKEND_PROC BACKEND_DNNL BACKEND_OPENVINO BACKEND_NUPHAR BACKEND_TVM BACKEND_OPENBLAS BACKEND_MIGRAPHX BACKEND_ACL BACKEND_ARMNN BACKEND_DML #include "core/session/onnxruntime_cxx_api.h" #include "core/providers/providers.h" #include "core/providers/cpu/cpu_execution_provider.h" @@ -86,10 +86,10 @@ struct OrtStatus { #define BACKEND_NUPHAR "" #endif -#ifdef USE_STVM -#define BACKEND_STVM "-STVM" +#ifdef USE_TVM +#define BACKEND_TVM "-TVM" #else -#define BACKEND_STVM "" +#define BACKEND_TVM "" #endif #if USE_VITISAI @@ -156,8 +156,8 @@ extern std::string nuphar_settings; } } // namespace onnxruntime #endif -#ifdef USE_STVM -#include "core/providers/stvm/stvm_execution_provider_info.h" +#ifdef USE_TVM +#include "core/providers/tvm/tvm_execution_provider_info.h" #endif #ifdef USE_VITISAI #include "core/providers/vitisai/vitisai_provider_factory.h" @@ -483,9 +483,9 @@ std::shared_ptr CreateExecutionProviderFactory_Cuda(c std::shared_ptr CreateExecutionProviderFactory_Dnnl(int use_arena); std::shared_ptr CreateExecutionProviderFactory_OpenVINO(const OrtOpenVINOProviderOptions* params); std::shared_ptr CreateExecutionProviderFactory_Nuphar(bool, const char*); -#ifdef USE_STVM -std::shared_ptr CreateExecutionProviderFactory_Stvm(const StvmExecutionProviderInfo& info); -std::shared_ptr CreateExecutionProviderFactory_Stvm(const char* params); +#ifdef USE_TVM +std::shared_ptr CreateExecutionProviderFactory_Tvm(const TvmExecutionProviderInfo& info); +std::shared_ptr CreateExecutionProviderFactory_Tvm(const char* params); #endif std::shared_ptr CreateExecutionProviderFactory_VITISAI(const char* backend_type, int device_id, const char* export_runtime_module, diff --git a/onnxruntime/python/providers/stvm/__init__.py b/onnxruntime/python/providers/tvm/__init__.py similarity index 100% rename from onnxruntime/python/providers/stvm/__init__.py rename to onnxruntime/python/providers/tvm/__init__.py diff --git a/onnxruntime/python/providers/stvm/ort.py b/onnxruntime/python/providers/tvm/ort.py similarity index 100% rename from onnxruntime/python/providers/stvm/ort.py rename to onnxruntime/python/providers/tvm/ort.py diff --git a/onnxruntime/test/contrib_ops/fused_conv_test.cc b/onnxruntime/test/contrib_ops/fused_conv_test.cc index 1e31f66e80cfb..756df9993db6a 100644 --- a/onnxruntime/test/contrib_ops/fused_conv_test.cc +++ b/onnxruntime/test/contrib_ops/fused_conv_test.cc @@ -26,7 +26,7 @@ static std::unordered_set providers_except_cpu = { kDnnlExecutionProvider, kOpenVINOExecutionProvider, kNupharExecutionProvider, - kStvmExecutionProvider, + kTvmExecutionProvider, kVitisAIExecutionProvider, kTensorrtExecutionProvider, kNnapiExecutionProvider, @@ -41,7 +41,7 @@ static std::unordered_set providers_except_cpu_cuda = { kDnnlExecutionProvider, kOpenVINOExecutionProvider, kNupharExecutionProvider, - kStvmExecutionProvider, + kTvmExecutionProvider, kVitisAIExecutionProvider, kTensorrtExecutionProvider, kNnapiExecutionProvider, @@ -169,7 +169,7 @@ static std::unordered_set providers_except_cuda = { kDnnlExecutionProvider, kOpenVINOExecutionProvider, kNupharExecutionProvider, - kStvmExecutionProvider, + kTvmExecutionProvider, kVitisAIExecutionProvider, kTensorrtExecutionProvider, kNnapiExecutionProvider, diff --git a/onnxruntime/test/tvm/tvm_basic_test.cc b/onnxruntime/test/nuphar_tvm/tvm_basic_test.cc similarity index 99% rename from onnxruntime/test/tvm/tvm_basic_test.cc rename to onnxruntime/test/nuphar_tvm/tvm_basic_test.cc index 6e299e7b48967..6fa1cf3b250c1 100644 --- a/onnxruntime/test/tvm/tvm_basic_test.cc +++ b/onnxruntime/test/nuphar_tvm/tvm_basic_test.cc @@ -13,7 +13,7 @@ #include "core/session/onnxruntime_cxx_api.h" #include "test/framework/test_utils.h" #include "test/test_environment.h" -#include "test/tvm/tvm_demo/demo_compiler.h" +#include "test/nuphar_tvm/tvm_demo/demo_compiler.h" #include @@ -363,7 +363,7 @@ TEST(TVMTest, Native_TVM) { auto args = Array({A, B, D, E}); std::unordered_map binds; auto config = build_config(); -#ifdef USE_TVM_WITH_LLVM +#ifdef USE_NUPHAR_TVM_WITH_LLVM auto target = target::llvm(); #else auto target = target::stackvm(); diff --git a/onnxruntime/test/tvm/tvm_demo/demo_compiler.cc b/onnxruntime/test/nuphar_tvm/tvm_demo/demo_compiler.cc similarity index 99% rename from onnxruntime/test/tvm/tvm_demo/demo_compiler.cc rename to onnxruntime/test/nuphar_tvm/tvm_demo/demo_compiler.cc index 0f0685cd14313..ff2f9076a794d 100644 --- a/onnxruntime/test/tvm/tvm_demo/demo_compiler.cc +++ b/onnxruntime/test/nuphar_tvm/tvm_demo/demo_compiler.cc @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -#include "test/tvm/tvm_demo/demo_compiler.h" +#include "test/nuphar_tvm/tvm_demo/demo_compiler.h" #include "core/codegen/passes/scheduler/schedule_utils.h" #include "core/codegen/passes/utils/ort_tvm_utils.h" diff --git a/onnxruntime/test/tvm/tvm_demo/demo_compiler.h b/onnxruntime/test/nuphar_tvm/tvm_demo/demo_compiler.h similarity index 100% rename from onnxruntime/test/tvm/tvm_demo/demo_compiler.h rename to onnxruntime/test/nuphar_tvm/tvm_demo/demo_compiler.h diff --git a/onnxruntime/test/platform/windows/stacktrace_test.cc b/onnxruntime/test/platform/windows/stacktrace_test.cc index 3f656c0e51054..de84211a1e492 100644 --- a/onnxruntime/test/platform/windows/stacktrace_test.cc +++ b/onnxruntime/test/platform/windows/stacktrace_test.cc @@ -15,7 +15,7 @@ namespace test { using namespace ::testing; //TVM is not working with StackTrace now. -#if !defined(USE_TVM) && !defined(ORT_NO_EXCEPTIONS) +#if !defined(USE_NUPHAR_TVM) && !defined(ORT_NO_EXCEPTIONS) TEST(StacktraceTests, BasicTests) { auto result = ::onnxruntime::GetStackTrace(); diff --git a/onnxruntime/test/providers/cpu/reduction/reduction_ops_test.cc b/onnxruntime/test/providers/cpu/reduction/reduction_ops_test.cc index 92177ef03f5dd..5dbfc40fcbae8 100644 --- a/onnxruntime/test/providers/cpu/reduction/reduction_ops_test.cc +++ b/onnxruntime/test/providers/cpu/reduction/reduction_ops_test.cc @@ -171,14 +171,14 @@ TEST(ReductionOpTest, ReduceL1_int32) { test.Run(); } -#if !(defined USE_TVM) +#if !(defined USE_NUPHAR_TVM) TEST(ReductionOpTest, ReduceL10DTensor) { OpTester test("ReduceL1"); test.AddInput("data", {}, {2}); test.AddOutput("reduced", {}, {2}); test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); } -#endif // !(defined USE_TVM) +#endif // !(defined USE_NUPHAR_TVM) TEST(ReductionOpTest, ReduceL2_default_axes_keepdims) { OpTester test("ReduceL2"); @@ -290,14 +290,14 @@ TEST(ReductionOpTest, ReduceL2_int32) { test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: Int32 not allowed as input to this layer } -#if !(defined USE_TVM) +#if !(defined USE_NUPHAR_TVM) TEST(ReductionOpTest, ReduceL20DTensor) { OpTester test("ReduceL2"); test.AddInput("data", {}, {2}); test.AddOutput("reduced", {}, {2}); test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); } -#endif // !(defined USE_TVM) +#endif // !(defined USE_NUPHAR_TVM) TEST(ReductionOpTest, ReduceLogSum) { OpTester test("ReduceLogSum"); @@ -363,14 +363,14 @@ TEST(ReductionOpTest, ReduceLogSumAxes01) { test.Run(); } -#if !(defined USE_TVM) +#if !(defined USE_NUPHAR_TVM) TEST(ReductionOpTest, ReduceLogSum0DTensor) { OpTester test("ReduceLogSum"); test.AddInput("data", {}, {2.f}); test.AddOutput("reduced", {}, {0.693147f}); test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); } -#endif // !(defined USE_TVM) +#endif // !(defined USE_NUPHAR_TVM) TEST(ReductionOpTest, ReduceLogSumExp_default_axes_keepdims) { OpTester test("ReduceLogSumExp"); @@ -616,7 +616,7 @@ TEST(ReductionOpTest, ReduceLogSumExp_int32) { test.Run(); } -#if !(defined USE_TVM) +#if !(defined USE_NUPHAR_TVM) TEST(ReductionOpTest, ReduceLogSumExp0DTensor) { OpTester test("ReduceLogSumExp"); test.AddInput("data", {}, {2}); @@ -630,7 +630,7 @@ TEST(ReductionOpTest, ReduceLogSumExp0DTensor_double) { test.AddOutput("reduced", {}, {2}); test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); } -#endif // !(defined USE_TVM) +#endif // !(defined USE_NUPHAR_TVM) TEST(ReductionOpTest, ReduceMax_default_axes_keepdims) { OpTester test("ReduceMax"); @@ -846,14 +846,14 @@ TEST(ReductionOpTest, ReduceMax_uint8) { #endif } -#if !(defined USE_TVM) +#if !(defined USE_NUPHAR_TVM) TEST(ReductionOpTest, ReduceMax0DTensor) { OpTester test("ReduceMax"); test.AddInput("data", {}, {2}); test.AddOutput("reduced", {}, {2}); test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); } -#endif // !(defined USE_TVM) +#endif // !(defined USE_NUPHAR_TVM) TEST(ReductionOpTest, ReduceMean_default_axes_keepdims) { OpTester test("ReduceMean"); @@ -1089,7 +1089,7 @@ TEST(ReductionOpTest, ReduceMean_int32) { test.Run(); } -#if !(defined USE_TVM) +#if !(defined USE_NUPHAR_TVM) TEST(ReductionOpTest, ReduceMean0DTensor) { OpTester test("ReduceMean"); test.AddInput("data", {}, {2}); @@ -1103,7 +1103,7 @@ TEST(ReductionOpTest, ReduceMean0DTensor_double) { test.AddOutput("reduced", {}, {2}); test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); } -#endif // !(defined USE_TVM) +#endif // !(defined USE_NUPHAR_TVM) TEST(ReductionOpTest, ReduceMean_keepdims_results_in_noop) { OpTester test("ReduceMean"); @@ -1330,14 +1330,14 @@ TEST(ReductionOpTest, ReduceMin_uint8) { test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); } -#if !(defined USE_TVM) +#if !(defined USE_NUPHAR_TVM) TEST(ReductionOpTest, ReduceMin0DTensor) { OpTester test("ReduceMin"); test.AddInput("data", {}, {2}); test.AddOutput("reduced", {}, {2}); test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); } -#endif // !(defined USE_TVM) +#endif // !(defined USE_NUPHAR_TVM) TEST(ReductionOpTest, ReduceSum) { OpTester test("ReduceSum"); @@ -1833,14 +1833,14 @@ TEST(ReductionOpTest, ReduceSum_noop_axes_input_initializer) { test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider, kOpenVINOExecutionProvider}); } -#if !(defined USE_TVM) +#if !(defined USE_NUPHAR_TVM) TEST(ReductionOpTest, ReduceSum0DTensor) { OpTester test("ReduceSum"); test.AddInput("data", {}, {2}); test.AddOutput("reduced", {}, {2}); test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); } -#endif // !(defined USE_TVM) +#endif // !(defined USE_NUPHAR_TVM) TEST(ReductionOpTest, ReduceSumSquare) { OpTester test("ReduceSumSquare"); @@ -1969,14 +1969,14 @@ TEST(ReductionOpTest, ReduceSumSquare_keepdims) { test.Run(); } -#if !(defined USE_TVM) +#if !(defined USE_NUPHAR_TVM) TEST(ReductionOpTest, ReduceSumSquare0DTensor) { OpTester test("ReduceSumSquare"); test.AddInput("data", {}, {2}); test.AddOutput("reduced", {}, {4}); test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); } -#endif // !(defined USE_TVM) +#endif // !(defined USE_NUPHAR_TVM) TEST(ReductionOpTest, ReduceProd_default_axes_keepdims) { OpTester test("ReduceProd"); @@ -2101,14 +2101,14 @@ TEST(ReductionOpTest, ReduceProd_int64) { test.Run(); } -#if !(defined USE_TVM) +#if !(defined USE_NUPHAR_TVM) TEST(ReductionOpTest, ReduceProd0DTensor) { OpTester test("ReduceProd"); test.AddInput("data", {}, {2}); test.AddOutput("reduced", {}, {2}); test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); } -#endif // (!defined USE_TVM) +#endif // (!defined USE_NUPHAR_TVM) TEST(ReductionOpTest, ArgMax) { OpTester test("ArgMax"); diff --git a/onnxruntime/test/providers/provider_test_utils.cc b/onnxruntime/test/providers/provider_test_utils.cc index b937cdca2ff7a..094ec955febb8 100644 --- a/onnxruntime/test/providers/provider_test_utils.cc +++ b/onnxruntime/test/providers/provider_test_utils.cc @@ -1122,7 +1122,7 @@ void OpTester::Run( if (provider_type == onnxruntime::kOpenVINOExecutionProvider || provider_type == onnxruntime::kTensorrtExecutionProvider || provider_type == onnxruntime::kNupharExecutionProvider || - // provider_type == onnxruntime::kStvmExecutionProvider || + // provider_type == onnxruntime::kTvmExecutionProvider || provider_type == onnxruntime::kNnapiExecutionProvider || provider_type == onnxruntime::kCoreMLExecutionProvider || provider_type == onnxruntime::kDnnlExecutionProvider) diff --git a/onnxruntime/test/python/onnxruntime_test_python.py b/onnxruntime/test/python/onnxruntime_test_python.py index b16f61903ccd8..5cbd998bfafd9 100644 --- a/onnxruntime/test/python/onnxruntime_test_python.py +++ b/onnxruntime/test/python/onnxruntime_test_python.py @@ -20,7 +20,7 @@ available_providers = [ provider for provider in onnxrt.get_available_providers() - if provider not in {'StvmExecutionProvider'}] + if provider not in {'TvmExecutionProvider'}] class TestInferenceSession(unittest.TestCase): @@ -33,7 +33,7 @@ def run_model(self, session_object, run_options): np.testing.assert_allclose(output_expected, res[0], rtol=1e-05, atol=1e-08) def testTvmImported(self): - if "StvmExecutionProvider" not in onnxrt.get_available_providers(): + if "TvmExecutionProvider" not in onnxrt.get_available_providers(): return import tvm self.assertTrue(tvm is not None) diff --git a/onnxruntime/test/python/onnxruntime_test_python_stvm.py b/onnxruntime/test/python/onnxruntime_test_python_tvm.py similarity index 89% rename from onnxruntime/test/python/onnxruntime_test_python_stvm.py rename to onnxruntime/test/python/onnxruntime_test_python_tvm.py index c44cc3ab34b2d..945bf2604f924 100644 --- a/onnxruntime/test/python/onnxruntime_test_python_stvm.py +++ b/onnxruntime/test/python/onnxruntime_test_python_tvm.py @@ -6,9 +6,9 @@ make_graph, make_tensor_value_info) import onnxruntime -if "StvmExecutionProvider" not in onnxruntime.get_available_providers(): +if "TvmExecutionProvider" not in onnxruntime.get_available_providers(): raise AssertionError( - "Unable to find 'StvmExecutionProvider' in %r." % onnxruntime.get_available_providers()) + "Unable to find 'TvmExecutionProvider' in %r." % onnxruntime.get_available_providers()) X = make_tensor_value_info('X', TensorProto.FLOAT, [None, None]) A = make_tensor_value_info('A', TensorProto.FLOAT, [None, None]) @@ -47,7 +47,7 @@ sess = onnxruntime.InferenceSession( onnx_model.SerializeToString(), so, - providers=["StvmExecutionProvider"], + providers=["TvmExecutionProvider"], provider_options=[provider_options]) y_tvm = sess.run(None, data)[0] diff --git a/onnxruntime/test/util/default_providers.cc b/onnxruntime/test/util/default_providers.cc index 5e0975d667fb9..d22a1627a1d11 100644 --- a/onnxruntime/test/util/default_providers.cc +++ b/onnxruntime/test/util/default_providers.cc @@ -124,9 +124,9 @@ std::unique_ptr DefaultNupharExecutionProvider(bool allow_un #endif } -// std::unique_ptr DefaultStvmExecutionProvider() { -// #ifdef USE_STVM -// return CreateExecutionProviderFactory_Stvm("")->CreateProvider(); +// std::unique_ptr DefaultTvmExecutionProvider() { +// #ifdef USE_TVM +// return CreateExecutionProviderFactory_Tvm("")->CreateProvider(); // #else // return nullptr; // #endif diff --git a/onnxruntime/test/util/include/default_providers.h b/onnxruntime/test/util/include/default_providers.h index 980129e95c7c4..655387cfbf000 100644 --- a/onnxruntime/test/util/include/default_providers.h +++ b/onnxruntime/test/util/include/default_providers.h @@ -17,7 +17,7 @@ std::shared_ptr CreateExecutionProviderFactory_MIGrap std::shared_ptr CreateExecutionProviderFactory_Nnapi( uint32_t flags, const optional& partitioning_stop_ops_list); std::shared_ptr CreateExecutionProviderFactory_Nuphar(bool, const char*); -//std::shared_ptr CreateExecutionProviderFactory_Stvm(const char*); +//std::shared_ptr CreateExecutionProviderFactory_Tvm(const char*); std::shared_ptr CreateExecutionProviderFactory_OpenVINO( const char* device_type, bool enable_vpu_fast_compile, const char* device_id, size_t num_of_threads, bool use_compiled_network, const char* blob_dump_path); std::shared_ptr CreateExecutionProviderFactory_OpenVINO(const OrtOpenVINOProviderOptions* params); @@ -36,7 +36,7 @@ std::unique_ptr DefaultCpuExecutionProvider(bool enable_aren std::unique_ptr DefaultCudaExecutionProvider(); std::unique_ptr DefaultDnnlExecutionProvider(bool enable_arena = true); std::unique_ptr DefaultNupharExecutionProvider(bool allow_unaligned_buffers = true); -//std::unique_ptr DefaultStvmExecutionProvider(); +//std::unique_ptr DefaultTvmExecutionProvider(); std::unique_ptr DefaultTensorrtExecutionProvider(); std::unique_ptr TensorrtExecutionProviderWithOptions(const OrtTensorRTProviderOptions* params); std::unique_ptr TensorrtExecutionProviderWithOptions(const OrtTensorRTProviderOptionsV2* params); diff --git a/onnxruntime/test/util/include/providers.h b/onnxruntime/test/util/include/providers.h index 3ec1b79a9b39b..7d1e3639b1bfa 100644 --- a/onnxruntime/test/util/include/providers.h +++ b/onnxruntime/test/util/include/providers.h @@ -10,8 +10,8 @@ #ifdef USE_NUPHAR #include "core/providers/nuphar/nuphar_provider_factory.h" #endif -#ifdef USE_STVM -#include "core/providers/stvm/stvm_provider_factory.h" +#ifdef USE_TVM +#include "core/providers/tvm/tvm_provider_factory.h" #endif #ifdef USE_TENSORRT #include "core/providers/tensorrt/tensorrt_provider_factory.h" diff --git a/server/CMakeLists.txt b/server/CMakeLists.txt index 1ce6ae8ec4d96..4d09cd4c7e1ef 100755 --- a/server/CMakeLists.txt +++ b/server/CMakeLists.txt @@ -11,7 +11,7 @@ option(onnxruntime_USE_OPENVINO "Build with OpenVINO support" OFF) option(onnxruntime_USE_NNAPI_BUILTIN "Build with builtin NNAPI lib for Android NNAPI support" OFF) option(onnxruntime_USE_DNNL "Build with DNNL support" OFF) option(onnxruntime_USE_NUPHAR "Build with Nuphar" OFF) -option(onnxruntime_USE_STVM "Build with Stvm" OFF) +option(onnxruntime_USE_TVM "Build with Tvm" OFF) option(onnxruntime_USE_TENSORRT "Build with TensorRT support" OFF) option(onnxruntime_USE_DML "Build with DirectML support" OFF) option(onnxruntime_USE_ACL "Build with ACL support" OFF) diff --git a/server/environment.cc b/server/environment.cc index ede884d1102be..9dd0b441bfccc 100644 --- a/server/environment.cc +++ b/server/environment.cc @@ -17,9 +17,9 @@ #endif -#ifdef USE_STVM +#ifdef USE_TVM -#include "core/providers/stvm/stvm_provider_factory.h" +#include "core/providers/tvm/tvm_provider_factory.h" #endif @@ -75,8 +75,8 @@ void ServerEnvironment::RegisterExecutionProviders(){ Ort::ThrowOnError(OrtSessionOptionsAppendExecutionProvider_Nuphar(options_, 1, "")); #endif - #ifdef USE_STVM - Ort::ThrowOnError(OrtSessionOptionsAppendExecutionProvider_Stvm(options_, "")); + #ifdef USE_TVM + Ort::ThrowOnError(OrtSessionOptionsAppendExecutionProvider_Tvm(options_, "")); #endif #ifdef USE_OPENVINO diff --git a/setup.py b/setup.py index 8a7fc5f935825..53b33afbe9ad3 100644 --- a/setup.py +++ b/setup.py @@ -66,8 +66,8 @@ def parse_arg_remove_string(argv, arg_name_equal): package_name = 'onnxruntime-dnnl' elif parse_arg_remove_boolean(sys.argv, '--use_nuphar'): package_name = 'onnxruntime-nuphar' -elif parse_arg_remove_boolean(sys.argv, '--use_stvm'): - package_name = 'onnxruntime-stvm' +elif parse_arg_remove_boolean(sys.argv, '--use_tvm'): + package_name = 'onnxruntime-tvm' elif parse_arg_remove_boolean(sys.argv, '--use_vitisai'): package_name = 'onnxruntime-vitisai' elif parse_arg_remove_boolean(sys.argv, '--use_acl'): @@ -165,7 +165,7 @@ def _rewrite_ld_preload_tvm(self): # the global table of functions. Registration is carried out through the JIT interface, # so it is necessary to call special functions for registration. # To do this, we need to make the following import. - import onnxruntime.providers.stvm + import onnxruntime.providers.tvm except ImportError as e: warnings.warn( f"WARNING: Failed to register python functions to work with TVM EP. More details: {e}" @@ -235,7 +235,7 @@ def run(self): self._rewrite_ld_preload(to_preload) self._rewrite_ld_preload_cuda(to_preload_cuda) self._rewrite_ld_preload_tensorrt(to_preload_tensorrt) - if package_name == 'onnxruntime-stvm': + if package_name == 'onnxruntime-tvm': self._rewrite_ld_preload_tvm() _bdist_wheel.run(self) if is_manylinux and not disable_auditwheel_repair: @@ -430,8 +430,8 @@ def run(self): packages += ["onnxruntime.nuphar"] extra += [path.join('nuphar', 'NUPHAR_CACHE_VERSION')] -if package_name == 'onnxruntime-stvm': - packages += ['onnxruntime.providers.stvm'] +if package_name == 'onnxruntime-tvm': + packages += ['onnxruntime.providers.tvm'] package_data["onnxruntime"] = data + examples + extra diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index 2b294cd73c79d..401c4b8bc4ff1 100644 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -438,7 +438,10 @@ def convert_arg_line_to_args(self, arg_line): parser.add_argument( "--use_nuphar", action='store_true', help="Build with nuphar") parser.add_argument( - "--use_stvm", action='store_true', help="Build with standalone TVM") + "--use_tvm", action='store_true', help="Build with TVM") + parser.add_argument( + "--tvm_cuda_runtime", action='store_true', default=False, + help="Build TVM with CUDA support") parser.add_argument( "--use_tensorrt", action='store_true', help="Build with TensorRT") parser.add_argument( @@ -761,15 +764,16 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home args.android or (args.ios and is_macOS()) or args.use_rknpu) else "OFF"), - "-Donnxruntime_USE_TVM=" + ("ON" if (args.use_nuphar or args.use_stvm) else "OFF"), - "-Donnxruntime_USE_LLVM=" + ("ON" if (args.use_nuphar or args.use_stvm) else "OFF"), + "-Donnxruntime_USE_NUPHAR_TVM=" + ("ON" if args.use_nuphar else "OFF"), + "-Donnxruntime_USE_LLVM=" + ("ON" if args.use_nuphar or args.use_tvm else "OFF"), "-Donnxruntime_ENABLE_MICROSOFT_INTERNAL=" + ("ON" if args.enable_msinternal else "OFF"), "-Donnxruntime_USE_VITISAI=" + ("ON" if args.use_vitisai else "OFF"), "-Donnxruntime_USE_NUPHAR=" + ("ON" if args.use_nuphar else "OFF"), "-Donnxruntime_USE_TENSORRT=" + ("ON" if args.use_tensorrt else "OFF"), "-Donnxruntime_TENSORRT_HOME=" + (tensorrt_home if args.use_tensorrt else ""), - # set vars for standalone TVM - "-Donnxruntime_USE_STVM=" + ("ON" if args.use_stvm else "OFF"), + # set vars for TVM + "-Donnxruntime_USE_TVM=" + ("ON" if args.use_tvm else "OFF"), + "-Donnxruntime_TVM_CUDA_RUNTIME=" + ("ON" if args.use_tvm and args.tvm_cuda_runtime else "OFF"), # set vars for migraphx "-Donnxruntime_USE_MIGRAPHX=" + ("ON" if args.use_migraphx else "OFF"), "-Donnxruntime_MIGRAPHX_HOME=" + (migraphx_home if args.use_migraphx else ""), @@ -929,7 +933,7 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home "-DProtobuf_USE_STATIC_LIBS=ON" ] - if (args.use_nuphar or args.use_stvm) and args.llvm_path is not None: + if (args.use_nuphar or args.use_tvm) and args.llvm_path is not None: cmake_args += ["-DLLVM_DIR=%s" % args.llvm_path] if args.use_cuda and not is_windows(): @@ -1125,7 +1129,7 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home for config in configs: config_build_dir = get_config_build_dir(build_dir, config) os.makedirs(config_build_dir, exist_ok=True) - if args.use_nuphar or args.use_stvm: + if args.use_nuphar or args.use_tvm: os.environ["PATH"] = ( os.path.join(config_build_dir, "_deps", "tvm-build") + os.pathsep + os.path.join(config_build_dir, "_deps", "tvm-src") + os.pathsep + @@ -1134,7 +1138,7 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home run_subprocess( cmake_args + [ "-Donnxruntime_ENABLE_MEMLEAK_CHECKER=" + - ("ON" if config.lower() == 'debug' and not (args.use_nuphar or args.use_stvm) and not + ("ON" if config.lower() == 'debug' and not (args.use_nuphar or args.use_tvm) and not args.use_openvino and not args.enable_msvc_static_runtime and not args.disable_memleak_checker @@ -1703,7 +1707,7 @@ def nuphar_run_python_tests(build_dir, configs): cwd=cwd, dll_path=dll_path) -def stvm_run_python_tests(build_dir, configs): +def tvm_run_python_tests(build_dir, configs): for config in configs: if config == 'Debug': continue @@ -1712,7 +1716,7 @@ def stvm_run_python_tests(build_dir, configs): cwd = os.path.join(cwd, config) dll_path = os.path.join(build_dir, config, "_deps", "tvm-build", config) run_subprocess( - [sys.executable, 'onnxruntime_test_python_stvm.py'], + [sys.executable, 'onnxruntime_test_python_tvm.py'], cwd=cwd, dll_path=dll_path) @@ -1725,7 +1729,7 @@ def run_nodejs_tests(nodejs_binding_dir): def build_python_wheel( source_dir, build_dir, configs, use_cuda, cuda_version, use_rocm, rocm_version, use_dnnl, - use_tensorrt, use_openvino, use_nuphar, use_stvm, use_vitisai, use_acl, use_armnn, use_dml, + use_tensorrt, use_openvino, use_nuphar, use_tvm, use_vitisai, use_acl, use_armnn, use_dml, wheel_name_suffix, enable_training, nightly_build=False, default_training_package_device=False, use_ninja=False, build_eager_mode=False): for config in configs: @@ -1764,8 +1768,8 @@ def build_python_wheel( args.append('--use_dnnl') elif use_nuphar: args.append('--use_nuphar') - elif use_stvm: - args.append('--use_stvm') + elif use_tvm: + args.append('--use_tvm') elif use_vitisai: args.append('--use_vitisai') elif use_acl: @@ -1786,7 +1790,7 @@ def derive_linux_build_property(): def build_nuget_package(source_dir, build_dir, configs, use_cuda, use_openvino, use_tensorrt, use_dnnl, use_nuphar, - use_stvm, use_winml): + use_tvm, use_winml): if not (is_windows() or is_linux()): raise BuildError( 'Currently csharp builds and nuget package creation is only supportted ' @@ -1820,8 +1824,8 @@ def build_nuget_package(source_dir, build_dir, configs, use_cuda, use_openvino, package_name = "/p:OrtPackageId=\"Microsoft.ML.OnnxRuntime.Gpu\"" elif use_nuphar: package_name = "/p:OrtPackageId=\"Microsoft.ML.OnnxRuntime.Nuphar\"" - elif use_stvm: - package_name = "/p:OrtPackageId=\"Microsoft.ML.OnnxRuntime.Stvm\"" + elif use_tvm: + package_name = "/p:OrtPackageId=\"Microsoft.ML.OnnxRuntime.Tvm\"" else: # use the solution file that includes Xamarin mobile targets sln = "OnnxRuntime.CSharp.sln" @@ -2334,8 +2338,8 @@ def gen_ort_ops(): if args.enable_pybind and not args.skip_onnx_tests and args.use_nuphar: nuphar_run_python_tests(build_dir, configs) - if args.enable_pybind and not args.skip_onnx_tests and args.use_stvm: - stvm_run_python_tests(build_dir, configs) + if args.enable_pybind and not args.skip_onnx_tests and args.use_tvm: + tvm_run_python_tests(build_dir, configs) # run node.js binding tests if args.build_nodejs and not args.skip_nodejs_tests: @@ -2362,7 +2366,7 @@ def gen_ort_ops(): args.use_tensorrt, args.use_openvino, args.use_nuphar, - args.use_stvm, + args.use_tvm, args.use_vitisai, args.use_acl, args.use_armnn, @@ -2384,7 +2388,7 @@ def gen_ort_ops(): args.use_tensorrt, args.use_dnnl, args.use_nuphar, - args.use_stvm, + args.use_tvm, args.use_winml, ) diff --git a/tools/ci_build/github/linux/run_build.sh b/tools/ci_build/github/linux/run_build.sh index 599da9fece459..ab54379901525 100755 --- a/tools/ci_build/github/linux/run_build.sh +++ b/tools/ci_build/github/linux/run_build.sh @@ -35,7 +35,7 @@ if [ $BUILD_OS = "yocto" ]; then cd build . /opt/fsl-imx-xwayland/$YOCTO_FOLDER/environment-setup-aarch64-poky-linux alias cmake="/usr/bin/cmake -DCMAKE_TOOLCHAIN_FILE=$OECORE_NATIVE_SYSROOT/usr/share/cmake/OEToolchainConfig.cmake" - cmake ../cmake -Donnxruntime_RUN_ONNX_TESTS=OFF -Donnxruntime_GENERATE_TEST_REPORTS=ON -Donnxruntime_DEV_MODE=ON -DPYTHON_EXECUTABLE=/usr/bin/python3 -Donnxruntime_USE_CUDA=OFF -Donnxruntime_USE_NSYNC=OFF -Donnxruntime_CUDNN_HOME= -Donnxruntime_USE_JEMALLOC=OFF -Donnxruntime_ENABLE_PYTHON=OFF -Donnxruntime_BUILD_CSHARP=OFF -Donnxruntime_USE_EIGEN_FOR_BLAS=ON -Donnxruntime_USE_OPENBLAS=OFF -Donnxruntime_USE_ACL=ON -Donnxruntime_USE_MKLDNN=OFF -Donnxruntime_USE_MKLML=OFF -Donnxruntime_USE_TVM=OFF -Donnxruntime_USE_LLVM=OFF -Donnxruntime_ENABLE_MICROSOFT_INTERNAL=OFF -Donnxruntime_USE_NUPHAR=OFF -Donnxruntime_USE_EIGEN_THREADPOOL=OFF -Donnxruntime_BUILD_UNIT_TESTS=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_IMPLICIT_INCLUDE_DIRECTORIES:PATH=/opt/fsl-imx-xwayland/$YOCTO_FOLDER/sysroots/aarch64-poky-linux/usr/include -DCMAKE_CXX_IMPLICIT_INCLUDE_DIRECTORIES:PATH=/opt/fsl-imx-xwayland/$YOCTO_FOLDER/sysroots/aarch64-poky-linux/usr/include -DONNX_CUSTOM_PROTOC_EXECUTABLE=/usr/bin/protoc + cmake ../cmake -Donnxruntime_RUN_ONNX_TESTS=OFF -Donnxruntime_GENERATE_TEST_REPORTS=ON -Donnxruntime_DEV_MODE=ON -DPYTHON_EXECUTABLE=/usr/bin/python3 -Donnxruntime_USE_CUDA=OFF -Donnxruntime_USE_NSYNC=OFF -Donnxruntime_CUDNN_HOME= -Donnxruntime_USE_JEMALLOC=OFF -Donnxruntime_ENABLE_PYTHON=OFF -Donnxruntime_BUILD_CSHARP=OFF -Donnxruntime_USE_EIGEN_FOR_BLAS=ON -Donnxruntime_USE_OPENBLAS=OFF -Donnxruntime_USE_ACL=ON -Donnxruntime_USE_MKLDNN=OFF -Donnxruntime_USE_MKLML=OFF -Donnxruntime_USE_NUPHAR_TVM=OFF -Donnxruntime_USE_LLVM=OFF -Donnxruntime_ENABLE_MICROSOFT_INTERNAL=OFF -Donnxruntime_USE_NUPHAR=OFF -Donnxruntime_USE_EIGEN_THREADPOOL=OFF -Donnxruntime_BUILD_UNIT_TESTS=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_IMPLICIT_INCLUDE_DIRECTORIES:PATH=/opt/fsl-imx-xwayland/$YOCTO_FOLDER/sysroots/aarch64-poky-linux/usr/include -DCMAKE_CXX_IMPLICIT_INCLUDE_DIRECTORIES:PATH=/opt/fsl-imx-xwayland/$YOCTO_FOLDER/sysroots/aarch64-poky-linux/usr/include -DONNX_CUSTOM_PROTOC_EXECUTABLE=/usr/bin/protoc make -j$(nproc) else