Rename flag to enable qdq optimizer; Fix bug in dst graph inputs orde…

…ring
intel · May 9, 2024 · c062cc2 · c062cc2
1 parent 7860793
commit c062cc2
Show file tree

Hide file tree

Showing 8 changed files with 46 additions and 22 deletions.
diff --git a/onnxruntime/core/providers/openvino/backend_manager.cc b/onnxruntime/core/providers/openvino/backend_manager.cc
@@ -256,7 +256,7 @@ BackendManager::GetModelProtoFromFusedNode(const onnxruntime::Node& fused_node,
                                            const logging::Logger& logger) const {
   // QDQ stripping enabled only for the NPU
   if (global_context_.device_type.find("NPU") != std::string::npos &&
-      global_context_.is_ptq) {
+      global_context_.enable_qdq_optimizer) {
     std::unique_ptr<onnxruntime::Model> model;
     Status status = CreateModelWithStrippedQDQNodes(subgraph, logger, model);
     auto model_proto = model->ToProto();

diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc
@@ -31,7 +31,7 @@ OpenVINOExecutionProvider::OpenVINOExecutionProvider(const OpenVINOExecutionProv
   global_context_->num_of_threads = info.num_of_threads_;
   global_context_->OpenVINO_Version = {OPENVINO_VERSION_MAJOR, OPENVINO_VERSION_MINOR};
   global_context_->export_ep_ctx_blob = info.export_ep_ctx_blob_;
-  global_context_->is_ptq = info.is_ptq_;
+  global_context_->enable_qdq_optimizer = info.enable_qdq_optimizer_;
 
   // to check if target device is available
   // using ie_core capability GetAvailableDevices to fetch list of devices plugged in

diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.h b/onnxruntime/core/providers/openvino/openvino_execution_provider.h
@@ -72,14 +72,14 @@ struct OpenVINOExecutionProviderInfo {
   bool enable_opencl_throttling_{false};
   bool disable_dynamic_shapes_{false};
   bool export_ep_ctx_blob_{false};
-  bool is_ptq_{false};
+  bool enable_qdq_optimizer_{false};
 
   OpenVINOExecutionProviderInfo() = delete;
 
   explicit OpenVINOExecutionProviderInfo(std::string dev_type, std::string precision, bool enable_npu_fast_compile,
                                          size_t num_of_threads, std::string cache_dir, std::string model_priority,
                                          int num_streams, void* context, bool enable_opencl_throttling,
-                                         bool disable_dynamic_shapes, bool export_ep_ctx_blob, bool is_ptq)
+                                         bool disable_dynamic_shapes, bool export_ep_ctx_blob, bool enable_qdq_optimizer)
       : precision_(precision),
         enable_npu_fast_compile_(enable_npu_fast_compile),
         num_of_threads_(num_of_threads),
@@ -90,7 +90,7 @@ struct OpenVINOExecutionProviderInfo {
         enable_opencl_throttling_(enable_opencl_throttling),
         disable_dynamic_shapes_(disable_dynamic_shapes),
         export_ep_ctx_blob_(export_ep_ctx_blob),
-        is_ptq_(is_ptq) {
+        enable_qdq_optimizer_(enable_qdq_optimizer) {
     std::set<std::string> ov_supported_device_types = {"CPU", "GPU",
                                                        "GPU.0", "GPU.1", "NPU"};
     if (dev_type == "") {

diff --git a/onnxruntime/core/providers/openvino/openvino_provider_factory.cc b/onnxruntime/core/providers/openvino/openvino_provider_factory.cc
@@ -13,7 +13,7 @@ struct OpenVINOProviderFactory : IExecutionProviderFactory {
                           const char* cache_dir, const char* model_priority,
                           int num_streams, void* context,
                           bool enable_opencl_throttling, bool disable_dynamic_shapes,
-                          bool export_ep_ctx_blob, bool is_ptq)
+                          bool export_ep_ctx_blob, bool enable_qdq_optimizer)
       : precision_(precision),
         enable_npu_fast_compile_(enable_npu_fast_compile),
         num_of_threads_(num_of_threads),
@@ -23,7 +23,7 @@ struct OpenVINOProviderFactory : IExecutionProviderFactory {
         enable_opencl_throttling_(enable_opencl_throttling),
         disable_dynamic_shapes_(disable_dynamic_shapes),
         export_ep_ctx_blob_(export_ep_ctx_blob),
-        is_ptq_(is_ptq) {
+        enable_qdq_optimizer_(enable_qdq_optimizer) {
     device_type_ = (device_type == nullptr) ? "" : device_type;
     cache_dir_ = (cache_dir == nullptr) ? "" : cache_dir;
   }
@@ -44,13 +44,13 @@ struct OpenVINOProviderFactory : IExecutionProviderFactory {
   bool enable_opencl_throttling_;
   bool disable_dynamic_shapes_;
   bool export_ep_ctx_blob_;
-  bool is_ptq_;
+  bool enable_qdq_optimizer_;
 };
 
 std::unique_ptr<IExecutionProvider> OpenVINOProviderFactory::CreateProvider() {
   OpenVINOExecutionProviderInfo info(device_type_, precision_, enable_npu_fast_compile_, num_of_threads_,
                                      cache_dir_, model_priority_, num_streams_, context_, enable_opencl_throttling_,
-                                     disable_dynamic_shapes_, export_ep_ctx_blob_, is_ptq_);
+                                     disable_dynamic_shapes_, export_ep_ctx_blob_, enable_qdq_optimizer_);
   return std::make_unique<OpenVINOExecutionProvider>(info);
 }
 
@@ -97,7 +97,7 @@ struct OpenVINO_Provider : Provider {
 
     void* context = nullptr;
 
-    bool is_ptq = false;
+    bool enable_qdq_optimizer = false;
 
     if (provider_options_map.find("device_type") != provider_options_map.end()) {
       device_type = provider_options_map.at("device_type").c_str();
@@ -218,12 +218,12 @@ struct OpenVINO_Provider : Provider {
       bool_flag = "";
     }
 
-    if (provider_options_map.find("is_ptq") != provider_options_map.end()) {
-      bool_flag = provider_options_map.at("is_ptq");
+    if (provider_options_map.find("enable_qdq_optimizer") != provider_options_map.end()) {
+      bool_flag = provider_options_map.at("enable_qdq_optimizer");
       if (bool_flag == "true" || bool_flag == "True")
-        is_ptq = true;
+        enable_qdq_optimizer = true;
       else if (bool_flag == "false" || bool_flag == "False")
-        is_ptq = false;
+        enable_qdq_optimizer = false;
       bool_flag = "";
     }
 
@@ -267,7 +267,7 @@ struct OpenVINO_Provider : Provider {
                                                      enable_opencl_throttling,
                                                      disable_dynamic_shapes,
                                                      export_ep_ctx_blob,
-                                                     is_ptq);
+                                                     enable_qdq_optimizer);
   }
 
   void Initialize() override {

diff --git a/onnxruntime/core/providers/openvino/qdq_transformations/qdq_stripping.cc b/onnxruntime/core/providers/openvino/qdq_transformations/qdq_stripping.cc
@@ -548,6 +548,29 @@ Status CreateModelWithStrippedQDQNodes(const GraphViewer& src_graph,
 
   auto& dst_graph = model->MainGraph();
 
+  // Set inputs outputs explicitly to make sure the order is same as the user model.
+  auto inputs = src_graph.GetInputs();
+  auto outputs = src_graph.GetOutputs();
+
+  InlinedVector<const NodeArg*> dst_graph_inputs;
+  dst_graph_inputs.reserve(inputs.size());
+  for (auto& input : inputs) {
+    auto input_arg = src_graph.GetNodeArg(input->Name());
+    auto& ep_graph_input_arg = dst_graph.GetOrCreateNodeArg(input_arg->Name(), input_arg->TypeAsProto());
+    dst_graph_inputs.push_back(&ep_graph_input_arg);
+  }
+
+  InlinedVector<const NodeArg*> dst_graph_outputs;
+  dst_graph_outputs.reserve(outputs.size());
+  for (auto& output : outputs) {
+    auto output_arg = src_graph.GetNodeArg(output->Name());
+    auto& ep_graph_output_arg = dst_graph.GetOrCreateNodeArg(output_arg->Name(), output_arg->TypeAsProto());
+    dst_graph_outputs.push_back(&ep_graph_output_arg);
+  }
+
+  dst_graph.SetInputs(dst_graph_inputs);
+  dst_graph.SetOutputs(dst_graph_outputs);
+
   // TODO: add Graph::SetName() provider api
   // dst_graph.SetName(src_graph.Name());
 
@@ -621,8 +644,8 @@ Status CreateModelWithStrippedQDQNodes(const GraphViewer& src_graph,
   for (auto& it : const_inits) {
     if (initializers_to_keep.count(it))
       dst_graph.AddInitializedTensor(*(initializers.at(it)));
-    current_scope_initializer_set.insert(it);
-  }
+      current_scope_initializer_set.insert(it);
+      }
 
   // handle outer scope value which is a constant initializer
   for (auto& node_idx : src_graph.GetNodesInTopologicalOrder()) {
@@ -634,8 +657,8 @@ Status CreateModelWithStrippedQDQNodes(const GraphViewer& src_graph,
       if (src_graph.IsConstantInitializer(input->Name(), true)) {
         if (initializers_to_keep.count(input->Name()))
           dst_graph.AddInitializedTensor(*(src_graph.GetConstantInitializer(input->Name(), true)));
-        current_scope_initializer_set.insert(input->Name());
-      }
+          current_scope_initializer_set.insert(input->Name());
+              }
     }
   }
 

diff --git a/onnxruntime/core/session/provider_bridge_ort.cc b/onnxruntime/core/session/provider_bridge_ort.cc
@@ -1772,6 +1772,7 @@ ProviderOptions OrtOpenVINOProviderOptionsToOrtOpenVINOProviderOptionsV2(const O
   ov_options_converted_map["num_streams"] = "1";
   ov_options_converted_map["export_ep_ctx_blob"] = "false";
   ov_options_converted_map["model_priority"] = "DEFAULT";
+  ov_options_converted_map["enable_qdq_optimizer"] = "false";
   return ov_options_converted_map;
 }
 

diff --git a/onnxruntime/python/onnxruntime_pybind_state.cc b/onnxruntime/python/onnxruntime_pybind_state.cc
@@ -981,7 +981,7 @@ std::unique_ptr<IExecutionProvider> CreateExecutionProviderInstance(
         } else if (option.first == "export_ep_ctx_blob") {
           OV_provider_options_map[option.first] = option.second;
           continue;
-        } else if (option.first == "is_ptq") {
+        } else if (option.first == "enable_qdq_optimizer") {
           OV_provider_options_map[option.first] = option.second;
           continue;
         } else {

diff --git a/onnxruntime/test/perftest/ort_test_session.cc b/onnxruntime/test/perftest/ort_test_session.cc
@@ -315,12 +315,12 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device
         } else {
           ORT_THROW("[ERROR] [OpenVINO] The value for the key 'enable_opencl_throttling' should be a boolean i.e. true or false. Default value is false.\n");
         }
-      } else if (key == "is_ptq") {
+      } else if (key == "enable_qdq_optimizer") {
         if (value == "true" || value == "True" ||
             value == "false" || value == "False") {
           ov_options[key] = value;
         } else {
-          ORT_THROW("[ERROR] [OpenVINO] The value for the key 'is_ptq' should be a boolean i.e. true or false. Default value is false.\n");
+          ORT_THROW("[ERROR] [OpenVINO] The value for the key 'enable_qdq_optimizer' should be a boolean i.e. true or false. Default value is false.\n");
         }
       } else if (key == "disable_dynamic_shapes") {
         if (value == "true" || value == "True" ||