Enable TRT provider option configuration for C#

csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.cs

@@ -192,6 +192,24 @@ public struct OrtApi
         public IntPtr ModelMetadataGetGraphDescription;
     }
 
+    #region ORT Provider options
+    [StructLayout(LayoutKind.Sequential)]
+    public struct OrtTensorRTProviderOptionsNative
+    {
+        public int device_id;                                  // cuda device id.
+        public int has_user_compute_stream;                    // indicator of user specified CUDA compute stream.
+        public IntPtr user_compute_stream;                     // user specified CUDA compute stream.
+        public int has_trt_options;                            // override environment variables with following TensorRT settings at runtime.
+        public UIntPtr trt_max_workspace_size;                 // maximum workspace size for TensorRT.
+        public int trt_fp16_enable;                            // enable TensorRT FP16 precision. Default 0 = false, nonzero = true
+        public int trt_int8_enable;                            // enable TensorRT INT8 precision. Default 0 = false, nonzero = true
+        public IntPtr trt_int8_calibration_table_name;         // TensorRT INT8 calibration table name.
+        public int trt_int8_use_native_calibration_table;      // use native TensorRT generated calibration table. Default 0 = false, nonzero = true
+    }
+    #endregion
+
+
+
     internal static class NativeMethods
     {
         private const string nativeLib = "onnxruntime";
@@ -574,6 +592,9 @@ IntPtr[] outputValues /* An array of output value pointers. Array must be alloca
         [DllImport(nativeLib, CharSet = charSet)]
         public static extern IntPtr /*(OrtStatus*)*/ OrtSessionOptionsAppendExecutionProvider_Tensorrt(IntPtr /*(OrtSessionOptions*)*/ options, int device_id);
 
+        [DllImport(nativeLib, CharSet = charSet)]
+        public static extern IntPtr /*(OrtStatus*)*/ SessionOptionsAppendExecutionProvider_TensorRT(IntPtr /*(OrtSessionOptions*)*/ options, ref OrtTensorRTProviderOptionsNative trt_options);
+
         [DllImport(nativeLib, CharSet = charSet)]
         public static extern IntPtr /*(OrtStatus*)*/ OrtSessionOptionsAppendExecutionProvider_MIGraphX(IntPtr /*(OrtSessionOptions*)*/ options, int device_id);
 

csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.cs

@@ -38,6 +38,7 @@ public class SessionOptions : SafeHandle
     {
         // Delay-loaded CUDA or cuDNN DLLs. Currently, delayload is disabled. See cmake/CMakeLists.txt for more information.
         private static string[] cudaDelayLoadedLibs = { };
+        private static string[] trtDelayLoadedLibs = { };
 
         #region Constructor and Factory methods
 
@@ -75,6 +76,63 @@ public static SessionOptions MakeSessionOptionWithCudaProvider(int deviceId = 0)
             return options;
         }
 
+        /// <summary>
+        /// A helper method to construct a SessionOptions object for TensorRT execution.
+        /// Use only if CUDA/TensorRT are installed and you have the onnxruntime package specific to this Execution Provider.
+        /// </summary>
+        /// <param name="deviceId"></param>
+        /// <returns>A SessionsOptions() object configured for execution on deviceId</returns>
+        public static SessionOptions MakeSessionOptionWithTensorrtProvider(int deviceId = 0)
+        {
+            CheckTensorrtExecutionProviderDLLs();
+            SessionOptions options = new SessionOptions();
+            NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionOptionsAppendExecutionProvider_Tensorrt(options.Handle, deviceId));
+            NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionOptionsAppendExecutionProvider_CUDA(options.Handle, deviceId));
+            NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionOptionsAppendExecutionProvider_CPU(options.Handle, 1));
+            return options;
+        }
+
+        /// <summary>
+        /// A helper method to construct a SessionOptions object for TensorRT execution.
+        /// Use only if CUDA/TensorRT are installed and you have the onnxruntime package specific to this Execution Provider.
+        /// </summary>
+        /// <param name="trt_options">Provider Options for TensorRT EP.</param>
+        /// <returns>A SessionsOptions() object configured for execution on deviceId</returns>
+        /// 
+        public static SessionOptions MakeSessionOptionWithTensorrtProvider(OrtTensorRTProviderOptions trt_options)
+        {
+            CheckTensorrtExecutionProviderDLLs();
+            SessionOptions options = new SessionOptions();
+
+            OrtTensorRTProviderOptionsNative trt_options_native;
+            trt_options_native.device_id = trt_options.device_id;
+            trt_options_native.has_user_compute_stream = 0;
+            trt_options_native.user_compute_stream = IntPtr.Zero;
+            trt_options_native.has_trt_options = trt_options.has_trt_options;
+            if ((ulong)trt_options.trt_max_workspace_size > (1 << 30))
+            {
+                trt_options_native.trt_max_workspace_size = (UIntPtr)(1 << 30);
+            }
+            else
+            {
+                trt_options_native.trt_max_workspace_size = trt_options.trt_max_workspace_size;
+            }
+            trt_options_native.trt_fp16_enable = trt_options.trt_fp16_enable;
+            trt_options_native.trt_int8_enable = trt_options.trt_int8_enable;
+            var tableNamePinned = GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(trt_options.trt_int8_calibration_table_name), GCHandleType.Pinned);
+            using (var pinnedSettingsName = new PinnedGCHandle(tableNamePinned))
+            {
+                trt_options_native.trt_int8_calibration_table_name = pinnedSettingsName.Pointer;
+            }
+            trt_options_native.trt_int8_use_native_calibration_table = trt_options.trt_int8_use_native_calibration_table;
+
+
+            NativeApiStatus.VerifySuccess(NativeMethods.SessionOptionsAppendExecutionProvider_TensorRT(options.Handle, ref trt_options_native));
+            NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionOptionsAppendExecutionProvider_CUDA(options.Handle, trt_options.device_id));
+            NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionOptionsAppendExecutionProvider_CPU(options.Handle, 1));
+            return options;
+        }
+
         /// <summary>
         /// A helper method to construct a SessionOptions object for Nuphar execution.
         /// Use only if you have the onnxruntime package specific to this Execution Provider.
@@ -592,6 +650,31 @@ public ExecutionMode ExecutionMode
         }
         private ExecutionMode _executionMode = ExecutionMode.ORT_SEQUENTIAL;
 
+
+        /// <summary>
+        /// Provider options for TensorRT.
+        /// </summary>
+        /// 
+        //  Example for setting:
+        //    SessionOptions.OrtTensorRTProviderOptions trt_options;
+        //    trt_options.device_id = 0;
+        //    trt_options.has_trt_options = 1;
+        //    trt_options.trt_max_workspace_size = (UIntPtr) (1<<30);
+        //    trt_options.trt_fp16_enable = 1;
+        //    trt_options.trt_int8_enable = 1;
+        //    trt_options.trt_int8_calibration_table_name = "C:\calibration.flatbuffers";
+        //    trt_options.trt_int8_use_native_calibration_table = 0;
+        public struct OrtTensorRTProviderOptions
+        {
+            public int device_id;                                  // cuda device id. Default is 0.
+            public int has_trt_options;                            // override environment variables with following TensorRT settings at runtime. Default 0 = false, nonzero = true.
+            public UIntPtr trt_max_workspace_size;                 // maximum workspace size for TensorRT. ORT C++ DLL has this field to be the type of size_t, hence using UIntPtr for conversion.
+            public int trt_fp16_enable;                            // enable TensorRT FP16 precision. Default 0 = false, nonzero = true.
+            public int trt_int8_enable;                            // enable TensorRT INT8 precision. Default 0 = false, nonzero = true.
+            public String trt_int8_calibration_table_name;         // TensorRT INT8 calibration table name.
+            public int trt_int8_use_native_calibration_table;      // use native TensorRT generated calibration table. Default 0 = false, nonzero = true
+        }
+
         #endregion
 
         #region Private Methods
@@ -624,6 +707,27 @@ private static bool CheckCudaExecutionProviderDLLs()
             return true;
         }
 
+        private static bool CheckTensorrtExecutionProviderDLLs()
+        {
+            if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
+            {
+                foreach (var dll in trtDelayLoadedLibs)
+                {
+                    IntPtr handle = LoadLibrary(dll);
+                    if (handle != IntPtr.Zero)
+                        continue;
+                    var sysdir = new StringBuilder(String.Empty, 2048);
+                    GetSystemDirectory(sysdir, (uint)sysdir.Capacity);
+                    throw new OnnxRuntimeException(
+                        ErrorCode.NoSuchFile,
+                        $"kernel32.LoadLibrary():'{dll}' not found. TensorRT/CUDA are required for GPU execution. " +
+                        $". Verify it is available in the system directory={sysdir}. Else copy it to the output folder."
+                        );
+                }
+            }
+            return true;
+        }
+
 
         #endregion
         #region SafeHandle

onnxruntime/core/providers/tensorrt/symbols.txt

@@ -1 +1,2 @@
 OrtSessionOptionsAppendExecutionProvider_Tensorrt
+SessionOptionsAppendExecutionProvider_TensorRT