Enable TRT provider option configuration for C# (updated version)

csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.cs

@@ -201,6 +201,11 @@ public struct OrtApi
         public IntPtr ReleasePrepackedWeightsContainer;
         public IntPtr CreateSessionWithPrepackedWeightsContainer;
         public IntPtr CreateSessionFromArrayWithPrepackedWeightsContainer;
+        public IntPtr SessionOptionsAppendExecutionProvider_TensorRT_V2;
+        public IntPtr CreateTensorRTProviderOptions;
+        public IntPtr UpdateTensorRTProviderOptions;
+        public IntPtr GetTensorRTProviderOptionsAsString;
+        public IntPtr ReleaseTensorRTProviderOptions;
     }
 
     internal static class NativeMethods
@@ -271,6 +276,8 @@ static NativeMethods()
             OrtRegisterCustomOpsLibrary = (DOrtRegisterCustomOpsLibrary)Marshal.GetDelegateForFunctionPointer(api_.RegisterCustomOpsLibrary, typeof(DOrtRegisterCustomOpsLibrary));
             OrtAddSessionConfigEntry = (DOrtAddSessionConfigEntry)Marshal.GetDelegateForFunctionPointer(api_.AddSessionConfigEntry, typeof(DOrtAddSessionConfigEntry));
             OrtAddInitializer = (DOrtAddInitializer)Marshal.GetDelegateForFunctionPointer(api_.AddInitializer, typeof(DOrtAddInitializer));
+            SessionOptionsAppendExecutionProvider_TensorRT = (DSessionOptionsAppendExecutionProvider_TensorRT)Marshal.GetDelegateForFunctionPointer(
+                                                             api_.SessionOptionsAppendExecutionProvider_TensorRT, typeof(DSessionOptionsAppendExecutionProvider_TensorRT));
 
             OrtCreateRunOptions = (DOrtCreateRunOptions)Marshal.GetDelegateForFunctionPointer(api_.CreateRunOptions, typeof(DOrtCreateRunOptions));
             OrtReleaseRunOptions = (DOrtReleaseRunOptions)Marshal.GetDelegateForFunctionPointer(api_.ReleaseRunOptions, typeof(DOrtReleaseRunOptions));
@@ -354,6 +361,12 @@ static NativeMethods()
             OrtCreatePrepackedWeightsContainer = (DOrtCreatePrepackedWeightsContainer)Marshal.GetDelegateForFunctionPointer(api_.CreatePrepackedWeightsContainer, typeof(DOrtCreatePrepackedWeightsContainer));
             OrtReleasePrepackedWeightsContainer = (DOrtReleasePrepackedWeightsContainer)Marshal.GetDelegateForFunctionPointer(api_.ReleasePrepackedWeightsContainer, typeof(DOrtReleasePrepackedWeightsContainer));
 
+            SessionOptionsAppendExecutionProvider_TensorRT_V2 = (DSessionOptionsAppendExecutionProvider_TensorRT_V2)Marshal.GetDelegateForFunctionPointer(
+                                                             api_.SessionOptionsAppendExecutionProvider_TensorRT_V2, typeof(DSessionOptionsAppendExecutionProvider_TensorRT_V2));
+            OrtCreateTensorRTProviderOptions = (DOrtCreateTensorRTProviderOptions)Marshal.GetDelegateForFunctionPointer(api_.CreateTensorRTProviderOptions, typeof(DOrtCreateTensorRTProviderOptions));
+            OrtUpdateTensorRTProviderOptions = (DOrtUpdateTensorRTProviderOptions)Marshal.GetDelegateForFunctionPointer(api_.UpdateTensorRTProviderOptions, typeof(DOrtUpdateTensorRTProviderOptions));
+            OrtGetTensorRTProviderOptionsAsString = (DOrtGetTensorRTProviderOptionsAsString)Marshal.GetDelegateForFunctionPointer(api_.GetTensorRTProviderOptionsAsString, typeof(DOrtGetTensorRTProviderOptionsAsString));
+            OrtReleaseTensorRTProviderOptions = (DOrtReleaseTensorRTProviderOptions)Marshal.GetDelegateForFunctionPointer(api_.ReleaseTensorRTProviderOptions, typeof(DOrtReleaseTensorRTProviderOptions));
         }
 
         [DllImport(nativeLib, CharSet = charSet)]
@@ -376,6 +389,50 @@ static NativeMethods()
 
         #endregion Runtime/Environment API
 
+        #region Provider Options API
+
+        /// <summary>
+        /// Creates native OrtTensorRTProviderOptions instance
+        /// </summary>
+        /// <param name="trtProviderOptionsInstance">(output) native instance of OrtTensorRTProviderOptions</param>
+        public delegate IntPtr /* OrtStatus* */DOrtCreateTensorRTProviderOptions(
+            out IntPtr /*(OrtTensorRTProviderOptions**)*/ trtProviderOptionsInstance);
+        public static DOrtCreateTensorRTProviderOptions OrtCreateTensorRTProviderOptions;
+
+        /// <summary>
+        /// Updates native OrtTensorRTProviderOptions instance using given key/value pairs
+        /// </summary>
+        /// <param name="trtProviderOptionsInstance">native instance of OrtTensorRTProviderOptions</param>
+        /// <param name="providerOptionsKeys">configuration keys of OrtTensorRTProviderOptions</param>
+        /// <param name="providerOptionsValues">configuration values of OrtTensorRTProviderOptions</param>
+        /// <param name="numKeys">number of configuration keys</param>
+        public delegate IntPtr /* OrtStatus* */DOrtUpdateTensorRTProviderOptions(
+            IntPtr /*(OrtTensorRTProviderOptions*)*/ trtProviderOptionsInstance,
+            IntPtr[] /*(const char* const *)*/ providerOptionsKeys,
+            IntPtr[] /*(const char* const *)*/ providerOptionsValues,
+            UIntPtr /*(size_t)*/ numKeys);
+        public static DOrtUpdateTensorRTProviderOptions OrtUpdateTensorRTProviderOptions;
+
+        /// <summary>
+        /// Get native OrtTensorRTProviderOptionsV2 in serialized string
+        /// </summary>
+        /// <param name="allocator">instance of OrtAllocator</param>
+        /// <param name="ptr">is a UTF-8 null terminated string allocated using 'allocator'</param>
+        public delegate IntPtr /* OrtStatus* */DOrtGetTensorRTProviderOptionsAsString(
+            IntPtr /*(OrtTensorRTProviderOptionsV2**)*/ trtProviderOptionsInstance,
+            IntPtr /*(OrtAllocator*)*/ allocator,
+            out IntPtr /*(char**)*/ptr);
+        public static DOrtGetTensorRTProviderOptionsAsString OrtGetTensorRTProviderOptionsAsString;
+
+        /// <summary>
+        /// Releases native OrtTensorRTProviderOptions instance
+        /// </summary>
+        /// <param name="trtProviderOptionsInstance">native instance of OrtTensorRTProviderOptions to be released</param>
+        public delegate void DOrtReleaseTensorRTProviderOptions(IntPtr /*(OrtTensorRTProviderOptions*)*/ trtProviderOptionsInstance);
+        public static DOrtReleaseTensorRTProviderOptions OrtReleaseTensorRTProviderOptions;
+
+        #endregion
+
         #region Status API
         public delegate ErrorCode DOrtGetErrorCode(IntPtr /*(OrtStatus*)*/status);
         public static DOrtGetErrorCode OrtGetErrorCode;
@@ -640,6 +697,26 @@ IntPtr[] outputValues /* An array of output value pointers. Array must be alloca
 
         //[DllImport(nativeLib, CharSet = charSet)]
         //public static extern void OrtAddCustomOp(IntPtr /*(OrtSessionOptions*)*/ options, string custom_op_path);
+        //
+        /// <summary>
+        /// Append a TensorRT EP instance (configured based on given provider options) to the native OrtSessionOptions instance
+        /// </summary>
+        /// <param name="options">Native OrtSessionOptions instance</param>
+        /// <param name="trtProviderOptions">Native OrtTensorRTProviderOptions instance</param>
+        public delegate IntPtr /*(OrtStatus*)*/DSessionOptionsAppendExecutionProvider_TensorRT(
+                                               IntPtr /*(OrtSessionOptions*)*/ options,
+                                               IntPtr /*(const OrtTensorRTProviderOptions*)*/ trtProviderOptions);
+        public static DSessionOptionsAppendExecutionProvider_TensorRT SessionOptionsAppendExecutionProvider_TensorRT;
+
+        /// <summary>
+        /// Append a TensorRT EP instance (configured based on given provider options) to the native OrtSessionOptions instance
+        /// </summary>
+        /// <param name="options">Native OrtSessionOptions instance</param>
+        /// <param name="trtProviderOptions">Native OrtTensorRTProviderOptionsV2 instance</param>
+        public delegate IntPtr /*(OrtStatus*)*/DSessionOptionsAppendExecutionProvider_TensorRT_V2(
+                                               IntPtr /*(OrtSessionOptions*)*/ options,
+                                               IntPtr /*(const OrtTensorRTProviderOptionsV2*)*/ trtProviderOptions);
+        public static DSessionOptionsAppendExecutionProvider_TensorRT_V2 SessionOptionsAppendExecutionProvider_TensorRT_V2;
 
         /// <summary>
         /// Free Dimension override (by denotation)

csharp/src/Microsoft.ML.OnnxRuntime/NativeOnnxValueHelper.cs

@@ -5,6 +5,8 @@
 using System;
 using System.Runtime.InteropServices;
 using System.Text;
+using System.Collections.Generic;
+using System.Linq;
 
 namespace Microsoft.ML.OnnxRuntime
 {
@@ -77,6 +79,31 @@ internal static string StringFromNativeUtf8(IntPtr nativeUtf8)
             Marshal.Copy(nativeUtf8, buffer, 0, len);
             return Encoding.UTF8.GetString(buffer, 0, buffer.Length);
         }
+
+        /// <summary>
+        /// Run helper
+        /// </summary>
+        /// <param name="names">names to convert to zero terminated utf8 and pin</param>
+        /// <param name="extractor">delegate for string extraction from inputs</param>
+        /// <param name="cleanupList">list to add pinned memory to for later disposal</param>
+        /// <returns></returns>
+        internal static IntPtr[] ConvertNamesToUtf8<T>(IReadOnlyCollection<T> names, NameExtractor<T> extractor,
+            DisposableList<IDisposable> cleanupList)
+        {
+            var result = new IntPtr[names.Count];
+            for (int i = 0; i < names.Count; ++i)
+            {
+                var name = extractor(names.ElementAt(i));
+                var utf8Name = NativeOnnxValueHelper.StringToZeroTerminatedUtf8(name);
+                var pinnedHandle = new PinnedGCHandle(GCHandle.Alloc(utf8Name, GCHandleType.Pinned));
+                result[i] = pinnedHandle.Pointer;
+                cleanupList.Add(pinnedHandle);
+            }
+            return result;
+        }
+
+        // Delegate for string extraction from an arbitrary input/output object
+        internal delegate string NameExtractor<in TInput>(TInput input);
     }
 
     internal static class TensorElementTypeConverter

csharp/src/Microsoft.ML.OnnxRuntime/ProviderOptions.cs

@@ -0,0 +1,149 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Runtime.InteropServices;
+
+namespace Microsoft.ML.OnnxRuntime
+{
+    /// <summary>
+    /// Holds the options for configuring a TensorRT Execution Provider instance
+    /// </summary>
+    public class OrtTensorRTProviderOptions : SafeHandle
+    {
+        internal IntPtr Handle
+        {
+            get
+            {
+                return handle;
+            }
+        }
+
+        private int _deviceId = 0;
+        private string _deviceIdStr = "device_id";
+
+        #region Constructor
+
+        /// <summary>
+        /// Constructs an empty OrtTensorRTProviderOptions instance
+        /// </summary>
+        public OrtTensorRTProviderOptions() : base(IntPtr.Zero, true)
+        {
+            NativeApiStatus.VerifySuccess(NativeMethods.OrtCreateTensorRTProviderOptions(out handle));
+        }
+
+        #endregion
+
+        #region Public Methods
+
+        /// <summary>
+        /// Get TensorRT EP provider options
+        /// </summary>
+        /// <returns> return C# UTF-16 encoded string </returns>
+        public string GetOptions()
+        {
+            var allocator = OrtAllocator.DefaultInstance;
+
+            // Process provider options string
+            IntPtr providerOptions = IntPtr.Zero;
+            NativeApiStatus.VerifySuccess(NativeMethods.OrtGetTensorRTProviderOptionsAsString(handle, allocator.Pointer, out providerOptions));
+            using (var ortAllocation = new OrtMemoryAllocation(allocator, providerOptions, 0))
+            {
+                return NativeOnnxValueHelper.StringFromNativeUtf8(providerOptions);
+            }
+        }
+
+        /// <summary>
+        /// Updates  the configuration knobs of OrtTensorRTProviderOptions that will eventually be used to configure a TensorRT EP
+        /// Please refer to the following on different key/value pairs to configure a TensorRT EP and their meaning:
+        /// https://www.onnxruntime.ai/docs/reference/execution-providers/TensorRT-ExecutionProvider.html
+        /// </summary>
+        /// <param name="providerOptions">key/value pairs used to configure a TensorRT Execution Provider</param>
+        public void UpdateOptions(Dictionary<string, string> providerOptions)
+        {
+
+            using (var cleanupList = new DisposableList<IDisposable>())
+            {
+                var keysArray = NativeOnnxValueHelper.ConvertNamesToUtf8(providerOptions.Keys.ToArray(), n => n, cleanupList);
+                var valuesArray = NativeOnnxValueHelper.ConvertNamesToUtf8(providerOptions.Values.ToArray(), n => n, cleanupList);
+
+                NativeApiStatus.VerifySuccess(NativeMethods.OrtUpdateTensorRTProviderOptions(handle, keysArray, valuesArray, (UIntPtr)providerOptions.Count));
+
+                if (providerOptions.ContainsKey(_deviceIdStr))
+                {
+                    _deviceId = Int32.Parse(providerOptions[_deviceIdStr]);
+                }
+            }
+        }
+
+        /// <summary>
+        /// Get device id of TensorRT EP.
+        /// </summary>
+        /// <returns> device id </returns>
+        public int GetDeviceId()
+        {
+            return _deviceId;
+        }
+
+        #endregion
+
+        #region Public Properties
+
+        /// <summary>
+        /// Overrides SafeHandle.IsInvalid
+        /// </summary>
+        /// <value>returns true if handle is equal to Zero</value>
+        public override bool IsInvalid { get { return handle == IntPtr.Zero; } }
+
+        #endregion
+
+        #region Private Methods
+
+
+        #endregion
+
+        #region SafeHandle
+        /// <summary>
+        /// Overrides SafeHandle.ReleaseHandle() to properly dispose of
+        /// the native instance of OrtTensorRTProviderOptions
+        /// </summary>
+        /// <returns>always returns true</returns>
+        protected override bool ReleaseHandle()
+        {
+            NativeMethods.OrtReleaseTensorRTProviderOptions(handle);
+            handle = IntPtr.Zero;
+            return true;
+        }
+
+        #endregion
+    }
+
+    /// <summary>
+    /// This helper class contains methods to handle values of provider options
+    /// </summary>
+    public class ProviderOptionsValueHelper
+    {
+        /// <summary>
+        /// Parse from string and save to dictionary
+        /// </summary>
+        /// <param name="s">C# string</param>
+        /// <param name="dict">Dictionary instance to store the parsing result of s</param>
+        public static void StringToDict(string s, Dictionary<string, string> dict)
+        {
+            string[] paris = s.Split(';');
+
+            foreach (var p in paris)
+            {
+                string[] keyValue = p.Split('=');
+                if (keyValue.Length != 2)
+                {
+                    throw new ArgumentException("Make sure input string contains key-value paris, e.g. key1=value1;key2=value2...", "s");
+                }
+                dict.Add(keyValue[0], keyValue[1]);
+            }
+        }
+    }
+
+}

csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.cs

@@ -4,6 +4,7 @@
 using System;
 using System.Runtime.InteropServices;
 using System.Text;
+using System.Collections.Generic;
 
 namespace Microsoft.ML.OnnxRuntime
 {
@@ -100,6 +101,33 @@ public static SessionOptions MakeSessionOptionWithTensorrtProvider(int deviceId
             }
         }
 
+        /// <summary>
+        /// A helper method to construct a SessionOptions object for TensorRT execution provider.
+        /// Use only if CUDA/TensorRT are installed and you have the onnxruntime package specific to this Execution Provider.
+        /// </summary>
+        /// <param name="trtProviderOptions">TensorRT EP provider options</param>
+        /// <returns>A SessionsOptions() object configured for execution on provider options</returns>
+        public static SessionOptions MakeSessionOptionWithTensorrtProvider(OrtTensorRTProviderOptions trtProviderOptions)
+        {
+            CheckTensorrtExecutionProviderDLLs();
+            SessionOptions options = new SessionOptions();
+            try
+            {
+                // Make sure that CUDA EP uses the same device id as TensorRT EP.
+                int deviceId = trtProviderOptions.GetDeviceId() ;
+
+                NativeApiStatus.VerifySuccess(NativeMethods.SessionOptionsAppendExecutionProvider_TensorRT(options.Handle, trtProviderOptions.Handle));
+                NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionOptionsAppendExecutionProvider_CUDA(options.Handle, deviceId));
+                NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionOptionsAppendExecutionProvider_CPU(options.Handle, 1));
+                return options;
+            }
+            catch (Exception e)
+            {
+                options.Dispose();
+                throw e;
+            }
+        }
+
         /// <summary>
         /// A helper method to construct a SessionOptions object for Nuphar execution.
         /// Use only if you have the onnxruntime package specific to this Execution Provider.
@@ -205,6 +233,16 @@ public void AppendExecutionProvider_Tensorrt(int deviceId)
             NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionOptionsAppendExecutionProvider_Tensorrt(handle, deviceId));
         }
 
+        /// <summary>
+        /// Append a TensorRT EP instance (based on specified configuration) to the SessionOptions instance.
+        /// Use only if you have the onnxruntime package specific to this Execution Provider.
+        /// </summary>
+        /// <param name="trtProviderOptions">TensorRT EP provider options</param>
+        public void AppendExecutionProvider_Tensorrt(OrtTensorRTProviderOptions trtProviderOptions)
+        {
+            NativeApiStatus.VerifySuccess(NativeMethods.SessionOptionsAppendExecutionProvider_TensorRT(handle, trtProviderOptions.Handle));
+        }
+
         /// <summary>
         /// Use only if you have the onnxruntime package specific to this Execution Provider.
         /// </summary>