Generic live command support (QuantConnect#8330)

* Generic command support

- Adding generic algorithm command support. Adding regression algorithms
- Allow PythonWrapper to validate classes too

* Minor improvements

Algorithm.CSharp/CallbackCommandRegressionAlgorithm.cs

@@ -0,0 +1,147 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using QuantConnect.Commands;
+using QuantConnect.Interfaces;
+using System.Collections.Generic;
+
+namespace QuantConnect.Algorithm.CSharp
+{
+    /// <summary>
+    /// Regression algorithm asserting the behavior of different callback commands call
+    /// </summary>
+    public class CallbackCommandRegressionAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition
+    {
+        /// <summary>
+        /// Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized.
+        /// </summary>
+        public override void Initialize()
+        {
+            SetStartDate(2013, 10, 07);
+            SetEndDate(2013, 10, 11);
+
+            AddEquity("SPY");
+            AddEquity("BAC");
+            AddEquity("IBM");
+            AddCommand<BoolCommand>();
+            AddCommand<VoidCommand>();
+        }
+
+        /// <summary>
+        /// Handle generic command callback
+        /// </summary>
+        public override bool? OnCommand(dynamic data)
+        {
+            Buy(data.Symbol, data.parameters["quantity"]);
+            return true;
+        }
+
+        private class VoidCommand : Command
+        {
+            public DateTime TargetTime { get; set; }
+            public string[] Target { get; set; }
+            public decimal Quantity { get; set; }
+            public Dictionary<string, string> Parameters { get; set; }
+            public override bool? Run(IAlgorithm algorithm)
+            {
+                if (TargetTime != algorithm.Time)
+                {
+                    return null;
+                }
+
+                ((QCAlgorithm)algorithm).Order(Target[0], Quantity, tag: Parameters["tag"]);
+                return null;
+            }
+        }
+        private class BoolCommand : Command
+        {
+            public bool? Result { get; set; }
+            public override bool? Run(IAlgorithm algorithm)
+            {
+                var shouldTrade = MyCustomMethod();
+                if (shouldTrade.HasValue && shouldTrade.Value)
+                {
+                    ((QCAlgorithm)algorithm).Buy("IBM", 1);
+                }
+                return shouldTrade;
+            }
+
+            private bool? MyCustomMethod()
+            {
+                return Result;
+            }
+        }
+
+        /// <summary>
+        /// This is used by the regression test system to indicate if the open source Lean repository has the required data to run this algorithm.
+        /// </summary>
+        public bool CanRunLocally { get; }
+
+        /// <summary>
+        /// This is used by the regression test system to indicate which languages this algorithm is written in.
+        /// </summary>
+        public List<Language> Languages { get; } = new() { Language.CSharp, Language.Python };
+
+        /// <summary>
+        /// Data Points count of all timeslices of algorithm
+        /// </summary>
+        public long DataPoints => 3943;
+
+        /// <summary>
+        /// Data Points count of the algorithm history
+        /// </summary>
+        public int AlgorithmHistoryDataPoints => 0;
+
+        /// <summary>
+        /// Final status of the algorithm
+        /// </summary>
+        public AlgorithmStatus AlgorithmStatus => AlgorithmStatus.Completed;
+
+        /// <summary>
+        /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm
+        /// </summary>
+        public Dictionary<string, string> ExpectedStatistics => new Dictionary<string, string>
+        {
+            {"Total Orders", "1"},
+            {"Average Win", "0%"},
+            {"Average Loss", "0%"},
+            {"Compounding Annual Return", "271.453%"},
+            {"Drawdown", "2.200%"},
+            {"Expectancy", "0"},
+            {"Start Equity", "100000"},
+            {"End Equity", "101691.92"},
+            {"Net Profit", "1.692%"},
+            {"Sharpe Ratio", "8.854"},
+            {"Sortino Ratio", "0"},
+            {"Probabilistic Sharpe Ratio", "67.609%"},
+            {"Loss Rate", "0%"},
+            {"Win Rate", "0%"},
+            {"Profit-Loss Ratio", "0"},
+            {"Alpha", "-0.005"},
+            {"Beta", "0.996"},
+            {"Annual Standard Deviation", "0.222"},
+            {"Annual Variance", "0.049"},
+            {"Information Ratio", "-14.565"},
+            {"Tracking Error", "0.001"},
+            {"Treynor Ratio", "1.97"},
+            {"Total Fees", "$3.44"},
+            {"Estimated Strategy Capacity", "$56000000.00"},
+            {"Lowest Capacity Asset", "SPY R735QTJ8XC9X"},
+            {"Portfolio Turnover", "19.93%"},
+            {"OrderListHash", "3da9fa60bf95b9ed148b95e02e0cfc9e"}
+        };
+    }
+}

Algorithm.Python/CallbackCommandRegressionAlgorithm.py

@@ -0,0 +1,73 @@
+# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+# Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from AlgorithmImports import *
+
+class InvalidCommand():
+    variable = 10
+
+class VoidCommand():
+    quantity = 0
+    target = []
+    parameters = {}
+    targettime = None
+
+    def run(self, algo: QCAlgorithm) -> bool | None:
+        if not self.targettime or self.targettime != algo.time:
+            return
+        tag = self.parameters["tag"]
+        algo.order(self.target[0], self.get_quantity(), tag=tag)
+
+    def get_quantity(self):
+        return self.quantity
+
+class BoolCommand(Command):
+    result = False
+
+    def run(self, algo: QCAlgorithm) -> bool | None:
+        trade_ibm = self.my_custom_method()
+        if trade_ibm:
+            algo.buy("IBM", 1)
+        return trade_ibm
+
+    def my_custom_method(self):
+        return self.result
+
+### <summary>
+### Regression algorithm asserting the behavior of different callback commands call
+### </summary>
+class CallbackCommandRegressionAlgorithm(QCAlgorithm):
+    def initialize(self):
+        '''Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized.'''
+
+        self.set_start_date(2013, 10, 7)
+        self.set_end_date(2013, 10, 11)
+
+        self.add_equity("SPY")
+        self.add_equity("IBM")
+        self.add_equity("BAC")
+
+        self.add_command(VoidCommand)
+        self.add_command(BoolCommand)
+
+        threw_exception = False
+        try:
+            self.add_command(InvalidCommand)
+        except:
+            threw_exception = True
+        if not threw_exception:
+            raise ValueError('InvalidCommand did not throw!')
+
+    def on_command(self, data):
+        self.buy(data.symbol, data.parameters["quantity"])
+        return True # False, None

Algorithm/QCAlgorithm.Python.cs

@@ -31,6 +31,7 @@
 using QuantConnect.Util;
 using QuantConnect.Interfaces;
 using QuantConnect.Orders;
+using QuantConnect.Commands;
 
 namespace QuantConnect.Algorithm
 {
@@ -1620,6 +1621,23 @@ public List<OrderTicket> Liquidate(PyObject symbols, bool asynchronous = false,
             return Liquidate(symbols.ConvertToSymbolEnumerable(), asynchronous, tag, orderProperties);
         }
 
+        /// <summary>
+        /// Register a command type to be used
+        /// </summary>
+        /// <param name="type">The command type</param>
+        public void AddCommand(PyObject type)
+        {
+            // create a test instance to validate interface is implemented accurate
+            var testInstance = new CommandPythonWrapper(type);
+
+            var wrappedType = Extensions.CreateType(type);
+            _registeredCommands[wrappedType.Name] = (CallbackCommand command) =>
+            {
+                var commandWrapper = new CommandPythonWrapper(type, command.Payload);
+                return commandWrapper.Run(this);
+            };
+        }
+
         /// <summary>
         /// Gets indicator base type
         /// </summary>

Algorithm/QCAlgorithm.cs

@@ -54,6 +54,8 @@
 using QuantConnect.Algorithm.Framework.Alphas.Analysis;
 using QuantConnect.Algorithm.Framework.Portfolio.SignalExports;
 using Python.Runtime;
+using QuantConnect.Commands;
+using Newtonsoft.Json;
 
 namespace QuantConnect.Algorithm
 {
@@ -115,6 +117,9 @@ public partial class QCAlgorithm : MarshalByRefObject, IAlgorithm
         private IStatisticsService _statisticsService;
         private IBrokerageModel _brokerageModel;
 
+        private readonly HashSet<string> _oneTimeCommandErrors = new();
+        private readonly Dictionary<string, Func<CallbackCommand, bool?>> _registeredCommands = new(StringComparer.InvariantCultureIgnoreCase);
+
         //Error tracking to avoid message flooding:
         private string _previousDebugMessage = "";
         private string _previousErrorMessage = "";
@@ -3385,6 +3390,62 @@ public DataHistory<OptionUniverse> OptionChain(Symbol symbol)
             return new DataHistory<OptionUniverse>(optionChain, new Lazy<PyObject>(() => PandasConverter.GetDataFrame(optionChain)));
         }
 
+        /// <summary>
+        /// Register a command type to be used
+        /// </summary>
+        /// <typeparam name="T">The command type</typeparam>
+        public void AddCommand<T>() where T : Command
+        {
+            _registeredCommands[typeof(T).Name] = (CallbackCommand command) =>
+            {
+                var commandInstance = JsonConvert.DeserializeObject<T>(command.Payload);
+                return commandInstance.Run(this);
+            };
+        }
+
+        /// <summary>
+        /// Run a callback command instance
+        /// </summary>
+        /// <param name="command">The callback command instance</param>
+        /// <returns>The command result</returns>
+        public CommandResultPacket RunCommand(CallbackCommand command)
+        {
+            bool? result = null;
+            if (_registeredCommands.TryGetValue(command.Type, out var target))
+            {
+                try
+                {
+                    result = target.Invoke(command);
+                }
+                catch (Exception ex)
+                {
+                    QuantConnect.Logging.Log.Error(ex);
+                    if (_oneTimeCommandErrors.Add(command.Type))
+                    {
+                        Log($"Unexpected error running command '{command.Type}' error: '{ex.Message}'");
+                    }
+                }
+            }
+            else
+            {
+                if (_oneTimeCommandErrors.Add(command.Type))
+                {
+                    Log($"Detected unregistered command type '{command.Type}', will be ignored");
+                }
+            }
+            return new CommandResultPacket(command, result) { CommandName = command.Type };
+        }
+
+        /// <summary>
+        /// Generic untyped command call handler
+        /// </summary>
+        /// <param name="data">The associated data</param>
+        /// <returns>True if success, false otherwise. Returning null will disable command feedback</returns>
+        public virtual bool? OnCommand(dynamic data)
+        {
+            return true;
+        }
+
         private static Symbol GetCanonicalOptionSymbol(Symbol symbol)
         {
             // We got the underlying

AlgorithmFactory/Python/Wrappers/AlgorithmPythonWrapper.cs

@@ -37,6 +37,7 @@
 using QuantConnect.Statistics;
 using QuantConnect.Data.Market;
 using QuantConnect.Algorithm.Framework.Alphas.Analysis;
+using QuantConnect.Commands;
 
 namespace QuantConnect.AlgorithmFactory.Python.Wrappers
 {
@@ -62,6 +63,7 @@ public class AlgorithmPythonWrapper : BasePythonWrapper<IAlgorithm>, IAlgorithm
         private dynamic _onEndOfDay;
         private dynamic _onMarginCallWarning;
         private dynamic _onOrderEvent;
+        private dynamic _onCommand;
         private dynamic _onAssignmentOrderEvent;
         private dynamic _onSecuritiesChanged;
         private dynamic _onFrameworkSecuritiesChanged;
@@ -153,6 +155,7 @@ public AlgorithmPythonWrapper(string moduleName)
                             _onDelistings = _algorithm.GetMethod("OnDelistings");
                             _onSymbolChangedEvents = _algorithm.GetMethod("OnSymbolChangedEvents");
                             _onEndOfDay = _algorithm.GetMethod("OnEndOfDay");
+                            _onCommand = _algorithm.GetMethod("OnCommand");
                             _onMarginCallWarning = _algorithm.GetMethod("OnMarginCallWarning");
                             _onOrderEvent = _algorithm.GetMethod("OnOrderEvent");
                             _onAssignmentOrderEvent = _algorithm.GetMethod("OnAssignmentOrderEvent");
@@ -921,6 +924,16 @@ public void OnOrderEvent(OrderEvent newEvent)
             _onOrderEvent(newEvent);
         }
 
+        /// <summary>
+        /// Generic untyped command call handler
+        /// </summary>
+        /// <param name="data">The associated data</param>
+        /// <returns>True if success, false otherwise. Returning null will disable command feedback</returns>
+        public bool? OnCommand(dynamic data)
+        {
+            return _onCommand(data);
+        }
+
         /// <summary>
         /// Will submit an order request to the algorithm
         /// </summary>
@@ -1242,5 +1255,13 @@ public void SetTags(HashSet<string> tags)
         {
             _baseAlgorithm.SetTags(tags);
         }
+
+        /// <summary>
+        /// Run a callback command instance
+        /// </summary>
+        /// <param name="command">The callback command instance</param>
+        /// <returns>The command result</returns>
+        public CommandResultPacket RunCommand(CallbackCommand command) => _baseAlgorithm.RunCommand(command);
+
     }
 }