From 82f5dd0f6edf42288329c26796062030ef2e89cc Mon Sep 17 00:00:00 2001 From: Martin-Molinero Date: Mon, 30 Sep 2024 16:01:08 -0300 Subject: [PATCH] Enhance command support (#8348) * Enhance command support - Enhance command support, adding link helper methods. Adding and expanding new tests - Minor improvement for command str representation * Minor test fix --- .../CallbackCommandRegressionAlgorithm.cs | 12 ++ .../CallbackCommandRegressionAlgorithm.py | 13 ++ Algorithm/QCAlgorithm.Python.cs | 18 +++ Algorithm/QCAlgorithm.cs | 25 ++++ Common/Api/Authentication.cs | 126 ++++++++++++++++++ Common/Commands/BaseCommandHandler.cs | 3 + Common/Commands/Command.cs | 56 +++++++- Common/Globals.cs | 5 + Common/Python/CommandPythonWrapper.cs | 36 ++++- Common/Python/PythonWrapper.cs | 4 + Tests/Api/AuthenticationTests.cs | 51 +++++++ Tests/Common/Commands/CallbackCommandTests.cs | 68 +++++++++- 12 files changed, 413 insertions(+), 4 deletions(-) create mode 100644 Common/Api/Authentication.cs create mode 100644 Tests/Api/AuthenticationTests.cs diff --git a/Algorithm.CSharp/CallbackCommandRegressionAlgorithm.cs b/Algorithm.CSharp/CallbackCommandRegressionAlgorithm.cs index db31b1a8dcdd..6523b26bbd55 100644 --- a/Algorithm.CSharp/CallbackCommandRegressionAlgorithm.cs +++ b/Algorithm.CSharp/CallbackCommandRegressionAlgorithm.cs @@ -38,6 +38,18 @@ public override void Initialize() AddEquity("IBM"); AddCommand(); AddCommand(); + + var potentialCommand = new VoidCommand + { + Target = new[] { "BAC" }, + Quantity = 10, + Parameters = new() { { "tag", "Signal X" } } + }; + var commandLink = Link(potentialCommand); + Notify.Email("email@address", "Trade Command Event", $"Signal X trade\nFollow link to trigger: {commandLink}"); + + var commandLink2 = Link(new { Symbol = "SPY", Parameters = new Dictionary() { { "Quantity", 10 } } }); + Notify.Email("email@address", "Untyped Command Event", $"Signal Y trade\nFollow link to trigger: {commandLink2}"); } /// diff --git a/Algorithm.Python/CallbackCommandRegressionAlgorithm.py b/Algorithm.Python/CallbackCommandRegressionAlgorithm.py index 2d495bb38b3f..8ecae11a1853 100644 --- a/Algorithm.Python/CallbackCommandRegressionAlgorithm.py +++ b/Algorithm.Python/CallbackCommandRegressionAlgorithm.py @@ -37,6 +37,7 @@ class BoolCommand(Command): def run(self, algo: QCAlgorithm) -> bool | None: trade_ibm = self.my_custom_method() if trade_ibm: + algo.debug(f"BoolCommand.run: {str(self)}") algo.buy("IBM", 1) return trade_ibm @@ -68,6 +69,18 @@ def initialize(self): if not threw_exception: raise ValueError('InvalidCommand did not throw!') + potential_command = VoidCommand() + potential_command.target = [ "BAC" ] + potential_command.quantity = 10 + potential_command.parameters = { "tag": "Signal X" } + + command_link = self.link(potential_command) + self.notify.email("email@address", "Trade Command Event", f"Signal X trade\nFollow link to trigger: {command_link}") + + untyped_command_link = self.link({ "symbol": "SPY", "parameters": { "quantity": 10 } }) + self.notify.email("email@address", "Untyped Command Event", f"Signal Y trade\nFollow link to trigger: {untyped_command_link}") + def on_command(self, data): + self.debug(f"on_command: {str(data)}") self.buy(data.symbol, data.parameters["quantity"]) return True # False, None diff --git a/Algorithm/QCAlgorithm.Python.cs b/Algorithm/QCAlgorithm.Python.cs index dbca952c68b3..0068de131f04 100644 --- a/Algorithm/QCAlgorithm.Python.cs +++ b/Algorithm/QCAlgorithm.Python.cs @@ -26,6 +26,7 @@ using QuantConnect.Data.UniverseSelection; using QuantConnect.Data.Fundamental; using System.Linq; +using Newtonsoft.Json; using QuantConnect.Brokerages; using QuantConnect.Scheduling; using QuantConnect.Util; @@ -1638,6 +1639,23 @@ public void AddCommand(PyObject type) }; } + /// + /// Get an authenticated link to execute the given command instance + /// + /// The target command + /// The authenticated link + public string Link(PyObject command) + { + using var _ = Py.GIL(); + + var strResult = CommandPythonWrapper.Serialize(command); + using var pyType = command.GetPythonType(); + var wrappedType = Extensions.CreateType(pyType); + + var payload = JsonConvert.DeserializeObject>(strResult); + return CommandLink(wrappedType.Name, payload); + } + /// /// Gets indicator base type /// diff --git a/Algorithm/QCAlgorithm.cs b/Algorithm/QCAlgorithm.cs index c1deaf6f1b24..c85f52f498be 100644 --- a/Algorithm/QCAlgorithm.cs +++ b/Algorithm/QCAlgorithm.cs @@ -3390,6 +3390,21 @@ public DataHistory OptionChain(Symbol symbol) return new DataHistory(optionChain, new Lazy(() => PandasConverter.GetDataFrame(optionChain))); } + /// + /// Get an authenticated link to execute the given command instance + /// + /// The target command + /// The authenticated link + public string Link(object command) + { + var typeName = command.GetType().Name; + if (command is Command || typeName.Contains("AnonymousType", StringComparison.InvariantCultureIgnoreCase)) + { + return CommandLink(typeName, command); + } + return string.Empty; + } + /// /// Register a command type to be used /// @@ -3446,6 +3461,16 @@ public CommandResultPacket RunCommand(CallbackCommand command) return true; } + private string CommandLink(string typeName, object command) + { + var payload = new Dictionary { { "projectId", ProjectId }, { "command", command } }; + if (_registeredCommands.ContainsKey(typeName)) + { + payload["command[$type]"] = typeName; + } + return Api.Authentication.Link("live/commands/create", payload); + } + private static Symbol GetCanonicalOptionSymbol(Symbol symbol) { // We got the underlying diff --git a/Common/Api/Authentication.cs b/Common/Api/Authentication.cs new file mode 100644 index 000000000000..1f3813962bf2 --- /dev/null +++ b/Common/Api/Authentication.cs @@ -0,0 +1,126 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +using System; +using System.Web; +using System.Text; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using System.Collections.Generic; +using System.Collections.Specialized; + +namespace QuantConnect.Api +{ + /// + /// Helper methods for api authentication and interaction + /// + public static class Authentication + { + /// + /// Generate a secure hash for the authorization headers. + /// + /// Time based hash of user token and timestamp. + public static string Hash(int timestamp) + { + return Hash(timestamp, Globals.UserToken); + } + + /// + /// Generate a secure hash for the authorization headers. + /// + /// Time based hash of user token and timestamp. + public static string Hash(int timestamp, string token) + { + // Create a new hash using current UTC timestamp. + // Hash must be generated fresh each time. + var data = $"{token}:{timestamp.ToStringInvariant()}"; + return data.ToSHA256(); + } + + /// + /// Create an authenticated link for the target endpoint using the optional given payload + /// + /// The endpoint + /// The payload + /// The authenticated link to trigger the request + public static string Link(string endpoint, IEnumerable> payload = null) + { + var queryString = HttpUtility.ParseQueryString(string.Empty); + + var timestamp = (int)Time.TimeStamp(); + queryString.Add("authorization", Convert.ToBase64String(Encoding.UTF8.GetBytes($"{Globals.UserId}:{Hash(timestamp)}"))); + queryString.Add("timestamp", timestamp.ToStringInvariant()); + + PopulateQueryString(queryString, payload); + + return $"{Globals.Api}{endpoint.RemoveFromStart("/").RemoveFromEnd("/")}?{queryString}"; + } + + /// + /// Helper method to populate a query string with the given payload + /// + /// Useful for testing purposes + public static void PopulateQueryString(NameValueCollection queryString, IEnumerable> payload = null) + { + if (payload != null) + { + foreach (var kv in payload) + { + AddToQuery(queryString, kv); + } + } + } + + /// + /// Will add the given key value pairs to the query encoded as xform data + /// + private static void AddToQuery(NameValueCollection queryString, KeyValuePair keyValuePairs) + { + var objectType = keyValuePairs.Value.GetType(); + if (objectType.IsValueType || objectType == typeof(string)) + { + // straight + queryString.Add(keyValuePairs.Key, keyValuePairs.Value.ToString()); + } + else + { + // let's take advantage of json to load the properties we should include + var serialized = JsonConvert.SerializeObject(keyValuePairs.Value); + foreach (var jObject in JObject.Parse(serialized)) + { + var subKey = $"{keyValuePairs.Key}[{jObject.Key}]"; + if (jObject.Value is JObject) + { + // inception + AddToQuery(queryString, new KeyValuePair(subKey, jObject.Value.ToObject())); + } + else if(jObject.Value is JArray jArray) + { + var counter = 0; + foreach (var value in jArray.ToObject>()) + { + queryString.Add($"{subKey}[{counter++}]", value.ToString()); + } + } + else + { + queryString.Add(subKey, jObject.Value.ToString()); + } + } + } + } + } +} diff --git a/Common/Commands/BaseCommandHandler.cs b/Common/Commands/BaseCommandHandler.cs index 20daceec3d19..9919c7d2b591 100644 --- a/Common/Commands/BaseCommandHandler.cs +++ b/Common/Commands/BaseCommandHandler.cs @@ -29,6 +29,9 @@ namespace QuantConnect.Commands /// public abstract class BaseCommandHandler : ICommandHandler { + /// + /// Command json settings + /// protected static readonly JsonSerializerSettings Settings = new() { TypeNameHandling = TypeNameHandling.All }; /// diff --git a/Common/Commands/Command.cs b/Common/Commands/Command.cs index d5e862058e08..bbf9ab501b9d 100644 --- a/Common/Commands/Command.cs +++ b/Common/Commands/Command.cs @@ -14,13 +14,15 @@ */ using System; +using System.Linq; using System.Dynamic; +using Newtonsoft.Json; using QuantConnect.Data; using System.Reflection; +using Newtonsoft.Json.Linq; using System.Linq.Expressions; using QuantConnect.Interfaces; using System.Collections.Generic; -using Newtonsoft.Json.Linq; namespace QuantConnect.Commands { @@ -34,12 +36,17 @@ public class Command : DynamicObject private readonly Dictionary _storage = new(StringComparer.InvariantCultureIgnoreCase); + /// + /// Useful to string representation in python + /// + protected string PayloadData { get; set; } + /// /// Get the metaObject required for Dynamism. /// public sealed override DynamicMetaObject GetMetaObject(Expression parameter) { - return new GetSetPropertyDynamicMetaObject(parameter, this, SetPropertyMethodInfo, GetPropertyMethodInfo); + return new SerializableDynamicMetaObject(parameter, this, SetPropertyMethodInfo, GetPropertyMethodInfo); } /// @@ -73,6 +80,20 @@ public object GetProperty(string name) { if (!_storage.TryGetValue(name, out var value)) { + var type = GetType(); + if (type != typeof(Command)) + { + var propertyInfo = type.GetProperty(name, BindingFlags.Public | BindingFlags.Instance); + if (propertyInfo != null) + { + return propertyInfo.GetValue(this, null); + } + var fieldInfo = type.GetField(name, BindingFlags.Public | BindingFlags.Instance); + if (fieldInfo != null) + { + return fieldInfo.GetValue(this); + } + } throw new KeyNotFoundException($"Property with name \'{name}\' does not exist. Properties: {string.Join(", ", _storage.Keys)}"); } return value; @@ -87,5 +108,36 @@ public object GetProperty(string name) { throw new NotImplementedException($"Please implement the 'def run(algorithm) -> bool | None:' method"); } + + /// + /// The string representation of this command + /// + public override string ToString() + { + if (!string.IsNullOrEmpty(PayloadData)) + { + return PayloadData; + } + return JsonConvert.SerializeObject(this); + } + + /// + /// Helper class so we can serialize a command + /// + private class SerializableDynamicMetaObject : GetSetPropertyDynamicMetaObject + { + private readonly Command _object; + public SerializableDynamicMetaObject(Expression expression, object value, MethodInfo setPropertyMethodInfo, MethodInfo getPropertyMethodInfo) + : base(expression, value, setPropertyMethodInfo, getPropertyMethodInfo) + { + _object = (Command)value; + } + public override IEnumerable GetDynamicMemberNames() + { + return _object._storage.Keys.Concat(_object.GetType() + .GetMembers(BindingFlags.Public | BindingFlags.Instance) + .Where(x => x.MemberType == MemberTypes.Field || x.MemberType == MemberTypes.Property).Select(x => x.Name)); + } + } } } diff --git a/Common/Globals.cs b/Common/Globals.cs index c137bf056b75..d36920697d3a 100644 --- a/Common/Globals.cs +++ b/Common/Globals.cs @@ -30,6 +30,11 @@ static Globals() Reset(); } + /// + /// The base api url address to use + /// + public static string Api { get; } = "https://www.quantconnect.com/api/v2/"; + /// /// The user Id /// diff --git a/Common/Python/CommandPythonWrapper.cs b/Common/Python/CommandPythonWrapper.cs index 4554725459ed..32b873df959a 100644 --- a/Common/Python/CommandPythonWrapper.cs +++ b/Common/Python/CommandPythonWrapper.cs @@ -27,6 +27,8 @@ namespace QuantConnect.Python /// public class CommandPythonWrapper : BasePythonWrapper { + private static PyObject _linkSerializationMethod; + /// /// Constructor for initialising the class with wrapped object /// @@ -40,8 +42,13 @@ public CommandPythonWrapper(PyObject type, string data = null) var instance = type.Invoke(); SetPythonInstance(instance); - if (data != null) + if (!string.IsNullOrEmpty(data)) { + if (HasAttr("PayloadData")) + { + SetProperty("PayloadData", data); + } + foreach (var kvp in JsonConvert.DeserializeObject>(data)) { if (kvp.Value is JArray jArray) @@ -70,5 +77,32 @@ public CommandPythonWrapper(PyObject type, string data = null) var result = InvokeMethod(nameof(Run), algorithm); return result.GetAndDispose(); } + + /// + /// Helper method to serialize a command instance + /// + public static string Serialize(PyObject command) + { + if (command == null) + { + return string.Empty; + } + + if (_linkSerializationMethod == null) + { + var module = PyModule.FromString("python_serialization", @"from json import dumps +def serialize(target): + if not hasattr(target, '__dict__'): + # for example dictionaries + return dumps(target) + return dumps(target.__dict__) +"); + _linkSerializationMethod = module.GetAttr("serialize"); + } + using var _ = Py.GIL(); + using var strResult = _linkSerializationMethod.Invoke(command); + + return strResult.As(); + } } } diff --git a/Common/Python/PythonWrapper.cs b/Common/Python/PythonWrapper.cs index fc20aac09af5..127f52ab13b7 100644 --- a/Common/Python/PythonWrapper.cs +++ b/Common/Python/PythonWrapper.cs @@ -55,6 +55,10 @@ public static PyObject ValidateImplementationOf(this PyObject model) { continue; } + else if (member.Name is "ToString") + { + continue; + } } missingMembers.Add(member.Name); } diff --git a/Tests/Api/AuthenticationTests.cs b/Tests/Api/AuthenticationTests.cs new file mode 100644 index 000000000000..495cc1969b46 --- /dev/null +++ b/Tests/Api/AuthenticationTests.cs @@ -0,0 +1,51 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System.Web; +using NUnit.Framework; +using QuantConnect.Api; +using Newtonsoft.Json.Linq; +using System.Collections.Generic; + +namespace QuantConnect.Tests.API +{ + [TestFixture] + public class AuthenticationTests + { + [Test, Explicit("Requires api creds")] + public void Link() + { + var link = Authentication.Link("authenticate"); + + var response = link.DownloadData(); + + Assert.IsNotNull(response); + + var jobject = JObject.Parse(response); + Assert.IsTrue(jobject["success"].ToObject()); + } + + [Test] + public void PopulateQueryString() + { + var payload = new { SomeArray = new[] { 1, 2, 3 }, Symbol = "SPY", Parameters = new Dictionary() { { "Quantity", 10 } } }; + + var queryString = HttpUtility.ParseQueryString(string.Empty); + Authentication.PopulateQueryString(queryString, new[] { new KeyValuePair("command", payload) }); + + Assert.AreEqual("command[SomeArray][0]=1&command[SomeArray][1]=2&command[SomeArray][2]=3&command[Symbol]=SPY&command[Parameters][Quantity]=10", queryString.ToString()); + } + } +} diff --git a/Tests/Common/Commands/CallbackCommandTests.cs b/Tests/Common/Commands/CallbackCommandTests.cs index 319109748574..dd0ae18b311e 100644 --- a/Tests/Common/Commands/CallbackCommandTests.cs +++ b/Tests/Common/Commands/CallbackCommandTests.cs @@ -13,21 +13,75 @@ * limitations under the License. */ +using System; using System.IO; +using System.Web; using NUnit.Framework; using Newtonsoft.Json; +using QuantConnect.Commands; using QuantConnect.Statistics; using QuantConnect.Configuration; using System.Collections.Generic; using QuantConnect.Algorithm.CSharp; using QuantConnect.Lean.Engine.Server; -using System; +using QuantConnect.Tests.Engine.DataFeeds; namespace QuantConnect.Tests.Common.Commands { [TestFixture] public class CallbackCommandTests { + [Test] + public void BaseTypedLink() + { + var algorithmStub = new AlgorithmStub + { + ProjectId = 19542033 + }; + algorithmStub.AddCommand(); + var commandInstance = new MyCommand + { + Quantity = 0.1m, + Target = "BTCUSD" + }; + var link = algorithmStub.Link(commandInstance); + + var parse = HttpUtility.ParseQueryString(link); + Assert.IsFalse(string.IsNullOrEmpty(link)); + } + + [Test] + public void ComplexTypedLink() + { + var algorithmStub = new AlgorithmStub + { + ProjectId = 19542033 + }; + algorithmStub.AddCommand(); + var commandInstance = new MyCommand2 + { + Parameters = new Dictionary { { "quantity", 0.1 } }, + Target = new[] { "BTCUSD", "AAAA" } + }; + var link = algorithmStub.Link(commandInstance); + + var parse = HttpUtility.ParseQueryString(link); + Assert.IsFalse(string.IsNullOrEmpty(link)); + } + + [Test] + public void UntypedLink() + { + var algorithmStub = new AlgorithmStub + { + ProjectId = 19542033 + }; + var link = algorithmStub.Link(new { quantity = -0.1, target = "BTCUSD" }); + + var parse = HttpUtility.ParseQueryString(link); + Assert.IsFalse(string.IsNullOrEmpty(link)); + } + [TestCase(Language.CSharp)] [TestCase(Language.Python)] public void CommanCallback(Language language) @@ -125,5 +179,17 @@ public override void OnAlgorithmStart() SetCommandHandler(); } } + + private class MyCommand2 : Command + { + public string[] Target { get; set; } + public Dictionary Parameters; + } + + private class MyCommand : Command + { + public string Target { get; set; } + public decimal Quantity; + } } }