diff --git a/Algorithm.CSharp/RuntimeCurrencyConversionSeedingRegressionAlgorithm.cs b/Algorithm.CSharp/RuntimeCurrencyConversionSeedingRegressionAlgorithm.cs new file mode 100644 index 000000000000..60c17b097f2b --- /dev/null +++ b/Algorithm.CSharp/RuntimeCurrencyConversionSeedingRegressionAlgorithm.cs @@ -0,0 +1,174 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System.Collections.Generic; +using System.Linq; +using QuantConnect.Brokerages; +using QuantConnect.Data; +using QuantConnect.Data.UniverseSelection; +using QuantConnect.Interfaces; + +namespace QuantConnect.Algorithm.CSharp +{ + /// + /// Regression algorithm asserting that a currency added at runtime (here BTCEUR, from a scheduled event) has its + /// conversion rate seeded right away, so using it immediately no longer throws because the rate is still 0. + /// + public class RuntimeCurrencyConversionSeedingRegressionAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition + { + private Symbol _ltcusd; + private bool _addedAtRuntime; + private bool _assertedSeeded; + + /// + /// Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized. + /// + public override void Initialize() + { + SetStartDate(2018, 4, 5); + SetEndDate(2018, 4, 5); + SetBrokerageModel(BrokerageName.GDAX, AccountType.Cash); + SetCash(100000); + + // Account currency asset that funds the loop + _ltcusd = AddCrypto("LTCUSD", Resolution.Minute).Symbol; + + // Add a non-account-currency asset at runtime, mirroring users that add assets from a scheduled event + Schedule.On(DateRules.EveryDay(), TimeRules.At(10, 0), () => + { + if (_addedAtRuntime) + { + return; + } + _addedAtRuntime = true; + AddCrypto("BTCEUR", Resolution.Minute); + }); + } + + /// + /// Runs right after the runtime-added security is wired up, the earliest point it can be used + /// + public override void OnSecuritiesChanged(SecurityChanges changes) + { + if (!changes.AddedSecurities.Any(security => security.Symbol.Value == "BTCEUR")) + { + return; + } + _assertedSeeded = true; + + // With the fix these are already seeded here. Without it they would still be 0 and the conversion below would throw. + var eur = Portfolio.CashBook["EUR"]; + var btc = Portfolio.CashBook["BTC"]; + if (eur.ConversionRate == 0 || btc.ConversionRate == 0) + { + throw new RegressionTestException( + $"Runtime-added currency conversion rates were not seeded (EUR={eur.ConversionRate}, BTC={btc.ConversionRate})"); + } + + if (Portfolio.CashBook.ConvertToAccountCurrency(100m, "EUR") <= 0) + { + throw new RegressionTestException("Expected a positive EUR -> account currency conversion"); + } + } + + /// + /// OnData event is the primary entry point for your algorithm. Each new data point will be pumped in here. + /// + /// Slice object keyed by symbol containing the stock data + public override void OnData(Slice slice) + { + if (!_addedAtRuntime || Portfolio.Invested) + { + return; + } + + if (Securities[_ltcusd].Price != 0) + { + SetHoldings(_ltcusd, 0.5); + } + } + + /// + /// Makes sure the seeding path was actually exercised so the test can't silently pass + /// + public override void OnEndOfAlgorithm() + { + if (!_assertedSeeded) + { + throw new RegressionTestException("BTCEUR was never added at runtime, the seeding path was not exercised"); + } + } + + /// + /// This is used by the regression test system to indicate if the open source Lean repository has the required data to run this algorithm. + /// + public bool CanRunLocally { get; } = true; + + /// + /// This is used by the regression test system to indicate which languages this algorithm is written in. + /// + public List Languages { get; } = new() { Language.CSharp }; + + /// + /// Data Points count of all timeslices of algorithm + /// + public long DataPoints => 6005; + + /// + /// Data Points count of the algorithm history + /// + public int AlgorithmHistoryDataPoints => 591; + + /// + /// Final status of the algorithm + /// + public AlgorithmStatus AlgorithmStatus => AlgorithmStatus.Completed; + + /// + /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm + /// + public Dictionary ExpectedStatistics => new Dictionary + { + {"Total Orders", "1"}, + {"Average Win", "0%"}, + {"Average Loss", "0%"}, + {"Compounding Annual Return", "0%"}, + {"Drawdown", "0%"}, + {"Expectancy", "0"}, + {"Start Equity", "100000.00"}, + {"End Equity", "99064.52"}, + {"Net Profit", "0%"}, + {"Sharpe Ratio", "0"}, + {"Sortino Ratio", "0"}, + {"Probabilistic Sharpe Ratio", "0%"}, + {"Loss Rate", "0%"}, + {"Win Rate", "0%"}, + {"Profit-Loss Ratio", "0"}, + {"Alpha", "0"}, + {"Beta", "0"}, + {"Annual Standard Deviation", "0"}, + {"Annual Variance", "0"}, + {"Information Ratio", "0"}, + {"Tracking Error", "0"}, + {"Treynor Ratio", "0"}, + {"Total Fees", "$149.18"}, + {"Estimated Strategy Capacity", "$160000.00"}, + {"Lowest Capacity Asset", "LTCUSD 2XR"}, + {"Portfolio Turnover", "50.20%"}, + {"Drawdown Recovery", "0"}, + {"OrderListHash", "69d27a394cffbd938ec23fbb451f37ae"} + }; + } +} diff --git a/Algorithm.CSharp/ScheduledUniverseSelectionModelRegressionAlgorithm.cs b/Algorithm.CSharp/ScheduledUniverseSelectionModelRegressionAlgorithm.cs index ff30b377cd86..4794355ea9d4 100644 --- a/Algorithm.CSharp/ScheduledUniverseSelectionModelRegressionAlgorithm.cs +++ b/Algorithm.CSharp/ScheduledUniverseSelectionModelRegressionAlgorithm.cs @@ -199,7 +199,7 @@ private void ExpectRemovals(SecurityChanges changes, params string[] tickers) /// /// Data Points count of the algorithm history /// - public int AlgorithmHistoryDataPoints => 0; + public int AlgorithmHistoryDataPoints => 50; /// /// Final status of the algorithm diff --git a/Engine/DataFeeds/CurrencySubscriptionDataConfigManager.cs b/Engine/DataFeeds/CurrencySubscriptionDataConfigManager.cs index a67eae1bd00b..c72c12e7845d 100644 --- a/Engine/DataFeeds/CurrencySubscriptionDataConfigManager.cs +++ b/Engine/DataFeeds/CurrencySubscriptionDataConfigManager.cs @@ -131,7 +131,9 @@ public IEnumerable GetPendingSubscriptionDataConfigs() /// /// Checks the current and adds new necessary currency pair feeds to provide real time conversion data /// - public void EnsureCurrencySubscriptionDataConfigs(SecurityChanges securityChanges, IBrokerageModel brokerageModel) + /// True if new currency conversion feeds were introduced, false otherwise. Lets callers skip + /// follow up work like seeding the new conversion rates when nothing was added + public bool EnsureCurrencySubscriptionDataConfigs(SecurityChanges securityChanges, IBrokerageModel brokerageModel) { _ensureCurrencyDataFeeds = false; // remove any 'to be added' if the security has already been added @@ -150,6 +152,8 @@ public void EnsureCurrencySubscriptionDataConfigs(SecurityChanges securityChange _toBeAddedCurrencySubscriptionDataConfigs.Add(config); } _pendingSubscriptionDataConfigs = _toBeAddedCurrencySubscriptionDataConfigs.Any(); + + return newConfigs.Count > 0; } } } diff --git a/Engine/DataFeeds/UniverseSelection.cs b/Engine/DataFeeds/UniverseSelection.cs index 6bb15037b785..6afee2ba7eb4 100644 --- a/Engine/DataFeeds/UniverseSelection.cs +++ b/Engine/DataFeeds/UniverseSelection.cs @@ -128,7 +128,7 @@ public SecurityChanges ApplyUniverseSelection(Universe universe, DateTime dateTi // if the input is already fundamental data we just need to filter it and pass it through var hasFundamentalData = universeData.Data.Count > 0 && universeData.Data[0] is Fundamental; - if(hasFundamentalData) + if (hasFundamentalData) { // Remove selected symbols that does not have fine fundamental data var anyDoesNotHaveFundamentalData = false; @@ -137,7 +137,8 @@ public SecurityChanges ApplyUniverseSelection(Universe universe, DateTime dateTi // which do not use coarse data as underlying, in which case it could happen that we try to load fine fundamental data that is missing, but no problem, // 'FineFundamentalSubscriptionEnumeratorFactory' won't emit it var set = selectSymbolsResult.ToHashSet(); - fineCollection.Data.AddRange(universeData.Data.OfType().Where(fundamental => { + fineCollection.Data.AddRange(universeData.Data.OfType().Where(fundamental => + { // we remove to we distict by symbol if (set.Remove(fundamental.Symbol)) { @@ -360,7 +361,7 @@ public bool AddPendingInternalDataFeeds(DateTime utcStart) resolution = supportedResolutions.OrderByDescending(x => x).First(); } - var subscriptionList = new List>() {subscriptionType}; + var subscriptionList = new List>() { subscriptionType }; var dataConfig = _algorithm.SubscriptionManager.SubscriptionDataConfigService.Add( securityBenchmark.Security.Symbol, resolution, @@ -418,9 +419,50 @@ public bool AddPendingInternalDataFeeds(DateTime utcStart) /// /// Checks the current subscriptions and adds necessary currency pair feeds to provide real time conversion data /// - public void EnsureCurrencyDataFeeds(SecurityChanges securityChanges) + /// The security changes to consume + /// Whether to seed the conversion rate of newly added currencies with their last + /// known price. The setup handler passes false because it performs its own (optionally white-listed) seeding + public void EnsureCurrencyDataFeeds(SecurityChanges securityChanges, bool seedNewCurrencies = true) { - _currencySubscriptionDataConfigManager.EnsureCurrencySubscriptionDataConfigs(securityChanges, _algorithm.BrokerageModel); + var newCurrencyFeedsAdded = _currencySubscriptionDataConfigManager.EnsureCurrencySubscriptionDataConfigs(securityChanges, _algorithm.BrokerageModel); + + // Only scan the cashbook and seed when a new conversion feed was actually introduced + if (!seedNewCurrencies || !newCurrencyFeedsAdded) + { + return; + } + + // Seed the new conversion rates with their last known price so they are non-zero right away, instead of + // waiting for the first conversion pair bar to arrive. Otherwise a conversion needed in that gap would + // throw. This is the same thing BaseSetupHandler does during setup, but for cashes added at runtime. + var cashToUpdate = _algorithm.Portfolio.CashBook.Values + .Where(cash => cash.CurrencyConversion != null && cash.ConversionRate == 0) + .ToList(); + + if (cashToUpdate.Count == 0) + { + return; + } + + var securitiesToUpdate = cashToUpdate + .SelectMany(cash => cash.CurrencyConversion.ConversionRateSecurities) + .Distinct() + .ToList(); + + try + { + AlgorithmUtils.SeedSecurities(securitiesToUpdate, _algorithm); + + foreach (var cash in cashToUpdate) + { + cash.Update(); + } + } + catch (Exception err) + { + // Seeding must never break the algorithm, the rate will be set on the first conversion pair bar + Log.Error($"UniverseSelection.EnsureCurrencyDataFeeds(): failed to seed runtime currency conversion rate(s): {err.Message}"); + } } /// diff --git a/Engine/Setup/BaseSetupHandler.cs b/Engine/Setup/BaseSetupHandler.cs index 44998da47fc9..3d9c763a50f6 100644 --- a/Engine/Setup/BaseSetupHandler.cs +++ b/Engine/Setup/BaseSetupHandler.cs @@ -84,8 +84,10 @@ public static void SetupCurrencyConversions( IReadOnlyCollection currenciesToUpdateWhiteList = null) { // this is needed to have non-zero currency conversion rates during warmup - // will also set the Cash.ConversionRateSecurity - universeSelection.EnsureCurrencyDataFeeds(SecurityChanges.None); + // will also set the Cash.ConversionRateSecurity. + // We don't let it seed the conversion rates here because we do that right below, + // where we can also limit the seeding to a specific white list of currencies + universeSelection.EnsureCurrencyDataFeeds(SecurityChanges.None, seedNewCurrencies: false); // now set conversion rates Func cashToUpdateFilter = currenciesToUpdateWhiteList == null diff --git a/Tests/Engine/Setup/BaseSetupHandlerTests.cs b/Tests/Engine/Setup/BaseSetupHandlerTests.cs index 01787c9c983e..c2a032b6b293 100644 --- a/Tests/Engine/Setup/BaseSetupHandlerTests.cs +++ b/Tests/Engine/Setup/BaseSetupHandlerTests.cs @@ -16,6 +16,7 @@ using NUnit.Framework; using QuantConnect.Data; +using QuantConnect.Data.UniverseSelection; using QuantConnect.Util; using QuantConnect.Data.Auxiliary; using QuantConnect.Lean.Engine.Setup; @@ -110,5 +111,48 @@ public void CurrencyConversionRateResolvedForWhiteListedCurrenciesOnly() Assert.AreEqual(0, algorithm.Portfolio.CashBook["EUR"].ConversionRate); Assert.AreEqual(0, algorithm.Portfolio.CashBook["USDT"].ConversionRate); } + + [Test] + public void RuntimeCurrencyConversionRateIsSeeded() + { + // When a currency requiring a conversion feed is introduced at runtime (a universe adding a security + // whose quote currency isn't in the cashbook yet), the runtime path used to wire up the conversion + // subscription without seeding its price, leaving the rate at 0 until the first pair bar. After the fix + // it seeds the new conversion security right away, just like BaseSetupHandler does during setup. + + var historyProvider = new SubscriptionDataReaderHistoryProvider(); + + var algorithm = new BrokerageSetupHandlerTests.TestAlgorithm { UniverseSettings = { Resolution = Resolution.Minute } }; + + historyProvider.Initialize(new HistoryProviderInitializeParameters( + null, + null, + TestGlobals.DataProvider, + TestGlobals.DataCacheProvider, + TestGlobals.MapFileProvider, + TestGlobals.FactorFileProvider, + null, + false, + new DataPermissionManager(), + algorithm.ObjectStore, + algorithm.Settings)); + algorithm.SetHistoryProvider(historyProvider); + + algorithm.SetStartDate(2015, 1, 24); + algorithm.SetCash("USD", 0); + + // Run setup so the engine is in the post-setup (runtime) state + BaseSetupHandler.SetupCurrencyConversions(algorithm, algorithm.DataManager.UniverseSelection); + + // Introduce a new currency at runtime and drive the runtime path that wires up its conversion feed + algorithm.SetCash("BTC", 10); + algorithm.DataManager.UniverseSelection.EnsureCurrencyDataFeeds(SecurityChanges.None); + + // The new currency should already have a non-zero rate, without waiting for a live bar + Assert.IsNotNull(algorithm.Portfolio.CashBook["BTC"].CurrencyConversion); + Assert.AreNotEqual(0, algorithm.Portfolio.CashBook["BTC"].ConversionRate); + Assert.IsTrue(algorithm.Portfolio.CashBook["BTC"].ValueInAccountCurrency > 0); + Assert.DoesNotThrow(() => algorithm.Portfolio.CashBook.ConvertToAccountCurrency(10, "BTC")); + } } }