diff --git a/Demonstration.cs b/Demonstration.cs deleted file mode 100644 index f98b935..0000000 --- a/Demonstration.cs +++ /dev/null @@ -1,94 +0,0 @@ -/* - * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * -*/ - -using QuantConnect.Algorithm; -using QuantConnect.Data.Market; -using QuantConnect.Interfaces; -using QuantConnect; -using QuantConnect.Data; -using QuantConnect.Securities.Future; -using QuantConnect.Util; -using System; -using System.Linq; - -namespace QuantConnect.Algorithm.CSharp -{ - public class DatabentoFuturesTestAlgorithm : QCAlgorithm - { - private Future _es; - - public override void Initialize() - { - Log("Algorithm Initialize"); - - SetStartDate(2025, 10, 1); - SetEndDate(2025, 10, 16); - SetCash(100000); - - var exp = new DateTime(2025, 12, 19); - var symbol = QuantConnect.Symbol.CreateFuture("ES", Market.CME, exp); - //_es = AddFutureContract(symbol, Resolution.Tick, true, 1, true); - _es = AddFutureContract(symbol, Resolution.Second, true, 1, true); - Log($"_es: {_es}"); - - var history = History(_es.Symbol, 10, Resolution.Minute).ToList(); - - Log($"History returned {history.Count} bars"); - - foreach (var bar in history) - { - Log($"History Bar: {bar.Time} - O:{bar.Open} H:{bar.High} L:{bar.Low} C:{bar.Close} V:{bar.Volume}"); - } - - } - - public override void OnData(Slice slice) - { - if (!slice.HasData) - { - Log("Slice has no data"); - return; - } - - Log($"OnData: Slice has {slice.Count} data points"); - - // For Tick resolution, check Ticks collection - if (slice.Ticks.ContainsKey(_es.Symbol)) - { - var ticks = slice.Ticks[_es.Symbol]; - Log($"Received {ticks.Count} ticks for {_es.Symbol}"); - - foreach (var tick in ticks) - { - if (tick.TickType == TickType.Trade) - { - Log($"Trade Tick - Price: {tick.Price}, Quantity: {tick.Quantity}, Time: {tick.Time}"); - } - else if (tick.TickType == TickType.Quote) - { - Log($"Quote Tick - Bid: {tick.BidPrice}x{tick.BidSize}, Ask: {tick.AskPrice}x{tick.AskSize}, Time: {tick.Time}"); - } - } - } - - // Access OHLCV bars - foreach (var bar in slice.Bars.Values) - { - Log($"OHLCV BAR: {bar.Symbol.Value} - O: {bar.Open}, H: {bar.High}, L: {bar.Low}, C: {bar.Close}, V: {bar.Volume}"); - } - } - } -} diff --git a/Lean.DataSource.DataBento.sln b/Lean.DataSource.DataBento.sln index f2f03d6..bfecccc 100644 --- a/Lean.DataSource.DataBento.sln +++ b/Lean.DataSource.DataBento.sln @@ -2,9 +2,9 @@ Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 17 VisualStudioVersion = 17.5.2.0 MinimumVisualStudioVersion = 10.0.40219.1 -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "QuantConnect.DataBento", "QuantConnect.DataBento\QuantConnect.DataSource.DataBento.csproj", "{367AEEDC-F0B3-7F47-539D-10E5EC242C2A}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "QuantConnect.DataSource.DataBento", "QuantConnect.DataBento\QuantConnect.DataSource.DataBento.csproj", "{367AEEDC-F0B3-7F47-539D-10E5EC242C2A}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "QuantConnect.DataBento.Tests", "QuantConnect.DataBento.Tests\QuantConnect.DataSource.DataBento.Tests.csproj", "{9CF47860-2CEA-F379-09D8-9AEF27965D12}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "QuantConnect.DataSource.DataBento.Tests", "QuantConnect.DataBento.Tests\QuantConnect.DataSource.DataBento.Tests.csproj", "{9CF47860-2CEA-F379-09D8-9AEF27965D12}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -16,10 +16,6 @@ Global {367AEEDC-F0B3-7F47-539D-10E5EC242C2A}.Debug|Any CPU.Build.0 = Debug|Any CPU {367AEEDC-F0B3-7F47-539D-10E5EC242C2A}.Release|Any CPU.ActiveCfg = Release|Any CPU {367AEEDC-F0B3-7F47-539D-10E5EC242C2A}.Release|Any CPU.Build.0 = Release|Any CPU - {4B379C8F-16CE-1972-73E3-C14F6410D428}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {4B379C8F-16CE-1972-73E3-C14F6410D428}.Debug|Any CPU.Build.0 = Debug|Any CPU - {4B379C8F-16CE-1972-73E3-C14F6410D428}.Release|Any CPU.ActiveCfg = Release|Any CPU - {4B379C8F-16CE-1972-73E3-C14F6410D428}.Release|Any CPU.Build.0 = Release|Any CPU {9CF47860-2CEA-F379-09D8-9AEF27965D12}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {9CF47860-2CEA-F379-09D8-9AEF27965D12}.Debug|Any CPU.Build.0 = Debug|Any CPU {9CF47860-2CEA-F379-09D8-9AEF27965D12}.Release|Any CPU.ActiveCfg = Release|Any CPU diff --git a/QuantConnect.DataBento.Tests/DataBentoDataDownloaderTests.cs b/QuantConnect.DataBento.Tests/DataBentoDataDownloaderTests.cs index d9d8071..940813c 100644 --- a/QuantConnect.DataBento.Tests/DataBentoDataDownloaderTests.cs +++ b/QuantConnect.DataBento.Tests/DataBentoDataDownloaderTests.cs @@ -1,6 +1,6 @@ /* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,167 +17,169 @@ using System; using System.Linq; using NUnit.Framework; -using QuantConnect.Data; -using QuantConnect.Data.Market; -using QuantConnect.Lean.DataSource.DataBento; +using QuantConnect.Util; using QuantConnect.Logging; -using QuantConnect.Configuration; +using QuantConnect.Securities; +using QuantConnect.Data.Market; + +namespace QuantConnect.Lean.DataSource.DataBento.Tests; -namespace QuantConnect.Lean.DataSource.DataBento.Tests +[TestFixture] +public class DataBentoDataDownloaderTests { - [TestFixture] - public class DataBentoDataDownloaderTests + private DataBentoDataDownloader _downloader; + + private readonly MarketHoursDatabase _marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); + + [SetUp] + public void SetUp() { - private DataBentoDataDownloader _downloader; - private readonly string _apiKey = Config.Get("databento-api-key"); + _downloader = new DataBentoDataDownloader(); + } - [SetUp] - public void SetUp() - { - _downloader = new DataBentoDataDownloader(_apiKey); - } + [TearDown] + public void TearDown() + { + _downloader?.DisposeSafely(); + } + + [TestCase(Resolution.Daily)] + [TestCase(Resolution.Hour)] + [TestCase(Resolution.Minute)] + [TestCase(Resolution.Second)] + [TestCase(Resolution.Tick)] + public void DownloadsTradeDataForLeanFuture(Resolution resolution) + { + var symbol = Symbol.CreateFuture("ES", Market.CME, new DateTime(2026, 3, 20)); + var exchangeTimeZone = _marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType).TimeZone; - [TearDown] - public void TearDown() + var startUtc = new DateTime(2026, 1, 18, 0, 0, 0, DateTimeKind.Utc); + var endUtc = new DateTime(2026, 1, 20, 0, 0, 0, DateTimeKind.Utc); + + if (resolution == Resolution.Tick) { - _downloader?.Dispose(); + startUtc = new DateTime(2026, 1, 21, 9, 30, 0, DateTimeKind.Utc); + endUtc = startUtc.AddMinutes(15); } - [Test] - [TestCase("ESM3", SecurityType.Future, Market.CME, Resolution.Daily, TickType.Trade)] - [TestCase("ESM3", SecurityType.Future, Market.CME, Resolution.Hour, TickType.Trade)] - [TestCase("ESM3", SecurityType.Future, Market.CME, Resolution.Minute, TickType.Trade)] - [TestCase("ESM3", SecurityType.Future, Market.CME, Resolution.Second, TickType.Trade)] - [TestCase("ESM3", SecurityType.Future, Market.CME, Resolution.Tick, TickType.Trade)] - [Explicit("This test requires a configured DataBento API key")] - public void DownloadsHistoricalData(string ticker, SecurityType securityType, string market, Resolution resolution, TickType tickType) - { - var symbol = Symbol.Create(ticker, securityType, market); - var startTime = new DateTime(2024, 1, 15); - var endTime = new DateTime(2024, 1, 16); - var param = new DataDownloaderGetParameters(symbol, resolution, startTime, endTime, tickType); + var parameters = new DataDownloaderGetParameters( + symbol, + resolution, + startUtc, + endUtc, + TickType.Trade + ); - var downloadResponse = _downloader.Get(param).ToList(); + var data = _downloader.Get(parameters).ToList(); - Log.Trace($"Downloaded {downloadResponse.Count} data points for {symbol} at {resolution} resolution"); + Log.Trace($"Downloaded {data.Count} trade points for {symbol} @ {resolution}"); - Assert.IsTrue(downloadResponse.Any(), "Expected to download at least one data point"); + Assert.IsNotEmpty(data); - foreach (var data in downloadResponse) + var startExchange = startUtc.ConvertFromUtc(exchangeTimeZone); + var endExchange = endUtc.ConvertFromUtc(exchangeTimeZone); + + foreach (var point in data) + { + Assert.AreEqual(symbol, point.Symbol); + Assert.That(point.Time, Is.InRange(startExchange, endExchange)); + + switch (point) { - Assert.IsNotNull(data, "Data point should not be null"); - Assert.AreEqual(symbol, data.Symbol, "Symbol should match requested symbol"); - Assert.IsTrue(data.Time >= startTime && data.Time <= endTime, "Data time should be within requested range"); - - if (data is TradeBar tradeBar) - { - Assert.Greater(tradeBar.Close, 0, "Close price should be positive"); - Assert.GreaterOrEqual(tradeBar.Volume, 0, "Volume should be non-negative"); - Assert.Greater(tradeBar.High, 0, "High price should be positive"); - Assert.Greater(tradeBar.Low, 0, "Low price should be positive"); - Assert.Greater(tradeBar.Open, 0, "Open price should be positive"); - Assert.GreaterOrEqual(tradeBar.High, tradeBar.Low, "High should be >= Low"); - Assert.GreaterOrEqual(tradeBar.High, tradeBar.Open, "High should be >= Open"); - Assert.GreaterOrEqual(tradeBar.High, tradeBar.Close, "High should be >= Close"); - Assert.LessOrEqual(tradeBar.Low, tradeBar.Open, "Low should be <= Open"); - Assert.LessOrEqual(tradeBar.Low, tradeBar.Close, "Low should be <= Close"); - } - else if (data is QuoteBar quoteBar) - { - Assert.Greater(quoteBar.Close, 0, "Quote close price should be positive"); - if (quoteBar.Bid != null) - { - Assert.Greater(quoteBar.Bid.Close, 0, "Bid price should be positive"); - } - if (quoteBar.Ask != null) - { - Assert.Greater(quoteBar.Ask.Close, 0, "Ask price should be positive"); - } - } - else if (data is Tick tick) - { - Assert.Greater(tick.Value, 0, "Tick value should be positive"); - Assert.GreaterOrEqual(tick.Quantity, 0, "Tick quantity should be non-negative"); - } + case TradeBar bar: + Assert.Greater(bar.Open, 0); + Assert.Greater(bar.High, 0); + Assert.Greater(bar.Low, 0); + Assert.Greater(bar.Close, 0); + Assert.GreaterOrEqual(bar.Volume, 0); + Assert.GreaterOrEqual(bar.High, bar.Low); + break; + + case Tick tick: + Assert.Greater(tick.Value, 0); + Assert.GreaterOrEqual(tick.Quantity, 0); + break; + + default: + Assert.Fail($"Unexpected data type {point.GetType()}"); + break; } } + } - [Test] - [TestCase("ZNM3", SecurityType.Future, Market.CME, Resolution.Daily, TickType.Trade)] - [TestCase("ZNM3", SecurityType.Future, Market.CME, Resolution.Hour, TickType.Trade)] - [Explicit("This test requires a configured DataBento API key")] - public void DownloadsFuturesHistoricalData(string ticker, SecurityType securityType, string market, Resolution resolution, TickType tickType) - { - var symbol = Symbol.Create(ticker, securityType, market); - var startTime = new DateTime(2024, 1, 15); - var endTime = new DateTime(2024, 1, 16); - var param = new DataDownloaderGetParameters(symbol, resolution, startTime, endTime, tickType); - - var downloadResponse = _downloader.Get(param).ToList(); + [Test] + public void DownloadsQuoteTicksForLeanFuture() + { + var symbol = Symbol.CreateFuture("ES", Market.CME, new DateTime(2026, 3, 20)); + var exchangeTimeZone = _marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType).TimeZone; - Log.Trace($"Downloaded {downloadResponse.Count} data points for futures {symbol}"); + var startUtc = new DateTime(2026, 1, 20, 9, 30, 0, DateTimeKind.Utc); + var endUtc = startUtc.AddMinutes(15); - Assert.IsTrue(downloadResponse.Any(), "Expected to download futures data"); + var parameters = new DataDownloaderGetParameters( + symbol, + Resolution.Tick, + startUtc, + endUtc, + TickType.Quote + ); - foreach (var data in downloadResponse) - { - Assert.AreEqual(symbol, data.Symbol, "Symbol should match requested futures symbol"); - Assert.Greater(data.Value, 0, "Data value should be positive"); - } - } + var data = _downloader.Get(parameters).ToList(); - [Test] - [TestCase("ESM3", SecurityType.Future, Market.CME, Resolution.Tick, TickType.Quote)] - [Explicit("This test requires a configured DataBento API key and advanced subscription")] - public void DownloadsQuoteData(string ticker, SecurityType securityType, string market, Resolution resolution, TickType tickType) - { - var symbol = Symbol.Create(ticker, securityType, market); - var startTime = new DateTime(2024, 1, 15, 9, 30, 0); - var endTime = new DateTime(2024, 1, 15, 9, 45, 0); - var param = new DataDownloaderGetParameters(symbol, resolution, startTime, endTime, tickType); + Log.Trace($"Downloaded {data.Count} quote ticks for {symbol}"); - var downloadResponse = _downloader.Get(param).ToList(); + Assert.IsNotEmpty(data); - Log.Trace($"Downloaded {downloadResponse.Count} quote data points for {symbol}"); + var startExchange = startUtc.ConvertFromUtc(exchangeTimeZone); + var endExchange = endUtc.ConvertFromUtc(exchangeTimeZone); - Assert.IsTrue(downloadResponse.Any(), "Expected to download quote data"); + foreach (var point in data) + { + Assert.AreEqual(symbol, point.Symbol); + Assert.That(point.Time, Is.InRange(startExchange, endExchange)); - foreach (var data in downloadResponse) + if (point is Tick tick) { - Assert.AreEqual(symbol, data.Symbol, "Symbol should match requested symbol"); - if (data is QuoteBar quoteBar) - { - Assert.IsTrue(quoteBar.Bid != null || quoteBar.Ask != null, "Quote should have bid or ask data"); - } + Assert.AreEqual(TickType.Quote, tick.TickType); + Assert.IsTrue( + tick.BidPrice > 0 || tick.AskPrice > 0, + "Quote tick must have bid or ask" + ); + } + else if (point is QuoteBar bar) + { + Assert.IsTrue(bar.Bid != null || bar.Ask != null); } } + } - [Test] - [Explicit("This test requires a configured DataBento API key")] - public void DataIsSortedByTime() - { - var symbol = Symbol.Create("ESM3", SecurityType.Future, Market.CME); - var startTime = new DateTime(2024, 1, 15); - var endTime = new DateTime(2024, 1, 16); - var param = new DataDownloaderGetParameters(symbol, Resolution.Minute, startTime, endTime, TickType.Trade); + [Test] + public void DataIsSortedByTime() + { + var symbol = Symbol.CreateFuture("ES", Market.CME, new DateTime(2026, 3, 20)); - var downloadResponse = _downloader.Get(param).ToList(); + var startUtc = new DateTime(2026, 1, 20, 0, 0, 0, DateTimeKind.Utc); + var endUtc = new DateTime(2024, 1, 21, 0, 0, 0, DateTimeKind.Utc); - Assert.IsTrue(downloadResponse.Any(), "Expected to download data for time sorting test"); + var parameters = new DataDownloaderGetParameters( + symbol, + Resolution.Minute, + startUtc, + endUtc, + TickType.Trade + ); - for (int i = 1; i < downloadResponse.Count; i++) - { - Assert.GreaterOrEqual(downloadResponse[i].Time, downloadResponse[i - 1].Time, - $"Data should be sorted by time. Item {i} time {downloadResponse[i].Time} should be >= item {i - 1} time {downloadResponse[i - 1].Time}"); - } - } + var data = _downloader.Get(parameters).ToList(); + + Assert.IsNotEmpty(data); - [Test] - public void DisposesCorrectly() + for (int i = 1; i < data.Count; i++) { - var downloader = new DataBentoDataDownloader(); - Assert.DoesNotThrow(() => downloader.Dispose(), "Dispose should not throw"); - Assert.DoesNotThrow(() => downloader.Dispose(), "Multiple dispose calls should not throw"); + Assert.GreaterOrEqual( + data[i].Time, + data[i - 1].Time, + $"Data not sorted at index {i}" + ); } } } diff --git a/QuantConnect.DataBento.Tests/DataBentoDataProviderHistoryTests.cs b/QuantConnect.DataBento.Tests/DataBentoDataProviderHistoryTests.cs index 2da5b8f..be19cbd 100644 --- a/QuantConnect.DataBento.Tests/DataBentoDataProviderHistoryTests.cs +++ b/QuantConnect.DataBento.Tests/DataBentoDataProviderHistoryTests.cs @@ -1,6 +1,6 @@ /* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,196 +15,103 @@ */ using System; -using System.Linq; using NUnit.Framework; using QuantConnect.Data; using QuantConnect.Util; -using QuantConnect.Lean.DataSource.DataBento; using QuantConnect.Securities; -using System.Collections.Generic; -using QuantConnect.Logging; using QuantConnect.Data.Market; -using QuantConnect.Configuration; +using System.Collections.Generic; + +namespace QuantConnect.Lean.DataSource.DataBento.Tests; -namespace QuantConnect.Lean.DataSource.DataBento.Tests +[TestFixture] +public class DataBentoDataProviderHistoryTests { - [TestFixture] - public class DataBentoDataProviderHistoryTests - { - private DataBentoProvider _historyDataProvider; - private readonly string _apiKey = Config.Get("databento-api-key"); + private DataBentoProvider _historyDataProvider; - [SetUp] - public void SetUp() - { - _historyDataProvider = new DataBentoProvider(_apiKey); - } + [SetUp] + public void SetUp() + { + _historyDataProvider = new DataBentoProvider(); + } - [TearDown] - public void TearDown() - { - _historyDataProvider?.Dispose(); - } + [TearDown] + public void TearDown() + { + _historyDataProvider?.DisposeSafely(); + } - internal static IEnumerable TestParameters + internal static IEnumerable TestParameters + { + get { - get - { - - // DataBento futures - var esMini = Symbol.Create("ESM3", SecurityType.Future, Market.CME); - var znNote = Symbol.Create("ZNM3", SecurityType.Future, Market.CME); - var gcGold = Symbol.Create("GCM3", SecurityType.Future, Market.CME); - - // test cases for supported futures - yield return new TestCaseData(esMini, Resolution.Daily, TickType.Trade, TimeSpan.FromDays(5), false) - .SetDescription("Valid ES futures - Daily resolution, 5 days period") - .SetCategory("Valid"); + var es = Symbol.CreateFuture("ES", Market.CME, new DateTime(2026, 3, 20)); - yield return new TestCaseData(esMini, Resolution.Hour, TickType.Trade, TimeSpan.FromDays(2), false) - .SetDescription("Valid ES futures - Hour resolution, 2 days period") - .SetCategory("Valid"); - - yield return new TestCaseData(esMini, Resolution.Minute, TickType.Trade, TimeSpan.FromHours(4), false) - .SetDescription("Valid ES futures - Minute resolution, 4 hours period") - .SetCategory("Valid"); - - yield return new TestCaseData(znNote, Resolution.Daily, TickType.Trade, TimeSpan.FromDays(3), false) - .SetDescription("Valid ZN futures - Daily resolution, 3 days period") - .SetCategory("Valid"); - - yield return new TestCaseData(gcGold, Resolution.Hour, TickType.Trade, TimeSpan.FromDays(1), false) - .SetDescription("Valid GC futures - Hour resolution, 1 day period") - .SetCategory("Valid"); - - // Test cases for quote data (may require advanced subscription) - yield return new TestCaseData(esMini, Resolution.Tick, TickType.Quote, TimeSpan.FromMinutes(15), false) - .SetDescription("ES futures quote data - Tick resolution") - .SetCategory("Quote"); + yield return new TestCaseData(es, Resolution.Daily, TickType.Trade, TimeSpan.FromDays(5), false); + yield return new TestCaseData(es, Resolution.Hour, TickType.Trade, TimeSpan.FromDays(2), false); + yield return new TestCaseData(es, Resolution.Minute, TickType.Trade, TimeSpan.FromHours(4), false); + yield return new TestCaseData(es, Resolution.Second, TickType.Trade, TimeSpan.FromHours(4), false); + yield return new TestCaseData(es, Resolution.Tick, TickType.Quote, TimeSpan.FromMinutes(15), true); + } + } - // Unsupported security types - var equity = Symbol.Create("SPY", SecurityType.Equity, Market.USA); - var option = Symbol.Create("SPY", SecurityType.Option, Market.USA); + [Test, TestCaseSource(nameof(TestParameters))] + public void GetsHistory(Symbol symbol, Resolution resolution, TickType tickType, TimeSpan period, bool expectsNoData) + { + var request = GetHistoryRequest(resolution, tickType, symbol, period); - yield return new TestCaseData(equity, Resolution.Daily, TickType.Trade, TimeSpan.FromDays(5), true) - .SetDescription("Invalid - Equity not supported by DataBento") - .SetCategory("Invalid"); + var history = _historyDataProvider.GetHistory(request); - yield return new TestCaseData(option, Resolution.Daily, TickType.Trade, TimeSpan.FromDays(5), true) - .SetDescription("Invalid - Option not supported by DataBento") - .SetCategory("Invalid"); - } - } + Assert.IsNotNull(history); - [Test, TestCaseSource(nameof(TestParameters))] - [Explicit("This test requires a configured DataBento API key")] - public void GetsHistory(Symbol symbol, Resolution resolution, TickType tickType, TimeSpan period, bool expectsNoData) + foreach (var point in history) { - var request = GetHistoryRequest(resolution, tickType, symbol, period); + Assert.AreEqual(symbol, point.Symbol); - try + if (point is TradeBar bar) { - var slices = _historyDataProvider.GetHistory(request)?.Select(data => new Slice(data.Time, new[] { data }, data.Time.ConvertToUtc(request.DataTimeZone))).ToList(); - - if (expectsNoData) - { - Assert.IsTrue(slices == null || !slices.Any(), - $"Expected no data for unsupported symbol/security type: {symbol}"); - } - else - { - Assert.IsNotNull(slices, "Expected to receive history data"); - - if (slices.Any()) - { - Log.Trace($"Received {slices.Count} slices for {symbol} at {resolution} resolution"); - - foreach (var slice in slices.Take(5)) // Check first 5 slices - { - Assert.IsNotNull(slice, "Slice should not be null"); - Assert.IsTrue(slice.Time >= request.StartTimeUtc && slice.Time <= request.EndTimeUtc, - "Slice time should be within requested range"); - - if (slice.Bars.ContainsKey(symbol)) - { - var bar = slice.Bars[symbol]; - Assert.Greater(bar.Close, 0, "Bar close price should be positive"); - Assert.GreaterOrEqual(bar.Volume, 0, "Bar volume should be non-negative"); - } - } - } - } + Assert.Greater(bar.Close, 0); + Assert.GreaterOrEqual(bar.Volume, 0); } - catch (Exception ex) - { - Log.Error($"Error getting history for {symbol}: {ex.Message}"); - - if (!expectsNoData) - { - throw; - } - } - } - - [Test] - [Explicit("This test requires a configured DataBento API key")] - public void GetHistoryWithMultipleSymbols() - { - var symbol1 = Symbol.Create("ESM3", SecurityType.Future, Market.CME); - var symbol2 = Symbol.Create("ZNM3", SecurityType.Future, Market.CME); - - var request1 = GetHistoryRequest(Resolution.Daily, TickType.Trade, symbol1, TimeSpan.FromDays(3)); - var request2 = GetHistoryRequest(Resolution.Daily, TickType.Trade, symbol2, TimeSpan.FromDays(3)); - var history1 = _historyDataProvider.GetHistory(request1); - var history2 = _historyDataProvider.GetHistory(request2); - - var allData = new List(); - if (history1 != null) allData.AddRange(history1); - if (history2 != null) allData.AddRange(history2); - - // timezone from the first request - var slices = allData.GroupBy(d => d.Time) - .Select(g => new Slice(g.Key, g.ToList(), g.Key.ConvertToUtc(request1.DataTimeZone))) - .ToList(); - - Assert.IsNotNull(slices, "Expected to receive history data for multiple symbols"); - - if (slices.Any()) + if (point is Tick tick && tickType == TickType.Quote) { - Log.Trace($"Received {slices.Count} slices for multiple symbols"); - - var hasSymbol1Data = slices.Any(s => s.Bars.ContainsKey(symbol1)); - var hasSymbol2Data = slices.Any(s => s.Bars.ContainsKey(symbol2)); - - Assert.IsTrue(hasSymbol1Data || hasSymbol2Data, - "Expected data for at least one of the requested symbols"); + Assert.IsTrue(tick.BidPrice > 0 || tick.AskPrice > 0); } } + } - internal static HistoryRequest GetHistoryRequest(Resolution resolution, TickType tickType, Symbol symbol, TimeSpan period) - { - var utcNow = DateTime.UtcNow; - var dataType = LeanData.GetDataType(resolution, tickType); - var marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); - - var exchangeHours = marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType); - var dataTimeZone = marketHoursDatabase.GetDataTimeZone(symbol.ID.Market, symbol, symbol.SecurityType); - - return new HistoryRequest( - startTimeUtc: utcNow.Add(-period), - endTimeUtc: utcNow, - dataType: dataType, - symbol: symbol, - resolution: resolution, - exchangeHours: exchangeHours, - dataTimeZone: dataTimeZone, - fillForwardResolution: resolution, - includeExtendedMarketHours: true, - isCustomData: false, - DataNormalizationMode.Raw, - tickType: tickType - ); - } + private static HistoryRequest GetHistoryRequest( + Resolution resolution, + TickType tickType, + Symbol symbol, + TimeSpan period) + { + var endUtc = new DateTime(2026, 1, 22); + var startUtc = endUtc - period; + + var dataType = LeanData.GetDataType(resolution, tickType); + var marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); + + var exchangeHours = marketHoursDatabase.GetExchangeHours( + symbol.ID.Market, symbol, symbol.SecurityType); + + var dataTimeZone = marketHoursDatabase.GetDataTimeZone( + symbol.ID.Market, symbol, symbol.SecurityType); + + return new HistoryRequest( + startTimeUtc: startUtc, + endTimeUtc: endUtc, + dataType: dataType, + symbol: symbol, + resolution: resolution, + exchangeHours: exchangeHours, + dataTimeZone: dataTimeZone, + fillForwardResolution: resolution, + includeExtendedMarketHours: true, + isCustomData: false, + DataNormalizationMode.Raw, + tickType: tickType + ); } } diff --git a/QuantConnect.DataBento.Tests/DataBentoDataQueueHandlerTests.cs b/QuantConnect.DataBento.Tests/DataBentoDataQueueHandlerTests.cs new file mode 100644 index 0000000..dd57c25 --- /dev/null +++ b/QuantConnect.DataBento.Tests/DataBentoDataQueueHandlerTests.cs @@ -0,0 +1,234 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + + +using System; +using System.Text; +using NUnit.Framework; +using System.Threading; +using QuantConnect.Util; +using QuantConnect.Data; +using QuantConnect.Logging; +using System.Threading.Tasks; +using QuantConnect.Securities; +using QuantConnect.Data.Market; +using System.Collections.Generic; +using QuantConnect.Lean.Engine.DataFeeds.Enumerators; + +namespace QuantConnect.Lean.DataSource.DataBento.Tests; + +[TestFixture] +public class DataBentoDataQueueHandlerTests +{ + private DataBentoProvider _dataProvider; + private CancellationTokenSource _cancellationTokenSource; + + [SetUp] + public void SetUp() + { + _cancellationTokenSource = new(); + _dataProvider = new(); + } + + [TearDown] + public void TearDown() + { + _cancellationTokenSource?.Cancel(); + _cancellationTokenSource.DisposeSafely(); + _dataProvider?.DisposeSafely(); + } + + private static IEnumerable TestParameters + { + get + { + var sp500EMiniMarch = Symbol.CreateFuture(Futures.Indices.SP500EMini, Market.CME, new(2026, 03, 20)); + + yield return new TestCaseData(new Symbol[] { sp500EMiniMarch }, Resolution.Second); + } + } + + [Test, TestCaseSource(nameof(TestParameters))] + public void CanSubscribeAndUnsubscribeOnDifferentResolution(Symbol[] symbols, Resolution resolution) + { + + var configs = new List(); + + var dataFromEnumerator = new Dictionary>(); + + foreach (var symbol in symbols) + { + dataFromEnumerator[symbol] = new Dictionary(); + foreach (var config in GetSubscriptionDataConfigs(symbol, resolution)) + { + configs.Add(config); + + var tickType = config.TickType switch + { + TickType.Quote => typeof(QuoteBar), + TickType.Trade => typeof(TradeBar), + _ => throw new NotImplementedException() + }; + + dataFromEnumerator[symbol][tickType] = 0; + } + } + + Assert.That(configs, Is.Not.Empty); + + Action callback = (dataPoint) => + { + if (dataPoint == null) + { + return; + } + + switch (dataPoint) + { + case TradeBar tb: + dataFromEnumerator[tb.Symbol][typeof(TradeBar)] += 1; + break; + case QuoteBar qb: + Assert.GreaterOrEqual(qb.Ask.Open, qb.Bid.Open, $"QuoteBar validation failed for {qb.Symbol}: Ask.Open ({qb.Ask.Open}) <= Bid.Open ({qb.Bid.Open}). Full data: {DisplayBaseData(qb)}"); + Assert.GreaterOrEqual(qb.Ask.High, qb.Bid.High, $"QuoteBar validation failed for {qb.Symbol}: Ask.High ({qb.Ask.High}) <= Bid.High ({qb.Bid.High}). Full data: {DisplayBaseData(qb)}"); + Assert.GreaterOrEqual(qb.Ask.Low, qb.Bid.Low, $"QuoteBar validation failed for {qb.Symbol}: Ask.Low ({qb.Ask.Low}) <= Bid.Low ({qb.Bid.Low}). Full data: {DisplayBaseData(qb)}"); + Assert.GreaterOrEqual(qb.Ask.Close, qb.Bid.Close, $"QuoteBar validation failed for {qb.Symbol}: Ask.Close ({qb.Ask.Close}) <= Bid.Close ({qb.Bid.Close}). Full data: {DisplayBaseData(qb)}"); + dataFromEnumerator[qb.Symbol][typeof(QuoteBar)] += 1; + break; + } + ; + }; + + foreach (var config in configs) + { + ProcessFeed(_dataProvider.Subscribe(config, (sender, args) => + { + var dataPoint = ((NewDataAvailableEventArgs)args).DataPoint; + Log.Trace($"{dataPoint}. Time span: {dataPoint.Time} - {dataPoint.EndTime}"); + }), _cancellationTokenSource.Token, callback: callback); + } + + Thread.Sleep(TimeSpan.FromSeconds(120)); + + Log.Trace("Unsubscribing symbols"); + foreach (var config in configs) + { + _dataProvider.Unsubscribe(config); + } + + Thread.Sleep(TimeSpan.FromSeconds(5)); + + _cancellationTokenSource.Cancel(); + + var str = new StringBuilder(); + + str.AppendLine($"{nameof(DataBentoDataQueueHandlerTests)}.{nameof(CanSubscribeAndUnsubscribeOnDifferentResolution)}: ***** Summary *****"); + + foreach (var symbol in symbols) + { + str.AppendLine($"Input parameters: ticker:{symbol} | securityType:{symbol.SecurityType} | resolution:{resolution}"); + + foreach (var tickType in dataFromEnumerator[symbol]) + { + str.AppendLine($"[{tickType.Key}] = {tickType.Value}"); + + if (symbol.SecurityType != SecurityType.Index) + { + Assert.Greater(tickType.Value, 0); + } + // The ThetaData returns TradeBar seldom. Perhaps should find more relevant ticker. + Assert.GreaterOrEqual(tickType.Value, 0); + } + str.AppendLine(new string('-', 30)); + } + + Log.Trace(str.ToString()); + } + + private static string DisplayBaseData(BaseData item) + { + switch (item) + { + case TradeBar tradeBar: + return $"Data Type: {item.DataType} | " + tradeBar.ToString() + $" Time: {tradeBar.Time}, EndTime: {tradeBar.EndTime}"; + default: + return $"DEFAULT: Data Type: {item.DataType} | Time: {item.Time} | End Time: {item.EndTime} | Symbol: {item.Symbol} | Price: {item.Price} | IsFillForward: {item.IsFillForward}"; + } + } + + private static IEnumerable GetSubscriptionDataConfigs(Symbol symbol, Resolution resolution) + { + yield return GetSubscriptionDataConfig(symbol, resolution); + yield return GetSubscriptionDataConfig(symbol, resolution); + } + + public static IEnumerable GetSubscriptionTickDataConfigs(Symbol symbol) + { + yield return new SubscriptionDataConfig(GetSubscriptionDataConfig(symbol, Resolution.Tick), tickType: TickType.Trade); + yield return new SubscriptionDataConfig(GetSubscriptionDataConfig(symbol, Resolution.Tick), tickType: TickType.Quote); + } + + private static SubscriptionDataConfig GetSubscriptionDataConfig(Symbol symbol, Resolution resolution) + { + return new SubscriptionDataConfig( + typeof(T), + symbol, + resolution, + TimeZones.Utc, + TimeZones.Utc, + true, + extendedHours: false, + false); + } + + private Task ProcessFeed( + IEnumerator enumerator, + CancellationToken cancellationToken, + int cancellationTokenDelayMilliseconds = 100, + Action callback = null, + Action throwExceptionCallback = null) + { + return Task.Factory.StartNew(() => + { + try + { + while (enumerator.MoveNext() && !cancellationToken.IsCancellationRequested) + { + BaseData tick = enumerator.Current; + + if (tick != null) + { + callback?.Invoke(tick); + } + + cancellationToken.WaitHandle.WaitOne(TimeSpan.FromMilliseconds(cancellationTokenDelayMilliseconds)); + } + } + catch (Exception ex) + { + Log.Debug($"{nameof(DataBentoDataQueueHandlerTests)}.{nameof(ProcessFeed)}.Exception: {ex.Message}"); + throw; + } + }, cancellationToken).ContinueWith(task => + { + if (throwExceptionCallback != null) + { + throwExceptionCallback(); + } + Log.Debug("The throwExceptionCallback is null."); + }, TaskContinuationOptions.OnlyOnFaulted); + } +} diff --git a/QuantConnect.DataBento.Tests/DataBentoHistoricalApiClientTests.cs b/QuantConnect.DataBento.Tests/DataBentoHistoricalApiClientTests.cs new file mode 100644 index 0000000..4623776 --- /dev/null +++ b/QuantConnect.DataBento.Tests/DataBentoHistoricalApiClientTests.cs @@ -0,0 +1,104 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System; +using NUnit.Framework; +using QuantConnect.Logging; +using QuantConnect.Configuration; +using QuantConnect.Lean.DataSource.DataBento.Api; + +namespace QuantConnect.Lean.DataSource.DataBento.Tests; + +[TestFixture] +public class DataBentoHistoricalApiClientTests +{ + private HistoricalAPIClient _client; + + /// + /// Dataset for CME Globex futures + /// https://databento.com/docs/venues-and-datasets has more information on datasets through DataBento + /// + /// + /// TODO: Hard coded for now. Later on can add equities and options with different mapping + /// + private const string Dataset = "GLBX.MDP3"; + + [OneTimeSetUp] + public void OneTimeSetUp() + { + var apiKey = Config.Get("databento-api-key"); + if (string.IsNullOrEmpty(apiKey)) + { + Assert.Inconclusive("Please set the 'databento-api-key' in your configuration to enable these tests."); + } + + _client = new HistoricalAPIClient(apiKey); + } + + [TestCase("ESH6", "2025/01/11", "2026/01/20", Resolution.Daily)] + [TestCase("ESH6", "2025/01/11", "2026/01/20", Resolution.Hour)] + [TestCase("ESH6", "2025/01/11", "2026/01/20", Resolution.Minute)] + [TestCase("ESH6", "2025/01/11", "2026/01/20", Resolution.Second)] + //[TestCase("ESH6", "2025/01/11", "2026/01/20", Resolution.Tick)] + [TestCase("ESH6 C6875", "2026/01/11", "2026/01/20", Resolution.Daily)] + public void CanInitializeHistoricalApiClient(string ticker, DateTime startDate, DateTime endDate, Resolution resolution) + { + var dataCounter = 0; + var previousEndTime = DateTime.MinValue; + foreach (var data in _client.GetHistoricalOhlcvBars(ticker, startDate, endDate, resolution, Dataset)) + { + Assert.IsNotNull(data); + + Assert.Greater(data.Open, 0m); + Assert.Greater(data.High, 0m); + Assert.Greater(data.Low, 0m); + Assert.Greater(data.Close, 0m); + Assert.Greater(data.Volume, 0m); + Assert.AreNotEqual(default(DateTime), data.Header.UtcTime); + + Assert.IsTrue(data.Header.UtcTime > previousEndTime, + $"Bar at {data.Header.UtcTime:o} is not after previous bar at {previousEndTime:o}"); + previousEndTime = data.Header.UtcTime; + + dataCounter++; + } + + Log.Trace($"{nameof(CanInitializeHistoricalApiClient)}: {ticker} | [{startDate} - {endDate}] | {resolution} = {dataCounter} (bars)"); + Assert.Greater(dataCounter, 0); + } + + [TestCase("ESH6 C6875", "2026/01/11", "2026/01/20", Resolution.Daily)] + public void ShouldFetchOpenInterest(string ticker, DateTime startDate, DateTime endDate, Resolution resolution) + { + var dataCounter = 0; + var previousEndTime = DateTime.MinValue; + foreach (var data in _client.GetOpenInterest(ticker, startDate, endDate, Dataset)) + { + Assert.IsNotNull(data); + + Assert.Greater(data.Quantity, 0m); + Assert.AreNotEqual(default(DateTime), data.Header.UtcTime); + + Assert.IsTrue(data.Header.UtcTime > previousEndTime, + $"Bar at {data.Header.UtcTime:o} is not after previous bar at {previousEndTime:o}"); + previousEndTime = data.Header.UtcTime; + + dataCounter++; + } + + Log.Trace($"{nameof(CanInitializeHistoricalApiClient)}: {ticker} | [{startDate} - {endDate}] | {resolution} = {dataCounter} (bars)"); + Assert.Greater(dataCounter, 0); + } +} diff --git a/QuantConnect.DataBento.Tests/DataBentoJsonConverterTests.cs b/QuantConnect.DataBento.Tests/DataBentoJsonConverterTests.cs new file mode 100644 index 0000000..d29d488 --- /dev/null +++ b/QuantConnect.DataBento.Tests/DataBentoJsonConverterTests.cs @@ -0,0 +1,287 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using NUnit.Framework; +using QuantConnect.Lean.DataSource.DataBento.Models; +using QuantConnect.Lean.DataSource.DataBento.Models.Enums; +using QuantConnect.Lean.DataSource.DataBento.Models.Live; + +namespace QuantConnect.Lean.DataSource.DataBento.Tests; + +[TestFixture] +public class DataBentoJsonConverterTests +{ + [Test] + public void DeserializeHistoricalOhlcvBar() + { + var json = @"{ + ""hd"": { + ""ts_event"": ""1738281600000000000"", + ""rtype"": 35, + ""publisher_id"": 1, + ""instrument_id"": 42140878 + }, + ""open"": ""6359.000000000"", + ""high"": ""6359.000000000"", + ""low"": ""6355.000000000"", + ""close"": ""6355.000000000"", + ""volume"": ""2"" +}"; + var res = json.DeserializeKebabCase(); + + Assert.IsNotNull(res); + + Assert.AreEqual(1738281600000000000m, res.Header.TsEvent); + Assert.AreEqual(RecordType.OpenHighLowCloseVolume1Day, res.Header.Rtype); + Assert.AreEqual(1, res.Header.PublisherId); + Assert.AreEqual(42140878, res.Header.InstrumentId); + + Assert.AreEqual(6359m, res.Open); + Assert.AreEqual(6359m, res.High); + Assert.AreEqual(6355m, res.Low); + Assert.AreEqual(6355m, res.Close); + Assert.AreEqual(2L, res.Volume); + } + + [Test] + public void DeserializeHistoricalLevelOneData() + { + var json = @"{ + ""ts_recv"": ""1768137063449660443"", + ""hd"": { + ""ts_event"": ""1768137063107829777"", + ""rtype"": 1, + ""publisher_id"": 1, + ""instrument_id"": 42140878 + }, + ""action"": ""A"", + ""side"": ""N"", + ""depth"": 0, + ""price"": ""7004.250000000"", + ""size"": 15, + ""flags"": 128, + ""ts_in_delta"": 17537, + ""sequence"": 811, + ""levels"": [ + { + ""bid_px"": ""7004.000000000"", + ""ask_px"": ""7004.250000000"", + ""bid_sz"": 11, + ""ask_sz"": 15, + ""bid_ct"": 1, + ""ask_ct"": 1 + } + ] +}"; + var res = json.DeserializeKebabCase(); + + Assert.IsNotNull(res); + + Assert.AreEqual(1768137063449660443, res.TsRecv); + + Assert.AreEqual(1768137063107829777, res.Header.TsEvent); + Assert.AreEqual(RecordType.MarketByPriceDepth1, res.Header.Rtype); + Assert.AreEqual(1, res.Header.PublisherId); + Assert.AreEqual(42140878, res.Header.InstrumentId); + + Assert.AreEqual('A', res.Action); + Assert.AreEqual('N', res.Side); + Assert.AreEqual(0, res.Depth); + Assert.AreEqual(7004.25m, res.Price); + Assert.AreEqual(15, res.Size); + Assert.AreEqual(128, res.Flags); + Assert.IsNotNull(res.Levels); + Assert.AreEqual(1, res.Levels.Count); + var level = res.Levels[0]; + Assert.AreEqual(7004.0m, level.BidPx); + Assert.AreEqual(7004.25m, level.AskPx); + Assert.AreEqual(11, level.BidSz); + Assert.AreEqual(15, level.AskSz); + Assert.AreEqual(1, level.BidCt); + Assert.AreEqual(1, level.AskCt); + } + + [Test] + public void DeserializeHistoricalStatisticsData() + { + var json = @"{ + ""ts_recv"": ""1768156232522711477"", + ""hd"": { + ""ts_event"": ""1768156232522476283"", + ""rtype"": 24, + ""publisher_id"": 1, + ""instrument_id"": 42566722 + }, + ""ts_ref"": ""1767916800000000000"", + ""price"": null, + ""quantity"": 470, + ""sequence"": 29232, + ""ts_in_delta"": 12477, + ""stat_type"": 9, + ""channel_id"": 1, + ""update_action"": 1, + ""stat_flags"": 0 +}"; + + var res = json.DeserializeKebabCase(); + + Assert.IsNotNull(res); + + + Assert.AreEqual(1768156232522476283, res.Header.TsEvent); + Assert.AreEqual(RecordType.Statistics, res.Header.Rtype); + Assert.AreEqual(1, res.Header.PublisherId); + Assert.AreEqual(42566722, res.Header.InstrumentId); + Assert.AreEqual(470m, res.Quantity); + Assert.AreEqual(StatisticType.OpenInterest, res.StatType); + } + + [Test] + public void DeserializeLiveHeartbeatMessage() + { + var json = @"{""hd"":{""ts_event"":""1769176693139629181"",""rtype"":23,""publisher_id"":0,""instrument_id"":0},""msg"":""Heartbeat""}"; + + var res = json.DeserializeKebabCase(); + + Assert.IsNotNull(res); + Assert.AreEqual(1769176693139629181, res.Header.TsEvent); + Assert.AreEqual(RecordType.System, res.Header.Rtype); + Assert.AreEqual(0, res.Header.PublisherId); + Assert.AreEqual(0, res.Header.InstrumentId); + Assert.AreEqual("Heartbeat", res.Msg); + } + + [TestCase("success=0|error=Unknown subscription param 'sssauth'", false)] + [TestCase("success=0|error=Authentication failed.", false)] + [TestCase("success=1|session_id=1769508116", true)] + public void ParsePotentialAuthenticationMessageResponses(string authenticationResponse, bool success) + { + var auth = new AuthenticationMessageResponse(authenticationResponse); + + if (success) + { + Assert.IsTrue(auth.Success); + Assert.AreNotEqual(0, auth.SessionId); + } + else + { + Assert.IsFalse(auth.Success); + Assert.That(auth.Error, Is.Not.Null.And.Not.Empty); + } + } + + [TestCase("cram=HCxTgxMcqglVMTMeaDZ2ICmcnrW8j92e", "auth=a6c5c23e06854dc0310e11ce6d3081509e415a5a37a323bb94bc90f64c9214d4-12345|dataset=GLBX.MDP3|pretty_px=1|encoding=json|heartbeat_interval_s=5")] + [TestCase("cram=HCxTgxMcqglVMTMeaDZ2ICmcnrW8j92e\n", "auth=a6c5c23e06854dc0310e11ce6d3081509e415a5a37a323bb94bc90f64c9214d4-12345|dataset=GLBX.MDP3|pretty_px=1|encoding=json|heartbeat_interval_s=5")] + public void ParsePotentialCramChallenges(string challenge, string expectedString) + { + var auth = new AuthenticationMessageRequest(challenge, "my-api-key-12345", "GLBX.MDP3"); + + var actualString = auth.ToString(); + + Assert.AreEqual(expectedString, actualString); + } + + [Test] + public void DeserializeSymbolMappingMessage() + { + var json = @"{ + ""hd"": { + ""ts_event"": ""1769546804979770503"", + ""rtype"": 22, + ""publisher_id"": 0, + ""instrument_id"": 42140878 + }, + ""stype_in_symbol"": ""ESH6"", + ""stype_out_symbol"": ""ESH6"", + ""start_ts"": ""18446744073709551615"", + ""end_ts"": ""18446744073709551615"" +}"; + + var marketData = json.DeserializeSnakeCaseLiveData(); + + Assert.IsNotNull(marketData); + Assert.AreEqual(1769546804979770503, marketData.Header.TsEvent); + Assert.AreEqual(RecordType.SymbolMapping, marketData.Header.Rtype); + Assert.AreEqual(0, marketData.Header.PublisherId); + Assert.AreEqual(42140878, marketData.Header.InstrumentId); + + Assert.IsInstanceOf(marketData); + + var sm = marketData as SymbolMappingMessage; + + Assert.AreEqual("ESH6", sm.StypeInSymbol); + Assert.AreEqual("ESH6", sm.StypeOutSymbol); + } + + + [Test] + public void DeserializeMarketByPriceMessage() + { + var json = @" +{ + ""ts_recv"": ""1769546804990938439"", + ""hd"": { + ""ts_event"": ""1769546804990833083"", + ""rtype"": 1, + ""publisher_id"": 1, + ""instrument_id"": 42005017 + }, + ""action"": ""A"", + ""side"": ""A"", + ""depth"": 0, + ""price"": ""2676.400000000"", + ""size"": 1, + ""flags"": 128, + ""ts_in_delta"": 17695, + ""sequence"": 126257483, + ""levels"": [ + { + ""bid_px"": ""2676.300000000"", + ""ask_px"": ""2676.400000000"", + ""bid_sz"": 14, + ""ask_sz"": 2, + ""bid_ct"": 8, + ""ask_ct"": 2 + } + ] +}"; + var marketData = json.DeserializeSnakeCaseLiveData(); + + Assert.IsNotNull(marketData); + Assert.AreEqual(1769546804990833083, marketData.Header.TsEvent); + Assert.AreEqual(RecordType.MarketByPriceDepth1, marketData.Header.Rtype); + Assert.AreEqual(1, marketData.Header.PublisherId); + Assert.AreEqual(42005017, marketData.Header.InstrumentId); + + Assert.IsInstanceOf(marketData); + + var mbp = marketData as LevelOneData; + Assert.AreEqual('A', mbp.Action); + Assert.AreEqual('A', mbp.Side); + Assert.AreEqual(0, mbp.Depth); + Assert.AreEqual(2676.4m, mbp.Price); + Assert.AreEqual(1, mbp.Size); + Assert.AreEqual(128, mbp.Flags); + Assert.IsNotNull(mbp.Levels); + Assert.AreEqual(1, mbp.Levels.Count); + var level = mbp.Levels[0]; + Assert.AreEqual(2676.3m, level.BidPx); + Assert.AreEqual(2676.4m, level.AskPx); + Assert.AreEqual(14, level.BidSz); + Assert.AreEqual(2, level.AskSz); + Assert.AreEqual(8, level.BidCt); + Assert.AreEqual(2, level.AskCt); + } +} diff --git a/QuantConnect.DataBento.Tests/DataBentoLiveAPIClientTests.cs b/QuantConnect.DataBento.Tests/DataBentoLiveAPIClientTests.cs new file mode 100644 index 0000000..69d5f1d --- /dev/null +++ b/QuantConnect.DataBento.Tests/DataBentoLiveAPIClientTests.cs @@ -0,0 +1,95 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System; +using NUnit.Framework; +using System.Threading; +using QuantConnect.Util; +using QuantConnect.Configuration; +using System.Collections.Generic; +using QuantConnect.Lean.DataSource.DataBento.Api; + +namespace QuantConnect.Lean.DataSource.DataBento.Tests; + +[TestFixture] +public class DataBentoLiveAPIClientTests +{ + /// + /// Dataset for CME Globex futures + /// https://databento.com/docs/venues-and-datasets has more information on datasets through DataBento + /// + /// + /// TODO: Hard coded for now. Later on can add equities and options with different mapping + /// + private const string Dataset = "GLBX.MDP3"; + + private LiveAPIClient _live; + + [OneTimeSetUp] + public void OneTimeSetUp() + { + var apiKey = Config.Get("databento-api-key"); + if (string.IsNullOrEmpty(apiKey)) + { + Assert.Inconclusive("Please set the 'databento-api-key' in your configuration to enable these tests."); + } + + _live = new LiveAPIClient(apiKey, null); + } + + [OneTimeTearDown] + public void OneTimeTearDown() + { + _live?.DisposeSafely(); + } + + [Test] + public void ShouldReceiveSymbolMappingConfirmation() + { + var dataAvailableEvent = new AutoResetEvent(false); + + var subs = new Dictionary() + { + { Securities.Futures.Indices.SP500EMini + "H6", 0 }, + { Securities.Futures.Indices.Russell2000EMini + "H6", 0 } + }; + + void OnSymbolMappingConfirmation(object sender, Models.Live.SymbolMappingConfirmationEventArgs e) + { + if (subs.ContainsKey(e.Symbol)) + { + subs[e.Symbol] = e.InstrumentId; + dataAvailableEvent.Set(); + } + } + + _live.SymbolMappingConfirmation += OnSymbolMappingConfirmation; + + foreach (var s in subs.Keys) + { + _live.Subscribe(Dataset, s); + dataAvailableEvent.WaitOne(TimeSpan.FromSeconds(5)); + } + + dataAvailableEvent.WaitOne(TimeSpan.FromSeconds(1)); + + foreach (var instrumentId in subs.Values) + { + Assert.Greater(instrumentId, 0); + } + + _live.SymbolMappingConfirmation -= OnSymbolMappingConfirmation; + } +} diff --git a/QuantConnect.DataBento.Tests/DataBentoRawLiveClientTests.cs b/QuantConnect.DataBento.Tests/DataBentoRawLiveClientTests.cs deleted file mode 100644 index 7df71a4..0000000 --- a/QuantConnect.DataBento.Tests/DataBentoRawLiveClientTests.cs +++ /dev/null @@ -1,253 +0,0 @@ -/* - * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * -*/ - -using System; -using System.Threading; -using System.Threading.Tasks; -using NUnit.Framework; -using QuantConnect.Data; -using QuantConnect.Data.Market; -using QuantConnect.Lean.DataSource.DataBento; -using QuantConnect.Logging; -using QuantConnect.Configuration; - -namespace QuantConnect.Lean.DataSource.DataBento.Tests -{ - [TestFixture] - public class DataBentoRawLiveClientTests - { - private DatabentoRawClient _client; - private readonly string _apiKey = Config.Get("databento-api-key"); - - [SetUp] - public void SetUp() - { - _client = new DatabentoRawClient(_apiKey); - } - - [TearDown] - public void TearDown() - { - _client?.Dispose(); - } - - [Test] - [Explicit("This test requires a configured DataBento API key and live connection")] - public async Task ConnectsToGateway() - { - if (string.IsNullOrEmpty(_apiKey)) - { - Assert.Ignore("DataBento API key not configured"); - return; - } - - var connected = await _client.ConnectAsync(); - - Assert.IsTrue(connected, "Should successfully connect to DataBento gateway"); - Assert.IsTrue(_client.IsConnected, "IsConnected should return true after successful connection"); - - Log.Trace("Successfully connected to DataBento gateway"); - } - - [Test] - [Explicit("This test requires a configured DataBento API key and live connection")] - public async Task SubscribesToSymbol() - { - if (string.IsNullOrEmpty(_apiKey)) - { - Assert.Ignore("DataBento API key not configured"); - return; - } - - var connected = await _client.ConnectAsync(); - Assert.IsTrue(connected, "Must be connected to test subscription"); - - var symbol = Symbol.Create("ESM3", SecurityType.Future, Market.CME); - var subscribed = _client.Subscribe(symbol, Resolution.Minute, TickType.Trade); - - Assert.IsTrue(subscribed, "Should successfully subscribe to symbol"); - - Log.Trace($"Successfully subscribed to {symbol}"); - - // Wait a moment to ensure subscription is active - await Task.Delay(2000); - - var unsubscribed = _client.Unsubscribe(symbol); - Assert.IsTrue(unsubscribed, "Should successfully unsubscribe from symbol"); - - Log.Trace($"Successfully unsubscribed from {symbol}"); - } - - [Test] - [Explicit("This test requires a configured DataBento API key and live connection")] - public async Task ReceivesLiveData() - { - if (string.IsNullOrEmpty(_apiKey)) - { - Assert.Ignore("DataBento API key not configured"); - return; - } - - var dataReceived = false; - var dataReceivedEvent = new ManualResetEventSlim(false); - BaseData receivedData = null; - - _client.DataReceived += (sender, data) => - { - receivedData = data; - dataReceived = true; - dataReceivedEvent.Set(); - Log.Trace($"Received data: {data}"); - }; - - var connected = await _client.ConnectAsync(); - Assert.IsTrue(connected, "Must be connected to test data reception"); - - var symbol = Symbol.Create("ESM3", SecurityType.Future, Market.CME); - var subscribed = _client.Subscribe(symbol, Resolution.Tick, TickType.Trade); - Assert.IsTrue(subscribed, "Must be subscribed to receive data"); - - // Wait for data with timeout - var dataReceiptTimeout = TimeSpan.FromMinutes(2); - var receivedWithinTimeout = dataReceivedEvent.Wait(dataReceiptTimeout); - - if (receivedWithinTimeout) - { - Assert.IsTrue(dataReceived, "Should have received data"); - Assert.IsNotNull(receivedData, "Received data should not be null"); - Assert.AreEqual(symbol, receivedData.Symbol, "Received data symbol should match subscription"); - Assert.Greater(receivedData.Value, 0, "Received data value should be positive"); - - Log.Trace($"Successfully received live data: {receivedData}"); - } - else - { - Log.Trace("No data received within timeout period - this may be expected during non-market hours"); - } - - _client.Unsubscribe(symbol); - } - - [Test] - [Explicit("This test requires a configured DataBento API key and live connection")] - public async Task HandlesConnectionEvents() - { - if (string.IsNullOrEmpty(_apiKey)) - { - Assert.Ignore("DataBento API key not configured"); - return; - } - - var connectionStatusChanged = false; - var connectionStatusEvent = new ManualResetEventSlim(false); - - _client.ConnectionStatusChanged += (sender, isConnected) => - { - connectionStatusChanged = true; - connectionStatusEvent.Set(); - Log.Trace($"Connection status changed: {isConnected}"); - }; - - var connected = await _client.ConnectAsync(); - Assert.IsTrue(connected, "Should connect successfully"); - - // Connection status event should fire on connect - var eventFiredWithinTimeout = connectionStatusEvent.Wait(TimeSpan.FromSeconds(10)); - Assert.IsTrue(eventFiredWithinTimeout || connectionStatusChanged, - "Connection status changed event should fire"); - - _client.Disconnect(); - Assert.IsFalse(_client.IsConnected, "Should be disconnected after calling Disconnect()"); - } - - [Test] - public void HandlesInvalidApiKey() - { - var invalidClient = new DatabentoRawClient("invalid-api-key"); - - // Connection with invalid API key should fail gracefully - Assert.DoesNotThrowAsync(async () => - { - var connected = await invalidClient.ConnectAsync(); - Assert.IsFalse(connected, "Connection should fail with invalid API key"); - }); - - invalidClient.Dispose(); - } - - [Test] - public void DisposesCorrectly() - { - var client = new DatabentoRawClient(_apiKey); - Assert.DoesNotThrow(() => client.Dispose(), "Dispose should not throw"); - Assert.DoesNotThrow(() => client.Dispose(), "Multiple dispose calls should not throw"); - } - - [Test] - public void SymbolMappingWorksCorrectly() - { - // Test that futures are mapped correctly to DataBento format - var esFuture = Symbol.Create("ESM3", SecurityType.Future, Market.CME); - - // Since the mapping method is private, we test indirectly through subscription - Assert.DoesNotThrowAsync(async () => - { - if (!string.IsNullOrEmpty(_apiKey)) - { - var connected = await _client.ConnectAsync(); - if (connected) - { - _client.Subscribe(esFuture, Resolution.Minute, TickType.Trade); - _client.Unsubscribe(esFuture); - } - } - }); - } - - [Test] - public void SchemaResolutionMappingWorksCorrectly() - { - // Test that resolution mappings work correctly - var symbol = Symbol.Create("ESM3", SecurityType.Future, Market.CME); - - Assert.DoesNotThrowAsync(async () => - { - if (!string.IsNullOrEmpty(_apiKey)) - { - var connected = await _client.ConnectAsync(); - if (connected) - { - // Test different resolutions - _client.Subscribe(symbol, Resolution.Tick, TickType.Trade); - _client.Unsubscribe(symbol); - - _client.Subscribe(symbol, Resolution.Second, TickType.Trade); - _client.Unsubscribe(symbol); - - _client.Subscribe(symbol, Resolution.Minute, TickType.Trade); - _client.Unsubscribe(symbol); - - _client.Subscribe(symbol, Resolution.Hour, TickType.Trade); - _client.Unsubscribe(symbol); - - _client.Subscribe(symbol, Resolution.Daily, TickType.Trade); - _client.Unsubscribe(symbol); - } - } - }); - } - } -} diff --git a/QuantConnect.DataBento.Tests/DataBentoSymbolMapperTests.cs.cs b/QuantConnect.DataBento.Tests/DataBentoSymbolMapperTests.cs.cs new file mode 100644 index 0000000..feb393a --- /dev/null +++ b/QuantConnect.DataBento.Tests/DataBentoSymbolMapperTests.cs.cs @@ -0,0 +1,66 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +using System; +using NUnit.Framework; +using System.Collections.Generic; + +namespace QuantConnect.Lean.DataSource.DataBento.Tests; + +[TestFixture] +public class DataBentoSymbolMapperTests +{ + /// + /// Provides the mapping between Lean symbols and brokerage specific symbols. + /// + private DataBentoSymbolMapper _symbolMapper; + + [OneTimeSetUp] + public void OneTimeSetUp() + { + _symbolMapper = new DataBentoSymbolMapper(); + } + private static IEnumerable LeanSymbolTestCases + { + get + { + // TSLA - Equity + var es = Symbol.CreateFuture(Securities.Futures.Indices.SP500EMini, Market.CME, new DateTime(2026, 3, 20)); + yield return new TestCaseData(es, "ESH6"); + + } + } + + [Test, TestCaseSource(nameof(LeanSymbolTestCases))] + public void ReturnsCorrectBrokerageSymbol(Symbol symbol, string expectedBrokerageSymbol) + { + var brokerageSymbol = _symbolMapper.GetBrokerageSymbol(symbol); + + Assert.IsNotNull(brokerageSymbol); + Assert.IsNotEmpty(brokerageSymbol); + Assert.AreEqual(expectedBrokerageSymbol, brokerageSymbol); + } + + [TestCase(Market.CME, "GLBX.MDP3", true)] + [TestCase(Market.EUREX, "XEUR.EOBI", true)] + [TestCase(Market.USA, null, false)] + public void ReturnsCorrectDataBentoDataSet(string market, string expectedDataSet, bool expectedExist) + { + var actualExist = _symbolMapper.DataBentoDataSetByLeanMarket.TryGetValue(market, out var actualDataSet); + Assert.AreEqual(expectedExist, actualExist); + Assert.AreEqual(expectedDataSet, actualDataSet); + } +} diff --git a/QuantConnect.DataBento.Tests/QuantConnect.DataSource.DataBento.Tests.csproj b/QuantConnect.DataBento.Tests/QuantConnect.DataSource.DataBento.Tests.csproj index d2d7ef1..c58ec16 100644 --- a/QuantConnect.DataBento.Tests/QuantConnect.DataSource.DataBento.Tests.csproj +++ b/QuantConnect.DataBento.Tests/QuantConnect.DataSource.DataBento.Tests.csproj @@ -1,34 +1,34 @@ - - net9.0 - QuantConnect.DataLibrary.Tests - - - - - - - - - all - - - - - - - - - - - - - - - - - - PreserveNewest - - + + Release + AnyCPU + net10.0 + false + UnitTest + bin\$(Configuration)\ + QuantConnect.Lean.DataSource.DataBento.Tests + QuantConnect.Lean.DataSource.DataBento.Tests + QuantConnect.Lean.DataSource.DataBento.Tests + QuantConnect.Lean.DataSource.DataBento.Tests + false + + + + + + all + + + + + + + + + + + + PreserveNewest + + diff --git a/QuantConnect.DataBento.Tests/TestSetup.cs b/QuantConnect.DataBento.Tests/TestSetup.cs index b0e07c1..e56a03d 100644 --- a/QuantConnect.DataBento.Tests/TestSetup.cs +++ b/QuantConnect.DataBento.Tests/TestSetup.cs @@ -1,10 +1,9 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. - * * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software @@ -15,69 +14,51 @@ */ using System; -using System.Collections; using System.IO; using NUnit.Framework; -using QuantConnect.Configuration; +using System.Collections; using QuantConnect.Logging; +using QuantConnect.Configuration; -namespace QuantConnect.Lean.DataSource.DataBento.Tests +namespace QuantConnect.Lean.DataSource.DataBento.Tests; + +[SetUpFixture] +public class TestSetup { - [SetUpFixture] - public class TestSetup + [OneTimeSetUp] + public void GlobalSetup() { - [OneTimeSetUp] - public void GlobalSetup() - { - Log.DebuggingEnabled = true; - Log.LogHandler = new CompositeLogHandler(); - Log.Trace("TestSetup(): starting..."); - ReloadConfiguration(); - } + Log.DebuggingEnabled = Config.GetBool("debug-mode"); + Log.LogHandler = new CompositeLogHandler(); + Log.Trace("TestSetup(): starting..."); + ReloadConfiguration(); + } + + private static void ReloadConfiguration() + { + // nunit 3 sets the current folder to a temp folder we need it to be the test bin output folder + var dir = TestContext.CurrentContext.TestDirectory; + Environment.CurrentDirectory = dir; + Directory.SetCurrentDirectory(dir); + // reload config from current path + Config.Reset(); - private static void ReloadConfiguration() + var environment = Environment.GetEnvironmentVariables(); + foreach (DictionaryEntry entry in environment) { - // nunit 3 sets the current folder to a temp folder we need it to be the test bin output folder - var dir = TestContext.CurrentContext.TestDirectory; - Environment.CurrentDirectory = dir; - Directory.SetCurrentDirectory(dir); - // reload config from current path - Config.Reset(); + var envKey = entry.Key.ToString(); + var value = entry.Value.ToString(); - var environment = Environment.GetEnvironmentVariables(); - foreach (DictionaryEntry entry in environment) + if (envKey.StartsWith("QC_")) { - var envKey = entry.Key.ToString(); - var value = entry.Value.ToString(); - - if (envKey.StartsWith("QC_")) - { - var key = envKey.Substring(3).Replace("_", "-").ToLower(); + var key = envKey.Substring(3).Replace("_", "-").ToLower(); - Log.Trace($"TestSetup(): Updating config setting '{key}' from environment var '{envKey}'"); - Config.Set(key, value); - } + Log.Trace($"TestSetup(): Updating config setting '{key}' from environment var '{envKey}'"); + Config.Set(key, value); } - - // resets the version among other things - Globals.Reset(); - } - - private static void SetUp() - { - Log.LogHandler = new CompositeLogHandler(); - Log.Trace("TestSetup(): starting..."); - ReloadConfiguration(); - Log.DebuggingEnabled = Config.GetBool("debug-mode"); } - private static TestCaseData[] TestParameters - { - get - { - SetUp(); - return new [] { new TestCaseData() }; - } - } + // resets the version among other things + Globals.Reset(); } } diff --git a/QuantConnect.DataBento.Tests/config.json b/QuantConnect.DataBento.Tests/config.json index ff861ca..0d61d83 100644 --- a/QuantConnect.DataBento.Tests/config.json +++ b/QuantConnect.DataBento.Tests/config.json @@ -1,9 +1,8 @@ { - "data-folder":"../../../../../Lean/Data/", - - "job-user-id": "0", + "data-folder": "../../../../Lean/Data/", + "job-user-id": "", "api-access-token": "", "job-organization-id": "", - - "databento-api-key":"" + "databento-api-key": "", + "debug-mode": false } \ No newline at end of file diff --git a/QuantConnect.DataBento/Api/HistoricalAPIClient.cs b/QuantConnect.DataBento/Api/HistoricalAPIClient.cs new file mode 100644 index 0000000..23c79ee --- /dev/null +++ b/QuantConnect.DataBento/Api/HistoricalAPIClient.cs @@ -0,0 +1,157 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System.Net; +using System.Text; +using QuantConnect.Util; +using QuantConnect.Logging; +using System.Net.Http.Headers; +using QuantConnect.Lean.DataSource.DataBento.Models; + +namespace QuantConnect.Lean.DataSource.DataBento.Api; + +public class HistoricalAPIClient : IDisposable +{ + private readonly HttpClient _httpClient = new() + { + BaseAddress = new Uri("https://hist.databento.com") + }; + + public HistoricalAPIClient(string apiKey) + { + _httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue( + AuthenticationSchemes.Basic.ToString(), + // Basic Auth expects "username:password". Using ":" means API key with an empty password. + Convert.ToBase64String(Encoding.UTF8.GetBytes($"{apiKey}:")) + ); + } + + public IEnumerable GetHistoricalOhlcvBars(string symbol, DateTime startDateTimeUtc, DateTime endDateTimeUtc, Resolution resolution, string dataSet) + { + string schema; + switch (resolution) + { + case Resolution.Second: + schema = "ohlcv-1s"; + break; + case Resolution.Minute: + schema = "ohlcv-1m"; + break; + case Resolution.Hour: + schema = "ohlcv-1h"; + break; + case Resolution.Daily: + schema = "ohlcv-1d"; + break; + default: + throw new ArgumentException($"Unsupported resolution {resolution} for OHLCV data."); + } + + return GetRange(symbol, startDateTimeUtc, endDateTimeUtc, schema, dataSet); + } + + public IEnumerable GetTickBars(string symbol, DateTime startDateTimeUtc, DateTime endDateTimeUtc, string dataSet) + { + return GetRange(symbol, startDateTimeUtc, endDateTimeUtc, "mbp-1", dataSet, useLimit: true); + } + + public IEnumerable GetOpenInterest(string symbol, DateTime startDateTimeUtc, DateTime endDateTimeUtc, string dataSet) + { + foreach (var statistics in GetRange(symbol, startDateTimeUtc, endDateTimeUtc, "statistics", dataSet)) + { + if (statistics.StatType == Models.Enums.StatisticType.OpenInterest) + { + yield return statistics; + } + } + } + + private IEnumerable GetRange(string symbol, DateTime startDateTimeUtc, DateTime endDateTimeUtc, string schema, string dataSet, bool useLimit = false) where T : MarketDataRecord + { + var formData = new Dictionary + { + { "dataset", dataSet }, + { "end", Time.DateTimeToUnixTimeStampNanoseconds(endDateTimeUtc).ToStringInvariant() }, + { "symbols", symbol }, + { "schema", schema }, + { "encoding", "json" }, + { "stype_in", "raw_symbol" }, + { "pretty_px", "true" }, + }; + + if (useLimit) + { + formData["limit"] = "10000"; + } + + var start = startDateTimeUtc; + var httpStatusCode = default(HttpStatusCode); + do + { + formData["start"] = Time.DateTimeToUnixTimeStampNanoseconds(start).ToStringInvariant(); + + using var content = new FormUrlEncodedContent(formData); + + using var requestMessage = new HttpRequestMessage(HttpMethod.Post, "/v0/timeseries.get_range") + { + Content = content + }; + + using var response = _httpClient.Send(requestMessage); + + if (response.Headers.TryGetValues("x-warning", out var warnings)) + { + foreach (var warning in warnings) + { + Log.Trace($"{nameof(HistoricalAPIClient)}.{nameof(GetRange)}: {warning}"); + } + } + + using var stream = response.Content.ReadAsStream(); + + if (stream.Length == 0) + { + continue; + } + + using var reader = new StreamReader(stream); + + var line = default(string); + if (response.StatusCode == HttpStatusCode.UnprocessableContent) + { + line = reader.ReadLine(); + Log.Trace($"{nameof(HistoricalAPIClient)}.{nameof(GetRange)}.Response: {line}. " + + $"Request: [{response.RequestMessage?.Method}]({response.RequestMessage?.RequestUri}), " + + $"Payload: {string.Join(", ", formData.Select(kvp => $"{kvp.Key}: {kvp.Value}"))}"); + yield break; + } + + httpStatusCode = response.EnsureSuccessStatusCode().StatusCode; + + var data = default(T); + while ((line = reader.ReadLine()) != null) + { + data = line.DeserializeKebabCase(); + yield return data; + } + start = data.Header.UtcTime.AddTicks(1); + } while (httpStatusCode == HttpStatusCode.PartialContent); + } + + public void Dispose() + { + _httpClient?.DisposeSafely(); + } +} diff --git a/QuantConnect.DataBento/Api/LiveAPIClient.cs b/QuantConnect.DataBento/Api/LiveAPIClient.cs new file mode 100644 index 0000000..3629b95 --- /dev/null +++ b/QuantConnect.DataBento/Api/LiveAPIClient.cs @@ -0,0 +1,128 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +using QuantConnect.Util; +using QuantConnect.Logging; +using QuantConnect.Lean.DataSource.DataBento.Models; +using QuantConnect.Lean.DataSource.DataBento.Models.Live; + +namespace QuantConnect.Lean.DataSource.DataBento.Api; + +public sealed class LiveAPIClient : IDisposable +{ + private readonly string _apiKey; + + private readonly Dictionary _tcpClientByDataSet = []; + + private readonly Action _levelOneDataHandler; + + public event EventHandler? SymbolMappingConfirmation; + + public event EventHandler? ConnectionLost; + + public bool IsConnected => _tcpClientByDataSet.Values.All(c => c.IsConnected); + + public LiveAPIClient(string apiKey, Action levelOneDataHandler) + { + _apiKey = apiKey; + _levelOneDataHandler = levelOneDataHandler; + } + + public void Dispose() + { + foreach (var tcpClient in _tcpClientByDataSet.Values) + { + tcpClient.DisposeSafely(); + } + _tcpClientByDataSet.Clear(); + } + + private LiveDataTcpClientWrapper EnsureDatasetConnection(string dataSet) + { + if (_tcpClientByDataSet.TryGetValue(dataSet, out var liveDataTcpClient) && liveDataTcpClient.IsConnected) + { + return liveDataTcpClient; + } + + LogTrace(nameof(EnsureDatasetConnection), "Starting connection to DataBento live API"); + + if (liveDataTcpClient == null) + { + liveDataTcpClient = new LiveDataTcpClientWrapper(dataSet, _apiKey, MessageReceived); + + liveDataTcpClient.ConnectionLost += (sender, message) => + { + LogError(nameof(EnsureDatasetConnection), $"Connection lost to DataBento live API (Dataset: {dataSet}). Reason: {message}"); + ConnectionLost?.Invoke(this, new ConnectionLostEventArgs(dataSet, message)); + }; + + _tcpClientByDataSet[dataSet] = liveDataTcpClient; + } + + liveDataTcpClient.Connect(); + + if (!liveDataTcpClient.IsConnected) + { + var msg = $"Unable to establish a connection to the DataBento Live API (Dataset: {dataSet})."; + LogError(nameof(EnsureDatasetConnection), msg); + throw new Exception(msg); + } + + LogTrace(nameof(EnsureDatasetConnection), $"Successfully connected to DataBento live API (Dataset: {dataSet})"); + + return liveDataTcpClient; + } + + public bool Subscribe(string dataSet, string symbol) + { + EnsureDatasetConnection(dataSet).SubscribeOnMarketBestPriceLevelOne(symbol); + return true; + } + + private void MessageReceived(string message) + { + var data = message.DeserializeSnakeCaseLiveData(); + + if (data == null) + { + LogError(nameof(MessageReceived), $"Failed to deserialize live data message: {message}"); + return; + } + + switch (data) + { + case SymbolMappingMessage smm: + SymbolMappingConfirmation?.Invoke(this, new(smm.StypeInSymbol, smm.Header.InstrumentId)); + break; + case LevelOneData lod: + _levelOneDataHandler?.Invoke(lod); + break; + default: + LogError(nameof(MessageReceived), $"Received unsupported record type: {data.Header.Rtype}. Message: {message}"); + break; + } + } + + private static void LogTrace(string method, string message) + { + Log.Trace($"LiveAPIClient.{method}: {message}"); + } + + private static void LogError(string method, string message) + { + Log.Error($"LiveAPIClient.{method}: {message}"); + } +} diff --git a/QuantConnect.DataBento/Api/LiveDataTcpClientWrapper.cs b/QuantConnect.DataBento/Api/LiveDataTcpClientWrapper.cs new file mode 100644 index 0000000..87f1581 --- /dev/null +++ b/QuantConnect.DataBento/Api/LiveDataTcpClientWrapper.cs @@ -0,0 +1,250 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +using System.Text; +using QuantConnect.Util; +using System.Net.Sockets; +using QuantConnect.Logging; +using QuantConnect.Lean.DataSource.DataBento.Models.Live; + +namespace QuantConnect.Lean.DataSource.DataBento.Api; + +public sealed class LiveDataTcpClientWrapper : IDisposable +{ + private const int DefaultPort = 13000; + + private readonly string _gateway; + private readonly string _dataSet; + private readonly string _apiKey; + private readonly TimeSpan _heartBeatInterval = TimeSpan.FromSeconds(10); + + private TcpClient _tcpClient; + private readonly CancellationTokenSource _cancellationTokenSource = new(); + + private NetworkStream? _stream; + private StreamReader? _reader; + private Task? _dataReceiverTask; + private bool _isConnected; + + private readonly Action MessageReceived; + + public event EventHandler? ConnectionLost; + + /// + /// Is client connected + /// + public bool IsConnected => _isConnected; + + public LiveDataTcpClientWrapper(string dataSet, string apiKey, Action messageReceived) + { + _apiKey = apiKey; + _dataSet = dataSet; + _gateway = DetermineGateway(dataSet); + MessageReceived = messageReceived; + } + + public void Connect() + { + var attemptToConnect = 1; + var error = default(string); + do + { + try + { + _tcpClient = new(_gateway, DefaultPort); + _stream = _tcpClient.GetStream(); + _reader = new StreamReader(_stream, Encoding.ASCII); + + if (!Authenticate(_dataSet).SynchronouslyAwaitTask()) + throw new Exception("Authentication failed"); + + _dataReceiverTask = new Task(async () => await DataReceiverAsync(_cancellationTokenSource.Token), _cancellationTokenSource.Token, TaskCreationOptions.LongRunning); + _dataReceiverTask.Start(); + + _isConnected = true; + } + catch (Exception ex) + { + error = ex.Message; + } + + var retryDelayMs = attemptToConnect * 2 * 1000; + LogError(nameof(Connect), $"Connection attempt #{attemptToConnect} failed. Retrying in {retryDelayMs} ms. Error: {error}"); + _cancellationTokenSource.Token.WaitHandle.WaitOne(attemptToConnect * 2 * 1000); + + } while (attemptToConnect++ < 5 && !_isConnected); + } + + private void Close() + { + _isConnected = false; + + _reader?.Close(); + _reader?.DisposeSafely(); + + _dataReceiverTask?.DisposeSafely(); + _tcpClient?.Close(); + _tcpClient?.DisposeSafely(); + } + + public void Dispose() + { + _isConnected = false; + + _reader?.Close(); + _reader?.DisposeSafely(); + + _cancellationTokenSource?.Cancel(); + _cancellationTokenSource?.DisposeSafely(); + + _dataReceiverTask?.DisposeSafely(); + _tcpClient?.Close(); + _tcpClient?.DisposeSafely(); + } + + public void SubscribeOnMarketBestPriceLevelOne(string symbol) + { + var request = $"schema=mbp-1|stype_in=raw_symbol|symbols={symbol}"; + WriteData(request); + } + + private async Task DataReceiverAsync(CancellationToken ct) + { + var methodName = nameof(DataReceiverAsync); + + var readTimeoutCts = CancellationTokenSource.CreateLinkedTokenSource(ct); + + var readTimeout = _heartBeatInterval.Add(TimeSpan.FromSeconds(5)); + + LogTrace(methodName, "Task Receiver started"); + + var errorMessage = string.Empty; + + try + { + while (!ct.IsCancellationRequested && IsConnected) + { + // Reset timeout + readTimeoutCts.CancelAfter(readTimeout); + + var line = await ReadDataAsync(readTimeoutCts.Token); + + if (line == null) + { + Log.Error("Remote closed connection"); + break; + } + + MessageReceived.Invoke(line); + } + } + catch (OperationCanceledException oce) + { + errorMessage = $"Read timeout exceeded: Outer CancellationToken: {ct.IsCancellationRequested}, Read Timeout: {readTimeoutCts.IsCancellationRequested}"; + LogTrace(methodName, errorMessage); + } + catch (Exception ex) + { + errorMessage += ex.Message; + LogError(methodName, $"Error processing messages: {ex.Message}\n{ex.StackTrace}"); + } + finally + { + LogTrace(methodName, "Task Receiver stopped"); + Close(); + readTimeoutCts.Dispose(); + ConnectionLost?.Invoke(this, new($"{errorMessage}. TcpConnected: {_tcpClient.Connected}")); + } + } + + private async Task ReadDataAsync(CancellationToken cancellationToken) + { + return await _reader.ReadLineAsync(cancellationToken); + } + + private void WriteData(string data) + { + if (!data.EndsWith('\n')) + { + data += '\n'; + } + var bytes = Encoding.ASCII.GetBytes(data); + _stream.Write(bytes, 0, bytes.Length); + } + + private async Task Authenticate(string dataSet) + { + var cts = CancellationTokenSource.CreateLinkedTokenSource(_cancellationTokenSource.Token); + + try + { + var versionLine = await ReadDataAsync(cts.Token); + var cramLine = await ReadDataAsync(cts.Token); + + if (Log.DebuggingEnabled) + { + LogDebug(nameof(Authenticate), $"Received initial message: {versionLine}, {cramLine}"); + } + + var request = new AuthenticationMessageRequest(cramLine, _apiKey, dataSet, _heartBeatInterval); + + LogTrace("Authenticate", $"Sending CRAM reply: {request}"); + + WriteData(request.ToString()); + + var authResponse = await ReadDataAsync(cts.Token); + + var authenticationResponse = new AuthenticationMessageResponse(authResponse); + + if (!authenticationResponse.Success) + { + LogError(nameof(Authenticate), $"Authentication response: {authResponse}"); + return false; + } + + LogTrace(nameof(Authenticate), $"Successfully authenticated with session ID: {authenticationResponse.SessionId}"); + + WriteData(request.GetStartSessionMessage()); // after start_session -> we get heartbeats and data + + return true; + } + finally + { + cts.DisposeSafely(); + } + } + + private static string DetermineGateway(string dataset) + { + dataset = dataset.Replace('.', '-').ToLowerInvariant(); + return dataset + ".lsg.databento.com"; + } + + private void LogTrace(string method, string message) + { + Log.Trace($"LiveDataTcpClientWrapper[{_dataSet}].{method}: {message}"); + } + + private void LogError(string method, string message) + { + Log.Error($"LiveDataTcpClientWrapper[{_dataSet}].{method}: {message}"); + } + + private void LogDebug(string method, string message) + { + Log.Debug($"LiveDataTcpClientWrapper[{_dataSet}].{method}: {message}"); + } +} diff --git a/QuantConnect.DataBento/Converters/LiveDataConverter.cs b/QuantConnect.DataBento/Converters/LiveDataConverter.cs new file mode 100644 index 0000000..395706f --- /dev/null +++ b/QuantConnect.DataBento/Converters/LiveDataConverter.cs @@ -0,0 +1,82 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using QuantConnect.Lean.DataSource.DataBento.Models; +using QuantConnect.Lean.DataSource.DataBento.Models.Enums; +using QuantConnect.Lean.DataSource.DataBento.Models.Live; + +namespace QuantConnect.Lean.DataSource.DataBento.Converters; + +public class LiveDataConverter : JsonConverter +{ + private const string headerIdentifier = "hd"; + + private const string recordTypeIdentifier = "rtype"; + + private static JsonSerializer _snakeSerializer = new JsonSerializer + { + ContractResolver = Serialization.SnakeCaseContractResolver.Instance + }; + + /// + /// Gets a value indicating whether this can write JSON. + /// + /// true if this can write JSON; otherwise, false. + public override bool CanWrite => false; + + /// + /// Gets a value indicating whether this can read JSON. + /// + /// true if this can read JSON; otherwise, false. + public override bool CanRead => true; + + /// + /// Reads the JSON representation of the object. + /// + /// The to read from. + /// Type of the object. + /// The existing property value of the JSON that is being converted. + /// The calling serializer. + /// The object value. + public override MarketDataRecord ReadJson(JsonReader reader, Type objectType, MarketDataRecord? existingValue, bool hasExistingValue, JsonSerializer serializer) + { + var jObject = JObject.Load(reader); + + var recordType = jObject[headerIdentifier]?[recordTypeIdentifier]?.ToObject(); + + switch (recordType) + { + case RecordType.SymbolMapping: + return jObject.ToObject(_snakeSerializer); + case RecordType.MarketByPriceDepth1: + return jObject.ToObject(_snakeSerializer); + default: + return null; + } + } + + /// + /// Writes the JSON representation of the object. + /// + /// The to write to. + /// The value. + /// The calling serializer. + public override void WriteJson(JsonWriter writer, MarketDataRecord? value, JsonSerializer serializer) + { + throw new NotImplementedException(); + } +} \ No newline at end of file diff --git a/QuantConnect.DataBento/DataBentoDataDownloader.cs b/QuantConnect.DataBento/DataBentoDataDownloader.cs index 2a44e55..9d78ce2 100644 --- a/QuantConnect.DataBento/DataBentoDataDownloader.cs +++ b/QuantConnect.DataBento/DataBentoDataDownloader.cs @@ -1,6 +1,6 @@ /* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,290 +14,84 @@ * */ -using System; -using System.IO; -using System.Text; -using System.Net.Http; -using System.Net.Http.Headers; -using System.Globalization; -using System.Collections.Generic; -using CsvHelper; -using CsvHelper.Configuration.Attributes; using QuantConnect.Data; -using QuantConnect.Data.Market; using QuantConnect.Util; -using QuantConnect.Configuration; -using QuantConnect.Interfaces; using QuantConnect.Securities; +using QuantConnect.Configuration; -namespace QuantConnect.Lean.DataSource.DataBento +namespace QuantConnect.Lean.DataSource.DataBento; + +/// +/// Data downloader class for pulling data from DataBento +/// +public class DataBentoDataDownloader : IDataDownloader, IDisposable { /// - /// Data downloader for historical data from DataBento's Raw HTTP API - /// Converts DataBento data to Lean data types + /// Provides access to historical market data via the DataBento service. /// - public class DataBentoDataDownloader : IDataDownloader, IDisposable - { - private readonly HttpClient _httpClient; - private readonly string _apiKey; - private const decimal PriceScaleFactor = 1e-9m; - - /// - /// Initializes a new instance of the - /// - public DataBentoDataDownloader(string apiKey) - { - _apiKey = apiKey; - _httpClient = new HttpClient(); - - // Set up HTTP Basic Authentication - var credentials = Convert.ToBase64String(Encoding.ASCII.GetBytes($"{_apiKey}:")); - _httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", credentials); - } - - public DataBentoDataDownloader() - : this(Config.Get("databento-api-key")) - { - } - - /// - /// Get historical data enumerable for a single symbol, type and resolution given this start and end time (in UTC). - /// - /// Parameters for the historical data request - /// Enumerable of base data for this symbol - /// - public IEnumerable Get(DataDownloaderGetParameters parameters) - { - var symbol = parameters.Symbol; - var resolution = parameters.Resolution; - var tickType = parameters.TickType; - - var dataset = "GLBX.MDP3"; // hard coded for now. Later on can add equities and options with different mapping - var schema = GetSchema(resolution, tickType); - var dbSymbol = MapSymbolToDataBento(symbol); - - // prepare body for Raw HTTP request - var body = new StringBuilder(); - body.Append($"dataset={dataset}"); - body.Append($"&symbols={dbSymbol}"); - body.Append($"&schema={schema}"); - body.Append($"&start={parameters.StartUtc:yyyy-MM-ddTHH:mm}"); - body.Append($"&end={parameters.EndUtc:yyyy-MM-ddTHH:mm}"); - body.Append("&stype_in=parent"); - body.Append("&encoding=csv"); - - var request = new HttpRequestMessage( - HttpMethod.Post, - "https://hist.databento.com/v0/timeseries.get_range") - { - Content = new StringContent(body.ToString(), Encoding.UTF8, "application/x-www-form-urlencoded") - }; - - // send the request with the get range url - var response = _httpClient.Send(request); - - // Add error handling to see the actual error message - if (!response.IsSuccessStatusCode) - { - var errorContent = response.Content.ReadAsStringAsync().Result; - throw new HttpRequestException($"DataBento API error ({response.StatusCode}): {errorContent}"); - } - - response.EnsureSuccessStatusCode(); - - using var stream = response.Content.ReadAsStream(); - using var reader = new StreamReader(stream); - using var csv = new CsvReader(reader, CultureInfo.InvariantCulture); - - if (tickType == TickType.Trade) - { - if (resolution == Resolution.Tick) - { - // For tick data, use the trades schema which returns individual trades - foreach (var record in csv.GetRecords()) - { - yield return new Tick - { - Time = record.Timestamp, - Symbol = symbol, - Value = record.Price, - Quantity = record.Size - }; - } - } - else - { - // For aggregated data, use the ohlcv schema which returns bars - foreach (var record in csv.GetRecords()) - { - yield return new TradeBar - { - Symbol = symbol, - Time = record.Timestamp, - Open = record.Open, - High = record.High, - Low = record.Low, - Close = record.Close, - Volume = record.Volume - }; - } - } - } - else if (tickType == TickType.Quote) - { - foreach (var record in csv.GetRecords()) - { - var bidPrice = record.BidPrice * PriceScaleFactor; - var askPrice = record.AskPrice * PriceScaleFactor; - - if (resolution == Resolution.Tick) - { - yield return new Tick - { - Time = record.Timestamp, - Symbol = symbol, - AskPrice = askPrice, - BidPrice = bidPrice, - AskSize = record.AskSize, - BidSize = record.BidSize, - TickType = TickType.Quote - }; - } - else - { - var bidBar = new Bar(bidPrice, bidPrice, bidPrice, bidPrice); - var askBar = new Bar(askPrice, askPrice, askPrice, askPrice); - yield return new QuoteBar( - record.Timestamp, - symbol, - bidBar, - record.BidSize, - askBar, - record.AskSize - ); - } - } - } - } - - /// - /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources. - /// - public void Dispose() - { - _httpClient?.DisposeSafely(); - } - - /// - /// Pick Databento schema from Lean resolution/ticktype - /// - private string GetSchema(Resolution resolution, TickType tickType) - { - if (tickType == TickType.Trade) - { - if (resolution == Resolution.Tick) - return "trades"; - if (resolution == Resolution.Second) - return "ohlcv-1s"; - if (resolution == Resolution.Minute) - return "ohlcv-1m"; - if (resolution == Resolution.Hour) - return "ohlcv-1h"; - if (resolution == Resolution.Daily) - return "ohlcv-1d"; - } - else if (tickType == TickType.Quote) - { - // top of book - if (resolution == Resolution.Tick || resolution == Resolution.Second || resolution == Resolution.Minute || resolution == Resolution.Hour || resolution == Resolution.Daily) - return "mbp-1"; - } - - throw new NotSupportedException($"Unsupported resolution {resolution} / {tickType}"); - } - - /// - /// Maps a LEAN symbol to DataBento symbol format - /// - private string MapSymbolToDataBento(Symbol symbol) - { - if (symbol.SecurityType == SecurityType.Future) - { - // For DataBento, use the root symbol with .FUT suffix for parent subscription - // ES19Z25 -> ES.FUT - var value = symbol.Value; + private readonly DataBentoProvider _historyProvider; - // Extract root by removing digits and month codes - var root = new string(value.TakeWhile(c => !char.IsDigit(c)).ToArray()); - - return $"{root}.FUT"; - } - - return symbol.Value; - } + /// + /// Provides exchange trading hours and market-specific time zone information. + /// + private readonly MarketHoursDatabase _marketHoursDatabase; - /// Class for parsing trade data from Databento - /// Really used as a map from the http request to then get it in QC data structures - private class DatabentoBar - { - [Name("ts_event")] - public long TimestampNanos { get; set; } + /// + /// Initializes a new instance of the + /// getting the DataBento API key from the configuration + /// + public DataBentoDataDownloader() + : this(Config.Get("databento-api-key")) + { - public DateTime Timestamp => DateTimeOffset.FromUnixTimeSeconds(TimestampNanos / 1_000_000_000) - .AddTicks((TimestampNanos % 1_000_000_000) / 100).UtcDateTime; + } - [Name("open")] - public decimal Open { get; set; } + /// + /// Initializes a new instance of the + /// + /// The DataBento API key. + public DataBentoDataDownloader(string apiKey) + { + _historyProvider = new DataBentoProvider(apiKey); + _marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); + } - [Name("high")] - public decimal High { get; set; } + /// + /// Get historical data enumerable for a single symbol, type and resolution given this start and end time (in UTC). + /// + /// Parameters for the historical data request + /// Enumerable of base data for this symbol + public IEnumerable? Get(DataDownloaderGetParameters parameters) + { + var symbol = parameters.Symbol; + var resolution = parameters.Resolution; + var startUtc = parameters.StartUtc; + var endUtc = parameters.EndUtc; + var tickType = parameters.TickType; - [Name("low")] - public decimal Low { get; set; } + var dataType = LeanData.GetDataType(resolution, tickType); + var exchangeHours = _marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType); + var dataTimeZone = _marketHoursDatabase.GetDataTimeZone(symbol.ID.Market, symbol, symbol.SecurityType); - [Name("close")] - public decimal Close { get; set; } + var historyRequest = new HistoryRequest(startUtc, endUtc, dataType, symbol, resolution, exchangeHours, dataTimeZone, resolution, + true, false, DataNormalizationMode.Raw, tickType); - [Name("volume")] - public decimal Volume { get; set; } - } + var historyData = _historyProvider.GetHistory(historyRequest); - private class DatabentoTrade + if (historyData == null) { - [Name("ts_event")] - public long TimestampNanos { get; set; } - - public DateTime Timestamp => DateTimeOffset.FromUnixTimeSeconds(TimestampNanos / 1_000_000_000) - .AddTicks((TimestampNanos % 1_000_000_000) / 100).UtcDateTime; - - [Name("price")] - public long PriceRaw { get; set; } - - public decimal Price => PriceRaw * PriceScaleFactor; - - [Name("size")] - public int Size { get; set; } + return null; } - private class DatabentoQuote - { - [Name("ts_event")] - public long TimestampNanos { get; set; } - - public DateTime Timestamp => DateTimeOffset.FromUnixTimeSeconds(TimestampNanos / 1_000_000_000) - .AddTicks((TimestampNanos % 1_000_000_000) / 100).UtcDateTime; - - [Name("bid_px_00")] - public long BidPrice { get; set; } - - [Name("bid_sz_00")] - public int BidSize { get; set; } + return historyData; + } - [Name("ask_px_00")] - public long AskPrice { get; set; } - [Name("ask_sz_00")] - public int AskSize { get; set; } - } + /// + /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources. + /// + public void Dispose() + { + _historyProvider.DisposeSafely(); } -} +} \ No newline at end of file diff --git a/QuantConnect.DataBento/DataBentoDataProvider.cs b/QuantConnect.DataBento/DataBentoDataProvider.cs index 8cf6015..23fd465 100644 --- a/QuantConnect.DataBento/DataBentoDataProvider.cs +++ b/QuantConnect.DataBento/DataBentoDataProvider.cs @@ -1,6 +1,6 @@ /* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,403 +14,412 @@ * */ -using System; -using System.Linq; -using NodaTime; -using QuantConnect.Data; -using QuantConnect.Data.Market; +using System.Net; +using System.Text; +using Newtonsoft.Json; +using QuantConnect.Api; using QuantConnect.Util; -using QuantConnect.Interfaces; -using System.Collections.Generic; -using QuantConnect.Configuration; +using QuantConnect.Data; +using Newtonsoft.Json.Linq; using QuantConnect.Logging; using QuantConnect.Packets; -using QuantConnect.Securities; +using QuantConnect.Interfaces; +using QuantConnect.Configuration; +using System.Security.Cryptography; +using System.Net.NetworkInformation; using System.Collections.Concurrent; - -namespace QuantConnect.Lean.DataSource.DataBento +using QuantConnect.Brokerages.LevelOneOrderBook; +using QuantConnect.Lean.DataSource.DataBento.Api; +using QuantConnect.Lean.DataSource.DataBento.Models; +using QuantConnect.Lean.DataSource.DataBento.Models.Live; + +namespace QuantConnect.Lean.DataSource.DataBento; + +/// +/// A data Provider for DataBento that provides live market data and historical data. +/// Handles Subscribing, Unsubscribing, and fetching historical data from DataBento. +/// It will handle if a symbol is subscribable and will log errors if it is not. +/// +public partial class DataBentoProvider : IDataQueueHandler { /// - /// Implementation of Custom Data Provider + /// Resolves map files to correctly handle current and historical ticker symbols. + /// + private readonly IMapFileProvider _mapFileProvider = Composer.Instance.GetPart(); + + private HistoricalAPIClient _historicalApiClient; + + private readonly DataBentoSymbolMapper _symbolMapper = new(); + + private readonly ConcurrentDictionary _pendingSubscriptions = []; + + private readonly Dictionary _subscribedSymbolsByDataBentoInstrumentId = []; + + /// + /// Manages Level 1 market data subscriptions and routing of updates to the shared . + /// Responsible for tracking and updating individual instances per symbol. + /// + private LevelOneServiceManager _levelOneServiceManager; + + private IDataAggregator _aggregator; + + private LiveAPIClient _liveApiClient; + + private bool _initialized; + + /// + /// Returns true if we're currently connected to the Data Provider + /// + public bool IsConnected => _liveApiClient.IsConnected; + + + /// + /// Initializes a new instance of the DataBentoProvider /// - public class DataBentoProvider : IDataQueueHandler + public DataBentoProvider() + : this(Config.Get("databento-api-key")) { - private readonly IDataAggregator _dataAggregator = Composer.Instance.GetExportedValueByTypeName( - Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager"), forceTypeNameOnExisting: false); - private EventBasedDataQueueHandlerSubscriptionManager _subscriptionManager = null!; - private readonly List _activeSubscriptionConfigs = new(); - private readonly ConcurrentDictionary _subscriptionConfigs = new(); - private DatabentoRawClient _client = null!; - private readonly string _apiKey; - private readonly DataBentoDataDownloader _dataDownloader; - private bool _potentialUnsupportedResolutionMessageLogged; - private bool _sessionStarted = false; - private readonly object _sessionLock = new object(); - private readonly MarketHoursDatabase _marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); - private readonly ConcurrentDictionary _symbolExchangeTimeZones = new(); - - /// - /// Returns true if we're currently connected to the Data Provider - /// - public bool IsConnected => _client?.IsConnected == true; - - /// - /// Initializes a new instance of the DataBentoProvider - /// - public DataBentoProvider() - { - _apiKey = Config.Get("databento-api-key"); - if (string.IsNullOrEmpty(_apiKey)) - { - throw new ArgumentException("DataBento API key is required. Set 'databento-api-key' in configuration."); - } + } - _dataDownloader = new DataBentoDataDownloader(_apiKey); - Initialize(); + public DataBentoProvider(string apiKey) + { + if (string.IsNullOrWhiteSpace(apiKey)) + { + // If the API key is not provided, we can't do anything. + // The handler might going to be initialized using a node packet job. + return; } - /// - /// Initializes a new instance of the DataBentoProvider with custom API key - /// - /// DataBento API key - public DataBentoProvider(string apiKey) + Initialize(apiKey); + } + + /// + /// Common initialization logic + /// DataBento API key from config file retrieved on constructor + /// + private void Initialize(string apiKey) + { + ValidateSubscription(); + + _aggregator = Composer.Instance.GetPart(); + if (_aggregator == null) { - _apiKey = apiKey ?? throw new ArgumentNullException(nameof(apiKey)); - _dataDownloader = new DataBentoDataDownloader(_apiKey); - Initialize(); + var aggregatorName = Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager"); + Log.Trace($"{nameof(DataBentoProvider)}.{nameof(Initialize)}: found no data aggregator instance, creating {aggregatorName}"); + _aggregator = Composer.Instance.GetExportedValueByTypeName(aggregatorName, forceTypeNameOnExisting: false); } - /// - /// Common initialization logic - /// - private void Initialize() - { - Log.Trace("DataBentoProvider.Initialize(): Starting initialization"); - _subscriptionManager = new EventBasedDataQueueHandlerSubscriptionManager(); - _subscriptionManager.SubscribeImpl = (symbols, tickType) => - { - Log.Trace($"DataBentoProvider.SubscribeImpl(): Received subscription request for {symbols.Count()} symbols, TickType={tickType}"); - foreach (var symbol in symbols) - { - Log.Trace($"DataBentoProvider.SubscribeImpl(): Processing symbol {symbol}"); - if (!_subscriptionConfigs.TryGetValue(symbol, out var config)) - { - Log.Error($"DataBentoProvider.SubscribeImpl(): No subscription config found for {symbol}"); - return false; - } - if (_client?.IsConnected != true) - { - Log.Error($"DataBentoProvider.SubscribeImpl(): Client is not connected. Cannot subscribe to {symbol}"); - return false; - } + _liveApiClient = new LiveAPIClient(apiKey, HandleLevelOneData); + _liveApiClient.SymbolMappingConfirmation += OnSymbolMappingConfirmation; + _liveApiClient.ConnectionLost += OnConnectionLost; - var resolution = config.Resolution > Resolution.Tick ? Resolution.Tick : config.Resolution; - if (!_client.Subscribe(config.Symbol, resolution, config.TickType)) - { - Log.Error($"Failed to subscribe to {config.Symbol}"); - return false; - } + _historicalApiClient = new(apiKey); - lock (_sessionLock) - { - if (!_sessionStarted) - _sessionStarted = _client.StartSession(); - } - } + _levelOneServiceManager = new LevelOneServiceManager( + _aggregator, + (symbols, _) => Subscribe(symbols), + (symbols, _) => Unsubscribe(symbols)); - return true; - }; + _initialized = true; + } - _subscriptionManager.UnsubscribeImpl = (symbols, tickType) => - { - foreach (var symbol in symbols) - { - Log.Trace($"DataBentoProvider.UnsubscribeImpl(): Processing symbol {symbol}"); - if (_client?.IsConnected != true) - { - throw new InvalidOperationException($"DataBentoProvider.UnsubscribeImpl(): Client is not connected. Cannot unsubscribe from {symbol}"); - } + private void OnConnectionLost(object? _, ConnectionLostEventArgs cle) + { + LogTrace(nameof(OnConnectionLost), "The connection was lost. Starting ReSubscription process"); - _client.Unsubscribe(symbol); - } + var symbols = _levelOneServiceManager.GetSubscribedSymbols(); - return true; - }; + Subscribe(symbols); - // Initialize the live client - Log.Trace("DataBentoProvider.Initialize(): Creating DatabentoRawClient"); - _client = new DatabentoRawClient(_apiKey); - _client.DataReceived += OnDataReceived; - _client.ConnectionStatusChanged += OnConnectionStatusChanged; + LogTrace(nameof(OnConnectionLost), $"Re-subscription completed successfully for {_levelOneServiceManager.Count} symbol(s)."); + } - // Connect to live gateway - Log.Trace("DataBentoProvider.Initialize(): Attempting connection to DataBento live gateway"); - Task.Run(() => - { - var connected = _client.Connect(); - Log.Trace($"DataBentoProvider.Initialize(): Connect() returned {connected}"); + private void OnSymbolMappingConfirmation(object? _, SymbolMappingConfirmationEventArgs smce) + { + if (_pendingSubscriptions.TryRemove(smce.Symbol, out var symbol)) + { + _subscribedSymbolsByDataBentoInstrumentId[smce.InstrumentId] = symbol; + } + } - if (connected) - { - Log.Trace("DataBentoProvider.Initialize(): Successfully connected to DataBento live gateway"); - } - else - { - Log.Error("DataBentoProvider.Initialize(): Failed to connect to DataBento live gateway"); - } - }); + private void HandleLevelOneData(LevelOneData levelOneData) + { + if (_subscribedSymbolsByDataBentoInstrumentId.TryGetValue(levelOneData.Header.InstrumentId, out var symbol)) + { + var time = levelOneData.Header.UtcTime; + _levelOneServiceManager.HandleLastTrade(symbol, time, levelOneData.Size, levelOneData.Price); - Log.Trace("DataBentoProvider.Initialize(): Initialization complete"); + foreach (var l in levelOneData.Levels) + { + _levelOneServiceManager.HandleQuote(symbol, time, l.BidPx, l.BidSz, l.AskPx, l.AskSz); + } } + } - /// - /// Subscribe to the specified configuration - /// - /// defines the parameters to subscribe to a data feed - /// handler to be fired on new data available - /// The new enumerator for this subscription request - public IEnumerator? Subscribe(SubscriptionDataConfig dataConfig, EventHandler newDataAvailableHandler) + /// + /// Attempts to resolve the DataBento dataset for the specified symbol based on its Lean market. + /// + /// The symbol whose market is used to determine the DataBento dataset. + /// + /// When this method returns true, contains the resolved DataBento dataset; otherwise, null. + /// + /// + /// true if a DataBento dataset mapping exists for the symbol's market; otherwise, false. + /// + private bool TryGetDataBentoDataSet(Symbol symbol, out string? dataSet) + { + return _symbolMapper.DataBentoDataSetByLeanMarket.TryGetValue(symbol.ID.Market, out dataSet); + } + + /// + /// Logic to subscribe to the specified symbols + /// + public bool Subscribe(IEnumerable symbols) + { + foreach (var symbol in symbols) { - Log.Trace($"DataBentoProvider.Subscribe(): Received subscription request for {dataConfig.Symbol}, Resolution={dataConfig.Resolution}, TickType={dataConfig.TickType}"); - if (!CanSubscribe(dataConfig)) + if (!TryGetDataBentoDataSet(symbol, out var dataSet)) { - Log.Error($"DataBentoProvider.Subscribe(): Cannot subscribe to {dataConfig.Symbol} with Resolution={dataConfig.Resolution}, TickType={dataConfig.TickType}"); - return null; + throw new ArgumentException($"No DataBento dataset mapping found for symbol {symbol} in market {symbol.ID.Market}. Cannot subscribe."); } - _subscriptionConfigs[dataConfig.Symbol] = dataConfig; - var enumerator = _dataAggregator.Add(dataConfig, newDataAvailableHandler); - _subscriptionManager.Subscribe(dataConfig); - _activeSubscriptionConfigs.Add(dataConfig); + var brokerageSymbol = _symbolMapper.GetBrokerageSymbol(symbol); + + _pendingSubscriptions[brokerageSymbol] = symbol; - return enumerator; + _liveApiClient.Subscribe(dataSet, brokerageSymbol); } - /// - /// Removes the specified configuration - /// - /// Subscription config to be removed - public void Unsubscribe(SubscriptionDataConfig dataConfig) + return true; + } + + public bool Unsubscribe(IEnumerable symbols) + { + // Please note there is no unsubscribe method. Subscriptions end when the TCP connection closes. + + var symbolsToRemove = symbols.ToHashSet(); + + foreach (var (instrumentId, symbol) in _subscribedSymbolsByDataBentoInstrumentId) { - Log.Trace($"DataBentoProvider.Unsubscribe(): Received unsubscription request for {dataConfig.Symbol}, Resolution={dataConfig.Resolution}, TickType={dataConfig.TickType}"); - _subscriptionConfigs.TryRemove(dataConfig.Symbol, out _); - _subscriptionManager.Unsubscribe(dataConfig); - var toRemove = _activeSubscriptionConfigs.FirstOrDefault(c => c.Symbol == dataConfig.Symbol && c.TickType == dataConfig.TickType); - if (toRemove != null) + if (symbolsToRemove.Contains(symbol)) { - Log.Trace($"DataBentoProvider.Unsubscribe(): Removing active subscription for {dataConfig.Symbol}, Resolution={dataConfig.Resolution}, TickType={dataConfig.TickType}"); - _activeSubscriptionConfigs.Remove(toRemove); + _subscribedSymbolsByDataBentoInstrumentId.Remove(instrumentId); } - _dataAggregator.Remove(dataConfig); } - /// - /// Sets the job we're subscribing for - /// - /// Job we're subscribing for - public void SetJob(LiveNodePacket job) - { - // No action required for DataBento since the job details are not used in the subscription process. - } + return true; + } - /// - /// Dispose of unmanaged resources. - /// - public void Dispose() + /// + /// Checks if this Data provider supports the specified symbol + /// + /// The symbol + /// returns true if Data Provider supports the specified symbol; otherwise false + private static bool CanSubscribe(Symbol symbol) + { + return !symbol.Value.Contains("universe", StringComparison.InvariantCultureIgnoreCase) && + !symbol.IsCanonical() && + symbol.SecurityType == SecurityType.Future; + } + + /// + /// Subscribe to the specified configuration + /// + /// defines the parameters to subscribe to a data feed + /// handler to be fired on new data available + /// The new enumerator for this subscription request + public IEnumerator? Subscribe(SubscriptionDataConfig dataConfig, EventHandler newDataAvailableHandler) + { + if (!CanSubscribe(dataConfig.Symbol)) { - _dataAggregator?.DisposeSafely(); - _subscriptionManager?.DisposeSafely(); - _client?.Dispose(); - _dataDownloader?.Dispose(); + return null; } - /// - /// Gets the history for the requested security - /// - /// The historical data request - /// An enumerable of BaseData points - public IEnumerable? GetHistory(Data.HistoryRequest request) - { - Log.Trace($"DataBentoProvider.GetHistory(): Received history request for {request.Symbol}, Resolution={request.Resolution}, TickType={request.TickType}"); - if (!CanSubscribe(request.Symbol)) - { - Log.Error($"DataBentoProvider.GetHistory(): Cannot provide history for {request.Symbol} with Resolution={request.Resolution}, TickType={request.TickType}"); - return null; - } + var enumerator = _aggregator.Add(dataConfig, newDataAvailableHandler); + _levelOneServiceManager.Subscribe(dataConfig); - try - { - // Use the data downloader to get historical data - var parameters = new DataDownloaderGetParameters( - request.Symbol, - request.Resolution, - request.StartTimeUtc, - request.EndTimeUtc, - request.TickType); - - return _dataDownloader.Get(parameters); - } - catch (Exception ex) - { - Log.Error($"DataBentoProvider.GetHistory(): Failed to get history for {request.Symbol}: {ex.Message}"); - return null; - } - } + return enumerator; + } - /// - /// Checks if this Data provider supports the specified symbol - /// - /// The symbol - /// returns true if Data Provider supports the specified symbol; otherwise false - private bool CanSubscribe(Symbol symbol) - { - return !symbol.Value.Contains("universe", StringComparison.InvariantCultureIgnoreCase) && - !symbol.IsCanonical() && - IsSecurityTypeSupported(symbol.SecurityType); - } + /// + /// Removes the specified configuration + /// + /// Subscription config to be removed + public void Unsubscribe(SubscriptionDataConfig dataConfig) + { + _levelOneServiceManager.Unsubscribe(dataConfig); + _aggregator.Remove(dataConfig); + } - /// - /// Determines whether or not the specified config can be subscribed to - /// - private bool CanSubscribe(SubscriptionDataConfig config) + /// + /// Sets the job we're subscribing for + /// + /// Job we're subscribing for + public void SetJob(LiveNodePacket job) + { + if (_initialized) { - return CanSubscribe(config.Symbol) && - IsSupported(config.SecurityType, config.Type, config.TickType, config.Resolution); + return; } - /// - /// Checks if the security type is supported - /// - /// Security type to check - /// True if supported - private bool IsSecurityTypeSupported(SecurityType securityType) + if (!job.BrokerageData.TryGetValue("databento-api-key", out var apiKey) || string.IsNullOrWhiteSpace(apiKey)) { - // DataBento primarily supports futures, but also has equity and option coverage - return securityType == SecurityType.Future; + throw new ArgumentException("The DataBento API key is missing from the brokerage data."); } - /// - /// Determines if the specified subscription is supported - /// - private bool IsSupported(SecurityType securityType, Type dataType, TickType tickType, Resolution resolution) - { - // Check supported security types - if (!IsSecurityTypeSupported(securityType)) - { - throw new NotSupportedException($"Unsupported security type: {securityType}"); - } + Initialize(apiKey); + } - // Check supported data types - if (dataType != typeof(TradeBar) && - dataType != typeof(QuoteBar) && - dataType != typeof(Tick) && - dataType != typeof(OpenInterest)) - { - throw new NotSupportedException($"Unsupported data type: {dataType}"); - } + /// + /// Dispose of unmanaged resources. + /// + public void Dispose() + { + _levelOneServiceManager?.DisposeSafely(); + _aggregator?.DisposeSafely(); + _liveApiClient?.DisposeSafely(); + _historicalApiClient?.DisposeSafely(); + } - // Warn about potential limitations for tick data - // I'm mimicing polygon implementation with this - if (!_potentialUnsupportedResolutionMessageLogged) - { - _potentialUnsupportedResolutionMessageLogged = true; - Log.Trace("DataBentoDataProvider.IsSupported(): " + - $"Subscription for {securityType}-{dataType}-{tickType}-{resolution} will be attempted. " + - $"An Advanced DataBento subscription plan is required to stream tick data."); - } + private class ModulesReadLicenseRead : RestResponse + { + [JsonProperty(PropertyName = "license")] + public string License; - return true; - } + [JsonProperty(PropertyName = "organizationId")] + public string OrganizationId; + } - /// - /// Converts the given UTC time into the symbol security exchange time zone - /// - private DateTime GetTickTime(Symbol symbol, DateTime utcTime) + /// + /// Validate the user of this project has permission to be using it via our web API. + /// + private static void ValidateSubscription() + { + try { - var exchangeTimeZone = _symbolExchangeTimeZones.GetOrAdd(symbol, sym => + const int productId = 306; + var userId = Globals.UserId; + var token = Globals.UserToken; + var organizationId = Globals.OrganizationID; + // Verify we can authenticate with this user and token + var api = new ApiConnection(userId, token); + if (!api.Connected) { - if (_marketHoursDatabase.TryGetEntry(sym.ID.Market, sym, sym.SecurityType, out var entry)) + throw new ArgumentException("Invalid api user id or token, cannot authenticate subscription."); + } + // Compile the information we want to send when validating + var information = new Dictionary() { - return entry.ExchangeHours.TimeZone; - } - // Futures default to Chicago - return TimeZones.Chicago; - }); - - return utcTime.ConvertFromUtc(exchangeTimeZone); - } - - // - /// Handles data received from the live client - /// - private void OnDataReceived(object? sender, BaseData data) - { + {"productId", productId}, + {"machineName", Environment.MachineName}, + {"userName", Environment.UserName}, + {"domainName", Environment.UserDomainName}, + {"os", Environment.OSVersion} + }; + // IP and Mac Address Information try { - if (data is Tick tick) - { - tick.Time = GetTickTime(tick.Symbol, tick.Time); - _dataAggregator.Update(tick); - - Log.Trace($"DataBentoProvider.OnDataReceived(): Updated tick - Symbol: {tick.Symbol}, " + - $"TickType: {tick.TickType}, Price: {tick.Value}, Quantity: {tick.Quantity}"); - } - else if (data is TradeBar tradeBar) - { - tradeBar.Time = GetTickTime(tradeBar.Symbol, tradeBar.Time); - tradeBar.EndTime = GetTickTime(tradeBar.Symbol, tradeBar.EndTime); - _dataAggregator.Update(tradeBar); - - Log.Trace($"DataBentoProvider.OnDataReceived(): Updated TradeBar - Symbol: {tradeBar.Symbol}, " + - $"O:{tradeBar.Open} H:{tradeBar.High} L:{tradeBar.Low} C:{tradeBar.Close} V:{tradeBar.Volume}"); - } - else + var interfaceDictionary = new List>(); + foreach (var nic in NetworkInterface.GetAllNetworkInterfaces().Where(nic => nic.OperationalStatus == OperationalStatus.Up)) { - data.Time = GetTickTime(data.Symbol, data.Time); - _dataAggregator.Update(data); + var interfaceInformation = new Dictionary(); + // Get UnicastAddresses + var addresses = nic.GetIPProperties().UnicastAddresses + .Select(uniAddress => uniAddress.Address) + .Where(address => !IPAddress.IsLoopback(address)).Select(x => x.ToString()); + // If this interface has non-loopback addresses, we will include it + if (!addresses.IsNullOrEmpty()) + { + interfaceInformation.Add("unicastAddresses", addresses); + // Get MAC address + interfaceInformation.Add("MAC", nic.GetPhysicalAddress().ToString()); + // Add Interface name + interfaceInformation.Add("name", nic.Name); + // Add these to our dictionary + interfaceDictionary.Add(interfaceInformation); + } } + information.Add("networkInterfaces", interfaceDictionary); } - catch (Exception ex) + catch (Exception) { - Log.Error($"DataBentoProvider.OnDataReceived(): Error updating data aggregator: {ex.Message}\n{ex.StackTrace}"); + // NOP, not necessary to crash if fails to extract and add this information + } + // Include our OrganizationId if specified + if (!string.IsNullOrEmpty(organizationId)) + { + information.Add("organizationId", organizationId); } - } - /// - /// Handles connection status changes from the live client - /// - private void OnConnectionStatusChanged(object? sender, bool isConnected) - { - Log.Trace($"DataBentoProvider.OnConnectionStatusChanged(): Connection status changed to: {isConnected}"); + // Create HTTP request + using var request = ApiUtils.CreateJsonPostRequest("modules/license/read", information); - if (isConnected) + api.TryRequest(request, out ModulesReadLicenseRead result); + if (!result.Success) { - // Reset session flag on reconnection - lock (_sessionLock) - { - _sessionStarted = false; - } + throw new InvalidOperationException($"Request for subscriptions from web failed, Response Errors : {string.Join(',', result.Errors)}"); + } - // Resubscribe to all active subscriptions - foreach (var config in _activeSubscriptionConfigs) + var encryptedData = result.License; + // Decrypt the data we received + DateTime? expirationDate = null; + long? stamp = null; + bool? isValid = null; + if (encryptedData != null) + { + // Fetch the org id from the response if it was not set, we need it to generate our validation key + if (string.IsNullOrEmpty(organizationId)) { - _client.Subscribe(config.Symbol, config.Resolution, config.TickType); + organizationId = result.OrganizationId; } - - // Start session after resubscribing - if (_activeSubscriptionConfigs.Any()) + // Create our combination key + var password = $"{token}-{organizationId}"; + var key = SHA256.HashData(Encoding.UTF8.GetBytes(password)); + // Split the data + var info = encryptedData.Split("::"); + var buffer = Convert.FromBase64String(info[0]); + var iv = Convert.FromBase64String(info[1]); + // Decrypt our information + using var aes = new AesManaged(); + var decryptor = aes.CreateDecryptor(key, iv); + using var memoryStream = new MemoryStream(buffer); + using var cryptoStream = new CryptoStream(memoryStream, decryptor, CryptoStreamMode.Read); + using var streamReader = new StreamReader(cryptoStream); + var decryptedData = streamReader.ReadToEnd(); + if (!decryptedData.IsNullOrEmpty()) { - lock (_sessionLock) - { - if (!_sessionStarted) - { - Log.Trace("DataBentoProvider.OnConnectionStatusChanged(): Starting session after reconnection"); - _sessionStarted = _client.StartSession(); - } - } + var jsonInfo = JsonConvert.DeserializeObject(decryptedData); + expirationDate = jsonInfo["expiration"]?.Value(); + isValid = jsonInfo["isValid"]?.Value(); + stamp = jsonInfo["stamped"]?.Value(); } } + // Validate our conditions + if (!expirationDate.HasValue || !isValid.HasValue || !stamp.HasValue) + { + throw new InvalidOperationException("Failed to validate subscription."); + } + + var nowUtc = DateTime.UtcNow; + var timeSpan = nowUtc - Time.UnixTimeStampToDateTime(stamp.Value); + if (timeSpan > TimeSpan.FromHours(12)) + { + throw new InvalidOperationException("Invalid API response."); + } + if (!isValid.Value) + { + throw new ArgumentException($"Your subscription is not valid, please check your product subscriptions on our website."); + } + if (expirationDate < nowUtc) + { + throw new ArgumentException($"Your subscription expired {expirationDate}, please renew in order to use this product."); + } + } + catch (Exception e) + { + Log.Error($"PolygonDataProvider.ValidateSubscription(): Failed during validation, shutting down. Error : {e.Message}"); + throw; } } } diff --git a/QuantConnect.DataBento/DataBentoHistoryProivder.cs b/QuantConnect.DataBento/DataBentoHistoryProivder.cs index a93b50e..fabf42e 100644 --- a/QuantConnect.DataBento/DataBentoHistoryProivder.cs +++ b/QuantConnect.DataBento/DataBentoHistoryProivder.cs @@ -1,6 +1,6 @@ /* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,275 +14,252 @@ * */ -using System; using NodaTime; using QuantConnect.Data; -using QuantConnect.Data.Market; -using QuantConnect.Lean.Engine.DataFeeds; -using QuantConnect.Lean.Engine.HistoricalData; -using QuantConnect.Logging; using QuantConnect.Util; -using QuantConnect.Lean.DataSource.DataBento; -using QuantConnect.Interfaces; -using System.Collections.Generic; -using QuantConnect.Configuration; +using QuantConnect.Logging; using QuantConnect.Securities; +using QuantConnect.Data.Market; using QuantConnect.Data.Consolidators; +using QuantConnect.Lean.Engine.DataFeeds; +using QuantConnect.Lean.Engine.HistoricalData; + +namespace QuantConnect.Lean.DataSource.DataBento; -namespace QuantConnect.Lean.DataSource.DataBento +/// +/// Implements a history provider for DataBento historical data. +/// Uses consolidators to produce the requested resolution when necessary. +/// +public partial class DataBentoProvider : SynchronizingHistoryProvider { + private static int _dataPointCount; + /// - /// DataBento implementation of + /// Indicates whether a error for an invalid start time has been fired, where the start time is greater than or equal to the end time in UTC. /// - public partial class DataBentoHistoryProvider : SynchronizingHistoryProvider - { - private int _dataPointCount; - private DataBentoDataDownloader _dataDownloader; - private volatile bool _invalidStartTimeErrorFired; - private volatile bool _invalidTickTypeAndResolutionErrorFired; - private volatile bool _unsupportedTickTypeMessagedLogged; - private MarketHoursDatabase _marketHoursDatabase; - private bool _unsupportedSecurityTypeMessageLogged; - private bool _unsupportedDataTypeMessageLogged; - private bool _potentialUnsupportedResolutionMessageLogged; - - /// - /// Gets the total number of data points emitted by this history provider - /// - public override int DataPointCount => _dataPointCount; + private volatile bool _invalidStartTimeErrorFired; - /// - /// Initializes this history provider to work for the specified job - /// - /// The initialization parameters - public override void Initialize(HistoryProviderInitializeParameters parameters) - { - _dataDownloader = new DataBentoDataDownloader(); - _marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); - } + /// + /// Indicates whether the warning for invalid has been fired. + /// + private volatile bool _invalidSecurityTypeWarningFired; - /// - /// Gets the history for the requested securities - /// - /// The historical data requests - /// The time zone used when time stamping the slice instances - /// An enumerable of the slices of data covering the span specified in each request - public override IEnumerable? GetHistory(IEnumerable requests, DateTimeZone sliceTimeZone) - { - var subscriptions = new List(); - foreach (var request in requests) - { - var history = GetHistory(request); - if (history == null) - { - continue; - } + /// + /// Indicates whether a DataBento dataset error has already been logged. + /// + private bool _dataBentoDatasetErrorFired; - var subscription = CreateSubscription(request, history); - if (!subscription.MoveNext()) - { - continue; - } + /// + /// Gets the total number of data points emitted by this history provider + /// + public override int DataPointCount => _dataPointCount; - subscriptions.Add(subscription); - } + /// + /// Initializes this history provider to work for the specified job + /// + /// The initialization parameters + public override void Initialize(HistoryProviderInitializeParameters parameters) + { + } + + /// + /// Gets the history for the requested securities + /// + /// The historical data requests + /// The time zone used when time stamping the slice instances + /// An enumerable of the slices of data covering the span specified in each request + public override IEnumerable? GetHistory(IEnumerable requests, DateTimeZone sliceTimeZone) + { + var subscriptions = new List(); + foreach (var request in requests) + { + var history = request.SplitHistoryRequestWithUpdatedMappedSymbol(_mapFileProvider).SelectMany(x => GetHistory(x) ?? []); - if (subscriptions.Count == 0) + var subscription = CreateSubscription(request, history); + if (!subscription.MoveNext()) { - return null; + continue; } - return CreateSliceEnumerableFromSubscriptions(subscriptions, sliceTimeZone); + + subscriptions.Add(subscription); } - /// - /// Gets the history for the requested security - /// - /// The historical data request - /// An enumerable of BaseData points - public IEnumerable? GetHistory(HistoryRequest request) + if (subscriptions.Count == 0) { - if (request.Symbol.IsCanonical() || - !IsSupported(request.Symbol.SecurityType, request.DataType, request.TickType, request.Resolution)) - { - // It is Logged in IsSupported(...) - return null; - } + return null; + } + return CreateSliceEnumerableFromSubscriptions(subscriptions, sliceTimeZone); + } - if (request.TickType == TickType.OpenInterest) + /// + /// Gets the history for the requested security + /// + /// The historical data request + /// An enumerable of BaseData points + public IEnumerable? GetHistory(HistoryRequest historyRequest) + { + if (!CanSubscribe(historyRequest.Symbol)) + { + if (!_invalidSecurityTypeWarningFired) { - if (!_unsupportedTickTypeMessagedLogged) - { - _unsupportedTickTypeMessagedLogged = true; - Log.Trace($"DataBentoHistoryProvider.GetHistory(): Unsupported tick type: {TickType.OpenInterest}"); - } - return null; + _invalidSecurityTypeWarningFired = true; + LogTrace(nameof(GetHistory), $"Unsupported SecurityType '{historyRequest.Symbol.SecurityType}' for symbol '{historyRequest.Symbol}'."); } + return null; + } - if (request.EndTimeUtc < request.StartTimeUtc) + if (historyRequest.EndTimeUtc < historyRequest.StartTimeUtc) + { + if (!_invalidStartTimeErrorFired) { - if (!_invalidStartTimeErrorFired) - { - _invalidStartTimeErrorFired = true; - Log.Error($"{nameof(DataBentoHistoryProvider)}.{nameof(GetHistory)}:InvalidDateRange. The history request start date must precede the end date, no history returned"); - } - return null; + _invalidStartTimeErrorFired = true; + Log.Error($"{nameof(DataBentoProvider)}.{nameof(GetHistory)}: Invalid date range: the start date must be earlier than the end date."); } + return null; + } - - // Use the trade aggregates API for resolutions above tick for fastest results - if (request.TickType == TickType.Trade && request.Resolution > Resolution.Tick) + if (!TryGetDataBentoDataSet(historyRequest.Symbol, out var dataSet) || dataSet == null) + { + if (!_dataBentoDatasetErrorFired) { - var data = GetAggregates(request); - - if (data == null) - { - return null; - } - - return data; + _dataBentoDatasetErrorFired = true; + Log.Error($"{nameof(DataBentoProvider)}.{nameof(GetHistory)}: " + + $"DataBento dataset not found for symbol '{historyRequest.Symbol.Value}, Market = {historyRequest.Symbol.ID.Market}." + ); } - - return GetHistoryThroughDataConsolidator(request); + return null; } - private IEnumerable? GetHistoryThroughDataConsolidator(HistoryRequest request) + var history = default(IEnumerable); + var brokerageSymbol = _symbolMapper.GetBrokerageSymbol(historyRequest.Symbol); + switch (historyRequest.TickType) { - IDataConsolidator consolidator; - IEnumerable history; + case TickType.Trade when historyRequest.Resolution == Resolution.Tick: + history = GetHistoryThroughDataConsolidator(historyRequest, brokerageSymbol, dataSet); + break; + case TickType.Trade: + history = GetAggregatedTradeBars(historyRequest, brokerageSymbol, dataSet); + break; + case TickType.Quote: + history = GetHistoryThroughDataConsolidator(historyRequest, brokerageSymbol, dataSet); + break; + case TickType.OpenInterest: + history = GetOpenInterestBars(historyRequest, brokerageSymbol, dataSet); + break; + default: + throw new ArgumentException(""); + } - if (request.TickType == TickType.Trade) - { - consolidator = request.Resolution != Resolution.Tick - ? new TickConsolidator(request.Resolution.ToTimeSpan()) - : FilteredIdentityDataConsolidator.ForTickType(request.TickType); - history = GetTrades(request); - } - else - { - consolidator = request.Resolution != Resolution.Tick - ? new TickQuoteBarConsolidator(request.Resolution.ToTimeSpan()) - : FilteredIdentityDataConsolidator.ForTickType(request.TickType); - history = GetQuotes(request); - } + if (history == null) + { + return null; + } - BaseData? consolidatedData = null; - DataConsolidatedHandler onDataConsolidated = (s, e) => - { - consolidatedData = (BaseData)e; - }; - consolidator.DataConsolidated += onDataConsolidated; + return FilterHistory(history, historyRequest, historyRequest.StartTimeLocal, historyRequest.EndTimeLocal); + } - foreach (var data in history) + private static IEnumerable FilterHistory(IEnumerable history, HistoryRequest request, DateTime startTimeLocal, DateTime endTimeLocal) + { + // cleaning the data before returning it back to user + foreach (var bar in history) + { + if (bar.Time >= startTimeLocal && bar.EndTime <= endTimeLocal) { - consolidator.Update(data); - if (consolidatedData != null) + if (request.ExchangeHours.IsOpen(bar.Time, bar.EndTime, request.IncludeExtendedMarketHours)) { Interlocked.Increment(ref _dataPointCount); - yield return consolidatedData; - consolidatedData = null; + yield return bar; } } - - consolidator.DataConsolidated -= onDataConsolidated; - consolidator.DisposeSafely(); } + } - /// - /// Gets the trade bars for the specified history request - /// - private IEnumerable GetAggregates(HistoryRequest request) + private IEnumerable GetOpenInterestBars(HistoryRequest request, string brokerageSymbol, string dataBentoDataSet) + { + foreach (var oi in _historicalApiClient.GetOpenInterest(brokerageSymbol, request.StartTimeUtc, request.EndTimeUtc, dataBentoDataSet)) { - var resolutionTimeSpan = request.Resolution.ToTimeSpan(); - foreach (var date in Time.EachDay(request.StartTimeUtc, request.EndTimeUtc)) - { - var start = date; - var end = date + Time.OneDay; - - var parameters = new DataDownloaderGetParameters(request.Symbol, request.Resolution, start, end, request.TickType); - var data = _dataDownloader.Get(parameters); - if (data == null) continue; - - foreach (var bar in data) - { - var tradeBar = (TradeBar)bar; - if (tradeBar.Time >= request.StartTimeUtc && tradeBar.EndTime <= request.EndTimeUtc) - { - yield return tradeBar; - } - } - } + yield return new OpenInterest(oi.Header.UtcTime.ConvertFromUtc(request.DataTimeZone), request.Symbol, oi.Quantity); } + } - /// - /// Gets the trade ticks that will potentially be aggregated for the specified history request - /// - private IEnumerable GetTrades(HistoryRequest request) + private IEnumerable GetAggregatedTradeBars(HistoryRequest request, string brokerageSymbol, string dataBentoDataSet) + { + var period = request.Resolution.ToTimeSpan(); + foreach (var b in _historicalApiClient.GetHistoricalOhlcvBars(brokerageSymbol, request.StartTimeUtc, request.EndTimeUtc, request.Resolution, dataBentoDataSet)) { - var parameters = new DataDownloaderGetParameters(request.Symbol, Resolution.Tick, request.StartTimeUtc, request.EndTimeUtc, request.TickType); - return _dataDownloader.Get(parameters); + yield return new TradeBar(b.Header.UtcTime.ConvertFromUtc(request.DataTimeZone), request.Symbol, b.Open, b.High, b.Low, b.Close, b.Volume, period); } + } + + private IEnumerable? GetHistoryThroughDataConsolidator(HistoryRequest request, string brokerageSymbol, string dataBentoDataSet) + { + IDataConsolidator consolidator; + IEnumerable history; - /// - /// Gets the quote ticks that will potentially be aggregated for the specified history request - /// - private IEnumerable GetQuotes(HistoryRequest request) + if (request.TickType == TickType.Trade) { - var parameters = new DataDownloaderGetParameters(request.Symbol, Resolution.Tick, request.StartTimeUtc, request.EndTimeUtc, request.TickType); - return _dataDownloader.Get(parameters); + consolidator = request.Resolution != Resolution.Tick + ? new TickConsolidator(request.Resolution.ToTimeSpan()) + : FilteredIdentityDataConsolidator.ForTickType(request.TickType); + history = GetTrades(request, brokerageSymbol, dataBentoDataSet); } - - /// - /// Checks if the security type is supported - /// - /// Security type to check - /// True if supported - private bool IsSecurityTypeSupported(SecurityType securityType) + else { - // DataBento primarily supports futures, but also has equity and option coverage - return securityType == SecurityType.Future; + consolidator = request.Resolution != Resolution.Tick + ? new TickQuoteBarConsolidator(request.Resolution.ToTimeSpan()) + : FilteredIdentityDataConsolidator.ForTickType(request.TickType); + history = GetQuotes(request, brokerageSymbol, dataBentoDataSet); } - /// - /// Determines if the specified subscription is supported - /// - private bool IsSupported(SecurityType securityType, Type dataType, TickType tickType, Resolution resolution) + BaseData? consolidatedData = null; + DataConsolidatedHandler onDataConsolidated = (s, e) => { - // Check supported security types - if (!IsSecurityTypeSupported(securityType)) - { - if (!_unsupportedSecurityTypeMessageLogged) - { - _unsupportedSecurityTypeMessageLogged = true; - Log.Trace($"DataBentoDataProvider.IsSupported(): Unsupported security type: {securityType}"); - } - return false; - } + consolidatedData = (BaseData)e; + }; + consolidator.DataConsolidated += onDataConsolidated; - // Check supported data types - if (dataType != typeof(TradeBar) && - dataType != typeof(QuoteBar) && - dataType != typeof(Tick) && - dataType != typeof(OpenInterest)) + foreach (var data in history) + { + consolidator.Update(data); + if (consolidatedData != null) { - if (!_unsupportedDataTypeMessageLogged) - { - _unsupportedDataTypeMessageLogged = true; - Log.Trace($"DataBentoDataProvider.IsSupported(): Unsupported data type: {dataType}"); - } - return false; + yield return consolidatedData; + consolidatedData = null; } + } - // Warn about potential limitations for tick data - // I'm mimicing polygon implementation with this - if (!_potentialUnsupportedResolutionMessageLogged) + consolidator.DataConsolidated -= onDataConsolidated; + consolidator.DisposeSafely(); + } + + /// + /// Gets the trade ticks that will potentially be aggregated for the specified history request + /// + private IEnumerable GetTrades(HistoryRequest request, string brokerageSymbol, string dataBentoDataSet) + { + foreach (var t in _historicalApiClient.GetTickBars(brokerageSymbol, request.StartTimeUtc, request.EndTimeUtc, dataBentoDataSet)) + { + yield return new Tick(t.Header.UtcTime.ConvertFromUtc(request.DataTimeZone), request.Symbol, "", "", t.Size, t.Price); + } + } + + /// + /// Gets the quote ticks that will potentially be aggregated for the specified history request + /// + private IEnumerable GetQuotes(HistoryRequest request, string brokerageSymbol, string dataBentoDataSet) + { + foreach (var quoteBar in _historicalApiClient.GetTickBars(brokerageSymbol, request.StartTimeUtc, request.EndTimeUtc, dataBentoDataSet)) + { + var time = quoteBar.Header.UtcTime.ConvertFromUtc(request.DataTimeZone); + foreach (var level in quoteBar.Levels) { - _potentialUnsupportedResolutionMessageLogged = true; - Log.Trace("DataBentoDataProvider.IsSupported(): " + - $"Subscription for {securityType}-{dataType}-{tickType}-{resolution} will be attempted. " + - $"An Advanced DataBento subscription plan is required to stream tick data."); + yield return new Tick(time, request.Symbol, level.BidSz, level.BidPx, level.AskSz, level.AskPx); } - - return true; } } + + private static void LogTrace(string methodName, string message) + { + Log.Trace($"{nameof(DataBentoProvider)}.{methodName}: {message}"); + } } diff --git a/QuantConnect.DataBento/DataBentoRawLiveClient.cs b/QuantConnect.DataBento/DataBentoRawLiveClient.cs deleted file mode 100644 index 2e7bb97..0000000 --- a/QuantConnect.DataBento/DataBentoRawLiveClient.cs +++ /dev/null @@ -1,752 +0,0 @@ -/* - * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * -*/ - -using System; -using System.IO; -using System.Text; -using System.Net.Sockets; -using System.Security.Cryptography; -using System.Collections.Concurrent; -using System.Text.Json; -using System.Linq; -using QuantConnect.Data; -using QuantConnect.Data.Market; -using QuantConnect.Logging; - -namespace QuantConnect.Lean.DataSource.DataBento -{ - /// - /// DataBento Raw TCP client for live streaming data - /// - public class DatabentoRawClient : IDisposable - { - private readonly string _apiKey; - private readonly string _gateway; - private readonly string _dataset; - private TcpClient? _tcpClient; - private NetworkStream? _stream; - private StreamReader? _reader; - private StreamWriter? _writer; - private CancellationTokenSource _cancellationTokenSource; - private readonly ConcurrentDictionary _subscriptions; - private readonly object _connectionLock = new object(); - private bool _isConnected; - private bool _disposed; - private const decimal PriceScaleFactor = 1e-9m; - private readonly ConcurrentDictionary _instrumentIdToSymbol = new ConcurrentDictionary(); - private readonly ConcurrentDictionary _lastTicks = new ConcurrentDictionary(); - - /// - /// Event fired when new data is received - /// - public event EventHandler? DataReceived; - - /// - /// Event fired when connection status changes - /// - public event EventHandler? ConnectionStatusChanged; - - /// - /// Gets whether the client is currently connected - /// - public bool IsConnected => _isConnected && _tcpClient?.Connected == true; - - /// - /// Initializes a new instance of the DatabentoRawClient - /// - public DatabentoRawClient(string apiKey, string gateway = "glbx-mdp3.lsg.databento.com:13000", string dataset = "GLBX.MDP3") - { - _apiKey = apiKey ?? throw new ArgumentNullException(nameof(apiKey)); - _gateway = gateway ?? throw new ArgumentNullException(nameof(gateway)); - _dataset = dataset; - _subscriptions = new ConcurrentDictionary(); - _cancellationTokenSource = new CancellationTokenSource(); - } - - /// - /// Connects to the DataBento live gateway - /// - public bool Connect() - { - Log.Trace("DatabentoRawClient.Connect(): Connecting to DataBento live gateway"); - if (_isConnected || _disposed) - { - return _isConnected; - } - - try - { - var parts = _gateway.Split(':'); - var host = parts[0]; - var port = parts.Length > 1 ? int.Parse(parts[1]) : 13000; - - _tcpClient = new TcpClient(); - _tcpClient.Connect(host, port); - _stream = _tcpClient.GetStream(); - _reader = new StreamReader(_stream, Encoding.ASCII); - _writer = new StreamWriter(_stream, Encoding.ASCII) { AutoFlush = true }; - - // Perform authentication handshake - if (Authenticate()) - { - _isConnected = true; - ConnectionStatusChanged?.Invoke(this, true); - - // Start message processing - ProcessMessages(); - - Log.Trace("DatabentoRawClient.Connect(): Connected and authenticated to DataBento live gateway"); - return true; - } - } - catch (Exception ex) - { - Log.Error($"DatabentoRawClient.Connect(): Failed to connect: {ex.Message}"); - Disconnect(); - } - - return false; - } - - /// - /// Authenticates with the DataBento gateway using CRAM-SHA256 - /// - private bool Authenticate() - { - if (_reader == null || _writer == null) - return false; - - try - { - // Read greeting and challenge - string? versionLine = _reader.ReadLine(); - string? cramLine = _reader.ReadLine(); - - if (string.IsNullOrEmpty(versionLine) || string.IsNullOrEmpty(cramLine)) - { - Log.Error("DatabentoRawClient.Authenticate(): Failed to receive greeting or challenge"); - return false; - } - - Log.Trace($"DatabentoRawClient.Authenticate(): Version: {versionLine}"); - Log.Trace($"DatabentoRawClient.Authenticate(): Challenge: {cramLine}"); - - // Parse challenge - string[] cramParts = cramLine.Split('='); - if (cramParts.Length != 2 || cramParts[0] != "cram") - { - Log.Error("DatabentoRawClient.Authenticate(): Invalid challenge format"); - return false; - } - string cram = cramParts[1].Trim(); - - // Compute auth hash - string concat = $"{cram}|{_apiKey}"; - string hashHex = ComputeSHA256(concat); - string bucketId = _apiKey.Length >= 5 ? _apiKey.Substring(_apiKey.Length - 5) : _apiKey; - string authString = $"{hashHex}-{bucketId}"; - - // Send auth message - string authMsg = $"auth={authString}|dataset={_dataset}|encoding=json|ts_out=0"; - Log.Trace($"DatabentoRawClient.Authenticate(): Sending auth"); - _writer.WriteLine(authMsg); - - // Read auth response - string? authResp = _reader.ReadLine(); - if (string.IsNullOrEmpty(authResp)) - { - Log.Error("DatabentoRawClient.Authenticate(): No authentication response received"); - return false; - } - - Log.Trace($"DatabentoRawClient.Authenticate(): Auth response: {authResp}"); - - if (!authResp.Contains("success=1")) - { - Log.Error($"DatabentoRawClient.Authenticate(): Authentication failed: {authResp}"); - return false; - } - - Log.Trace("DatabentoRawClient.Authenticate(): Authentication successful"); - return true; - } - catch (Exception ex) - { - Log.Error($"DatabentoRawClient.Authenticate(): Authentication failed: {ex.Message}"); - return false; - } - } - private static string ComputeSHA256(string input) - { - using var sha = SHA256.Create(); - byte[] hash = sha.ComputeHash(Encoding.UTF8.GetBytes(input)); - var sb = new StringBuilder(); - foreach (byte b in hash) - { - sb.Append(b.ToString("x2")); - } - return sb.ToString(); - } - - /// - /// Subscribes to live data for a symbol - /// - public bool Subscribe(Symbol symbol, Resolution resolution, TickType tickType) - { - if (!IsConnected || _writer == null) - { - Log.Error("DatabentoRawClient.Subscribe(): Not connected to gateway"); - return false; - } - - try - { - // Get the databento symbol form LEAN symbol - // Get schema from the resolution - var databentoSymbol = MapSymbolToDataBento(symbol); - var schema = GetSchema(resolution, tickType); - - // subscribe - var subscribeMessage = $"schema={schema}|stype_in=parent|symbols={databentoSymbol}"; - Log.Trace($"DatabentoRawClient.Subscribe(): Subscribing with message: {subscribeMessage}"); - - // Send subscribe message - _writer.WriteLine(subscribeMessage); - - // Store subscription - _subscriptions.TryAdd(symbol, (resolution, tickType)); - Log.Trace($"DatabentoRawClient.Subscribe(): Subscribed to {symbol} ({databentoSymbol}) at {resolution} resolution for {tickType}"); - - // If subscribing to quote ticks, also subscribe to trade ticks - if (tickType == TickType.Quote && resolution == Resolution.Tick) - { - var tradeSchema = GetSchema(resolution, TickType.Trade); - var tradeSubscribeMessage = $"schema={tradeSchema}|stype_in=parent|symbols={databentoSymbol}"; - Log.Trace($"DatabentoRawClient.Subscribe(): Also subscribing to trades with message: {tradeSubscribeMessage}"); - _writer.WriteLine(tradeSubscribeMessage); - } - - return true; - } - catch (Exception ex) - { - Log.Error($"DatabentoRawClient.Subscribe(): Failed to subscribe to {symbol}: {ex.Message}"); - return false; - } - } - - /// - /// Starts the session to begin receiving data - /// - public bool StartSession() - { - if (!IsConnected || _writer == null) - { - Log.Error("DatabentoRawClient.StartSession(): Not connected"); - return false; - } - - try - { - Log.Trace("DatabentoRawClient.StartSession(): Starting session"); - _writer.WriteLine("start_session=1"); - return true; - } - catch (Exception ex) - { - Log.Error($"DatabentoRawClient.StartSession(): Failed to start session: {ex.Message}"); - return false; - } - } - - /// - /// Unsubscribes from live data for a symbol - /// - public bool Unsubscribe(Symbol symbol) - { - try - { - if (_subscriptions.TryRemove(symbol, out _)) - { - Log.Trace($"DatabentoRawClient.Unsubscribe(): Unsubscribed from {symbol}"); - } - return true; - } - catch (Exception ex) - { - Log.Error($"DatabentoRawClient.Unsubscribe(): Failed to unsubscribe from {symbol}: {ex.Message}"); - return false; - } - } - - /// - /// Processes incoming messages from the DataBento gateway - /// - private void ProcessMessages() - { - Log.Trace("DatabentoRawClient.ProcessMessages(): Starting message processing"); - if (_reader == null) - { - Log.Error("DatabentoRawClient.ProcessMessages(): No reader available"); - return; - } - - var messageCount = 0; - - try - { - while (!_cancellationTokenSource.IsCancellationRequested && IsConnected) - { - var line = _reader.ReadLine(); - if (line == null) - { - Log.Trace("DatabentoRawClient.ProcessMessages(): Connection closed by server"); - break; - } - - if (string.IsNullOrWhiteSpace(line)) - continue; - - messageCount++; - if (messageCount <= 50 || messageCount % 100 == 0) - { - Log.Trace($"DatabentoRawClient.ProcessMessages(): Message #{messageCount}: {line.Substring(0, Math.Min(150, line.Length))}..."); - } - - ProcessSingleMessage(line); - } - } - catch (OperationCanceledException) - { - Log.Trace("DatabentoRawClient.ProcessMessages(): Message processing cancelled"); - } - catch (IOException ex) when (ex.InnerException is SocketException) - { - Log.Trace($"DatabentoRawClient.ProcessMessages(): Socket exception: {ex.Message}"); - } - catch (Exception ex) - { - Log.Error($"DatabentoRawClient.ProcessMessages(): Error processing messages: {ex.Message}\n{ex.StackTrace}"); - } - finally - { - Log.Trace($"DatabentoRawClient.ProcessMessages(): Exiting. Total messages processed: {messageCount}"); - Disconnect(); - } - } - - /// - /// Processes a single message from DataBento - /// - private void ProcessSingleMessage(string message) - { - try - { - using var document = JsonDocument.Parse(message); - var root = document.RootElement; - - // Check for error messages - if (root.TryGetProperty("hd", out var headerElement)) - { - if (headerElement.TryGetProperty("rtype", out var rtypeElement)) - { - var rtype = rtypeElement.GetInt32(); - - if (rtype == 23) - { - if (root.TryGetProperty("msg", out var msgElement)) - { - Log.Trace($"DatabentoRawClient: System message: {msgElement.GetString()}"); - } - return; - } - else if (rtype == 22) - { - // Symbol mapping message - if (root.TryGetProperty("stype_in_symbol", out var inSymbol) && - root.TryGetProperty("stype_out_symbol", out var outSymbol) && - headerElement.TryGetProperty("instrument_id", out var instId)) - { - var instrumentId = instId.GetInt64(); - var outSymbolStr = outSymbol.GetString(); - - Log.Trace($"DatabentoRawClient: Symbol mapping: {inSymbol.GetString()} -> {outSymbolStr} (instrument_id: {instrumentId})"); - - // Find the subscription that matches this symbol - foreach (var kvp in _subscriptions) - { - var leanSymbol = kvp.Key; - if (outSymbolStr != null) - { - _instrumentIdToSymbol[instrumentId] = leanSymbol; - Log.Trace($"DatabentoRawClient: Mapped instrument_id {instrumentId} to {leanSymbol}"); - break; - } - } - } - return; - } - else if (rtype == 1) - { - // MBP-1 (Market By Price) - Quote ticks - HandleMBPMessage(root, headerElement); - return; - } - else if (rtype == 0) - { - // Trade messages - Trade ticks - HandleTradeTickMessage(root, headerElement); - return; - } - else if (rtype == 32 || rtype == 33 || rtype == 34 || rtype == 35) - { - // OHLCV bar messages - HandleOHLCVMessage(root, headerElement); - return; - } - } - } - - // Handle other message types if needed - if (root.TryGetProperty("error", out var errorElement)) - { - Log.Error($"DatabentoRawClient: Server error: {errorElement.GetString()}"); - } - } - catch (JsonException ex) - { - Log.Error($"DatabentoRawClient.ProcessSingleMessage(): JSON parse error: {ex.Message}"); - } - catch (Exception ex) - { - Log.Error($"DatabentoRawClient.ProcessSingleMessage(): Error: {ex.Message}"); - } - } - - /// - /// Handles OHLCV messages and converts to LEAN TradeBar data - /// - private void HandleOHLCVMessage(JsonElement root, JsonElement header) - { - try - { - if (!header.TryGetProperty("ts_event", out var tsElement) || - !header.TryGetProperty("instrument_id", out var instIdElement)) - { - return; - } - - // Convert timestamp from nanoseconds to DateTime - var timestampNs = long.Parse(tsElement.GetString()!); - var unixEpoch = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc); - var timestamp = unixEpoch.AddTicks(timestampNs / 100); - - var instrumentId = instIdElement.GetInt64(); - - if (!_instrumentIdToSymbol.TryGetValue(instrumentId, out var matchedSymbol)) - { - Log.Trace($"DatabentoRawClient: No mapping for instrument_id {instrumentId} in OHLCV message."); - return; - } - - // Get the resolution for this symbol - if (!_subscriptions.TryGetValue(matchedSymbol, out var subscription)) - { - return; - } - - var resolution = subscription.Item1; - - // Extract OHLCV data - if (root.TryGetProperty("open", out var openElement) && - root.TryGetProperty("high", out var highElement) && - root.TryGetProperty("low", out var lowElement) && - root.TryGetProperty("close", out var closeElement) && - root.TryGetProperty("volume", out var volumeElement)) - { - // Parse prices - var openRaw = long.Parse(openElement.GetString()!); - var highRaw = long.Parse(highElement.GetString()!); - var lowRaw = long.Parse(lowElement.GetString()!); - var closeRaw = long.Parse(closeElement.GetString()!); - var volume = volumeElement.GetInt64(); - - var open = openRaw * PriceScaleFactor; - var high = highRaw * PriceScaleFactor; - var low = lowRaw * PriceScaleFactor; - var close = closeRaw * PriceScaleFactor; - - // Determine the period based on resolution - TimeSpan period = resolution switch - { - Resolution.Second => TimeSpan.FromSeconds(1), - Resolution.Minute => TimeSpan.FromMinutes(1), - Resolution.Hour => TimeSpan.FromHours(1), - Resolution.Daily => TimeSpan.FromDays(1), - _ => TimeSpan.FromMinutes(1) - }; - - // Create TradeBar - var tradeBar = new TradeBar( - timestamp, - matchedSymbol, - open, - high, - low, - close, - volume, - period - ); - - Log.Trace($"DatabentoRawClient: OHLCV bar: {matchedSymbol} O={open} H={high} L={low} C={close} V={volume} at {timestamp}"); - DataReceived?.Invoke(this, tradeBar); - } - } - catch (Exception ex) - { - Log.Error($"DatabentoRawClient.HandleOHLCVMessage(): Error: {ex.Message}"); - } - } - - /// - /// Handles MBP messages for quote ticks - /// - private void HandleMBPMessage(JsonElement root, JsonElement header) - { - try - { - if (!header.TryGetProperty("ts_event", out var tsElement) || - !header.TryGetProperty("instrument_id", out var instIdElement)) - { - return; - } - - // Convert timestamp from nanoseconds to DateTime - var timestampNs = long.Parse(tsElement.GetString()!); - var unixEpoch = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc); - var timestamp = unixEpoch.AddTicks(timestampNs / 100); - - var instrumentId = instIdElement.GetInt64(); - - if (!_instrumentIdToSymbol.TryGetValue(instrumentId, out var matchedSymbol)) - { - Log.Trace($"DatabentoRawClient: No mapping for instrument_id {instrumentId} in MBP message."); - return; - } - - // For MBP-1, bid/ask data is in the levels array at index 0 - if (root.TryGetProperty("levels", out var levelsElement) && - levelsElement.GetArrayLength() > 0) - { - var level0 = levelsElement[0]; - - var quoteTick = new Tick - { - Symbol = matchedSymbol, - Time = timestamp, - TickType = TickType.Quote - }; - - if (level0.TryGetProperty("ask_px", out var askPxElement) && - level0.TryGetProperty("ask_sz", out var askSzElement)) - { - var askPriceRaw = long.Parse(askPxElement.GetString()!); - quoteTick.AskPrice = askPriceRaw * PriceScaleFactor; - quoteTick.AskSize = askSzElement.GetInt32(); - } - - if (level0.TryGetProperty("bid_px", out var bidPxElement) && - level0.TryGetProperty("bid_sz", out var bidSzElement)) - { - var bidPriceRaw = long.Parse(bidPxElement.GetString()!); - quoteTick.BidPrice = bidPriceRaw * PriceScaleFactor; - quoteTick.BidSize = bidSzElement.GetInt32(); - } - - // Set the tick value to the mid price - quoteTick.Value = (quoteTick.BidPrice + quoteTick.AskPrice) / 2; - - // QuantConnect convention: Quote ticks should have zero Price and Quantity - quoteTick.Quantity = 0; - - Log.Trace($"DatabentoRawClient: Quote tick: {matchedSymbol} Bid={quoteTick.BidPrice}x{quoteTick.BidSize} Ask={quoteTick.AskPrice}x{quoteTick.AskSize}"); - DataReceived?.Invoke(this, quoteTick); - } - } - catch (Exception ex) - { - Log.Error($"DatabentoRawClient.HandleMBPMessage(): Error: {ex.Message}"); - } - } - - /// - /// Handles trade tick messages. Aggressor fills - /// - private void HandleTradeTickMessage(JsonElement root, JsonElement header) - { - try - { - if (!header.TryGetProperty("ts_event", out var tsElement) || - !header.TryGetProperty("instrument_id", out var instIdElement)) - { - return; - } - - // Convert timestamp from nanoseconds to DateTime - var timestampNs = long.Parse(tsElement.GetString()!); - var unixEpoch = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc); - var timestamp = unixEpoch.AddTicks(timestampNs / 100); - - var instrumentId = instIdElement.GetInt64(); - - if (!_instrumentIdToSymbol.TryGetValue(instrumentId, out var matchedSymbol)) - { - Log.Trace($"DatabentoRawClient: No mapping for instrument_id {instrumentId} in trade message."); - return; - } - - if (root.TryGetProperty("price", out var priceElement) && - root.TryGetProperty("size", out var sizeElement)) - { - var priceRaw = long.Parse(priceElement.GetString()!); - var size = sizeElement.GetInt32(); - var price = priceRaw * PriceScaleFactor; - - var tradeTick = new Tick - { - Symbol = matchedSymbol, - Time = timestamp, - Value = price, - Quantity = size, - TickType = TickType.Trade, - // Trade ticks should have zero bid/ask values - BidPrice = 0, - BidSize = 0, - AskPrice = 0, - AskSize = 0 - }; - - Log.Trace($"DatabentoRawClient: Trade tick: {matchedSymbol} Price={price} Quantity={size}"); - DataReceived?.Invoke(this, tradeTick); - } - } - catch (Exception ex) - { - Log.Error($"DatabentoRawClient.HandleTradeTickMessage(): Error: {ex.Message}"); - } - } - - /// - /// Maps a LEAN symbol to DataBento symbol format - /// - private string MapSymbolToDataBento(Symbol symbol) - { - if (symbol.SecurityType == SecurityType.Future) - { - // For DataBento, use the root symbol with .FUT suffix for parent subscription - // ES19Z25 -> ES.FUT - var value = symbol.Value; - - // Extract root by removing digits and month codes - var root = new string(value.TakeWhile(c => !char.IsDigit(c)).ToArray()); - - return $"{root}.FUT"; - } - - return symbol.Value; - } - - /// - /// Pick Databento schema from Lean resolution/ticktype - /// - private string GetSchema(Resolution resolution, TickType tickType) - { - if (tickType == TickType.Trade) - { - if (resolution == Resolution.Tick) - return "trades"; - if (resolution == Resolution.Second) - return "ohlcv-1s"; - if (resolution == Resolution.Minute) - return "ohlcv-1m"; - if (resolution == Resolution.Hour) - return "ohlcv-1h"; - if (resolution == Resolution.Daily) - return "ohlcv-1d"; - } - else if (tickType == TickType.Quote) - { - // top of book - if (resolution == Resolution.Tick || resolution == Resolution.Second || resolution == Resolution.Minute || resolution == Resolution.Hour || resolution == Resolution.Daily) - return "mbp-1"; - } - else if (tickType == TickType.OpenInterest) - { - return "statistics"; - } - - throw new NotSupportedException($"Unsupported resolution {resolution} / {tickType}"); - } - - /// - /// Disconnects from the DataBento gateway - /// - public void Disconnect() - { - lock (_connectionLock) - { - if (!_isConnected) - return; - - _isConnected = false; - _cancellationTokenSource?.Cancel(); - - try - { - _reader?.Dispose(); - _writer?.Dispose(); - _stream?.Close(); - _tcpClient?.Close(); - } - catch (Exception ex) - { - Log.Trace($"DatabentoRawClient.Disconnect(): Error during disconnect: {ex.Message}"); - } - - ConnectionStatusChanged?.Invoke(this, false); - Log.Trace("DatabentoRawClient.Disconnect(): Disconnected from DataBento gateway"); - } - } - - /// - /// Disposes of resources - /// - public void Dispose() - { - if (_disposed) - return; - - _disposed = true; - Disconnect(); - - _cancellationTokenSource?.Dispose(); - _reader?.Dispose(); - _writer?.Dispose(); - _stream?.Dispose(); - _tcpClient?.Dispose(); - } - } -} diff --git a/QuantConnect.DataBento/DataBentoSymbolMapper.cs b/QuantConnect.DataBento/DataBentoSymbolMapper.cs new file mode 100644 index 0000000..a0b8913 --- /dev/null +++ b/QuantConnect.DataBento/DataBentoSymbolMapper.cs @@ -0,0 +1,72 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using QuantConnect.Brokerages; +using System.Collections.Frozen; + +namespace QuantConnect.Lean.DataSource.DataBento; + +/// +/// Provides the mapping between Lean symbols and DataBento symbols. +/// +public class DataBentoSymbolMapper : ISymbolMapper +{ + /// + /// Dataset for CME Globex futures + /// https://databento.com/docs/venues-and-datasets has more information on datasets through DataBento + /// + public FrozenDictionary DataBentoDataSetByLeanMarket = new Dictionary + { + { Market.EUREX, "XEUR.EOBI" }, + + { Market.CBOT, "GLBX.MDP3" }, + { Market.CME, "GLBX.MDP3" }, + { Market.COMEX, "GLBX.MDP3" }, + { Market.NYMEX, "GLBX.MDP3" }, + + { Market.ICE, "IFUS.IMPACT" }, + { Market.NYSELIFFE, "IFUS.IMPACT" } + }.ToFrozenDictionary(); + + /// + /// Converts a Lean symbol instance to a brokerage symbol + /// + /// A Lean symbol instance + /// The brokerage symbol + public string GetBrokerageSymbol(Symbol symbol) + { + switch (symbol.SecurityType) + { + case SecurityType.Future: + return SymbolRepresentation.GenerateFutureTicker(symbol.ID.Symbol, symbol.ID.Date, doubleDigitsYear: false, includeExpirationDate: false); + default: + throw new Exception($"The unsupported security type: {symbol.SecurityType}"); + } + } + + /// + /// Converts a brokerage symbol to a Lean symbol instance + /// + /// The brokerage symbol + /// The security type + /// The market + /// Expiration date of the security(if applicable) + /// A new Lean Symbol instance + public Symbol GetLeanSymbol(string brokerageSymbol, SecurityType securityType, string market, + DateTime expirationDate = new DateTime(), decimal strike = 0, OptionRight optionRight = 0) + { + throw new NotImplementedException("This method is not used in the current implementation."); + } +} diff --git a/QuantConnect.DataBento/Extensions.cs b/QuantConnect.DataBento/Extensions.cs new file mode 100644 index 0000000..7240d6a --- /dev/null +++ b/QuantConnect.DataBento/Extensions.cs @@ -0,0 +1,49 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using Newtonsoft.Json; +using QuantConnect.Lean.DataSource.DataBento.Models; +using QuantConnect.Lean.DataSource.DataBento.Serialization; + +namespace QuantConnect.Lean.DataSource.DataBento; + +public static class Extensions +{ + /// + /// Deserializes the specified JSON string to an object of type + /// using snake-case property name resolution. + /// + /// The target type of the deserialized object. + /// The JSON string to deserialize. + /// The deserialized object of type . + public static T? DeserializeKebabCase(this string json) + { + return JsonConvert.DeserializeObject(json, JsonSettings.SnakeCase); + } + + /// + /// Deserializes the specified JSON string to a + /// using snake-case property name resolution. + /// + /// The JSON string to deserialize. + /// + /// The deserialized object, + /// or null if deserialization fails or the input is null or empty. + /// + public static MarketDataRecord? DeserializeSnakeCaseLiveData(this string json) + { + return JsonConvert.DeserializeObject(json, JsonSettings.LiveDataSnakeCase); + } +} diff --git a/QuantConnect.DataBento/Models/Enums/RecordType.cs b/QuantConnect.DataBento/Models/Enums/RecordType.cs new file mode 100644 index 0000000..67226e1 --- /dev/null +++ b/QuantConnect.DataBento/Models/Enums/RecordType.cs @@ -0,0 +1,86 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +namespace QuantConnect.Lean.DataSource.DataBento.Models.Enums; + +/// +/// Record type identifier (rtype) used in market data messages. +/// +public enum RecordType : byte +{ + /// Market-by-price record with book depth 0 (trades). + MarketByPriceDepth0 = 0, + + /// Market-by-price record with book depth 1 (TBBO, MBP-1). + MarketByPriceDepth1 = 1, + + /// Market-by-price record with book depth 10. + MarketByPriceDepth10 = 10, + + /// Exchange status record. + Status = 18, + + /// Instrument definition record. + Definition = 19, + + /// Order imbalance record. + Imbalance = 20, + + /// Error record from the live gateway. + Error = 21, + + /// Symbol mapping record from the live gateway. + SymbolMapping = 22, + + /// System record from the live gateway (e.g. heartbeat). + System = 23, + + /// Statistics record from the publisher. + Statistics = 24, + + /// OHLCV record at 1-second cadence. + OpenHighLowCloseVolume1Second = 32, + + /// OHLCV record at 1-minute cadence. + OpenHighLowCloseVolume1Minute = 33, + + /// OHLCV record at hourly cadence. + OpenHighLowCloseVolume1Hour = 34, + + /// OHLCV record at daily cadence. + OpenHighLowCloseVolume1Day = 35, + + /// Market-by-order record. + MarketByOrder = 160, + + /// Consolidated market-by-price record with book depth 1. + ConsolidatedMarketByPriceDepth1 = 177, + + /// Consolidated BBO at 1-second cadence. + ConsolidatedBestBidAndOffer1Second = 192, + + /// Consolidated BBO at 1-minute cadence. + ConsolidatedBestBidAndOffer1Minute = 193, + + /// Consolidated BBO with trades only. + TradeWithConsolidatedBestBidAndOffer = 194, + + /// Market-by-price BBO at 1-second cadence. + BBO1Second = 195, + + /// Market-by-price BBO at 1-minute cadence. + BBO1Minute = 196 +} diff --git a/QuantConnect.DataBento/Models/Enums/StatisticType.cs b/QuantConnect.DataBento/Models/Enums/StatisticType.cs new file mode 100644 index 0000000..51be1c2 --- /dev/null +++ b/QuantConnect.DataBento/Models/Enums/StatisticType.cs @@ -0,0 +1,117 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +namespace QuantConnect.Lean.DataSource.DataBento.Models.Enums; + +/// +/// Identifies the type of statistical market data reported for an instrument. +/// +/// +/// The statistic type defines how price, quantity, stat_flags, +/// and ts_ref should be interpreted for a given statistics record. +/// +public enum StatisticType +{ + /// + /// The price of the first trade of an instrument. + /// + OpeningPrice = 1, + + /// + /// The probable price and quantity of the first trade of an instrument, + /// published during the pre-open phase. + /// + IndicativeOpeningPrice = 2, + + /// + /// The settlement price of an instrument. + /// + /// + /// stat_flags indicate whether the settlement price is final or preliminary, + /// and whether it is actual or theoretical. + /// + SettlementPrice = 3, + + /// + /// The lowest trade price of an instrument during the trading session. + /// + TradingSessionLowPrice = 4, + + /// + /// The highest trade price of an instrument during the trading session. + /// + TradingSessionHighPrice = 5, + + /// + /// The number of contracts cleared for an instrument on the previous trading date. + /// + ClearedVolume = 6, + + /// + /// The lowest offer price for an instrument during the trading session. + /// + LowestOffer = 7, + + /// + /// The highest bid price for an instrument during the trading session. + /// + HighestBid = 8, + + /// + /// The current number of outstanding contracts of an instrument. + /// + OpenInterest = 9, + + /// + /// The volume-weighted average price (VWAP) for a fixing period. + /// + FixingPrice = 10, + + /// + /// The last trade price and quantity during a trading session. + /// + ClosePrice = 11, + + /// + /// The change in price from the previous session's close price + /// to the most recent close price. + /// + NetChange = 12, + + /// + /// The volume-weighted average price (VWAP) during the trading session. + /// + VolumeWeightedAveragePrice = 13, + + /// + /// The implied volatility associated with the settlement price. + /// + Volatility = 14, + + /// + /// The options delta associated with the settlement price. + /// + Delta = 15, + + /// + /// The auction uncrossing price and quantity. + /// + /// + /// Used for auctions that are neither the official opening auction + /// nor the official closing auction. + /// + UncrossingPrice = 16 +} + diff --git a/QuantConnect.DataBento/Models/Header.cs b/QuantConnect.DataBento/Models/Header.cs new file mode 100644 index 0000000..08b585d --- /dev/null +++ b/QuantConnect.DataBento/Models/Header.cs @@ -0,0 +1,50 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using QuantConnect.Lean.DataSource.DataBento.Models.Enums; + +namespace QuantConnect.Lean.DataSource.DataBento.Models; + +/// +/// Metadata header for a historical market data record. +/// Contains event timing, record type, data source, and instrument identifiers. +/// +public sealed class Header +{ + /// + /// The matching-engine-received timestamp expressed as the number of nanoseconds since the UNIX epoch. + /// + public long TsEvent { get; set; } + + /// + /// Record type identifier defining the data schema (e.g. trade, quote, bar). + /// + public RecordType Rtype { get; set; } + + /// + /// DataBento publisher (exchange / data source) identifier. + /// + public int PublisherId { get; set; } + + /// + /// Internal instrument identifier for the symbol. + /// + public uint InstrumentId { get; set; } + + /// + /// Event time converted to UTC . + /// + public DateTime UtcTime => Time.UnixNanosecondTimeStampToDateTime(TsEvent); +} diff --git a/QuantConnect.DataBento/Models/LevelOneBookLevel.cs b/QuantConnect.DataBento/Models/LevelOneBookLevel.cs new file mode 100644 index 0000000..3eb51f5 --- /dev/null +++ b/QuantConnect.DataBento/Models/LevelOneBookLevel.cs @@ -0,0 +1,49 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +namespace QuantConnect.Lean.DataSource.DataBento.Models; + +public sealed class LevelOneBookLevel +{ + /// + /// Bid price at this book level. + /// + public decimal BidPx { get; set; } + + /// + /// Ask price at this book level. + /// + public decimal AskPx { get; set; } + + /// + /// Total bid size at this level. + /// + public int BidSz { get; set; } + + /// + /// Total ask size at this level. + /// + public int AskSz { get; set; } + + /// + /// Number of bid orders at this level. + /// + public int BidCt { get; set; } + + /// + /// Number of ask orders at this level. + /// + public int AskCt { get; set; } +} diff --git a/QuantConnect.DataBento/Models/LevelOneData.cs b/QuantConnect.DataBento/Models/LevelOneData.cs new file mode 100644 index 0000000..cc68dd4 --- /dev/null +++ b/QuantConnect.DataBento/Models/LevelOneData.cs @@ -0,0 +1,68 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +namespace QuantConnect.Lean.DataSource.DataBento.Models; + +/// +/// Represents a level-one market data update containing best bid and ask information. +/// +public sealed class LevelOneData : MarketDataRecord +{ + /// + /// The capture-server-received timestamp expressed as the number of nanoseconds since the UNIX epoch. + /// + public long TsRecv { get; set; } + + /// + /// The event type or order book operation. Can be Add, Cancel, Modify, cleaR book, Trade, Fill, or None. + /// + public char Action { get; set; } + + /// + /// Side of the book affected by the update. + /// + public char Side { get; set; } + + /// + /// Book depth level affected by this update. + /// + public int Depth { get; set; } + + /// + /// Price associated with the update. + /// + public decimal Price { get; set; } + + /// + /// The side that initiates the event. + /// + /// + /// Can be: + /// - Ask for a sell order (or sell aggressor in a trade); + /// - Bid for a buy order (or buy aggressor in a trade); + /// - None where no side is specified by the original source. + /// + public int Size { get; set; } + + /// + /// A bit field indicating event end, message characteristics, and data quality. + /// + public int Flags { get; set; } + + /// + /// Snapshot of level-one bid and ask data. + /// + public IReadOnlyList Levels { get; set; } = []; +} \ No newline at end of file diff --git a/QuantConnect.DataBento/Models/Live/AuthenticationMessageRequest.cs b/QuantConnect.DataBento/Models/Live/AuthenticationMessageRequest.cs new file mode 100644 index 0000000..79d0715 --- /dev/null +++ b/QuantConnect.DataBento/Models/Live/AuthenticationMessageRequest.cs @@ -0,0 +1,69 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +namespace QuantConnect.Lean.DataSource.DataBento.Models.Live; + +public readonly struct AuthenticationMessageRequest +{ + private const int BucketIdLength = 5; + + private readonly string _dataset; + + private readonly string _auth; + + private readonly TimeSpan _heartBeatInterval; + + + public AuthenticationMessageRequest(string cramLine, string apiKey, string dataSet, TimeSpan? heartBeatInterval = default) + { + _dataset = dataSet; + + var newLineIndex = cramLine.IndexOf('\n'); + if (newLineIndex >= 0) + { + cramLine = cramLine[..newLineIndex]; + } + + cramLine = cramLine[(cramLine.IndexOf('=') + 1)..]; + + var challengeKey = cramLine + '|' + apiKey; + var bucketId = apiKey[^BucketIdLength..]; + + _auth = $"{QuantConnect.Extensions.ToSHA256(challengeKey)}-{bucketId}"; + + switch (heartBeatInterval) + { + case null: + _heartBeatInterval = TimeSpan.FromSeconds(5); + break; + case { TotalSeconds: < 5 }: + throw new ArgumentOutOfRangeException(nameof(heartBeatInterval), "The Heartbeat interval must be not les 5 seconds."); + default: + _heartBeatInterval = heartBeatInterval.Value; + break; + } + } + + public override string ToString() + { + return $"auth={_auth}|dataset={_dataset}|pretty_px=1|encoding=json|heartbeat_interval_s={_heartBeatInterval.TotalSeconds}"; + } + + public string GetStartSessionMessage() + { + return "start_session"; + } +} diff --git a/QuantConnect.DataBento/Models/Live/AuthenticationMessageResponse.cs b/QuantConnect.DataBento/Models/Live/AuthenticationMessageResponse.cs new file mode 100644 index 0000000..51720a3 --- /dev/null +++ b/QuantConnect.DataBento/Models/Live/AuthenticationMessageResponse.cs @@ -0,0 +1,54 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +using QuantConnect.Logging; + +namespace QuantConnect.Lean.DataSource.DataBento.Models.Live; + +public readonly struct AuthenticationMessageResponse +{ + public bool Success { get; } + public string? Error { get; } + public int SessionId { get; } + + public AuthenticationMessageResponse(string input) + { + if (Log.DebuggingEnabled) + { + Log.Debug($"{nameof(AuthenticationMessageResponse)}.ctor: Authentication response: {input}"); + } + + var parts = input.Split('|', StringSplitOptions.RemoveEmptyEntries); + + foreach (var part in parts) + { + var kv = part.Split('=', 2); + + switch (kv[0]) + { + case "success": + Success = kv[1] == "1"; + break; + case "error": + Error = kv[1]; + break; + case "session_id": + SessionId = int.Parse(kv[1]); + break; + } + } + } +} diff --git a/QuantConnect.DataBento/Models/Live/ConnectionLostEventArgs.cs b/QuantConnect.DataBento/Models/Live/ConnectionLostEventArgs.cs new file mode 100644 index 0000000..6994e98 --- /dev/null +++ b/QuantConnect.DataBento/Models/Live/ConnectionLostEventArgs.cs @@ -0,0 +1,34 @@ +namespace QuantConnect.Lean.DataSource.DataBento.Models.Live; + +/// +/// Provides data for the event that is raised when a connection is lost. +/// +public class ConnectionLostEventArgs : EventArgs +{ + /// + /// Gets the identifier of the data set or logical stream + /// associated with the lost connection. + /// + public string DataSet { get; } + + /// + /// Gets a human-readable description of the reason + /// why the connection was lost. + /// + public string Reason { get; } + + /// + /// Initializes a new instance of the class. + /// + /// + /// The identifier of the data set or logical stream related to the connection. + /// + /// + /// A human-readable explanation describing why the connection was lost. + /// + public ConnectionLostEventArgs(string message, string reason) + { + DataSet = message; + Reason = reason; + } +} diff --git a/QuantConnect.DataBento/Models/Live/HeartbeatMessage.cs b/QuantConnect.DataBento/Models/Live/HeartbeatMessage.cs new file mode 100644 index 0000000..4f1b5b7 --- /dev/null +++ b/QuantConnect.DataBento/Models/Live/HeartbeatMessage.cs @@ -0,0 +1,22 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +namespace QuantConnect.Lean.DataSource.DataBento.Models.Live; + +public sealed class HeartbeatMessage : MarketDataRecord +{ + public required string Msg { get; set; } +} diff --git a/QuantConnect.DataBento/Models/Live/SymbolMappingConfirmationEventArgs.cs b/QuantConnect.DataBento/Models/Live/SymbolMappingConfirmationEventArgs.cs new file mode 100644 index 0000000..461d0a2 --- /dev/null +++ b/QuantConnect.DataBento/Models/Live/SymbolMappingConfirmationEventArgs.cs @@ -0,0 +1,43 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +namespace QuantConnect.Lean.DataSource.DataBento.Models.Live; + +/// +/// Provides data for the symbol-mapping confirmation event. +/// +public sealed class SymbolMappingConfirmationEventArgs : EventArgs +{ + /// + /// Gets the original symbol that was mapped. + /// + public string Symbol { get; } + + /// + /// Gets the internal instrument identifier associated with the symbol. + /// + public uint InstrumentId { get; } + + /// + /// Initializes a new instance of the class. + /// + /// The symbol that was mapped. + /// The internal instrument identifier. + public SymbolMappingConfirmationEventArgs(string symbol, uint instrumentId) + { + Symbol = symbol; + InstrumentId = instrumentId; + } +} diff --git a/QuantConnect.DataBento/Models/Live/SymbolMappingMessage.cs b/QuantConnect.DataBento/Models/Live/SymbolMappingMessage.cs new file mode 100644 index 0000000..f0ffdfc --- /dev/null +++ b/QuantConnect.DataBento/Models/Live/SymbolMappingMessage.cs @@ -0,0 +1,30 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +namespace QuantConnect.Lean.DataSource.DataBento.Models.Live; + +public class SymbolMappingMessage : MarketDataRecord +{ + /// + /// The input symbol from the subscription + /// + public string? StypeInSymbol { get; set; } + + /// + /// The output symbol from the subscription + /// + public string? StypeOutSymbol { get; set; } +} diff --git a/QuantConnect.DataBento/Models/MarketDataRecord.cs b/QuantConnect.DataBento/Models/MarketDataRecord.cs new file mode 100644 index 0000000..31501d2 --- /dev/null +++ b/QuantConnect.DataBento/Models/MarketDataRecord.cs @@ -0,0 +1,30 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using Newtonsoft.Json; + +namespace QuantConnect.Lean.DataSource.DataBento.Models; + +/// +/// Base class for all market data records containing a standard metadata header. +/// +public abstract class MarketDataRecord +{ + /// + /// Gets or sets the standard metadata header for this market data record. + /// + [JsonProperty("hd")] + public required Header Header { get; set; } +} diff --git a/QuantConnect.DataBento/Models/OhlcvBar.cs b/QuantConnect.DataBento/Models/OhlcvBar.cs new file mode 100644 index 0000000..a7ce501 --- /dev/null +++ b/QuantConnect.DataBento/Models/OhlcvBar.cs @@ -0,0 +1,48 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +namespace QuantConnect.Lean.DataSource.DataBento.Models; + +/// +/// Open-High-Low-Close-Volume (OHLCV) bar representing aggregated market data +/// for a specific instrument and time interval. +/// +public sealed class OhlcvBar : MarketDataRecord +{ + /// + /// Opening price of the bar. + /// + public decimal Open { get; set; } + + /// + /// Highest traded price during the bar interval. + /// + public decimal High { get; set; } + + /// + /// Lowest traded price during the bar interval. + /// + public decimal Low { get; set; } + + /// + /// Closing price of the bar. + /// + public decimal Close { get; set; } + + /// + /// Total traded volume during the bar interval. + /// + public decimal Volume { get; set; } +} diff --git a/QuantConnect.DataBento/Models/StatisticsData.cs b/QuantConnect.DataBento/Models/StatisticsData.cs new file mode 100644 index 0000000..749127e --- /dev/null +++ b/QuantConnect.DataBento/Models/StatisticsData.cs @@ -0,0 +1,31 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using QuantConnect.Lean.DataSource.DataBento.Models.Enums; + +namespace QuantConnect.Lean.DataSource.DataBento.Models; + +public sealed class StatisticsData : MarketDataRecord +{ + /// + /// Quantity or value associated with the statistic. + /// + public decimal Quantity { get; set; } + + /// + /// Type of statistic represented by this record. + /// + public StatisticType StatType { get; set; } +} diff --git a/QuantConnect.DataBento/QuantConnect.DataSource.DataBento.csproj b/QuantConnect.DataBento/QuantConnect.DataSource.DataBento.csproj index 718002c..ada272c 100644 --- a/QuantConnect.DataBento/QuantConnect.DataSource.DataBento.csproj +++ b/QuantConnect.DataBento/QuantConnect.DataSource.DataBento.csproj @@ -1,9 +1,8 @@ - Release AnyCPU - net9.0 + net10.0 QuantConnect.Lean.DataSource.DataBento QuantConnect.Lean.DataSource.DataBento QuantConnect.Lean.DataSource.DataBento @@ -29,14 +28,14 @@ bin\Release\ - + - + diff --git a/QuantConnect.DataBento/Serialization/JsonSettings.cs b/QuantConnect.DataBento/Serialization/JsonSettings.cs new file mode 100644 index 0000000..4badd80 --- /dev/null +++ b/QuantConnect.DataBento/Serialization/JsonSettings.cs @@ -0,0 +1,45 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using Newtonsoft.Json; + +namespace QuantConnect.Lean.DataSource.DataBento.Serialization; + +/// +/// Provides globally accessible instances of +/// preconfigured with custom contract resolvers, such as snake-case formatting. +/// +public static class JsonSettings +{ + /// + /// Gets a reusable instance of that uses + /// for snake-case property name formatting. + /// + public static readonly JsonSerializerSettings SnakeCase = new() + { + ContractResolver = SnakeCaseContractResolver.Instance + }; + + /// + /// Gets a reusable instance of that uses + /// for snake-case property name formatting + /// and custom converter. + /// + public static readonly JsonSerializerSettings LiveDataSnakeCase = new() + { + ContractResolver = SnakeCaseContractResolver.Instance, + Converters = { new Converters.LiveDataConverter() } + }; +} diff --git a/QuantConnect.DataBento/Serialization/SnakeCaseContractResolver.cs b/QuantConnect.DataBento/Serialization/SnakeCaseContractResolver.cs new file mode 100644 index 0000000..cdc2319 --- /dev/null +++ b/QuantConnect.DataBento/Serialization/SnakeCaseContractResolver.cs @@ -0,0 +1,39 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using Newtonsoft.Json.Serialization; + +namespace QuantConnect.Lean.DataSource.DataBento.Serialization; + +/// +/// A singleton implementation that applies +/// a to JSON property names. +/// +public sealed class SnakeCaseContractResolver : DefaultContractResolver +{ + /// + /// Gets the singleton instance of the . + /// + public static readonly SnakeCaseContractResolver Instance = new(); + + /// + /// Initializes a new instance of the class + /// with a . + /// + private SnakeCaseContractResolver() + { + NamingStrategy = new SnakeCaseNamingStrategy(); + } +}