From 31d18f37629b26971d69b1d48f8a635baa786b7a Mon Sep 17 00:00:00 2001 From: Joseph Scorsone Date: Mon, 29 Dec 2025 14:34:13 -0500 Subject: [PATCH 01/13] Improvements from previous PR. Adhere to QC code conventions better. More robust code. Refactored and properly passing tests. --- .../DataBentoDataDownloaderTests.cs | 221 +++++----- .../DataBentoDataProviderHistoryTests.cs | 170 +++----- .../DataBentoRawLiveClientTests.cs | 227 +++------- ...tConnect.DataSource.DataBento.Tests.csproj | 4 +- QuantConnect.DataBento.Tests/TestSetup.cs | 17 - QuantConnect.DataBento.Tests/config.json | 2 +- .../DataBentoDataDownloader.cs | 332 ++++++--------- .../DataBentoDataProvider.cs | 383 +++++++---------- .../DataBentoHistoryProivder.cs | 85 +--- .../DataBentoRawLiveClient.cs | 402 ++++++++---------- .../DataBentoSymbolMapper.cs | 174 ++++++++ .../QuantConnect.DataSource.DataBento.csproj | 4 + models/DataBentoTypes.cs | 112 +++++ 13 files changed, 1012 insertions(+), 1121 deletions(-) create mode 100644 QuantConnect.DataBento/DataBentoSymbolMapper.cs create mode 100644 models/DataBentoTypes.cs diff --git a/QuantConnect.DataBento.Tests/DataBentoDataDownloaderTests.cs b/QuantConnect.DataBento.Tests/DataBentoDataDownloaderTests.cs index d9d8071..a5063df 100644 --- a/QuantConnect.DataBento.Tests/DataBentoDataDownloaderTests.cs +++ b/QuantConnect.DataBento.Tests/DataBentoDataDownloaderTests.cs @@ -17,11 +17,13 @@ using System; using System.Linq; using NUnit.Framework; +using QuantConnect.Configuration; using QuantConnect.Data; using QuantConnect.Data.Market; using QuantConnect.Lean.DataSource.DataBento; using QuantConnect.Logging; -using QuantConnect.Configuration; +using QuantConnect.Securities; +using QuantConnect.Util; namespace QuantConnect.Lean.DataSource.DataBento.Tests { @@ -29,12 +31,20 @@ namespace QuantConnect.Lean.DataSource.DataBento.Tests public class DataBentoDataDownloaderTests { private DataBentoDataDownloader _downloader; - private readonly string _apiKey = Config.Get("databento-api-key"); + private MarketHoursDatabase _marketHoursDatabase; + protected readonly string ApiKey = Config.Get("databento-api-key"); + + private static Symbol CreateEsFuture() + { + var expiration = new DateTime(2026, 3, 20); + return Symbol.CreateFuture("ES", Market.CME, expiration); + } [SetUp] public void SetUp() { - _downloader = new DataBentoDataDownloader(_apiKey); + _marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); + _downloader = new DataBentoDataDownloader(ApiKey, _marketHoursDatabase); } [TearDown] @@ -43,141 +53,154 @@ public void TearDown() _downloader?.Dispose(); } - [Test] - [TestCase("ESM3", SecurityType.Future, Market.CME, Resolution.Daily, TickType.Trade)] - [TestCase("ESM3", SecurityType.Future, Market.CME, Resolution.Hour, TickType.Trade)] - [TestCase("ESM3", SecurityType.Future, Market.CME, Resolution.Minute, TickType.Trade)] - [TestCase("ESM3", SecurityType.Future, Market.CME, Resolution.Second, TickType.Trade)] - [TestCase("ESM3", SecurityType.Future, Market.CME, Resolution.Tick, TickType.Trade)] - [Explicit("This test requires a configured DataBento API key")] - public void DownloadsHistoricalData(string ticker, SecurityType securityType, string market, Resolution resolution, TickType tickType) + [TestCase(Resolution.Daily)] + [TestCase(Resolution.Hour)] + [TestCase(Resolution.Minute)] + [TestCase(Resolution.Second)] + [TestCase(Resolution.Tick)] + public void DownloadsTradeDataForLeanFuture(Resolution resolution) { - var symbol = Symbol.Create(ticker, securityType, market); - var startTime = new DateTime(2024, 1, 15); - var endTime = new DateTime(2024, 1, 16); - var param = new DataDownloaderGetParameters(symbol, resolution, startTime, endTime, tickType); + var symbol = CreateEsFuture(); + var exchangeTimeZone = _marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType).TimeZone; - var downloadResponse = _downloader.Get(param).ToList(); + var startUtc = new DateTime(2024, 5, 1, 0, 0, 0, DateTimeKind.Utc); + var endUtc = new DateTime(2024, 5, 2, 0, 0, 0, DateTimeKind.Utc); - Log.Trace($"Downloaded {downloadResponse.Count} data points for {symbol} at {resolution} resolution"); + if (resolution == Resolution.Tick) + { + startUtc = new DateTime(2024, 5, 1, 9, 30, 0, DateTimeKind.Utc); + endUtc = startUtc.AddMinutes(15); + } - Assert.IsTrue(downloadResponse.Any(), "Expected to download at least one data point"); + var parameters = new DataDownloaderGetParameters( + symbol, + resolution, + startUtc, + endUtc, + TickType.Trade + ); - foreach (var data in downloadResponse) + var data = _downloader.Get(parameters).ToList(); + + Log.Trace($"Downloaded {data.Count} trade points for {symbol} @ {resolution}"); + + Assert.IsNotEmpty(data); + + var startExchange = startUtc.ConvertFromUtc(exchangeTimeZone); + var endExchange = endUtc.ConvertFromUtc(exchangeTimeZone); + + foreach (var point in data) { - Assert.IsNotNull(data, "Data point should not be null"); - Assert.AreEqual(symbol, data.Symbol, "Symbol should match requested symbol"); - Assert.IsTrue(data.Time >= startTime && data.Time <= endTime, "Data time should be within requested range"); + Assert.AreEqual(symbol, point.Symbol); + Assert.That(point.Time, Is.InRange(startExchange, endExchange)); - if (data is TradeBar tradeBar) + switch (point) { - Assert.Greater(tradeBar.Close, 0, "Close price should be positive"); - Assert.GreaterOrEqual(tradeBar.Volume, 0, "Volume should be non-negative"); - Assert.Greater(tradeBar.High, 0, "High price should be positive"); - Assert.Greater(tradeBar.Low, 0, "Low price should be positive"); - Assert.Greater(tradeBar.Open, 0, "Open price should be positive"); - Assert.GreaterOrEqual(tradeBar.High, tradeBar.Low, "High should be >= Low"); - Assert.GreaterOrEqual(tradeBar.High, tradeBar.Open, "High should be >= Open"); - Assert.GreaterOrEqual(tradeBar.High, tradeBar.Close, "High should be >= Close"); - Assert.LessOrEqual(tradeBar.Low, tradeBar.Open, "Low should be <= Open"); - Assert.LessOrEqual(tradeBar.Low, tradeBar.Close, "Low should be <= Close"); - } - else if (data is QuoteBar quoteBar) - { - Assert.Greater(quoteBar.Close, 0, "Quote close price should be positive"); - if (quoteBar.Bid != null) - { - Assert.Greater(quoteBar.Bid.Close, 0, "Bid price should be positive"); - } - if (quoteBar.Ask != null) - { - Assert.Greater(quoteBar.Ask.Close, 0, "Ask price should be positive"); - } - } - else if (data is Tick tick) - { - Assert.Greater(tick.Value, 0, "Tick value should be positive"); - Assert.GreaterOrEqual(tick.Quantity, 0, "Tick quantity should be non-negative"); + case TradeBar bar: + Assert.Greater(bar.Open, 0); + Assert.Greater(bar.High, 0); + Assert.Greater(bar.Low, 0); + Assert.Greater(bar.Close, 0); + Assert.GreaterOrEqual(bar.Volume, 0); + Assert.GreaterOrEqual(bar.High, bar.Low); + break; + + case Tick tick: + Assert.Greater(tick.Value, 0); + Assert.GreaterOrEqual(tick.Quantity, 0); + break; + + default: + Assert.Fail($"Unexpected data type {point.GetType()}"); + break; } } } [Test] - [TestCase("ZNM3", SecurityType.Future, Market.CME, Resolution.Daily, TickType.Trade)] - [TestCase("ZNM3", SecurityType.Future, Market.CME, Resolution.Hour, TickType.Trade)] - [Explicit("This test requires a configured DataBento API key")] - public void DownloadsFuturesHistoricalData(string ticker, SecurityType securityType, string market, Resolution resolution, TickType tickType) + public void DownloadsQuoteTicksForLeanFuture() { - var symbol = Symbol.Create(ticker, securityType, market); - var startTime = new DateTime(2024, 1, 15); - var endTime = new DateTime(2024, 1, 16); - var param = new DataDownloaderGetParameters(symbol, resolution, startTime, endTime, tickType); - - var downloadResponse = _downloader.Get(param).ToList(); + var symbol = CreateEsFuture(); + var exchangeTimeZone = _marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType).TimeZone; - Log.Trace($"Downloaded {downloadResponse.Count} data points for futures {symbol}"); + var startUtc = new DateTime(2024, 5, 1, 9, 30, 0, DateTimeKind.Utc); + var endUtc = startUtc.AddMinutes(15); - Assert.IsTrue(downloadResponse.Any(), "Expected to download futures data"); + var parameters = new DataDownloaderGetParameters( + symbol, + Resolution.Tick, + startUtc, + endUtc, + TickType.Quote + ); - foreach (var data in downloadResponse) - { - Assert.AreEqual(symbol, data.Symbol, "Symbol should match requested futures symbol"); - Assert.Greater(data.Value, 0, "Data value should be positive"); - } - } - - [Test] - [TestCase("ESM3", SecurityType.Future, Market.CME, Resolution.Tick, TickType.Quote)] - [Explicit("This test requires a configured DataBento API key and advanced subscription")] - public void DownloadsQuoteData(string ticker, SecurityType securityType, string market, Resolution resolution, TickType tickType) - { - var symbol = Symbol.Create(ticker, securityType, market); - var startTime = new DateTime(2024, 1, 15, 9, 30, 0); - var endTime = new DateTime(2024, 1, 15, 9, 45, 0); - var param = new DataDownloaderGetParameters(symbol, resolution, startTime, endTime, tickType); + var data = _downloader.Get(parameters).ToList(); - var downloadResponse = _downloader.Get(param).ToList(); + Log.Trace($"Downloaded {data.Count} quote ticks for {symbol}"); - Log.Trace($"Downloaded {downloadResponse.Count} quote data points for {symbol}"); + Assert.IsNotEmpty(data); - Assert.IsTrue(downloadResponse.Any(), "Expected to download quote data"); + var startExchange = startUtc.ConvertFromUtc(exchangeTimeZone); + var endExchange = endUtc.ConvertFromUtc(exchangeTimeZone); - foreach (var data in downloadResponse) + foreach (var point in data) { - Assert.AreEqual(symbol, data.Symbol, "Symbol should match requested symbol"); - if (data is QuoteBar quoteBar) + Assert.AreEqual(symbol, point.Symbol); + Assert.That(point.Time, Is.InRange(startExchange, endExchange)); + + if (point is Tick tick) + { + Assert.AreEqual(TickType.Quote, tick.TickType); + Assert.IsTrue( + tick.BidPrice > 0 || tick.AskPrice > 0, + "Quote tick must have bid or ask" + ); + } + else if (point is QuoteBar bar) { - Assert.IsTrue(quoteBar.Bid != null || quoteBar.Ask != null, "Quote should have bid or ask data"); + Assert.IsTrue(bar.Bid != null || bar.Ask != null); } } } [Test] - [Explicit("This test requires a configured DataBento API key")] public void DataIsSortedByTime() { - var symbol = Symbol.Create("ESM3", SecurityType.Future, Market.CME); - var startTime = new DateTime(2024, 1, 15); - var endTime = new DateTime(2024, 1, 16); - var param = new DataDownloaderGetParameters(symbol, Resolution.Minute, startTime, endTime, TickType.Trade); + var symbol = CreateEsFuture(); - var downloadResponse = _downloader.Get(param).ToList(); + var startUtc = new DateTime(2024, 5, 1, 0, 0, 0, DateTimeKind.Utc); + var endUtc = new DateTime(2024, 5, 2, 0, 0, 0, DateTimeKind.Utc); - Assert.IsTrue(downloadResponse.Any(), "Expected to download data for time sorting test"); + var parameters = new DataDownloaderGetParameters( + symbol, + Resolution.Minute, + startUtc, + endUtc, + TickType.Trade + ); - for (int i = 1; i < downloadResponse.Count; i++) + var data = _downloader.Get(parameters).ToList(); + + Assert.IsNotEmpty(data); + + for (int i = 1; i < data.Count; i++) { - Assert.GreaterOrEqual(downloadResponse[i].Time, downloadResponse[i - 1].Time, - $"Data should be sorted by time. Item {i} time {downloadResponse[i].Time} should be >= item {i - 1} time {downloadResponse[i - 1].Time}"); + Assert.GreaterOrEqual( + data[i].Time, + data[i - 1].Time, + $"Data not sorted at index {i}" + ); } } [Test] - public void DisposesCorrectly() + public void DisposeIsIdempotent() { - var downloader = new DataBentoDataDownloader(); - Assert.DoesNotThrow(() => downloader.Dispose(), "Dispose should not throw"); - Assert.DoesNotThrow(() => downloader.Dispose(), "Multiple dispose calls should not throw"); + var downloader = new DataBentoDataDownloader(ApiKey, + MarketHoursDatabase.FromDataFolder()); + + Assert.DoesNotThrow(downloader.Dispose); + Assert.DoesNotThrow(downloader.Dispose); } } } diff --git a/QuantConnect.DataBento.Tests/DataBentoDataProviderHistoryTests.cs b/QuantConnect.DataBento.Tests/DataBentoDataProviderHistoryTests.cs index 2da5b8f..94c78f8 100644 --- a/QuantConnect.DataBento.Tests/DataBentoDataProviderHistoryTests.cs +++ b/QuantConnect.DataBento.Tests/DataBentoDataProviderHistoryTests.cs @@ -32,12 +32,19 @@ namespace QuantConnect.Lean.DataSource.DataBento.Tests public class DataBentoDataProviderHistoryTests { private DataBentoProvider _historyDataProvider; - private readonly string _apiKey = Config.Get("databento-api-key"); + private MarketHoursDatabase _marketHoursDatabase; + protected readonly string ApiKey = Config.Get("databento-api-key"); + + private static Symbol CreateEsFuture() + { + var expiration = new DateTime(2026, 3, 20); + return Symbol.CreateFuture("ES", Market.CME, expiration); + } [SetUp] public void SetUp() { - _historyDataProvider = new DataBentoProvider(_apiKey); + _historyDataProvider = new DataBentoProvider(); } [TearDown] @@ -50,150 +57,99 @@ internal static IEnumerable TestParameters { get { + var es = CreateEsFuture(); - // DataBento futures - var esMini = Symbol.Create("ESM3", SecurityType.Future, Market.CME); - var znNote = Symbol.Create("ZNM3", SecurityType.Future, Market.CME); - var gcGold = Symbol.Create("GCM3", SecurityType.Future, Market.CME); - - // test cases for supported futures - yield return new TestCaseData(esMini, Resolution.Daily, TickType.Trade, TimeSpan.FromDays(5), false) - .SetDescription("Valid ES futures - Daily resolution, 5 days period") - .SetCategory("Valid"); - - yield return new TestCaseData(esMini, Resolution.Hour, TickType.Trade, TimeSpan.FromDays(2), false) - .SetDescription("Valid ES futures - Hour resolution, 2 days period") - .SetCategory("Valid"); - - yield return new TestCaseData(esMini, Resolution.Minute, TickType.Trade, TimeSpan.FromHours(4), false) - .SetDescription("Valid ES futures - Minute resolution, 4 hours period") + yield return new TestCaseData(es, Resolution.Daily, TickType.Trade, TimeSpan.FromDays(5), false) + .SetDescription("ES futures daily trade history") .SetCategory("Valid"); - yield return new TestCaseData(znNote, Resolution.Daily, TickType.Trade, TimeSpan.FromDays(3), false) - .SetDescription("Valid ZN futures - Daily resolution, 3 days period") + yield return new TestCaseData(es, Resolution.Hour, TickType.Trade, TimeSpan.FromDays(2), false) + .SetDescription("ES futures hourly trade history") .SetCategory("Valid"); - yield return new TestCaseData(gcGold, Resolution.Hour, TickType.Trade, TimeSpan.FromDays(1), false) - .SetDescription("Valid GC futures - Hour resolution, 1 day period") + yield return new TestCaseData(es, Resolution.Minute, TickType.Trade, TimeSpan.FromHours(4), false) + .SetDescription("ES futures minute trade history") .SetCategory("Valid"); - // Test cases for quote data (may require advanced subscription) - yield return new TestCaseData(esMini, Resolution.Tick, TickType.Quote, TimeSpan.FromMinutes(15), false) - .SetDescription("ES futures quote data - Tick resolution") + yield return new TestCaseData(es, Resolution.Tick, TickType.Quote, TimeSpan.FromMinutes(15), false) + .SetDescription("ES futures quote ticks") .SetCategory("Quote"); - - // Unsupported security types - var equity = Symbol.Create("SPY", SecurityType.Equity, Market.USA); - var option = Symbol.Create("SPY", SecurityType.Option, Market.USA); - - yield return new TestCaseData(equity, Resolution.Daily, TickType.Trade, TimeSpan.FromDays(5), true) - .SetDescription("Invalid - Equity not supported by DataBento") - .SetCategory("Invalid"); - - yield return new TestCaseData(option, Resolution.Daily, TickType.Trade, TimeSpan.FromDays(5), true) - .SetDescription("Invalid - Option not supported by DataBento") - .SetCategory("Invalid"); } } [Test, TestCaseSource(nameof(TestParameters))] - [Explicit("This test requires a configured DataBento API key")] public void GetsHistory(Symbol symbol, Resolution resolution, TickType tickType, TimeSpan period, bool expectsNoData) { var request = GetHistoryRequest(resolution, tickType, symbol, period); - try + var history = _historyDataProvider.GetHistory(request); + + if (expectsNoData) { - var slices = _historyDataProvider.GetHistory(request)?.Select(data => new Slice(data.Time, new[] { data }, data.Time.ConvertToUtc(request.DataTimeZone))).ToList(); + Assert.IsTrue(history == null || !history.Any(), + $"Expected no data for unsupported symbol: {symbol}"); + return; + } - if (expectsNoData) - { - Assert.IsTrue(slices == null || !slices.Any(), - $"Expected no data for unsupported symbol/security type: {symbol}"); - } - else + Assert.IsNotNull(history); + var data = history.ToList(); + Assert.IsNotEmpty(data); + + Log.Trace($"Received {data.Count} data points for {symbol} @ {resolution}"); + + foreach (var point in data.Take(5)) + { + Assert.AreEqual(symbol, point.Symbol); + + if (point is TradeBar bar) { - Assert.IsNotNull(slices, "Expected to receive history data"); - - if (slices.Any()) - { - Log.Trace($"Received {slices.Count} slices for {symbol} at {resolution} resolution"); - - foreach (var slice in slices.Take(5)) // Check first 5 slices - { - Assert.IsNotNull(slice, "Slice should not be null"); - Assert.IsTrue(slice.Time >= request.StartTimeUtc && slice.Time <= request.EndTimeUtc, - "Slice time should be within requested range"); - - if (slice.Bars.ContainsKey(symbol)) - { - var bar = slice.Bars[symbol]; - Assert.Greater(bar.Close, 0, "Bar close price should be positive"); - Assert.GreaterOrEqual(bar.Volume, 0, "Bar volume should be non-negative"); - } - } - } + Assert.Greater(bar.Close, 0); + Assert.GreaterOrEqual(bar.Volume, 0); } - } - catch (Exception ex) - { - Log.Error($"Error getting history for {symbol}: {ex.Message}"); - if (!expectsNoData) + if (point is Tick tick && tickType == TickType.Quote) { - throw; + Assert.IsTrue(tick.BidPrice > 0 || tick.AskPrice > 0); } } } [Test] - [Explicit("This test requires a configured DataBento API key")] public void GetHistoryWithMultipleSymbols() { - var symbol1 = Symbol.Create("ESM3", SecurityType.Future, Market.CME); - var symbol2 = Symbol.Create("ZNM3", SecurityType.Future, Market.CME); - - var request1 = GetHistoryRequest(Resolution.Daily, TickType.Trade, symbol1, TimeSpan.FromDays(3)); - var request2 = GetHistoryRequest(Resolution.Daily, TickType.Trade, symbol2, TimeSpan.FromDays(3)); - - var history1 = _historyDataProvider.GetHistory(request1); - var history2 = _historyDataProvider.GetHistory(request2); + var es = CreateEsFuture(); - var allData = new List(); - if (history1 != null) allData.AddRange(history1); - if (history2 != null) allData.AddRange(history2); + var request = GetHistoryRequest(Resolution.Daily, TickType.Trade, es, TimeSpan.FromDays(3)); - // timezone from the first request - var slices = allData.GroupBy(d => d.Time) - .Select(g => new Slice(g.Key, g.ToList(), g.Key.ConvertToUtc(request1.DataTimeZone))) - .ToList(); + var history = _historyDataProvider.GetHistory(request)?.ToList(); - Assert.IsNotNull(slices, "Expected to receive history data for multiple symbols"); - - if (slices.Any()) - { - Log.Trace($"Received {slices.Count} slices for multiple symbols"); - - var hasSymbol1Data = slices.Any(s => s.Bars.ContainsKey(symbol1)); - var hasSymbol2Data = slices.Any(s => s.Bars.ContainsKey(symbol2)); - - Assert.IsTrue(hasSymbol1Data || hasSymbol2Data, - "Expected data for at least one of the requested symbols"); - } + Assert.IsTrue( + history != null && history.Any(), + "Expected history for ES" + ); } - internal static HistoryRequest GetHistoryRequest(Resolution resolution, TickType tickType, Symbol symbol, TimeSpan period) + internal static HistoryRequest GetHistoryRequest( + Resolution resolution, + TickType tickType, + Symbol symbol, + TimeSpan period) { - var utcNow = DateTime.UtcNow; + var endUtc = new DateTime(2024, 5, 10, 0, 0, 0, DateTimeKind.Utc); + var startUtc = endUtc - period; + var dataType = LeanData.GetDataType(resolution, tickType); var marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); - var exchangeHours = marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType); - var dataTimeZone = marketHoursDatabase.GetDataTimeZone(symbol.ID.Market, symbol, symbol.SecurityType); + var exchangeHours = marketHoursDatabase.GetExchangeHours( + symbol.ID.Market, symbol, symbol.SecurityType); + + var dataTimeZone = marketHoursDatabase.GetDataTimeZone( + symbol.ID.Market, symbol, symbol.SecurityType); return new HistoryRequest( - startTimeUtc: utcNow.Add(-period), - endTimeUtc: utcNow, + startTimeUtc: startUtc, + endTimeUtc: endUtc, dataType: dataType, symbol: symbol, resolution: resolution, @@ -204,7 +160,7 @@ internal static HistoryRequest GetHistoryRequest(Resolution resolution, TickType isCustomData: false, DataNormalizationMode.Raw, tickType: tickType - ); + ); } } } diff --git a/QuantConnect.DataBento.Tests/DataBentoRawLiveClientTests.cs b/QuantConnect.DataBento.Tests/DataBentoRawLiveClientTests.cs index 7df71a4..22e7f04 100644 --- a/QuantConnect.DataBento.Tests/DataBentoRawLiveClientTests.cs +++ b/QuantConnect.DataBento.Tests/DataBentoRawLiveClientTests.cs @@ -16,26 +16,32 @@ using System; using System.Threading; -using System.Threading.Tasks; using NUnit.Framework; +using QuantConnect.Configuration; using QuantConnect.Data; using QuantConnect.Data.Market; using QuantConnect.Lean.DataSource.DataBento; using QuantConnect.Logging; -using QuantConnect.Configuration; namespace QuantConnect.Lean.DataSource.DataBento.Tests { [TestFixture] - public class DataBentoRawLiveClientTests + public class DataBentoRawLiveClientSyncTests { - private DatabentoRawClient _client; - private readonly string _apiKey = Config.Get("databento-api-key"); + private DataBentoRawLiveClient _client; + protected readonly string ApiKey = Config.Get("databento-api-key"); + + private static Symbol CreateEsFuture() + { + var expiration = new DateTime(2026, 3, 20); + return Symbol.CreateFuture("ES", Market.CME, expiration); + } [SetUp] public void SetUp() { - _client = new DatabentoRawClient(_apiKey); + Log.Trace("DataBentoLiveClientTests: Using API Key: " + ApiKey); + _client = new DataBentoRawLiveClient(ApiKey); } [TearDown] @@ -45,208 +51,97 @@ public void TearDown() } [Test] - [Explicit("This test requires a configured DataBento API key and live connection")] - public async Task ConnectsToGateway() + public void Connects() { - if (string.IsNullOrEmpty(_apiKey)) - { - Assert.Ignore("DataBento API key not configured"); - return; - } - - var connected = await _client.ConnectAsync(); + var connected = _client.Connect(); - Assert.IsTrue(connected, "Should successfully connect to DataBento gateway"); - Assert.IsTrue(_client.IsConnected, "IsConnected should return true after successful connection"); + Assert.IsTrue(connected); + Assert.IsTrue(_client.IsConnected); - Log.Trace("Successfully connected to DataBento gateway"); + Log.Trace("Connected successfully"); } [Test] - [Explicit("This test requires a configured DataBento API key and live connection")] - public async Task SubscribesToSymbol() + public void SubscribesToLeanFutureSymbol() { - if (string.IsNullOrEmpty(_apiKey)) - { - Assert.Ignore("DataBento API key not configured"); - return; - } - - var connected = await _client.ConnectAsync(); - Assert.IsTrue(connected, "Must be connected to test subscription"); + Assert.IsTrue(_client.Connect()); - var symbol = Symbol.Create("ESM3", SecurityType.Future, Market.CME); - var subscribed = _client.Subscribe(symbol, Resolution.Minute, TickType.Trade); + var symbol = CreateEsFuture(); - Assert.IsTrue(subscribed, "Should successfully subscribe to symbol"); + Assert.IsTrue(_client.Subscribe(symbol, TickType.Trade)); + Assert.IsTrue(_client.StartSession()); - Log.Trace($"Successfully subscribed to {symbol}"); + Thread.Sleep(1000); - // Wait a moment to ensure subscription is active - await Task.Delay(2000); - - var unsubscribed = _client.Unsubscribe(symbol); - Assert.IsTrue(unsubscribed, "Should successfully unsubscribe from symbol"); - - Log.Trace($"Successfully unsubscribed from {symbol}"); + Assert.IsTrue(_client.Unsubscribe(symbol)); } [Test] - [Explicit("This test requires a configured DataBento API key and live connection")] - public async Task ReceivesLiveData() + public void ReceivesTradeOrQuoteTicks() { - if (string.IsNullOrEmpty(_apiKey)) - { - Assert.Ignore("DataBento API key not configured"); - return; - } - - var dataReceived = false; - var dataReceivedEvent = new ManualResetEventSlim(false); - BaseData receivedData = null; + var receivedEvent = new ManualResetEventSlim(false); + BaseData received = null; - _client.DataReceived += (sender, data) => + _client.DataReceived += (_, data) => { - receivedData = data; - dataReceived = true; - dataReceivedEvent.Set(); - Log.Trace($"Received data: {data}"); + received = data; + receivedEvent.Set(); }; - var connected = await _client.ConnectAsync(); - Assert.IsTrue(connected, "Must be connected to test data reception"); + Assert.IsTrue(_client.Connect()); - var symbol = Symbol.Create("ESM3", SecurityType.Future, Market.CME); - var subscribed = _client.Subscribe(symbol, Resolution.Tick, TickType.Trade); - Assert.IsTrue(subscribed, "Must be subscribed to receive data"); + var symbol = CreateEsFuture(); - // Wait for data with timeout - var dataReceiptTimeout = TimeSpan.FromMinutes(2); - var receivedWithinTimeout = dataReceivedEvent.Wait(dataReceiptTimeout); + Assert.IsTrue(_client.Subscribe(symbol, TickType.Trade)); + Assert.IsTrue(_client.StartSession()); - if (receivedWithinTimeout) - { - Assert.IsTrue(dataReceived, "Should have received data"); - Assert.IsNotNull(receivedData, "Received data should not be null"); - Assert.AreEqual(symbol, receivedData.Symbol, "Received data symbol should match subscription"); - Assert.Greater(receivedData.Value, 0, "Received data value should be positive"); + var gotData = receivedEvent.Wait(TimeSpan.FromMinutes(2)); - Log.Trace($"Successfully received live data: {receivedData}"); - } - else + if (!gotData) { - Log.Trace("No data received within timeout period - this may be expected during non-market hours"); + Assert.Inconclusive("No data received (likely outside market hours)"); + return; } - _client.Unsubscribe(symbol); - } + Assert.NotNull(received); + Assert.AreEqual(symbol, received.Symbol); - [Test] - [Explicit("This test requires a configured DataBento API key and live connection")] - public async Task HandlesConnectionEvents() - { - if (string.IsNullOrEmpty(_apiKey)) + if (received is Tick tick) { - Assert.Ignore("DataBento API key not configured"); - return; + Assert.Greater(tick.Time, DateTime.MinValue); + Assert.Greater(tick.Value, 0); } - - var connectionStatusChanged = false; - var connectionStatusEvent = new ManualResetEventSlim(false); - - _client.ConnectionStatusChanged += (sender, isConnected) => + else if (received is TradeBar bar) { - connectionStatusChanged = true; - connectionStatusEvent.Set(); - Log.Trace($"Connection status changed: {isConnected}"); - }; - - var connected = await _client.ConnectAsync(); - Assert.IsTrue(connected, "Should connect successfully"); - - // Connection status event should fire on connect - var eventFiredWithinTimeout = connectionStatusEvent.Wait(TimeSpan.FromSeconds(10)); - Assert.IsTrue(eventFiredWithinTimeout || connectionStatusChanged, - "Connection status changed event should fire"); - - _client.Disconnect(); - Assert.IsFalse(_client.IsConnected, "Should be disconnected after calling Disconnect()"); - } - - [Test] - public void HandlesInvalidApiKey() - { - var invalidClient = new DatabentoRawClient("invalid-api-key"); - - // Connection with invalid API key should fail gracefully - Assert.DoesNotThrowAsync(async () => + Assert.Greater(bar.Close, 0); + } + else { - var connected = await invalidClient.ConnectAsync(); - Assert.IsFalse(connected, "Connection should fail with invalid API key"); - }); - - invalidClient.Dispose(); + Assert.Fail($"Unexpected data type: {received.GetType()}"); + } } [Test] - public void DisposesCorrectly() + public void DisposeIsIdempotent() { - var client = new DatabentoRawClient(_apiKey); - Assert.DoesNotThrow(() => client.Dispose(), "Dispose should not throw"); - Assert.DoesNotThrow(() => client.Dispose(), "Multiple dispose calls should not throw"); + var client = new DataBentoRawLiveClient(ApiKey); + Assert.DoesNotThrow(client.Dispose); + Assert.DoesNotThrow(client.Dispose); } [Test] - public void SymbolMappingWorksCorrectly() + public void SymbolMappingDoesNotThrow() { - // Test that futures are mapped correctly to DataBento format - var esFuture = Symbol.Create("ESM3", SecurityType.Future, Market.CME); + Assert.IsTrue(_client.Connect()); - // Since the mapping method is private, we test indirectly through subscription - Assert.DoesNotThrowAsync(async () => - { - if (!string.IsNullOrEmpty(_apiKey)) - { - var connected = await _client.ConnectAsync(); - if (connected) - { - _client.Subscribe(esFuture, Resolution.Minute, TickType.Trade); - _client.Unsubscribe(esFuture); - } - } - }); - } - - [Test] - public void SchemaResolutionMappingWorksCorrectly() - { - // Test that resolution mappings work correctly - var symbol = Symbol.Create("ESM3", SecurityType.Future, Market.CME); + var symbol = CreateEsFuture(); - Assert.DoesNotThrowAsync(async () => + Assert.DoesNotThrow(() => { - if (!string.IsNullOrEmpty(_apiKey)) - { - var connected = await _client.ConnectAsync(); - if (connected) - { - // Test different resolutions - _client.Subscribe(symbol, Resolution.Tick, TickType.Trade); - _client.Unsubscribe(symbol); - - _client.Subscribe(symbol, Resolution.Second, TickType.Trade); - _client.Unsubscribe(symbol); - - _client.Subscribe(symbol, Resolution.Minute, TickType.Trade); - _client.Unsubscribe(symbol); - - _client.Subscribe(symbol, Resolution.Hour, TickType.Trade); - _client.Unsubscribe(symbol); - - _client.Subscribe(symbol, Resolution.Daily, TickType.Trade); - _client.Unsubscribe(symbol); - } - } + _client.Subscribe(symbol, TickType.Trade); + _client.StartSession(); + Thread.Sleep(500); + _client.Unsubscribe(symbol); }); } } diff --git a/QuantConnect.DataBento.Tests/QuantConnect.DataSource.DataBento.Tests.csproj b/QuantConnect.DataBento.Tests/QuantConnect.DataSource.DataBento.Tests.csproj index d2d7ef1..f5fce66 100644 --- a/QuantConnect.DataBento.Tests/QuantConnect.DataSource.DataBento.Tests.csproj +++ b/QuantConnect.DataBento.Tests/QuantConnect.DataSource.DataBento.Tests.csproj @@ -15,7 +15,6 @@ - @@ -24,10 +23,9 @@ - - + PreserveNewest diff --git a/QuantConnect.DataBento.Tests/TestSetup.cs b/QuantConnect.DataBento.Tests/TestSetup.cs index b0e07c1..39cb9de 100644 --- a/QuantConnect.DataBento.Tests/TestSetup.cs +++ b/QuantConnect.DataBento.Tests/TestSetup.cs @@ -62,22 +62,5 @@ private static void ReloadConfiguration() // resets the version among other things Globals.Reset(); } - - private static void SetUp() - { - Log.LogHandler = new CompositeLogHandler(); - Log.Trace("TestSetup(): starting..."); - ReloadConfiguration(); - Log.DebuggingEnabled = Config.GetBool("debug-mode"); - } - - private static TestCaseData[] TestParameters - { - get - { - SetUp(); - return new [] { new TestCaseData() }; - } - } } } diff --git a/QuantConnect.DataBento.Tests/config.json b/QuantConnect.DataBento.Tests/config.json index ff861ca..7256aba 100644 --- a/QuantConnect.DataBento.Tests/config.json +++ b/QuantConnect.DataBento.Tests/config.json @@ -1,5 +1,5 @@ { - "data-folder":"../../../../../Lean/Data/", + "data-folder":"../../../../../Data/", "job-user-id": "0", "api-access-token": "", diff --git a/QuantConnect.DataBento/DataBentoDataDownloader.cs b/QuantConnect.DataBento/DataBentoDataDownloader.cs index 2a44e55..841d070 100644 --- a/QuantConnect.DataBento/DataBentoDataDownloader.cs +++ b/QuantConnect.DataBento/DataBentoDataDownloader.cs @@ -15,19 +15,19 @@ */ using System; +using System.Collections.Generic; +using System.Globalization; +using NodaTime; using System.IO; -using System.Text; +using System.Linq; using System.Net.Http; -using System.Net.Http.Headers; -using System.Globalization; -using System.Collections.Generic; +using System.Text; using CsvHelper; -using CsvHelper.Configuration.Attributes; +using QuantConnect.Configuration; using QuantConnect.Data; using QuantConnect.Data.Market; +using QuantConnect.Lean.DataSource.DataBento.Models; using QuantConnect.Util; -using QuantConnect.Configuration; -using QuantConnect.Interfaces; using QuantConnect.Securities; namespace QuantConnect.Lean.DataSource.DataBento @@ -38,32 +38,28 @@ namespace QuantConnect.Lean.DataSource.DataBento /// public class DataBentoDataDownloader : IDataDownloader, IDisposable { - private readonly HttpClient _httpClient; + private readonly HttpClient _httpClient = new(); private readonly string _apiKey; - private const decimal PriceScaleFactor = 1e-9m; + private readonly DataBentoSymbolMapper _symbolMapper; + private readonly MarketHoursDatabase _marketHoursDatabase; + private readonly Dictionary _symbolExchangeTimeZones = new(); /// /// Initializes a new instance of the /// - public DataBentoDataDownloader(string apiKey) + /// The DataBento API key. + public DataBentoDataDownloader(string apiKey, MarketHoursDatabase marketHoursDatabase) { + _marketHoursDatabase = marketHoursDatabase; _apiKey = apiKey; - _httpClient = new HttpClient(); - - // Set up HTTP Basic Authentication - var credentials = Convert.ToBase64String(Encoding.ASCII.GetBytes($"{_apiKey}:")); - _httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", credentials); - } - - public DataBentoDataDownloader() - : this(Config.Get("databento-api-key")) - { + _httpClient.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.UTF8.GetBytes($"{_apiKey}:"))); + _symbolMapper = new DataBentoSymbolMapper(); } /// /// Get historical data enumerable for a single symbol, type and resolution given this start and end time (in UTC). /// - /// Parameters for the historical data request + /// Parameters for the historical data request /// Enumerable of base data for this symbol /// public IEnumerable Get(DataDownloaderGetParameters parameters) @@ -72,25 +68,29 @@ public IEnumerable Get(DataDownloaderGetParameters parameters) var resolution = parameters.Resolution; var tickType = parameters.TickType; - var dataset = "GLBX.MDP3"; // hard coded for now. Later on can add equities and options with different mapping + /// + /// Dataset for CME Globex futures + /// https://databento.com/docs/venues-and-datasets has more information on datasets through DataBento + /// + const string dataset = "GLBX.MDP3"; // hard coded for now. Later on can add equities and options with different mapping var schema = GetSchema(resolution, tickType); - var dbSymbol = MapSymbolToDataBento(symbol); + var databentoSymbol = _symbolMapper.GetBrokerageSymbol(symbol); // prepare body for Raw HTTP request - var body = new StringBuilder(); - body.Append($"dataset={dataset}"); - body.Append($"&symbols={dbSymbol}"); - body.Append($"&schema={schema}"); - body.Append($"&start={parameters.StartUtc:yyyy-MM-ddTHH:mm}"); - body.Append($"&end={parameters.EndUtc:yyyy-MM-ddTHH:mm}"); - body.Append("&stype_in=parent"); - body.Append("&encoding=csv"); - - var request = new HttpRequestMessage( - HttpMethod.Post, + var body = new StringBuilder() + .Append($"dataset={dataset}") + .Append($"&symbols={databentoSymbol}") + .Append($"&schema={schema}") + .Append($"&start={parameters.StartUtc:yyyy-MM-ddTHH:mm}") + .Append($"&end={parameters.EndUtc:yyyy-MM-ddTHH:mm}") + .Append("&stype_in=parent") + .Append("&encoding=csv") + .ToString(); + + using var request = new HttpRequestMessage(HttpMethod.Post, "https://hist.databento.com/v0/timeseries.get_range") { - Content = new StringContent(body.ToString(), Encoding.UTF8, "application/x-www-form-urlencoded") + Content = new StringContent(body, Encoding.UTF8, "application/x-www-form-urlencoded") }; // send the request with the get range url @@ -99,85 +99,70 @@ public IEnumerable Get(DataDownloaderGetParameters parameters) // Add error handling to see the actual error message if (!response.IsSuccessStatusCode) { - var errorContent = response.Content.ReadAsStringAsync().Result; + var errorContent = response.Content.ReadAsStringAsync().SynchronouslyAwaitTaskResult(); throw new HttpRequestException($"DataBento API error ({response.StatusCode}): {errorContent}"); } - response.EnsureSuccessStatusCode(); - - using var stream = response.Content.ReadAsStream(); - using var reader = new StreamReader(stream); - using var csv = new CsvReader(reader, CultureInfo.InvariantCulture); + using var csv = new CsvReader( + new StreamReader(response.Content.ReadAsStream()), + CultureInfo.InvariantCulture + ); - if (tickType == TickType.Trade) + return (tickType, resolution) switch { - if (resolution == Resolution.Tick) - { - // For tick data, use the trades schema which returns individual trades - foreach (var record in csv.GetRecords()) - { - yield return new Tick - { - Time = record.Timestamp, - Symbol = symbol, - Value = record.Price, - Quantity = record.Size - }; - } - } - else - { - // For aggregated data, use the ohlcv schema which returns bars - foreach (var record in csv.GetRecords()) - { - yield return new TradeBar - { - Symbol = symbol, - Time = record.Timestamp, - Open = record.Open, - High = record.High, - Low = record.Low, - Close = record.Close, - Volume = record.Volume - }; - } - } - } - else if (tickType == TickType.Quote) - { - foreach (var record in csv.GetRecords()) - { - var bidPrice = record.BidPrice * PriceScaleFactor; - var askPrice = record.AskPrice * PriceScaleFactor; - - if (resolution == Resolution.Tick) - { - yield return new Tick + (TickType.Trade, Resolution.Tick) => + csv.ForEach(dt => + new Tick( + GetTickTime(symbol, dt.Timestamp), + symbol, + string.Empty, + string.Empty, + dt.Size, + dt.Price + ) + ), + + (TickType.Trade, _) => + csv.ForEach(bar => + new TradeBar( + GetTickTime(symbol, bar.Timestamp), + symbol, + bar.Open, + bar.High, + bar.Low, + bar.Close, + bar.Volume + ) + ), + + (TickType.Quote, Resolution.Tick) => + csv.ForEach(q => + new Tick( + GetTickTime(symbol, q.Timestamp), + symbol, + bidPrice: q.BidPrice, + askPrice: q.AskPrice, + bidSize: q.BidSize, + askSize: q.AskSize + ) { - Time = record.Timestamp, - Symbol = symbol, - AskPrice = askPrice, - BidPrice = bidPrice, - AskSize = record.AskSize, - BidSize = record.BidSize, TickType = TickType.Quote - }; - } - else - { - var bidBar = new Bar(bidPrice, bidPrice, bidPrice, bidPrice); - var askBar = new Bar(askPrice, askPrice, askPrice, askPrice); - yield return new QuoteBar( - record.Timestamp, + } + ), + + (TickType.Quote, _) => + csv.ForEach(q => + new QuoteBar( + GetTickTime(symbol, q.Timestamp), symbol, - bidBar, - record.BidSize, - askBar, - record.AskSize - ); - } - } - } + new Bar(q.BidPrice, q.BidPrice, q.BidPrice, q.BidPrice), q.BidSize, + new Bar(q.AskPrice, q.AskPrice, q.AskPrice, q.AskPrice), q.AskSize + ) + ), + + _ => throw new NotSupportedException( + $"Unsupported tickType={tickType} resolution={resolution}") + }; } /// @@ -191,113 +176,60 @@ public void Dispose() /// /// Pick Databento schema from Lean resolution/ticktype /// - private string GetSchema(Resolution resolution, TickType tickType) + private static string GetSchema(Resolution resolution, TickType tickType) { - if (tickType == TickType.Trade) + return (tickType, resolution) switch { - if (resolution == Resolution.Tick) - return "trades"; - if (resolution == Resolution.Second) - return "ohlcv-1s"; - if (resolution == Resolution.Minute) - return "ohlcv-1m"; - if (resolution == Resolution.Hour) - return "ohlcv-1h"; - if (resolution == Resolution.Daily) - return "ohlcv-1d"; - } - else if (tickType == TickType.Quote) - { - // top of book - if (resolution == Resolution.Tick || resolution == Resolution.Second || resolution == Resolution.Minute || resolution == Resolution.Hour || resolution == Resolution.Daily) - return "mbp-1"; - } + (TickType.Trade, Resolution.Tick) => "mbp-1", + (TickType.Trade, Resolution.Second) => "ohlcv-1s", + (TickType.Trade, Resolution.Minute) => "ohlcv-1m", + (TickType.Trade, Resolution.Hour) => "ohlcv-1h", + (TickType.Trade, Resolution.Daily) => "ohlcv-1d", - throw new NotSupportedException($"Unsupported resolution {resolution} / {tickType}"); + (TickType.Quote, _) => "mbp-1", + + _ => throw new NotSupportedException( + $"Unsupported resolution {resolution} / {tickType}" + ) + }; } /// - /// Maps a LEAN symbol to DataBento symbol format + /// Converts the given UTC time into the symbol security exchange time zone /// - private string MapSymbolToDataBento(Symbol symbol) + private DateTime GetTickTime(Symbol symbol, DateTime utcTime) { - if (symbol.SecurityType == SecurityType.Future) + DateTimeZone exchangeTimeZone; + lock (_symbolExchangeTimeZones) { - // For DataBento, use the root symbol with .FUT suffix for parent subscription - // ES19Z25 -> ES.FUT - var value = symbol.Value; - - // Extract root by removing digits and month codes - var root = new string(value.TakeWhile(c => !char.IsDigit(c)).ToArray()); + if (!_symbolExchangeTimeZones.TryGetValue(symbol, out exchangeTimeZone)) + { + // read the exchange time zone from market-hours-database + if (_marketHoursDatabase.TryGetEntry(symbol.ID.Market, symbol, symbol.SecurityType, out var entry)) + { + exchangeTimeZone = entry.ExchangeHours.TimeZone; + } + // If there is no entry for the given Symbol, default to New York + else + { + exchangeTimeZone = TimeZones.NewYork; + } - return $"{root}.FUT"; + _symbolExchangeTimeZones.Add(symbol, exchangeTimeZone); + } } - return symbol.Value; - } - - /// Class for parsing trade data from Databento - /// Really used as a map from the http request to then get it in QC data structures - private class DatabentoBar - { - [Name("ts_event")] - public long TimestampNanos { get; set; } - - public DateTime Timestamp => DateTimeOffset.FromUnixTimeSeconds(TimestampNanos / 1_000_000_000) - .AddTicks((TimestampNanos % 1_000_000_000) / 100).UtcDateTime; - - [Name("open")] - public decimal Open { get; set; } - - [Name("high")] - public decimal High { get; set; } - - [Name("low")] - public decimal Low { get; set; } - - [Name("close")] - public decimal Close { get; set; } - - [Name("volume")] - public decimal Volume { get; set; } - } - - private class DatabentoTrade - { - [Name("ts_event")] - public long TimestampNanos { get; set; } - - public DateTime Timestamp => DateTimeOffset.FromUnixTimeSeconds(TimestampNanos / 1_000_000_000) - .AddTicks((TimestampNanos % 1_000_000_000) / 100).UtcDateTime; - - [Name("price")] - public long PriceRaw { get; set; } - - public decimal Price => PriceRaw * PriceScaleFactor; - - [Name("size")] - public int Size { get; set; } + return utcTime.ConvertFromUtc(exchangeTimeZone); } + } +} - private class DatabentoQuote - { - [Name("ts_event")] - public long TimestampNanos { get; set; } - - public DateTime Timestamp => DateTimeOffset.FromUnixTimeSeconds(TimestampNanos / 1_000_000_000) - .AddTicks((TimestampNanos % 1_000_000_000) / 100).UtcDateTime; - - [Name("bid_px_00")] - public long BidPrice { get; set; } - - [Name("bid_sz_00")] - public int BidSize { get; set; } - - [Name("ask_px_00")] - public long AskPrice { get; set; } - - [Name("ask_sz_00")] - public int AskSize { get; set; } - } +public static class CsvReaderExtensions +{ + public static IEnumerable ForEach( + this CsvReader csv, + Func map) + { + return csv.GetRecords().Select(map).ToList(); } } diff --git a/QuantConnect.DataBento/DataBentoDataProvider.cs b/QuantConnect.DataBento/DataBentoDataProvider.cs index 8cf6015..aa9dd32 100644 --- a/QuantConnect.DataBento/DataBentoDataProvider.cs +++ b/QuantConnect.DataBento/DataBentoDataProvider.cs @@ -14,14 +14,11 @@ * */ -using System; -using System.Linq; using NodaTime; using QuantConnect.Data; using QuantConnect.Data.Market; using QuantConnect.Util; using QuantConnect.Interfaces; -using System.Collections.Generic; using QuantConnect.Configuration; using QuantConnect.Logging; using QuantConnect.Packets; @@ -31,23 +28,24 @@ namespace QuantConnect.Lean.DataSource.DataBento { /// - /// Implementation of Custom Data Provider + /// A data Provider for DataBento that provides live market data and historical data. + /// Handles Subscribing, Unsubscribing, and fetching historical data from DataBento. + /// It will handle if a symbol is subscribable and will log errors if it is not. /// - public class DataBentoProvider : IDataQueueHandler + public partial class DataBentoProvider : IDataQueueHandler { private readonly IDataAggregator _dataAggregator = Composer.Instance.GetExportedValueByTypeName( Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager"), forceTypeNameOnExisting: false); - private EventBasedDataQueueHandlerSubscriptionManager _subscriptionManager = null!; - private readonly List _activeSubscriptionConfigs = new(); - private readonly ConcurrentDictionary _subscriptionConfigs = new(); - private DatabentoRawClient _client = null!; - private readonly string _apiKey; + private EventBasedDataQueueHandlerSubscriptionManager _subscriptionManager; + private DataBentoRawLiveClient _client; private readonly DataBentoDataDownloader _dataDownloader; private bool _potentialUnsupportedResolutionMessageLogged; private bool _sessionStarted = false; - private readonly object _sessionLock = new object(); + private readonly object _sessionLock = new(); private readonly MarketHoursDatabase _marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); private readonly ConcurrentDictionary _symbolExchangeTimeZones = new(); + private bool _initialized; + private bool _unsupportedTickTypeMessagedLogged; /// /// Returns true if we're currently connected to the Data Provider @@ -59,109 +57,120 @@ public class DataBentoProvider : IDataQueueHandler /// public DataBentoProvider() { - _apiKey = Config.Get("databento-api-key"); - if (string.IsNullOrEmpty(_apiKey)) - { - throw new ArgumentException("DataBento API key is required. Set 'databento-api-key' in configuration."); - } - - _dataDownloader = new DataBentoDataDownloader(_apiKey); - Initialize(); - } - - /// - /// Initializes a new instance of the DataBentoProvider with custom API key - /// - /// DataBento API key - public DataBentoProvider(string apiKey) - { - _apiKey = apiKey ?? throw new ArgumentNullException(nameof(apiKey)); - _dataDownloader = new DataBentoDataDownloader(_apiKey); - Initialize(); + var apiKey = Config.Get("databento-api-key"); + _dataDownloader = new DataBentoDataDownloader(apiKey, _marketHoursDatabase); + Initialize(apiKey); } /// /// Common initialization logic + /// DataBento API key from config file retrieved on constructor /// - private void Initialize() + private void Initialize(string apiKey) { - Log.Trace("DataBentoProvider.Initialize(): Starting initialization"); - _subscriptionManager = new EventBasedDataQueueHandlerSubscriptionManager(); - _subscriptionManager.SubscribeImpl = (symbols, tickType) => + Log.Debug("DataBentoProvider.Initialize(): Starting initialization"); + _subscriptionManager = new EventBasedDataQueueHandlerSubscriptionManager() { - Log.Trace($"DataBentoProvider.SubscribeImpl(): Received subscription request for {symbols.Count()} symbols, TickType={tickType}"); - foreach (var symbol in symbols) + SubscribeImpl = (symbols, tickType) => { - Log.Trace($"DataBentoProvider.SubscribeImpl(): Processing symbol {symbol}"); - if (!_subscriptionConfigs.TryGetValue(symbol, out var config)) - { - Log.Error($"DataBentoProvider.SubscribeImpl(): No subscription config found for {symbol}"); - return false; - } - if (_client?.IsConnected != true) - { - Log.Error($"DataBentoProvider.SubscribeImpl(): Client is not connected. Cannot subscribe to {symbol}"); - return false; - } + return SubscriptionLogic(symbols, tickType); + }, + UnsubscribeImpl = (symbols, tickType) => + { + return UnsubscribeLogic(symbols, tickType); + } + }; + + // Initialize the live client + _client = new DataBentoRawLiveClient(apiKey); + _client.DataReceived += OnDataReceived; - var resolution = config.Resolution > Resolution.Tick ? Resolution.Tick : config.Resolution; - if (!_client.Subscribe(config.Symbol, resolution, config.TickType)) + // Connect to live gateway + Log.Debug("DataBentoProvider.Initialize(): Attempting connection to DataBento live gateway"); + var cancellationTokenSource = new CancellationTokenSource(); + Task.Factory.StartNew(() => + { + try + { + var connected = _client.Connect(); + Log.Debug($"DataBentoProvider.Initialize(): Connect() returned {connected}"); + + if (connected) { - Log.Error($"Failed to subscribe to {config.Symbol}"); - return false; + Log.Debug("DataBentoProvider.Initialize(): Successfully connected to DataBento live gateway"); } - - lock (_sessionLock) + else { - if (!_sessionStarted) - _sessionStarted = _client.StartSession(); + Log.Error("DataBentoProvider.Initialize(): Failed to connect to DataBento live gateway"); } } + catch (Exception ex) + { + Log.Error($"DataBentoProvider.Initialize(): Exception during Connect(): {ex.Message}\n{ex.StackTrace}"); + } + }, + cancellationTokenSource.Token, + TaskCreationOptions.LongRunning, + TaskScheduler.Default); + _initialized = true; - return true; - }; + Log.Debug("DataBentoProvider.Initialize(): Initialization complete"); + } - _subscriptionManager.UnsubscribeImpl = (symbols, tickType) => + /// + /// Logic to unsubscribe from the specified symbols + /// + public bool UnsubscribeLogic(IEnumerable symbols, TickType tickType) + { + foreach (var symbol in symbols) { - foreach (var symbol in symbols) + Log.Debug($"DataBentoProvider.UnsubscribeImpl(): Processing symbol {symbol}"); + if (_client?.IsConnected != true) { - Log.Trace($"DataBentoProvider.UnsubscribeImpl(): Processing symbol {symbol}"); - if (_client?.IsConnected != true) - { - throw new InvalidOperationException($"DataBentoProvider.UnsubscribeImpl(): Client is not connected. Cannot unsubscribe from {symbol}"); - } - - _client.Unsubscribe(symbol); + throw new InvalidOperationException($"DataBentoProvider.UnsubscribeImpl(): Client is not connected. Cannot unsubscribe from {symbol}"); } - return true; - }; + _client.Unsubscribe(symbol); + } - // Initialize the live client - Log.Trace("DataBentoProvider.Initialize(): Creating DatabentoRawClient"); - _client = new DatabentoRawClient(_apiKey); - _client.DataReceived += OnDataReceived; - _client.ConnectionStatusChanged += OnConnectionStatusChanged; + return true; + } - // Connect to live gateway - Log.Trace("DataBentoProvider.Initialize(): Attempting connection to DataBento live gateway"); - Task.Run(() => + /// + /// Logic to subscribe to the specified symbols + /// + public bool SubscriptionLogic(IEnumerable symbols, TickType tickType) + { + if (_client?.IsConnected != true) { - var connected = _client.Connect(); - Log.Trace($"DataBentoProvider.Initialize(): Connect() returned {connected}"); + Log.Error("DataBentoProvider.SubscriptionLogic(): Client is not connected. Cannot subscribe to symbols"); + return false; + } - if (connected) - { - Log.Trace("DataBentoProvider.Initialize(): Successfully connected to DataBento live gateway"); - } - else + foreach (var symbol in symbols) + { + if (!CanSubscribe(symbol)) { - Log.Error("DataBentoProvider.Initialize(): Failed to connect to DataBento live gateway"); + Log.Error($"DataBentoProvider.SubscriptionLogic(): Unsupported subscription: {symbol}"); + return false; } - }); + _client.Subscribe(symbol, tickType); + } - Log.Trace("DataBentoProvider.Initialize(): Initialization complete"); + return true; + } + + /// + /// Checks if this Data provider supports the specified symbol + /// + /// The symbol + /// returns true if Data Provider supports the specified symbol; otherwise false + private bool CanSubscribe(Symbol symbol) + { + return !symbol.Value.Contains("universe", StringComparison.InvariantCultureIgnoreCase) && + !symbol.IsCanonical() && + IsSecurityTypeSupported(symbol.SecurityType); } /// @@ -172,17 +181,22 @@ private void Initialize() /// The new enumerator for this subscription request public IEnumerator? Subscribe(SubscriptionDataConfig dataConfig, EventHandler newDataAvailableHandler) { - Log.Trace($"DataBentoProvider.Subscribe(): Received subscription request for {dataConfig.Symbol}, Resolution={dataConfig.Resolution}, TickType={dataConfig.TickType}"); - if (!CanSubscribe(dataConfig)) + if (!IsSupported(dataConfig.SecurityType, dataConfig.Type, dataConfig.TickType, dataConfig.Resolution)) { - Log.Error($"DataBentoProvider.Subscribe(): Cannot subscribe to {dataConfig.Symbol} with Resolution={dataConfig.Resolution}, TickType={dataConfig.TickType}"); return null; } - _subscriptionConfigs[dataConfig.Symbol] = dataConfig; + lock (_sessionLock) + { + if (!_sessionStarted) + { + Log.Debug("DataBentoProvider.SubscriptionLogic(): Starting session"); + _sessionStarted = _client.StartSession(); + } + } + var enumerator = _dataAggregator.Add(dataConfig, newDataAvailableHandler); _subscriptionManager.Subscribe(dataConfig); - _activeSubscriptionConfigs.Add(dataConfig); return enumerator; } @@ -193,15 +207,8 @@ private void Initialize() /// Subscription config to be removed public void Unsubscribe(SubscriptionDataConfig dataConfig) { - Log.Trace($"DataBentoProvider.Unsubscribe(): Received unsubscription request for {dataConfig.Symbol}, Resolution={dataConfig.Resolution}, TickType={dataConfig.TickType}"); - _subscriptionConfigs.TryRemove(dataConfig.Symbol, out _); + Log.Debug($"DataBentoProvider.Unsubscribe(): Received unsubscription request for {dataConfig.Symbol}, Resolution={dataConfig.Resolution}, TickType={dataConfig.TickType}"); _subscriptionManager.Unsubscribe(dataConfig); - var toRemove = _activeSubscriptionConfigs.FirstOrDefault(c => c.Symbol == dataConfig.Symbol && c.TickType == dataConfig.TickType); - if (toRemove != null) - { - Log.Trace($"DataBentoProvider.Unsubscribe(): Removing active subscription for {dataConfig.Symbol}, Resolution={dataConfig.Resolution}, TickType={dataConfig.TickType}"); - _activeSubscriptionConfigs.Remove(toRemove); - } _dataAggregator.Remove(dataConfig); } @@ -211,7 +218,10 @@ public void Unsubscribe(SubscriptionDataConfig dataConfig) /// Job we're subscribing for public void SetJob(LiveNodePacket job) { - // No action required for DataBento since the job details are not used in the subscription process. + if (_initialized) + { + return; + } } /// @@ -221,62 +231,8 @@ public void Dispose() { _dataAggregator?.DisposeSafely(); _subscriptionManager?.DisposeSafely(); - _client?.Dispose(); - _dataDownloader?.Dispose(); - } - - /// - /// Gets the history for the requested security - /// - /// The historical data request - /// An enumerable of BaseData points - public IEnumerable? GetHistory(Data.HistoryRequest request) - { - Log.Trace($"DataBentoProvider.GetHistory(): Received history request for {request.Symbol}, Resolution={request.Resolution}, TickType={request.TickType}"); - if (!CanSubscribe(request.Symbol)) - { - Log.Error($"DataBentoProvider.GetHistory(): Cannot provide history for {request.Symbol} with Resolution={request.Resolution}, TickType={request.TickType}"); - return null; - } - - try - { - // Use the data downloader to get historical data - var parameters = new DataDownloaderGetParameters( - request.Symbol, - request.Resolution, - request.StartTimeUtc, - request.EndTimeUtc, - request.TickType); - - return _dataDownloader.Get(parameters); - } - catch (Exception ex) - { - Log.Error($"DataBentoProvider.GetHistory(): Failed to get history for {request.Symbol}: {ex.Message}"); - return null; - } - } - - /// - /// Checks if this Data provider supports the specified symbol - /// - /// The symbol - /// returns true if Data Provider supports the specified symbol; otherwise false - private bool CanSubscribe(Symbol symbol) - { - return !symbol.Value.Contains("universe", StringComparison.InvariantCultureIgnoreCase) && - !symbol.IsCanonical() && - IsSecurityTypeSupported(symbol.SecurityType); - } - - /// - /// Determines whether or not the specified config can be subscribed to - /// - private bool CanSubscribe(SubscriptionDataConfig config) - { - return CanSubscribe(config.Symbol) && - IsSupported(config.SecurityType, config.Type, config.TickType, config.Resolution); + _client?.DisposeSafely(); + _dataDownloader?.DisposeSafely(); } /// @@ -295,12 +251,6 @@ private bool IsSecurityTypeSupported(SecurityType securityType) /// private bool IsSupported(SecurityType securityType, Type dataType, TickType tickType, Resolution resolution) { - // Check supported security types - if (!IsSecurityTypeSupported(securityType)) - { - throw new NotSupportedException($"Unsupported security type: {securityType}"); - } - // Check supported data types if (dataType != typeof(TradeBar) && dataType != typeof(QuoteBar) && @@ -328,47 +278,66 @@ private bool IsSupported(SecurityType securityType, Type dataType, TickType tick /// private DateTime GetTickTime(Symbol symbol, DateTime utcTime) { - var exchangeTimeZone = _symbolExchangeTimeZones.GetOrAdd(symbol, sym => + DateTimeZone exchangeTimeZone; + lock (_symbolExchangeTimeZones) { - if (_marketHoursDatabase.TryGetEntry(sym.ID.Market, sym, sym.SecurityType, out var entry)) + if (!_symbolExchangeTimeZones.TryGetValue(symbol, out exchangeTimeZone)) { - return entry.ExchangeHours.TimeZone; + // read the exchange time zone from market-hours-database + if (_marketHoursDatabase.TryGetEntry(symbol.ID.Market, symbol, symbol.SecurityType, out var entry)) + { + exchangeTimeZone = entry.ExchangeHours.TimeZone; + } + // If there is no entry for the given Symbol, default to New York + else + { + exchangeTimeZone = TimeZones.NewYork; + } + + _symbolExchangeTimeZones[symbol] = exchangeTimeZone; } - // Futures default to Chicago - return TimeZones.Chicago; - }); + } return utcTime.ConvertFromUtc(exchangeTimeZone); } - // + /// /// Handles data received from the live client /// - private void OnDataReceived(object? sender, BaseData data) + private void OnDataReceived(object _, BaseData data) { try { - if (data is Tick tick) + switch (data) { - tick.Time = GetTickTime(tick.Symbol, tick.Time); - _dataAggregator.Update(tick); - - Log.Trace($"DataBentoProvider.OnDataReceived(): Updated tick - Symbol: {tick.Symbol}, " + - $"TickType: {tick.TickType}, Price: {tick.Value}, Quantity: {tick.Quantity}"); - } - else if (data is TradeBar tradeBar) - { - tradeBar.Time = GetTickTime(tradeBar.Symbol, tradeBar.Time); - tradeBar.EndTime = GetTickTime(tradeBar.Symbol, tradeBar.EndTime); - _dataAggregator.Update(tradeBar); + case Tick tick: + tick.Time = GetTickTime(tick.Symbol, tick.Time); + lock (_dataAggregator) + { + _dataAggregator.Update(tick); + } + // Log.Trace($"DataBentoProvider.OnDataReceived(): Updated tick - Symbol: {tick.Symbol}, " + + // $"TickType: {tick.TickType}, Price: {tick.Value}, Quantity: {tick.Quantity}"); + break; + + case TradeBar tradeBar: + tradeBar.Time = GetTickTime(tradeBar.Symbol, tradeBar.Time); + tradeBar.EndTime = GetTickTime(tradeBar.Symbol, tradeBar.EndTime); + lock (_dataAggregator) + { + _dataAggregator.Update(tradeBar); + } + // Log.Trace($"DataBentoProvider.OnDataReceived(): Updated TradeBar - Symbol: {tradeBar.Symbol}, " + + // $"O:{tradeBar.Open} H:{tradeBar.High} L:{tradeBar.Low} C:{tradeBar.Close} V:{tradeBar.Volume}"); + break; - Log.Trace($"DataBentoProvider.OnDataReceived(): Updated TradeBar - Symbol: {tradeBar.Symbol}, " + - $"O:{tradeBar.Open} H:{tradeBar.High} L:{tradeBar.Low} C:{tradeBar.Close} V:{tradeBar.Volume}"); - } - else - { - data.Time = GetTickTime(data.Symbol, data.Time); - _dataAggregator.Update(data); + default: + data.Time = GetTickTime(data.Symbol, data.Time); + lock (_dataAggregator) + { + _dataAggregator.Update(data); + } + break; } } catch (Exception ex) @@ -376,41 +345,5 @@ private void OnDataReceived(object? sender, BaseData data) Log.Error($"DataBentoProvider.OnDataReceived(): Error updating data aggregator: {ex.Message}\n{ex.StackTrace}"); } } - - /// - /// Handles connection status changes from the live client - /// - private void OnConnectionStatusChanged(object? sender, bool isConnected) - { - Log.Trace($"DataBentoProvider.OnConnectionStatusChanged(): Connection status changed to: {isConnected}"); - - if (isConnected) - { - // Reset session flag on reconnection - lock (_sessionLock) - { - _sessionStarted = false; - } - - // Resubscribe to all active subscriptions - foreach (var config in _activeSubscriptionConfigs) - { - _client.Subscribe(config.Symbol, config.Resolution, config.TickType); - } - - // Start session after resubscribing - if (_activeSubscriptionConfigs.Any()) - { - lock (_sessionLock) - { - if (!_sessionStarted) - { - Log.Trace("DataBentoProvider.OnConnectionStatusChanged(): Starting session after reconnection"); - _sessionStarted = _client.StartSession(); - } - } - } - } - } } } diff --git a/QuantConnect.DataBento/DataBentoHistoryProivder.cs b/QuantConnect.DataBento/DataBentoHistoryProivder.cs index a93b50e..eb23154 100644 --- a/QuantConnect.DataBento/DataBentoHistoryProivder.cs +++ b/QuantConnect.DataBento/DataBentoHistoryProivder.cs @@ -14,7 +14,6 @@ * */ -using System; using NodaTime; using QuantConnect.Data; using QuantConnect.Data.Market; @@ -22,30 +21,25 @@ using QuantConnect.Lean.Engine.HistoricalData; using QuantConnect.Logging; using QuantConnect.Util; -using QuantConnect.Lean.DataSource.DataBento; using QuantConnect.Interfaces; -using System.Collections.Generic; -using QuantConnect.Configuration; using QuantConnect.Securities; using QuantConnect.Data.Consolidators; namespace QuantConnect.Lean.DataSource.DataBento { /// - /// DataBento implementation of + /// Impleements a history provider for DataBento historical data. + /// Uses consolidators to produce the requested resolution when necessary. /// - public partial class DataBentoHistoryProvider : SynchronizingHistoryProvider + public partial class DataBentoProvider : SynchronizingHistoryProvider { private int _dataPointCount; - private DataBentoDataDownloader _dataDownloader; + + /// + /// Indicates whether a error for an invalid start time has been fired, where the start time is greater than or equal to the end time in UTC. + /// private volatile bool _invalidStartTimeErrorFired; - private volatile bool _invalidTickTypeAndResolutionErrorFired; - private volatile bool _unsupportedTickTypeMessagedLogged; - private MarketHoursDatabase _marketHoursDatabase; - private bool _unsupportedSecurityTypeMessageLogged; - private bool _unsupportedDataTypeMessageLogged; - private bool _potentialUnsupportedResolutionMessageLogged; - + /// /// Gets the total number of data points emitted by this history provider /// @@ -57,8 +51,6 @@ public partial class DataBentoHistoryProvider : SynchronizingHistoryProvider /// The initialization parameters public override void Initialize(HistoryProviderInitializeParameters parameters) { - _dataDownloader = new DataBentoDataDownloader(); - _marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); } /// @@ -101,8 +93,7 @@ public override void Initialize(HistoryProviderInitializeParameters parameters) /// An enumerable of BaseData points public IEnumerable? GetHistory(HistoryRequest request) { - if (request.Symbol.IsCanonical() || - !IsSupported(request.Symbol.SecurityType, request.DataType, request.TickType, request.Resolution)) + if (!CanSubscribe(request.Symbol)) { // It is Logged in IsSupported(...) return null; @@ -113,7 +104,7 @@ public override void Initialize(HistoryProviderInitializeParameters parameters) if (!_unsupportedTickTypeMessagedLogged) { _unsupportedTickTypeMessagedLogged = true; - Log.Trace($"DataBentoHistoryProvider.GetHistory(): Unsupported tick type: {TickType.OpenInterest}"); + Log.Trace($"DataBentoProvider.GetHistory(): Unsupported tick type: {TickType.OpenInterest}"); } return null; } @@ -123,7 +114,7 @@ public override void Initialize(HistoryProviderInitializeParameters parameters) if (!_invalidStartTimeErrorFired) { _invalidStartTimeErrorFired = true; - Log.Error($"{nameof(DataBentoHistoryProvider)}.{nameof(GetHistory)}:InvalidDateRange. The history request start date must precede the end date, no history returned"); + Log.Error($"{nameof(DataBentoProvider)}.{nameof(GetHistory)}:InvalidDateRange. The history request start date must precede the end date, no history returned"); } return null; } @@ -230,59 +221,5 @@ private IEnumerable GetQuotes(HistoryRequest request) var parameters = new DataDownloaderGetParameters(request.Symbol, Resolution.Tick, request.StartTimeUtc, request.EndTimeUtc, request.TickType); return _dataDownloader.Get(parameters); } - - /// - /// Checks if the security type is supported - /// - /// Security type to check - /// True if supported - private bool IsSecurityTypeSupported(SecurityType securityType) - { - // DataBento primarily supports futures, but also has equity and option coverage - return securityType == SecurityType.Future; - } - - /// - /// Determines if the specified subscription is supported - /// - private bool IsSupported(SecurityType securityType, Type dataType, TickType tickType, Resolution resolution) - { - // Check supported security types - if (!IsSecurityTypeSupported(securityType)) - { - if (!_unsupportedSecurityTypeMessageLogged) - { - _unsupportedSecurityTypeMessageLogged = true; - Log.Trace($"DataBentoDataProvider.IsSupported(): Unsupported security type: {securityType}"); - } - return false; - } - - // Check supported data types - if (dataType != typeof(TradeBar) && - dataType != typeof(QuoteBar) && - dataType != typeof(Tick) && - dataType != typeof(OpenInterest)) - { - if (!_unsupportedDataTypeMessageLogged) - { - _unsupportedDataTypeMessageLogged = true; - Log.Trace($"DataBentoDataProvider.IsSupported(): Unsupported data type: {dataType}"); - } - return false; - } - - // Warn about potential limitations for tick data - // I'm mimicing polygon implementation with this - if (!_potentialUnsupportedResolutionMessageLogged) - { - _potentialUnsupportedResolutionMessageLogged = true; - Log.Trace("DataBentoDataProvider.IsSupported(): " + - $"Subscription for {securityType}-{dataType}-{tickType}-{resolution} will be attempted. " + - $"An Advanced DataBento subscription plan is required to stream tick data."); - } - - return true; - } } } diff --git a/QuantConnect.DataBento/DataBentoRawLiveClient.cs b/QuantConnect.DataBento/DataBentoRawLiveClient.cs index 2e7bb97..2dab1d2 100644 --- a/QuantConnect.DataBento/DataBentoRawLiveClient.cs +++ b/QuantConnect.DataBento/DataBentoRawLiveClient.cs @@ -14,14 +14,12 @@ * */ -using System; -using System.IO; using System.Text; using System.Net.Sockets; using System.Security.Cryptography; using System.Collections.Concurrent; using System.Text.Json; -using System.Linq; +using System.Threading.Tasks; using QuantConnect.Data; using QuantConnect.Data.Market; using QuantConnect.Logging; @@ -31,33 +29,44 @@ namespace QuantConnect.Lean.DataSource.DataBento /// /// DataBento Raw TCP client for live streaming data /// - public class DatabentoRawClient : IDisposable + public class DataBentoRawLiveClient : IDisposable { + /// + /// The DataBento API key for authentication + /// private readonly string _apiKey; - private readonly string _gateway; + /// + /// The DataBento live gateway address to receive data from + /// + private const string _gateway = "glbx-mdp3.lsg.databento.com:13000"; + /// + /// The dataset to subscribe to + /// private readonly string _dataset; - private TcpClient? _tcpClient; + private readonly TcpClient? _tcpClient; + private readonly string _host; + private readonly int _port; private NetworkStream? _stream; - private StreamReader? _reader; - private StreamWriter? _writer; - private CancellationTokenSource _cancellationTokenSource; + private StreamReader _reader; + private StreamWriter _writer; + private readonly CancellationTokenSource _cancellationTokenSource; private readonly ConcurrentDictionary _subscriptions; private readonly object _connectionLock = new object(); private bool _isConnected; private bool _disposed; private const decimal PriceScaleFactor = 1e-9m; private readonly ConcurrentDictionary _instrumentIdToSymbol = new ConcurrentDictionary(); - private readonly ConcurrentDictionary _lastTicks = new ConcurrentDictionary(); + private readonly DataBentoSymbolMapper _symbolMapper; /// /// Event fired when new data is received /// - public event EventHandler? DataReceived; + public event EventHandler DataReceived; /// /// Event fired when connection status changes /// - public event EventHandler? ConnectionStatusChanged; + public event EventHandler ConnectionStatusChanged; /// /// Gets whether the client is currently connected @@ -65,15 +74,21 @@ public class DatabentoRawClient : IDisposable public bool IsConnected => _isConnected && _tcpClient?.Connected == true; /// - /// Initializes a new instance of the DatabentoRawClient + /// Initializes a new instance of the DataBentoRawLiveClient + /// The DataBento API key. /// - public DatabentoRawClient(string apiKey, string gateway = "glbx-mdp3.lsg.databento.com:13000", string dataset = "GLBX.MDP3") + public DataBentoRawLiveClient(string apiKey, string dataset = "GLBX.MDP3") { _apiKey = apiKey ?? throw new ArgumentNullException(nameof(apiKey)); - _gateway = gateway ?? throw new ArgumentNullException(nameof(gateway)); _dataset = dataset; + _tcpClient = new TcpClient(); _subscriptions = new ConcurrentDictionary(); _cancellationTokenSource = new CancellationTokenSource(); + _symbolMapper = new DataBentoSymbolMapper(); + + var parts = _gateway.Split(':'); + _host = parts[0]; + _port = parts.Length > 1 ? int.Parse(parts[1]) : 13000; } /// @@ -81,20 +96,15 @@ public DatabentoRawClient(string apiKey, string gateway = "glbx-mdp3.lsg.databen /// public bool Connect() { - Log.Trace("DatabentoRawClient.Connect(): Connecting to DataBento live gateway"); - if (_isConnected || _disposed) + Log.Trace("DataBentoRawLiveClient.Connect(): Connecting to DataBento live gateway"); + if (_isConnected) { return _isConnected; } try { - var parts = _gateway.Split(':'); - var host = parts[0]; - var port = parts.Length > 1 ? int.Parse(parts[1]) : 13000; - - _tcpClient = new TcpClient(); - _tcpClient.Connect(host, port); + _tcpClient.Connect(_host, _port); _stream = _tcpClient.GetStream(); _reader = new StreamReader(_stream, Encoding.ASCII); _writer = new StreamWriter(_stream, Encoding.ASCII) { AutoFlush = true }; @@ -106,15 +116,15 @@ public bool Connect() ConnectionStatusChanged?.Invoke(this, true); // Start message processing - ProcessMessages(); + Task.Run(ProcessMessages, _cancellationTokenSource.Token); - Log.Trace("DatabentoRawClient.Connect(): Connected and authenticated to DataBento live gateway"); + Log.Trace("DataBentoRawLiveClient.Connect(): Connected and authenticated to DataBento live gateway"); return true; } } catch (Exception ex) { - Log.Error($"DatabentoRawClient.Connect(): Failed to connect: {ex.Message}"); + Log.Error($"DataBentoRawLiveClient.Connect(): Failed to connect: {ex.Message}"); Disconnect(); } @@ -126,124 +136,97 @@ public bool Connect() /// private bool Authenticate() { - if (_reader == null || _writer == null) - return false; - try { // Read greeting and challenge - string? versionLine = _reader.ReadLine(); - string? cramLine = _reader.ReadLine(); + var versionLine = _reader.ReadLine(); + var cramLine = _reader.ReadLine(); if (string.IsNullOrEmpty(versionLine) || string.IsNullOrEmpty(cramLine)) { - Log.Error("DatabentoRawClient.Authenticate(): Failed to receive greeting or challenge"); + Log.Error("DataBentoRawLiveClient.Authenticate(): Failed to receive greeting or challenge"); return false; } - Log.Trace($"DatabentoRawClient.Authenticate(): Version: {versionLine}"); - Log.Trace($"DatabentoRawClient.Authenticate(): Challenge: {cramLine}"); - // Parse challenge - string[] cramParts = cramLine.Split('='); + var cramParts = cramLine.Split('='); if (cramParts.Length != 2 || cramParts[0] != "cram") { - Log.Error("DatabentoRawClient.Authenticate(): Invalid challenge format"); - return false; - } - string cram = cramParts[1].Trim(); - - // Compute auth hash - string concat = $"{cram}|{_apiKey}"; - string hashHex = ComputeSHA256(concat); - string bucketId = _apiKey.Length >= 5 ? _apiKey.Substring(_apiKey.Length - 5) : _apiKey; - string authString = $"{hashHex}-{bucketId}"; - - // Send auth message - string authMsg = $"auth={authString}|dataset={_dataset}|encoding=json|ts_out=0"; - Log.Trace($"DatabentoRawClient.Authenticate(): Sending auth"); - _writer.WriteLine(authMsg); - - // Read auth response - string? authResp = _reader.ReadLine(); - if (string.IsNullOrEmpty(authResp)) - { - Log.Error("DatabentoRawClient.Authenticate(): No authentication response received"); + Log.Error("DataBentoRawLiveClient.Authenticate(): Invalid challenge format"); return false; } + var cram = cramParts[1].Trim(); - Log.Trace($"DatabentoRawClient.Authenticate(): Auth response: {authResp}"); - + // Auth + _writer.WriteLine($"auth={GetAuthStringFromCram(cram)}|dataset={_dataset}|encoding=json|ts_out=0"); + var authResp = _reader.ReadLine(); if (!authResp.Contains("success=1")) { - Log.Error($"DatabentoRawClient.Authenticate(): Authentication failed: {authResp}"); + Log.Error($"DataBentoRawLiveClient.Authenticate(): Authentication failed: {authResp}"); return false; } - Log.Trace("DatabentoRawClient.Authenticate(): Authentication successful"); + Log.Trace("DataBentoRawLiveClient.Authenticate(): Authentication successful"); return true; } catch (Exception ex) { - Log.Error($"DatabentoRawClient.Authenticate(): Authentication failed: {ex.Message}"); + Log.Error($"DataBentoRawLiveClient.Authenticate(): Authentication failed: {ex.Message}"); return false; } } - private static string ComputeSHA256(string input) + + /// + /// Handles the DataBento authentication string from a CRAM challenge + /// + /// The CRAM challenge string + /// The auth string to send to the server + private string GetAuthStringFromCram(string cram) { - using var sha = SHA256.Create(); - byte[] hash = sha.ComputeHash(Encoding.UTF8.GetBytes(input)); - var sb = new StringBuilder(); - foreach (byte b in hash) - { - sb.Append(b.ToString("x2")); - } - return sb.ToString(); + if (string.IsNullOrWhiteSpace(cram)) + throw new ArgumentException("CRAM challenge cannot be null or empty", nameof(cram)); + + string concat = $"{cram}|{_apiKey}"; + string hashHex = ComputeSHA256(concat); + string bucketId = _apiKey.Substring(_apiKey.Length - 5); + + return $"{hashHex}-{bucketId}"; } /// /// Subscribes to live data for a symbol /// - public bool Subscribe(Symbol symbol, Resolution resolution, TickType tickType) + public bool Subscribe(Symbol symbol, TickType tickType) { - if (!IsConnected || _writer == null) + if (!IsConnected) { - Log.Error("DatabentoRawClient.Subscribe(): Not connected to gateway"); + Log.Error("DataBentoRawLiveClient.Subscribe(): Not connected to gateway"); return false; } try { // Get the databento symbol form LEAN symbol - // Get schema from the resolution - var databentoSymbol = MapSymbolToDataBento(symbol); - var schema = GetSchema(resolution, tickType); + var databentoSymbol = _symbolMapper.GetBrokerageSymbol(symbol); + var schema = "mbp-1"; + var resolution = Resolution.Tick; // subscribe var subscribeMessage = $"schema={schema}|stype_in=parent|symbols={databentoSymbol}"; - Log.Trace($"DatabentoRawClient.Subscribe(): Subscribing with message: {subscribeMessage}"); + Log.Debug($"DataBentoRawLiveClient.Subscribe(): Subscribing with message: {subscribeMessage}"); // Send subscribe message _writer.WriteLine(subscribeMessage); // Store subscription _subscriptions.TryAdd(symbol, (resolution, tickType)); - Log.Trace($"DatabentoRawClient.Subscribe(): Subscribed to {symbol} ({databentoSymbol}) at {resolution} resolution for {tickType}"); - - // If subscribing to quote ticks, also subscribe to trade ticks - if (tickType == TickType.Quote && resolution == Resolution.Tick) - { - var tradeSchema = GetSchema(resolution, TickType.Trade); - var tradeSubscribeMessage = $"schema={tradeSchema}|stype_in=parent|symbols={databentoSymbol}"; - Log.Trace($"DatabentoRawClient.Subscribe(): Also subscribing to trades with message: {tradeSubscribeMessage}"); - _writer.WriteLine(tradeSubscribeMessage); - } + Log.Debug($"DataBentoRawLiveClient.Subscribe(): Subscribed to {symbol} ({databentoSymbol}) at {resolution} resolution for {tickType}"); return true; } catch (Exception ex) { - Log.Error($"DatabentoRawClient.Subscribe(): Failed to subscribe to {symbol}: {ex.Message}"); + Log.Error($"DataBentoRawLiveClient.Subscribe(): Failed to subscribe to {symbol}: {ex.Message}"); return false; } } @@ -253,21 +236,21 @@ public bool Subscribe(Symbol symbol, Resolution resolution, TickType tickType) /// public bool StartSession() { - if (!IsConnected || _writer == null) + if (!IsConnected) { - Log.Error("DatabentoRawClient.StartSession(): Not connected"); + Log.Error("DataBentoRawLiveClient.StartSession(): Not connected"); return false; } try { - Log.Trace("DatabentoRawClient.StartSession(): Starting session"); + Log.Trace("DataBentoRawLiveClient.StartSession(): Starting session"); _writer.WriteLine("start_session=1"); return true; } catch (Exception ex) { - Log.Error($"DatabentoRawClient.StartSession(): Failed to start session: {ex.Message}"); + Log.Error($"DataBentoRawLiveClient.StartSession(): Failed to start session: {ex.Message}"); return false; } } @@ -281,13 +264,13 @@ public bool Unsubscribe(Symbol symbol) { if (_subscriptions.TryRemove(symbol, out _)) { - Log.Trace($"DatabentoRawClient.Unsubscribe(): Unsubscribed from {symbol}"); + Log.Debug($"DataBentoRawLiveClient.Unsubscribe(): Unsubscribed from {symbol}"); } return true; } catch (Exception ex) { - Log.Error($"DatabentoRawClient.Unsubscribe(): Failed to unsubscribe from {symbol}: {ex.Message}"); + Log.Error($"DataBentoRawLiveClient.Unsubscribe(): Failed to unsubscribe from {symbol}: {ex.Message}"); return false; } } @@ -297,33 +280,17 @@ public bool Unsubscribe(Symbol symbol) /// private void ProcessMessages() { - Log.Trace("DatabentoRawClient.ProcessMessages(): Starting message processing"); - if (_reader == null) - { - Log.Error("DatabentoRawClient.ProcessMessages(): No reader available"); - return; - } - - var messageCount = 0; + Log.Debug("DataBentoRawLiveClient.ProcessMessages(): Starting message processing"); try { while (!_cancellationTokenSource.IsCancellationRequested && IsConnected) { var line = _reader.ReadLine(); - if (line == null) - { - Log.Trace("DatabentoRawClient.ProcessMessages(): Connection closed by server"); - break; - } - if (string.IsNullOrWhiteSpace(line)) - continue; - - messageCount++; - if (messageCount <= 50 || messageCount % 100 == 0) { - Log.Trace($"DatabentoRawClient.ProcessMessages(): Message #{messageCount}: {line.Substring(0, Math.Min(150, line.Length))}..."); + Log.Trace("DataBentoRawLiveClient.ProcessMessages(): Line is null or empty. Issue receiving data."); + break; } ProcessSingleMessage(line); @@ -331,19 +298,18 @@ private void ProcessMessages() } catch (OperationCanceledException) { - Log.Trace("DatabentoRawClient.ProcessMessages(): Message processing cancelled"); + Log.Trace("DataBentoRawLiveClient.ProcessMessages(): Message processing cancelled"); } catch (IOException ex) when (ex.InnerException is SocketException) { - Log.Trace($"DatabentoRawClient.ProcessMessages(): Socket exception: {ex.Message}"); + Log.Trace($"DataBentoRawLiveClient.ProcessMessages(): Socket exception: {ex.Message}"); } catch (Exception ex) { - Log.Error($"DatabentoRawClient.ProcessMessages(): Error processing messages: {ex.Message}\n{ex.StackTrace}"); + Log.Error($"DataBentoRawLiveClient.ProcessMessages(): Error processing messages: {ex.Message}\n{ex.StackTrace}"); } finally { - Log.Trace($"DatabentoRawClient.ProcessMessages(): Exiting. Total messages processed: {messageCount}"); Disconnect(); } } @@ -365,57 +331,71 @@ private void ProcessSingleMessage(string message) { var rtype = rtypeElement.GetInt32(); - if (rtype == 23) - { - if (root.TryGetProperty("msg", out var msgElement)) - { - Log.Trace($"DatabentoRawClient: System message: {msgElement.GetString()}"); - } - return; - } - else if (rtype == 22) + switch (rtype) { - // Symbol mapping message - if (root.TryGetProperty("stype_in_symbol", out var inSymbol) && - root.TryGetProperty("stype_out_symbol", out var outSymbol) && - headerElement.TryGetProperty("instrument_id", out var instId)) - { - var instrumentId = instId.GetInt64(); - var outSymbolStr = outSymbol.GetString(); - - Log.Trace($"DatabentoRawClient: Symbol mapping: {inSymbol.GetString()} -> {outSymbolStr} (instrument_id: {instrumentId})"); - - // Find the subscription that matches this symbol - foreach (var kvp in _subscriptions) + case 23: + // System message + if (root.TryGetProperty("msg", out var msgElement)) { - var leanSymbol = kvp.Key; + Log.Debug($"DataBentoRawLiveClient: System message: {msgElement.GetString()}"); + } + return; + + case 22: + // Symbol mapping message + if (root.TryGetProperty("stype_in_symbol", out var inSymbol) && + root.TryGetProperty("stype_out_symbol", out var outSymbol) && + headerElement.TryGetProperty("instrument_id", out var instId)) + { + var instrumentId = instId.GetInt64(); + var outSymbolStr = outSymbol.GetString(); + + Log.Debug($"DataBentoRawLiveClient: Symbol mapping: {inSymbol.GetString()} -> {outSymbolStr} (instrument_id: {instrumentId})"); + if (outSymbolStr != null) { - _instrumentIdToSymbol[instrumentId] = leanSymbol; - Log.Trace($"DatabentoRawClient: Mapped instrument_id {instrumentId} to {leanSymbol}"); - break; + // Let's find the subscribed symbol to get the market and security type + var inSymbolStr = inSymbol.GetString(); + var subscription = _subscriptions.Keys.FirstOrDefault(s => _symbolMapper.GetBrokerageSymbol(s) == inSymbolStr); + if (subscription != null) + { + if (subscription.SecurityType == SecurityType.Future) + { + var leanSymbol = _symbolMapper.GetLeanSymbolForFuture(outSymbolStr); + if (leanSymbol == null) + { + Log.Trace($"DataBentoRawLiveClient: Future spreads are not supported: {outSymbolStr}. Skipping mapping."); + return; + } + _instrumentIdToSymbol[instrumentId] = leanSymbol; + Log.Debug($"DataBentoRawLiveClient: Mapped instrument_id {instrumentId} to {leanSymbol}"); + } + } } } - } - return; - } - else if (rtype == 1) - { - // MBP-1 (Market By Price) - Quote ticks - HandleMBPMessage(root, headerElement); - return; - } - else if (rtype == 0) - { - // Trade messages - Trade ticks - HandleTradeTickMessage(root, headerElement); - return; - } - else if (rtype == 32 || rtype == 33 || rtype == 34 || rtype == 35) - { - // OHLCV bar messages - HandleOHLCVMessage(root, headerElement); - return; + return; + + case 1: + // MBP-1 (Market By Price) + HandleMBPMessage(root, headerElement); + return; + + case 0: + // Trade messages + HandleTradeTickMessage(root, headerElement); + return; + + case 32: + case 33: + case 34: + case 35: + // OHLCV bar messages + HandleOHLCVMessage(root, headerElement); + return; + + default: + Log.Error($"DataBentoRawLiveClient: Unknown rtype {rtype} in message"); + return; } } } @@ -423,16 +403,16 @@ private void ProcessSingleMessage(string message) // Handle other message types if needed if (root.TryGetProperty("error", out var errorElement)) { - Log.Error($"DatabentoRawClient: Server error: {errorElement.GetString()}"); + Log.Error($"DataBentoRawLiveClient: Server error: {errorElement.GetString()}"); } } catch (JsonException ex) { - Log.Error($"DatabentoRawClient.ProcessSingleMessage(): JSON parse error: {ex.Message}"); + Log.Error($"DataBentoRawLiveClient.ProcessSingleMessage(): JSON parse error: {ex.Message}"); } catch (Exception ex) { - Log.Error($"DatabentoRawClient.ProcessSingleMessage(): Error: {ex.Message}"); + Log.Error($"DataBentoRawLiveClient.ProcessSingleMessage(): Error: {ex.Message}"); } } @@ -458,7 +438,7 @@ private void HandleOHLCVMessage(JsonElement root, JsonElement header) if (!_instrumentIdToSymbol.TryGetValue(instrumentId, out var matchedSymbol)) { - Log.Trace($"DatabentoRawClient: No mapping for instrument_id {instrumentId} in OHLCV message."); + Log.Debug($"DataBentoRawLiveClient: No mapping for instrument_id {instrumentId} in OHLCV message."); return; } @@ -511,13 +491,13 @@ private void HandleOHLCVMessage(JsonElement root, JsonElement header) period ); - Log.Trace($"DatabentoRawClient: OHLCV bar: {matchedSymbol} O={open} H={high} L={low} C={close} V={volume} at {timestamp}"); + // Log.Trace($"DataBentoRawLiveClient: OHLCV bar: {matchedSymbol} O={open} H={high} L={low} C={close} V={volume} at {timestamp}"); DataReceived?.Invoke(this, tradeBar); } } catch (Exception ex) { - Log.Error($"DatabentoRawClient.HandleOHLCVMessage(): Error: {ex.Message}"); + Log.Error($"DataBentoRawLiveClient.HandleOHLCVMessage(): Error: {ex.Message}"); } } @@ -543,7 +523,7 @@ private void HandleMBPMessage(JsonElement root, JsonElement header) if (!_instrumentIdToSymbol.TryGetValue(instrumentId, out var matchedSymbol)) { - Log.Trace($"DatabentoRawClient: No mapping for instrument_id {instrumentId} in MBP message."); + Log.Trace($"DataBentoRawLiveClient: No mapping for instrument_id {instrumentId} in MBP message."); return; } @@ -582,13 +562,13 @@ private void HandleMBPMessage(JsonElement root, JsonElement header) // QuantConnect convention: Quote ticks should have zero Price and Quantity quoteTick.Quantity = 0; - Log.Trace($"DatabentoRawClient: Quote tick: {matchedSymbol} Bid={quoteTick.BidPrice}x{quoteTick.BidSize} Ask={quoteTick.AskPrice}x{quoteTick.AskSize}"); + // Log.Trace($"DataBentoRawLiveClient: Quote tick: {matchedSymbol} Bid={quoteTick.BidPrice}x{quoteTick.BidSize} Ask={quoteTick.AskPrice}x{quoteTick.AskSize}"); DataReceived?.Invoke(this, quoteTick); } } catch (Exception ex) { - Log.Error($"DatabentoRawClient.HandleMBPMessage(): Error: {ex.Message}"); + Log.Error($"DataBentoRawLiveClient.HandleMBPMessage(): Error: {ex.Message}"); } } @@ -614,7 +594,7 @@ private void HandleTradeTickMessage(JsonElement root, JsonElement header) if (!_instrumentIdToSymbol.TryGetValue(instrumentId, out var matchedSymbol)) { - Log.Trace($"DatabentoRawClient: No mapping for instrument_id {instrumentId} in trade message."); + Log.Trace($"DataBentoRawLiveClient: No mapping for instrument_id {instrumentId} in trade message."); return; } @@ -639,66 +619,14 @@ private void HandleTradeTickMessage(JsonElement root, JsonElement header) AskSize = 0 }; - Log.Trace($"DatabentoRawClient: Trade tick: {matchedSymbol} Price={price} Quantity={size}"); + // Log.Trace($"DataBentoRawLiveClient: Trade tick: {matchedSymbol} Price={price} Quantity={size}"); DataReceived?.Invoke(this, tradeTick); } } catch (Exception ex) { - Log.Error($"DatabentoRawClient.HandleTradeTickMessage(): Error: {ex.Message}"); - } - } - - /// - /// Maps a LEAN symbol to DataBento symbol format - /// - private string MapSymbolToDataBento(Symbol symbol) - { - if (symbol.SecurityType == SecurityType.Future) - { - // For DataBento, use the root symbol with .FUT suffix for parent subscription - // ES19Z25 -> ES.FUT - var value = symbol.Value; - - // Extract root by removing digits and month codes - var root = new string(value.TakeWhile(c => !char.IsDigit(c)).ToArray()); - - return $"{root}.FUT"; + Log.Error($"DataBentoRawLiveClient.HandleTradeTickMessage(): Error: {ex.Message}"); } - - return symbol.Value; - } - - /// - /// Pick Databento schema from Lean resolution/ticktype - /// - private string GetSchema(Resolution resolution, TickType tickType) - { - if (tickType == TickType.Trade) - { - if (resolution == Resolution.Tick) - return "trades"; - if (resolution == Resolution.Second) - return "ohlcv-1s"; - if (resolution == Resolution.Minute) - return "ohlcv-1m"; - if (resolution == Resolution.Hour) - return "ohlcv-1h"; - if (resolution == Resolution.Daily) - return "ohlcv-1d"; - } - else if (tickType == TickType.Quote) - { - // top of book - if (resolution == Resolution.Tick || resolution == Resolution.Second || resolution == Resolution.Minute || resolution == Resolution.Hour || resolution == Resolution.Daily) - return "mbp-1"; - } - else if (tickType == TickType.OpenInterest) - { - return "statistics"; - } - - throw new NotSupportedException($"Unsupported resolution {resolution} / {tickType}"); } /// @@ -723,11 +651,11 @@ public void Disconnect() } catch (Exception ex) { - Log.Trace($"DatabentoRawClient.Disconnect(): Error during disconnect: {ex.Message}"); + Log.Trace($"DataBentoRawLiveClient.Disconnect(): Error during disconnect: {ex.Message}"); } ConnectionStatusChanged?.Invoke(this, false); - Log.Trace("DatabentoRawClient.Disconnect(): Disconnected from DataBento gateway"); + Log.Trace("DataBentoRawLiveClient.Disconnect(): Disconnected from DataBento gateway"); } } @@ -748,5 +676,21 @@ public void Dispose() _stream?.Dispose(); _tcpClient?.Dispose(); } + + /// + /// Computes the SHA-256 hash of the input string + /// + private static string ComputeSHA256(string input) + { + using var sha = SHA256.Create(); + var hash = sha.ComputeHash(Encoding.UTF8.GetBytes(input)); + var sb = new StringBuilder(); + foreach (byte b in hash) + { + sb.Append(b.ToString("x2")); + } + return sb.ToString(); + } + } } diff --git a/QuantConnect.DataBento/DataBentoSymbolMapper.cs b/QuantConnect.DataBento/DataBentoSymbolMapper.cs new file mode 100644 index 0000000..3c6b8b3 --- /dev/null +++ b/QuantConnect.DataBento/DataBentoSymbolMapper.cs @@ -0,0 +1,174 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using QuantConnect; +using QuantConnect.Brokerages; +using System.Globalization; + +namespace QuantConnect.Lean.DataSource.DataBento +{ + /// + /// Provides the mapping between Lean symbols and DataBento symbols. + /// + public class DataBentoSymbolMapper : ISymbolMapper + { + private readonly Dictionary _leanSymbolsCache = new(); + private readonly Dictionary _brokerageSymbolsCache = new(); + private readonly object _locker = new(); + + /// + /// Converts a Lean symbol instance to a brokerage symbol + /// + /// A Lean symbol instance + /// The brokerage symbol + public string GetBrokerageSymbol(Symbol symbol) + { + if (symbol == null || string.IsNullOrWhiteSpace(symbol.Value)) + { + throw new ArgumentException($"Invalid symbol: {(symbol == null ? "null" : symbol.ToString())}"); + } + + return GetBrokerageSymbol(symbol, false); + } + + /// + /// Converts a Lean symbol instance to a brokerage symbol with updating of cached symbol collection + /// + /// + /// + /// + /// + public string GetBrokerageSymbol(Symbol symbol, bool isUpdateCachedSymbol) + { + lock (_locker) + { + if (!_brokerageSymbolsCache.TryGetValue(symbol, out var brokerageSymbol) || isUpdateCachedSymbol) + { + switch (symbol.SecurityType) + { + case SecurityType.Future: + brokerageSymbol = $"{symbol.ID.Symbol}.FUT"; + break; + + case SecurityType.Equity: + brokerageSymbol = symbol.Value; + break; + + default: + throw new Exception($"DataBentoSymbolMapper.GetBrokerageSymbol(): unsupported security type: {symbol.SecurityType}"); + } + + // Lean-to-DataBento symbol conversion is accurate, so we can cache it both ways + _brokerageSymbolsCache[symbol] = brokerageSymbol; + _leanSymbolsCache[brokerageSymbol] = symbol; + } + + return brokerageSymbol; + } + } + + /// + /// Converts a brokerage symbol to a Lean symbol instance + /// + /// The brokerage symbol + /// The security type + /// The market + /// Expiration date of the security(if applicable) + /// A new Lean Symbol instance + public Symbol GetLeanSymbol(string brokerageSymbol, SecurityType securityType, string market, + DateTime expirationDate = new DateTime(), decimal strike = 0, OptionRight optionRight = 0) + { + if (string.IsNullOrWhiteSpace(brokerageSymbol)) + { + throw new ArgumentException("Invalid symbol: " + brokerageSymbol); + } + + lock (_locker) + { + if (!_leanSymbolsCache.TryGetValue(brokerageSymbol, out var leanSymbol)) + { + switch (securityType) + { + case SecurityType.Future: + leanSymbol = Symbol.CreateFuture(brokerageSymbol, market, expirationDate); + break; + + default: + throw new Exception($"DataBentoSymbolMapper.GetLeanSymbol(): unsupported security type: {securityType}"); + } + + _leanSymbolsCache[brokerageSymbol] = leanSymbol; + _brokerageSymbolsCache[leanSymbol] = brokerageSymbol; + } + + return leanSymbol; + } + } + + /// + /// Gets the Lean symbol for the specified DataBento symbol + /// + /// The databento symbol + /// The corresponding Lean symbol + public Symbol GetLeanSymbol(string databentoSymbol) + { + lock (_locker) + { + if (!_leanSymbolsCache.TryGetValue(databentoSymbol, out var symbol)) + { + symbol = GetLeanSymbol(databentoSymbol, SecurityType.Equity, Market.USA); + } + + return symbol; + } + } + + /// + /// Converts a brokerage future symbol to a Lean symbol instance + /// + /// The brokerage symbol + /// A new Lean Symbol instance + public Symbol GetLeanSymbolForFuture(string brokerageSymbol) + { + if (string.IsNullOrWhiteSpace(brokerageSymbol)) + { + throw new ArgumentException("Invalid symbol: " + brokerageSymbol); + } + + // ignore futures spreads + if (brokerageSymbol.Contains("-")) + { + return null; + } + + lock (_locker) + { + if (!_leanSymbolsCache.TryGetValue(brokerageSymbol, out var leanSymbol)) + { + leanSymbol = SymbolRepresentation.ParseFutureSymbol(brokerageSymbol); + + if (leanSymbol == null) + { + throw new ArgumentException("Invalid future symbol: " + brokerageSymbol); + } + + _leanSymbolsCache[brokerageSymbol] = leanSymbol; + } + + return leanSymbol; + } + } + } +} diff --git a/QuantConnect.DataBento/QuantConnect.DataSource.DataBento.csproj b/QuantConnect.DataBento/QuantConnect.DataSource.DataBento.csproj index 718002c..249ad0a 100644 --- a/QuantConnect.DataBento/QuantConnect.DataSource.DataBento.csproj +++ b/QuantConnect.DataBento/QuantConnect.DataSource.DataBento.csproj @@ -39,4 +39,8 @@ + + + + diff --git a/models/DataBentoTypes.cs b/models/DataBentoTypes.cs new file mode 100644 index 0000000..b7fa58c --- /dev/null +++ b/models/DataBentoTypes.cs @@ -0,0 +1,112 @@ +using System; +using CsvHelper.Configuration.Attributes; +using QuantConnect; + +namespace QuantConnect.Lean.DataSource.DataBento.Models +{ + /// + /// Provides a constant for scaling price values from DataBento. + /// + public static class PriceScaling + { + /// + /// price scale factor is needed to find the true price from the message + /// Due to compression each "1 unit corresponds to 1e-9, i.e. 1/1,000,000,000 or 0.000000001" + /// https://databento.com/docs/api-reference-live/basics/schemas-and-conventions?historical=raw&live=raw&reference=raw + /// + public const decimal PriceScaleFactor = 1e-9m; + } + + /// + /// Represents a single bar of historical data from DataBento. + /// This class is used to map CSV data from HTTP requests into a structured format. + /// + public class DatabentoBar + { + [Name("ts_event")] + public long TimestampNanos { get; set; } + + public DateTime Timestamp => Time.UnixNanosecondTimeStampToDateTime(TimestampNanos); + + [Name("open")] + public long RawOpen { get; set; } + + [Name("high")] + public long RawHigh { get; set; } + + [Name("low")] + public long RawLow { get; set; } + + + + [Name("close")] + public long RawClose { get; set; } + + [Ignore] + public decimal Open => RawOpen == long.MaxValue ? 0m : RawOpen * PriceScaling.PriceScaleFactor; + + [Ignore] + public decimal High => RawHigh == long.MaxValue ? 0m : RawHigh * PriceScaling.PriceScaleFactor; + + [Ignore] + public decimal Low => RawLow == long.MaxValue ? 0m : RawLow * PriceScaling.PriceScaleFactor; + + [Ignore] + public decimal Close => RawClose == long.MaxValue ? 0m : RawClose * PriceScaling.PriceScaleFactor; + + [Name("volume")] + public long RawVolume { get; set; } + + [Ignore] + public decimal Volume => RawVolume == long.MaxValue ? 0m : RawVolume; + } + + /// + /// Represents a single trade event from DataBento. + /// + public class DatabentoTrade + { + [Name("ts_event")] + public long TimestampNanos { get; set; } + + public DateTime Timestamp => Time.UnixNanosecondTimeStampToDateTime(TimestampNanos); + + [Name("price")] + public long RawPrice { get; set; } + + [Ignore] + public decimal Price => RawPrice == long.MaxValue ? 0m : RawPrice * PriceScaling.PriceScaleFactor; + + [Name("size")] + public int Size { get; set; } + } + + /// + /// Represents a single quote from DataBento. + /// + public class DatabentoQuote + { + [Name("ts_event")] + public long TimestampNanos { get; set; } + + public DateTime Timestamp => Time.UnixNanosecondTimeStampToDateTime(TimestampNanos); + + [Name("bid_px_00")] + public long RawBidPrice { get; set; } + + [Ignore] + public decimal BidPrice => RawBidPrice == long.MaxValue ? 0m : RawBidPrice * PriceScaling.PriceScaleFactor; + + [Name("bid_sz_00")] + public int BidSize { get; set; } + + [Name("ask_px_00")] + public long RawAskPrice { get; set; } + + [Ignore] + public decimal AskPrice => RawAskPrice == long.MaxValue ? 0m : RawAskPrice * PriceScaling.PriceScaleFactor; + + [Name("ask_sz_00")] + public int AskSize { get; set; } + } +} From 68fd7f6f471dfe293d086bb63bb01a952993bddd Mon Sep 17 00:00:00 2001 From: Romazes Date: Tue, 20 Jan 2026 11:09:12 +0200 Subject: [PATCH 02/13] update: .net 10 --- .../QuantConnect.DataSource.DataBento.Tests.csproj | 2 +- QuantConnect.DataBento/QuantConnect.DataSource.DataBento.csproj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/QuantConnect.DataBento.Tests/QuantConnect.DataSource.DataBento.Tests.csproj b/QuantConnect.DataBento.Tests/QuantConnect.DataSource.DataBento.Tests.csproj index f5fce66..82c36da 100644 --- a/QuantConnect.DataBento.Tests/QuantConnect.DataSource.DataBento.Tests.csproj +++ b/QuantConnect.DataBento.Tests/QuantConnect.DataSource.DataBento.Tests.csproj @@ -1,6 +1,6 @@ - net9.0 + net10.0 QuantConnect.DataLibrary.Tests diff --git a/QuantConnect.DataBento/QuantConnect.DataSource.DataBento.csproj b/QuantConnect.DataBento/QuantConnect.DataSource.DataBento.csproj index 249ad0a..0c83d24 100644 --- a/QuantConnect.DataBento/QuantConnect.DataSource.DataBento.csproj +++ b/QuantConnect.DataBento/QuantConnect.DataSource.DataBento.csproj @@ -3,7 +3,7 @@ Release AnyCPU - net9.0 + net10.0 QuantConnect.Lean.DataSource.DataBento QuantConnect.Lean.DataSource.DataBento QuantConnect.Lean.DataSource.DataBento From c23c1851289e8e01ea6ed239c5109ea61ca7ccb0 Mon Sep 17 00:00:00 2001 From: Romazes Date: Tue, 20 Jan 2026 11:31:49 +0200 Subject: [PATCH 03/13] feat: setup project configurations --- ...tConnect.DataSource.DataBento.Tests.csproj | 65 ++++++++++--------- .../Models}/DataBentoTypes.cs | 0 .../QuantConnect.DataSource.DataBento.csproj | 9 +-- 3 files changed, 37 insertions(+), 37 deletions(-) rename {models => QuantConnect.DataBento/Models}/DataBentoTypes.cs (100%) diff --git a/QuantConnect.DataBento.Tests/QuantConnect.DataSource.DataBento.Tests.csproj b/QuantConnect.DataBento.Tests/QuantConnect.DataSource.DataBento.Tests.csproj index 82c36da..b4c0ccd 100644 --- a/QuantConnect.DataBento.Tests/QuantConnect.DataSource.DataBento.Tests.csproj +++ b/QuantConnect.DataBento.Tests/QuantConnect.DataSource.DataBento.Tests.csproj @@ -1,32 +1,37 @@ - - net10.0 - QuantConnect.DataLibrary.Tests - - - - - - - - - all - - - - - - - - - - - - - - - - PreserveNewest - - + + Release + AnyCPU + net10.0 + false + UnitTest + bin\$(Configuration)\ + QuantConnect.Lean.DataSource.DataBento.Tests + QuantConnect.Lean.DataSource.DataBento.Tests + QuantConnect.Lean.DataSource.DataBento.Tests + QuantConnect.Lean.DataSource.DataBento.Tests + false + + + + + + + + + all + + + + + + + + + + + + PreserveNewest + + diff --git a/models/DataBentoTypes.cs b/QuantConnect.DataBento/Models/DataBentoTypes.cs similarity index 100% rename from models/DataBentoTypes.cs rename to QuantConnect.DataBento/Models/DataBentoTypes.cs diff --git a/QuantConnect.DataBento/QuantConnect.DataSource.DataBento.csproj b/QuantConnect.DataBento/QuantConnect.DataSource.DataBento.csproj index 0c83d24..ada272c 100644 --- a/QuantConnect.DataBento/QuantConnect.DataSource.DataBento.csproj +++ b/QuantConnect.DataBento/QuantConnect.DataSource.DataBento.csproj @@ -1,4 +1,3 @@ - Release @@ -29,18 +28,14 @@ bin\Release\ - + - - - - From f660594837b0fc72cd5746cfe1b09d9cb48e3986 Mon Sep 17 00:00:00 2001 From: Romazes Date: Tue, 20 Jan 2026 11:36:17 +0200 Subject: [PATCH 04/13] remove: demonstration file --- Demonstration.cs | 94 ------------------- ...tConnect.DataSource.DataBento.Tests.csproj | 3 - 2 files changed, 97 deletions(-) delete mode 100644 Demonstration.cs diff --git a/Demonstration.cs b/Demonstration.cs deleted file mode 100644 index f98b935..0000000 --- a/Demonstration.cs +++ /dev/null @@ -1,94 +0,0 @@ -/* - * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * -*/ - -using QuantConnect.Algorithm; -using QuantConnect.Data.Market; -using QuantConnect.Interfaces; -using QuantConnect; -using QuantConnect.Data; -using QuantConnect.Securities.Future; -using QuantConnect.Util; -using System; -using System.Linq; - -namespace QuantConnect.Algorithm.CSharp -{ - public class DatabentoFuturesTestAlgorithm : QCAlgorithm - { - private Future _es; - - public override void Initialize() - { - Log("Algorithm Initialize"); - - SetStartDate(2025, 10, 1); - SetEndDate(2025, 10, 16); - SetCash(100000); - - var exp = new DateTime(2025, 12, 19); - var symbol = QuantConnect.Symbol.CreateFuture("ES", Market.CME, exp); - //_es = AddFutureContract(symbol, Resolution.Tick, true, 1, true); - _es = AddFutureContract(symbol, Resolution.Second, true, 1, true); - Log($"_es: {_es}"); - - var history = History(_es.Symbol, 10, Resolution.Minute).ToList(); - - Log($"History returned {history.Count} bars"); - - foreach (var bar in history) - { - Log($"History Bar: {bar.Time} - O:{bar.Open} H:{bar.High} L:{bar.Low} C:{bar.Close} V:{bar.Volume}"); - } - - } - - public override void OnData(Slice slice) - { - if (!slice.HasData) - { - Log("Slice has no data"); - return; - } - - Log($"OnData: Slice has {slice.Count} data points"); - - // For Tick resolution, check Ticks collection - if (slice.Ticks.ContainsKey(_es.Symbol)) - { - var ticks = slice.Ticks[_es.Symbol]; - Log($"Received {ticks.Count} ticks for {_es.Symbol}"); - - foreach (var tick in ticks) - { - if (tick.TickType == TickType.Trade) - { - Log($"Trade Tick - Price: {tick.Price}, Quantity: {tick.Quantity}, Time: {tick.Time}"); - } - else if (tick.TickType == TickType.Quote) - { - Log($"Quote Tick - Bid: {tick.BidPrice}x{tick.BidSize}, Ask: {tick.AskPrice}x{tick.AskSize}, Time: {tick.Time}"); - } - } - } - - // Access OHLCV bars - foreach (var bar in slice.Bars.Values) - { - Log($"OHLCV BAR: {bar.Symbol.Value} - O: {bar.Open}, H: {bar.High}, L: {bar.Low}, C: {bar.Close}, V: {bar.Volume}"); - } - } - } -} diff --git a/QuantConnect.DataBento.Tests/QuantConnect.DataSource.DataBento.Tests.csproj b/QuantConnect.DataBento.Tests/QuantConnect.DataSource.DataBento.Tests.csproj index b4c0ccd..c58ec16 100644 --- a/QuantConnect.DataBento.Tests/QuantConnect.DataSource.DataBento.Tests.csproj +++ b/QuantConnect.DataBento.Tests/QuantConnect.DataSource.DataBento.Tests.csproj @@ -12,9 +12,6 @@ QuantConnect.Lean.DataSource.DataBento.Tests false - - - From f85939dd148305e82a7c5ce8616314aac96bf12c Mon Sep 17 00:00:00 2001 From: Romazes Date: Tue, 20 Jan 2026 11:40:55 +0200 Subject: [PATCH 05/13] test:refactor: config file --- QuantConnect.DataBento.Tests/config.json | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/QuantConnect.DataBento.Tests/config.json b/QuantConnect.DataBento.Tests/config.json index 7256aba..2464c3e 100644 --- a/QuantConnect.DataBento.Tests/config.json +++ b/QuantConnect.DataBento.Tests/config.json @@ -1,9 +1,7 @@ { - "data-folder":"../../../../../Data/", - - "job-user-id": "0", + "data-folder": "../../../../Lean/Data/", + "job-user-id": "", "api-access-token": "", "job-organization-id": "", - - "databento-api-key":"" + "databento-api-key": "" } \ No newline at end of file From 991d8bf46d495ba17d663c369ef7392b9b288230 Mon Sep 17 00:00:00 2001 From: Romazes Date: Thu, 22 Jan 2026 01:00:36 +0200 Subject: [PATCH 06/13] refactor: HistoryProvider, DataDownloader, SymbolMapper --- Lean.DataSource.DataBento.sln | 8 +- .../DataBentoDataDownloaderTests.cs | 48 +--- .../DataBentoDataProviderHistoryTests.cs | 191 +++++-------- .../DataBentoHistoricalApiClientTests.cs | 95 +++++++ .../DataBentoJsonConverterTests.cs | 135 +++++++++ .../DataBentoRawLiveClientTests.cs | 2 +- .../DataBentoSymbolMapperTests.cs.cs | 56 ++++ QuantConnect.DataBento.Tests/TestSetup.cs | 2 +- .../Api/HistoricalAPIClient.cs | 168 ++++++++++++ .../DataBentoDataDownloader.cs | 256 ++++-------------- .../DataBentoDataProvider.cs | 46 ++-- .../DataBentoHistoryProivder.cs | 158 ++++++----- .../DataBentoRawLiveClient.cs | 2 +- .../DataBentoSymbolMapper.cs | 120 +------- QuantConnect.DataBento/Extensions.cs | 35 +++ .../Models/DataBentoTypes.cs | 112 -------- .../Models/Enums/StatisticType.cs | 117 ++++++++ QuantConnect.DataBento/Models/Header.cs | 48 ++++ .../Models/LevelOneBookLevel.cs | 49 ++++ QuantConnect.DataBento/Models/LevelOneData.cs | 69 +++++ .../Models/MarketDataRecord.cs | 30 ++ QuantConnect.DataBento/Models/OhlcvBar.cs | 48 ++++ .../Models/StatisticsData.cs | 31 +++ .../Serialization/JsonSettings.cs | 34 +++ .../SnakeCaseContractResolver.cs | 39 +++ 25 files changed, 1234 insertions(+), 665 deletions(-) create mode 100644 QuantConnect.DataBento.Tests/DataBentoHistoricalApiClientTests.cs create mode 100644 QuantConnect.DataBento.Tests/DataBentoJsonConverterTests.cs create mode 100644 QuantConnect.DataBento.Tests/DataBentoSymbolMapperTests.cs.cs create mode 100644 QuantConnect.DataBento/Api/HistoricalAPIClient.cs create mode 100644 QuantConnect.DataBento/Extensions.cs delete mode 100644 QuantConnect.DataBento/Models/DataBentoTypes.cs create mode 100644 QuantConnect.DataBento/Models/Enums/StatisticType.cs create mode 100644 QuantConnect.DataBento/Models/Header.cs create mode 100644 QuantConnect.DataBento/Models/LevelOneBookLevel.cs create mode 100644 QuantConnect.DataBento/Models/LevelOneData.cs create mode 100644 QuantConnect.DataBento/Models/MarketDataRecord.cs create mode 100644 QuantConnect.DataBento/Models/OhlcvBar.cs create mode 100644 QuantConnect.DataBento/Models/StatisticsData.cs create mode 100644 QuantConnect.DataBento/Serialization/JsonSettings.cs create mode 100644 QuantConnect.DataBento/Serialization/SnakeCaseContractResolver.cs diff --git a/Lean.DataSource.DataBento.sln b/Lean.DataSource.DataBento.sln index f2f03d6..bfecccc 100644 --- a/Lean.DataSource.DataBento.sln +++ b/Lean.DataSource.DataBento.sln @@ -2,9 +2,9 @@ Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 17 VisualStudioVersion = 17.5.2.0 MinimumVisualStudioVersion = 10.0.40219.1 -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "QuantConnect.DataBento", "QuantConnect.DataBento\QuantConnect.DataSource.DataBento.csproj", "{367AEEDC-F0B3-7F47-539D-10E5EC242C2A}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "QuantConnect.DataSource.DataBento", "QuantConnect.DataBento\QuantConnect.DataSource.DataBento.csproj", "{367AEEDC-F0B3-7F47-539D-10E5EC242C2A}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "QuantConnect.DataBento.Tests", "QuantConnect.DataBento.Tests\QuantConnect.DataSource.DataBento.Tests.csproj", "{9CF47860-2CEA-F379-09D8-9AEF27965D12}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "QuantConnect.DataSource.DataBento.Tests", "QuantConnect.DataBento.Tests\QuantConnect.DataSource.DataBento.Tests.csproj", "{9CF47860-2CEA-F379-09D8-9AEF27965D12}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -16,10 +16,6 @@ Global {367AEEDC-F0B3-7F47-539D-10E5EC242C2A}.Debug|Any CPU.Build.0 = Debug|Any CPU {367AEEDC-F0B3-7F47-539D-10E5EC242C2A}.Release|Any CPU.ActiveCfg = Release|Any CPU {367AEEDC-F0B3-7F47-539D-10E5EC242C2A}.Release|Any CPU.Build.0 = Release|Any CPU - {4B379C8F-16CE-1972-73E3-C14F6410D428}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {4B379C8F-16CE-1972-73E3-C14F6410D428}.Debug|Any CPU.Build.0 = Debug|Any CPU - {4B379C8F-16CE-1972-73E3-C14F6410D428}.Release|Any CPU.ActiveCfg = Release|Any CPU - {4B379C8F-16CE-1972-73E3-C14F6410D428}.Release|Any CPU.Build.0 = Release|Any CPU {9CF47860-2CEA-F379-09D8-9AEF27965D12}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {9CF47860-2CEA-F379-09D8-9AEF27965D12}.Debug|Any CPU.Build.0 = Debug|Any CPU {9CF47860-2CEA-F379-09D8-9AEF27965D12}.Release|Any CPU.ActiveCfg = Release|Any CPU diff --git a/QuantConnect.DataBento.Tests/DataBentoDataDownloaderTests.cs b/QuantConnect.DataBento.Tests/DataBentoDataDownloaderTests.cs index a5063df..7525823 100644 --- a/QuantConnect.DataBento.Tests/DataBentoDataDownloaderTests.cs +++ b/QuantConnect.DataBento.Tests/DataBentoDataDownloaderTests.cs @@ -1,6 +1,6 @@ /* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,13 +17,10 @@ using System; using System.Linq; using NUnit.Framework; -using QuantConnect.Configuration; -using QuantConnect.Data; -using QuantConnect.Data.Market; -using QuantConnect.Lean.DataSource.DataBento; +using QuantConnect.Util; using QuantConnect.Logging; using QuantConnect.Securities; -using QuantConnect.Util; +using QuantConnect.Data.Market; namespace QuantConnect.Lean.DataSource.DataBento.Tests { @@ -31,20 +28,13 @@ namespace QuantConnect.Lean.DataSource.DataBento.Tests public class DataBentoDataDownloaderTests { private DataBentoDataDownloader _downloader; - private MarketHoursDatabase _marketHoursDatabase; - protected readonly string ApiKey = Config.Get("databento-api-key"); - private static Symbol CreateEsFuture() - { - var expiration = new DateTime(2026, 3, 20); - return Symbol.CreateFuture("ES", Market.CME, expiration); - } + private readonly MarketHoursDatabase _marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); [SetUp] public void SetUp() { - _marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); - _downloader = new DataBentoDataDownloader(ApiKey, _marketHoursDatabase); + _downloader = new DataBentoDataDownloader(); } [TearDown] @@ -60,15 +50,15 @@ public void TearDown() [TestCase(Resolution.Tick)] public void DownloadsTradeDataForLeanFuture(Resolution resolution) { - var symbol = CreateEsFuture(); + var symbol = Symbol.CreateFuture("ES", Market.CME, new DateTime(2026, 3, 20)); var exchangeTimeZone = _marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType).TimeZone; - var startUtc = new DateTime(2024, 5, 1, 0, 0, 0, DateTimeKind.Utc); - var endUtc = new DateTime(2024, 5, 2, 0, 0, 0, DateTimeKind.Utc); + var startUtc = new DateTime(2026, 1, 18, 0, 0, 0, DateTimeKind.Utc); + var endUtc = new DateTime(2026, 1, 20, 0, 0, 0, DateTimeKind.Utc); if (resolution == Resolution.Tick) { - startUtc = new DateTime(2024, 5, 1, 9, 30, 0, DateTimeKind.Utc); + startUtc = new DateTime(2026, 1, 21, 9, 30, 0, DateTimeKind.Utc); endUtc = startUtc.AddMinutes(15); } @@ -120,10 +110,10 @@ public void DownloadsTradeDataForLeanFuture(Resolution resolution) [Test] public void DownloadsQuoteTicksForLeanFuture() { - var symbol = CreateEsFuture(); + var symbol = Symbol.CreateFuture("ES", Market.CME, new DateTime(2026, 3, 20)); var exchangeTimeZone = _marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType).TimeZone; - var startUtc = new DateTime(2024, 5, 1, 9, 30, 0, DateTimeKind.Utc); + var startUtc = new DateTime(2026, 1, 20, 9, 30, 0, DateTimeKind.Utc); var endUtc = startUtc.AddMinutes(15); var parameters = new DataDownloaderGetParameters( @@ -166,10 +156,10 @@ public void DownloadsQuoteTicksForLeanFuture() [Test] public void DataIsSortedByTime() { - var symbol = CreateEsFuture(); + var symbol = Symbol.CreateFuture("ES", Market.CME, new DateTime(2026, 3, 20)); - var startUtc = new DateTime(2024, 5, 1, 0, 0, 0, DateTimeKind.Utc); - var endUtc = new DateTime(2024, 5, 2, 0, 0, 0, DateTimeKind.Utc); + var startUtc = new DateTime(2026, 1, 20, 0, 0, 0, DateTimeKind.Utc); + var endUtc = new DateTime(2024, 1, 21, 0, 0, 0, DateTimeKind.Utc); var parameters = new DataDownloaderGetParameters( symbol, @@ -192,15 +182,5 @@ public void DataIsSortedByTime() ); } } - - [Test] - public void DisposeIsIdempotent() - { - var downloader = new DataBentoDataDownloader(ApiKey, - MarketHoursDatabase.FromDataFolder()); - - Assert.DoesNotThrow(downloader.Dispose); - Assert.DoesNotThrow(downloader.Dispose); - } } } diff --git a/QuantConnect.DataBento.Tests/DataBentoDataProviderHistoryTests.cs b/QuantConnect.DataBento.Tests/DataBentoDataProviderHistoryTests.cs index 94c78f8..f0d2e2f 100644 --- a/QuantConnect.DataBento.Tests/DataBentoDataProviderHistoryTests.cs +++ b/QuantConnect.DataBento.Tests/DataBentoDataProviderHistoryTests.cs @@ -1,6 +1,6 @@ /* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,148 +19,101 @@ using NUnit.Framework; using QuantConnect.Data; using QuantConnect.Util; -using QuantConnect.Lean.DataSource.DataBento; using QuantConnect.Securities; using System.Collections.Generic; using QuantConnect.Logging; using QuantConnect.Data.Market; -using QuantConnect.Configuration; -namespace QuantConnect.Lean.DataSource.DataBento.Tests +namespace QuantConnect.Lean.DataSource.DataBento.Tests; + +[TestFixture] +public class DataBentoDataProviderHistoryTests { - [TestFixture] - public class DataBentoDataProviderHistoryTests + private DataBentoProvider _historyDataProvider; + + [SetUp] + public void SetUp() { - private DataBentoProvider _historyDataProvider; - private MarketHoursDatabase _marketHoursDatabase; - protected readonly string ApiKey = Config.Get("databento-api-key"); + _historyDataProvider = new DataBentoProvider(); + } - private static Symbol CreateEsFuture() - { - var expiration = new DateTime(2026, 3, 20); - return Symbol.CreateFuture("ES", Market.CME, expiration); - } + [TearDown] + public void TearDown() + { + _historyDataProvider?.Dispose(); + } - [SetUp] - public void SetUp() + internal static IEnumerable TestParameters + { + get { - _historyDataProvider = new DataBentoProvider(); - } + var es = Symbol.CreateFuture("ES", Market.CME, new DateTime(2026, 3, 20)); - [TearDown] - public void TearDown() - { - _historyDataProvider?.Dispose(); + yield return new TestCaseData(es, Resolution.Daily, TickType.Trade, TimeSpan.FromDays(5), false); + yield return new TestCaseData(es, Resolution.Hour, TickType.Trade, TimeSpan.FromDays(2), false); + yield return new TestCaseData(es, Resolution.Minute, TickType.Trade, TimeSpan.FromHours(4), false); + yield return new TestCaseData(es, Resolution.Second, TickType.Trade, TimeSpan.FromHours(4), false); + yield return new TestCaseData(es, Resolution.Tick, TickType.Quote, TimeSpan.FromMinutes(15), true); } + } - internal static IEnumerable TestParameters - { - get - { - var es = CreateEsFuture(); - - yield return new TestCaseData(es, Resolution.Daily, TickType.Trade, TimeSpan.FromDays(5), false) - .SetDescription("ES futures daily trade history") - .SetCategory("Valid"); + [Test, TestCaseSource(nameof(TestParameters))] + public void GetsHistory(Symbol symbol, Resolution resolution, TickType tickType, TimeSpan period, bool expectsNoData) + { + var request = GetHistoryRequest(resolution, tickType, symbol, period); - yield return new TestCaseData(es, Resolution.Hour, TickType.Trade, TimeSpan.FromDays(2), false) - .SetDescription("ES futures hourly trade history") - .SetCategory("Valid"); + var history = _historyDataProvider.GetHistory(request); - yield return new TestCaseData(es, Resolution.Minute, TickType.Trade, TimeSpan.FromHours(4), false) - .SetDescription("ES futures minute trade history") - .SetCategory("Valid"); + Assert.IsNotNull(history); - yield return new TestCaseData(es, Resolution.Tick, TickType.Quote, TimeSpan.FromMinutes(15), false) - .SetDescription("ES futures quote ticks") - .SetCategory("Quote"); - } - } - - [Test, TestCaseSource(nameof(TestParameters))] - public void GetsHistory(Symbol symbol, Resolution resolution, TickType tickType, TimeSpan period, bool expectsNoData) + foreach (var point in history) { - var request = GetHistoryRequest(resolution, tickType, symbol, period); + Assert.AreEqual(symbol, point.Symbol); - var history = _historyDataProvider.GetHistory(request); - - if (expectsNoData) + if (point is TradeBar bar) { - Assert.IsTrue(history == null || !history.Any(), - $"Expected no data for unsupported symbol: {symbol}"); - return; + Assert.Greater(bar.Close, 0); + Assert.GreaterOrEqual(bar.Volume, 0); } - Assert.IsNotNull(history); - var data = history.ToList(); - Assert.IsNotEmpty(data); - - Log.Trace($"Received {data.Count} data points for {symbol} @ {resolution}"); - - foreach (var point in data.Take(5)) + if (point is Tick tick && tickType == TickType.Quote) { - Assert.AreEqual(symbol, point.Symbol); - - if (point is TradeBar bar) - { - Assert.Greater(bar.Close, 0); - Assert.GreaterOrEqual(bar.Volume, 0); - } - - if (point is Tick tick && tickType == TickType.Quote) - { - Assert.IsTrue(tick.BidPrice > 0 || tick.AskPrice > 0); - } + Assert.IsTrue(tick.BidPrice > 0 || tick.AskPrice > 0); } } + } - [Test] - public void GetHistoryWithMultipleSymbols() - { - var es = CreateEsFuture(); - - var request = GetHistoryRequest(Resolution.Daily, TickType.Trade, es, TimeSpan.FromDays(3)); - - var history = _historyDataProvider.GetHistory(request)?.ToList(); - - Assert.IsTrue( - history != null && history.Any(), - "Expected history for ES" - ); - } - - internal static HistoryRequest GetHistoryRequest( - Resolution resolution, - TickType tickType, - Symbol symbol, - TimeSpan period) - { - var endUtc = new DateTime(2024, 5, 10, 0, 0, 0, DateTimeKind.Utc); - var startUtc = endUtc - period; - - var dataType = LeanData.GetDataType(resolution, tickType); - var marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); - - var exchangeHours = marketHoursDatabase.GetExchangeHours( - symbol.ID.Market, symbol, symbol.SecurityType); - - var dataTimeZone = marketHoursDatabase.GetDataTimeZone( - symbol.ID.Market, symbol, symbol.SecurityType); - - return new HistoryRequest( - startTimeUtc: startUtc, - endTimeUtc: endUtc, - dataType: dataType, - symbol: symbol, - resolution: resolution, - exchangeHours: exchangeHours, - dataTimeZone: dataTimeZone, - fillForwardResolution: resolution, - includeExtendedMarketHours: true, - isCustomData: false, - DataNormalizationMode.Raw, - tickType: tickType - ); - } + private static HistoryRequest GetHistoryRequest( + Resolution resolution, + TickType tickType, + Symbol symbol, + TimeSpan period) + { + var endUtc = new DateTime(2026, 1, 22); + var startUtc = endUtc - period; + + var dataType = LeanData.GetDataType(resolution, tickType); + var marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); + + var exchangeHours = marketHoursDatabase.GetExchangeHours( + symbol.ID.Market, symbol, symbol.SecurityType); + + var dataTimeZone = marketHoursDatabase.GetDataTimeZone( + symbol.ID.Market, symbol, symbol.SecurityType); + + return new HistoryRequest( + startTimeUtc: startUtc, + endTimeUtc: endUtc, + dataType: dataType, + symbol: symbol, + resolution: resolution, + exchangeHours: exchangeHours, + dataTimeZone: dataTimeZone, + fillForwardResolution: resolution, + includeExtendedMarketHours: true, + isCustomData: false, + DataNormalizationMode.Raw, + tickType: tickType + ); } } diff --git a/QuantConnect.DataBento.Tests/DataBentoHistoricalApiClientTests.cs b/QuantConnect.DataBento.Tests/DataBentoHistoricalApiClientTests.cs new file mode 100644 index 0000000..7edac6d --- /dev/null +++ b/QuantConnect.DataBento.Tests/DataBentoHistoricalApiClientTests.cs @@ -0,0 +1,95 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System; +using NUnit.Framework; +using QuantConnect.Logging; +using QuantConnect.Configuration; +using QuantConnect.Lean.DataSource.DataBento.Api; + +namespace QuantConnect.Lean.DataSource.DataBento.Tests; + +[TestFixture] +public class DataBentoHistoricalApiClientTests +{ + private HistoricalAPIClient _client; + + [OneTimeSetUp] + public void OneTimeSetUp() + { + var apiKey = Config.Get("databento-api-key"); + if (string.IsNullOrEmpty(apiKey)) + { + Assert.Inconclusive("Please set the 'databento-api-key' in your configuration to enable these tests."); + } + + _client = new HistoricalAPIClient(apiKey); + } + + [TestCase("ESH6", "2025/01/11", "2026/01/20", Resolution.Daily)] + [TestCase("ESH6", "2025/01/11", "2026/01/20", Resolution.Hour)] + [TestCase("ESH6", "2025/01/11", "2026/01/20", Resolution.Minute)] + [TestCase("ESH6", "2025/01/11", "2026/01/20", Resolution.Second)] + //[TestCase("ESH6", "2025/01/11", "2026/01/20", Resolution.Tick)] + [TestCase("ESH6 C6875", "2026/01/11", "2026/01/20", Resolution.Daily)] + public void CanInitializeHistoricalApiClient(string ticker, DateTime startDate, DateTime endDate, Resolution resolution) + { + var dataCounter = 0; + var previousEndTime = DateTime.MinValue; + foreach (var data in _client.GetHistoricalOhlcvBars(ticker, startDate, endDate, resolution, TickType.Trade)) + { + Assert.IsNotNull(data); + + Assert.Greater(data.Open, 0m); + Assert.Greater(data.High, 0m); + Assert.Greater(data.Low, 0m); + Assert.Greater(data.Close, 0m); + Assert.Greater(data.Volume, 0m); + Assert.AreNotEqual(default(DateTime), data.Header.UtcTime); + + Assert.IsTrue(data.Header.UtcTime > previousEndTime, + $"Bar at {data.Header.UtcTime:o} is not after previous bar at {previousEndTime:o}"); + previousEndTime = data.Header.UtcTime; + + dataCounter++; + } + + Log.Trace($"{nameof(CanInitializeHistoricalApiClient)}: {ticker} | [{startDate} - {endDate}] | {resolution} = {dataCounter} (bars)"); + Assert.Greater(dataCounter, 0); + } + + [TestCase("ESH6 C6875", "2026/01/11", "2026/01/20", Resolution.Daily)] + public void ShouldFetchOpenInterest(string ticker, DateTime startDate, DateTime endDate, Resolution resolution) + { + var dataCounter = 0; + var previousEndTime = DateTime.MinValue; + foreach (var data in _client.GetOpenInterest(ticker, startDate, endDate)) + { + Assert.IsNotNull(data); + + Assert.Greater(data.Quantity, 0m); + Assert.AreNotEqual(default(DateTime), data.Header.UtcTime); + + Assert.IsTrue(data.Header.UtcTime > previousEndTime, + $"Bar at {data.Header.UtcTime:o} is not after previous bar at {previousEndTime:o}"); + previousEndTime = data.Header.UtcTime; + + dataCounter++; + } + + Log.Trace($"{nameof(CanInitializeHistoricalApiClient)}: {ticker} | [{startDate} - {endDate}] | {resolution} = {dataCounter} (bars)"); + Assert.Greater(dataCounter, 0); + } +} diff --git a/QuantConnect.DataBento.Tests/DataBentoJsonConverterTests.cs b/QuantConnect.DataBento.Tests/DataBentoJsonConverterTests.cs new file mode 100644 index 0000000..74d1126 --- /dev/null +++ b/QuantConnect.DataBento.Tests/DataBentoJsonConverterTests.cs @@ -0,0 +1,135 @@ +using Newtonsoft.Json; +using NUnit.Framework; +using QuantConnect.Lean.DataSource.DataBento.Models; +using QuantConnect.Lean.DataSource.DataBento.Models.Enums; + +namespace QuantConnect.Lean.DataSource.DataBento.Tests; + +[TestFixture] +public class DataBentoJsonConverterTests +{ + [Test] + public void DeserializeHistoricalOhlcvBar() + { + var json = @"{ + ""hd"": { + ""ts_event"": ""1738281600000000000"", + ""rtype"": 35, + ""publisher_id"": 1, + ""instrument_id"": 42140878 + }, + ""open"": ""6359.000000000"", + ""high"": ""6359.000000000"", + ""low"": ""6355.000000000"", + ""close"": ""6355.000000000"", + ""volume"": ""2"" +}"; + var res = json.DeserializeKebabCase(); + + Assert.IsNotNull(res); + + Assert.AreEqual(1738281600000000000m, res.Header.TsEvent); + Assert.AreEqual(35, res.Header.Rtype); + Assert.AreEqual(1, res.Header.PublisherId); + Assert.AreEqual(42140878, res.Header.InstrumentId); + + Assert.AreEqual(6359m, res.Open); + Assert.AreEqual(6359m, res.High); + Assert.AreEqual(6355m, res.Low); + Assert.AreEqual(6355m, res.Close); + Assert.AreEqual(2L, res.Volume); + } + + [Test] + public void DeserializeHistoricalLevelOneData() + { + var json = @"{ + ""ts_recv"": ""1768137063449660443"", + ""hd"": { + ""ts_event"": ""1768137063107829777"", + ""rtype"": 1, + ""publisher_id"": 1, + ""instrument_id"": 42140878 + }, + ""action"": ""A"", + ""side"": ""N"", + ""depth"": 0, + ""price"": ""7004.250000000"", + ""size"": 15, + ""flags"": 128, + ""ts_in_delta"": 17537, + ""sequence"": 811, + ""levels"": [ + { + ""bid_px"": ""7004.000000000"", + ""ask_px"": ""7004.250000000"", + ""bid_sz"": 11, + ""ask_sz"": 15, + ""bid_ct"": 1, + ""ask_ct"": 1 + } + ] +}"; + var res = json.DeserializeKebabCase(); + + Assert.IsNotNull(res); + + Assert.AreEqual(1768137063449660443, res.TsRecv); + + Assert.AreEqual(1768137063107829777, res.Header.TsEvent); + Assert.AreEqual(1, res.Header.Rtype); + Assert.AreEqual(1, res.Header.PublisherId); + Assert.AreEqual(42140878, res.Header.InstrumentId); + + Assert.AreEqual('A', res.Action); + Assert.AreEqual('N', res.Side); + Assert.AreEqual(0, res.Depth); + Assert.AreEqual(7004.25m, res.Price); + Assert.AreEqual(15, res.Size); + Assert.AreEqual(128, res.Flags); + Assert.IsNotNull(res.Levels); + Assert.AreEqual(1, res.Levels.Count); + var level = res.Levels[0]; + Assert.AreEqual(7004.0m, level.BidPx); + Assert.AreEqual(7004.25m, level.AskPx); + Assert.AreEqual(11, level.BidSz); + Assert.AreEqual(15, level.AskSz); + Assert.AreEqual(1, level.BidCt); + Assert.AreEqual(1, level.AskCt); + } + + [Test] + public void DeserializeHistoricalStatisticsData() + { + var json = @"{ + ""ts_recv"": ""1768156232522711477"", + ""hd"": { + ""ts_event"": ""1768156232522476283"", + ""rtype"": 24, + ""publisher_id"": 1, + ""instrument_id"": 42566722 + }, + ""ts_ref"": ""1767916800000000000"", + ""price"": null, + ""quantity"": 470, + ""sequence"": 29232, + ""ts_in_delta"": 12477, + ""stat_type"": 9, + ""channel_id"": 1, + ""update_action"": 1, + ""stat_flags"": 0 +}"; + + var res = json.DeserializeKebabCase(); + + Assert.IsNotNull(res); + + + Assert.AreEqual(1768156232522476283, res.Header.TsEvent); + Assert.AreEqual(24, res.Header.Rtype); + Assert.AreEqual(1, res.Header.PublisherId); + Assert.AreEqual(42566722, res.Header.InstrumentId); + Assert.AreEqual(470m, res.Quantity); + Assert.AreEqual(StatisticType.OpenInterest, res.StatType); + } +} diff --git a/QuantConnect.DataBento.Tests/DataBentoRawLiveClientTests.cs b/QuantConnect.DataBento.Tests/DataBentoRawLiveClientTests.cs index 22e7f04..10536a3 100644 --- a/QuantConnect.DataBento.Tests/DataBentoRawLiveClientTests.cs +++ b/QuantConnect.DataBento.Tests/DataBentoRawLiveClientTests.cs @@ -1,6 +1,6 @@ /* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/QuantConnect.DataBento.Tests/DataBentoSymbolMapperTests.cs.cs b/QuantConnect.DataBento.Tests/DataBentoSymbolMapperTests.cs.cs new file mode 100644 index 0000000..471519a --- /dev/null +++ b/QuantConnect.DataBento.Tests/DataBentoSymbolMapperTests.cs.cs @@ -0,0 +1,56 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +using NUnit.Framework; +using System; +using System.Collections.Generic; + +namespace QuantConnect.Lean.DataSource.DataBento.Tests; + +[TestFixture] +public class DataBentoSymbolMapperTests +{ + /// + /// Provides the mapping between Lean symbols and brokerage specific symbols. + /// + private DataBentoSymbolMapper _symbolMapper; + + [OneTimeSetUp] + public void OneTimeSetUp() + { + _symbolMapper = new DataBentoSymbolMapper(); + } + private static IEnumerable LeanSymbolTestCases + { + get + { + // TSLA - Equity + var es = Symbol.CreateFuture(Securities.Futures.Indices.SP500EMini, Market.CME, new DateTime(2026, 3, 20)); + yield return new TestCaseData(es, "ESH6"); + + } + } + + [Test, TestCaseSource(nameof(LeanSymbolTestCases))] + public void ReturnsCorrectBrokerageSymbol(Symbol symbol, string expectedBrokerageSymbol) + { + var brokerageSymbol = _symbolMapper.GetBrokerageSymbol(symbol); + + Assert.IsNotNull(brokerageSymbol); + Assert.IsNotEmpty(brokerageSymbol); + Assert.AreEqual(expectedBrokerageSymbol, brokerageSymbol); + } +} diff --git a/QuantConnect.DataBento.Tests/TestSetup.cs b/QuantConnect.DataBento.Tests/TestSetup.cs index 39cb9de..023eae0 100644 --- a/QuantConnect.DataBento.Tests/TestSetup.cs +++ b/QuantConnect.DataBento.Tests/TestSetup.cs @@ -1,6 +1,6 @@ /* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/QuantConnect.DataBento/Api/HistoricalAPIClient.cs b/QuantConnect.DataBento/Api/HistoricalAPIClient.cs new file mode 100644 index 0000000..92dc868 --- /dev/null +++ b/QuantConnect.DataBento/Api/HistoricalAPIClient.cs @@ -0,0 +1,168 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System.Net; +using System.Text; +using QuantConnect.Util; +using QuantConnect.Logging; +using System.Net.Http.Headers; +using QuantConnect.Lean.DataSource.DataBento.Models; + +namespace QuantConnect.Lean.DataSource.DataBento.Api; + +public class HistoricalAPIClient : IDisposable +{ + //private const string + + /// + /// Dataset for CME Globex futures + /// https://databento.com/docs/venues-and-datasets has more information on datasets through DataBento + /// + /// + /// TODO: Hard coded for now. Later on can add equities and options with different mapping + /// + private const string Dataset = "GLBX.MDP3"; + + private readonly HttpClient _httpClient = new() + { + BaseAddress = new Uri("https://hist.databento.com") + }; + + public HistoricalAPIClient(string apiKey) + { + _httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue( + AuthenticationSchemes.Basic.ToString(), + // Basic Auth expects "username:password". Using ":" means API key with an empty password. + Convert.ToBase64String(Encoding.UTF8.GetBytes($"{apiKey}:")) + ); + } + + public IEnumerable GetHistoricalOhlcvBars(string symbol, DateTime startDateTimeUtc, DateTime endDateTimeUtc, Resolution resolution, TickType tickType) + { + string schema; + switch (resolution) + { + case Resolution.Second: + schema = "ohlcv-1s"; + break; + case Resolution.Minute: + schema = "ohlcv-1m"; + break; + case Resolution.Hour: + schema = "ohlcv-1h"; + break; + case Resolution.Daily: + schema = "ohlcv-1d"; + break; + default: + throw new ArgumentException($"Unsupported resolution {resolution} for OHLCV data."); + } + + return GetRange(symbol, startDateTimeUtc, endDateTimeUtc, schema); + } + + public IEnumerable GetTickBars(string symbol, DateTime startDateTimeUtc, DateTime endDateTimeUtc) + { + return GetRange(symbol, startDateTimeUtc, endDateTimeUtc, "mbp-1", useLimit: true); + } + + public IEnumerable GetOpenInterest(string symbol, DateTime startDateTimeUtc, DateTime endDateTimeUtc) + { + foreach (var statistics in GetRange(symbol, startDateTimeUtc, endDateTimeUtc, "statistics")) + { + if (statistics.StatType == Models.Enums.StatisticType.OpenInterest) + { + yield return statistics; + } + } + } + + private IEnumerable GetRange(string symbol, DateTime startDateTimeUtc, DateTime endDateTimeUtc, string schema, bool useLimit = false) where T : MarketDataRecord + { + var formData = new Dictionary + { + { "dataset", Dataset }, + { "end", Time.DateTimeToUnixTimeStampNanoseconds(endDateTimeUtc).ToStringInvariant() }, + { "symbols", symbol }, + { "schema", schema }, + { "encoding", "json" }, + { "stype_in", "raw_symbol" }, + { "pretty_px", "true" }, + }; + + if (useLimit) + { + formData["limit"] = "10000"; + } + + var start = startDateTimeUtc; + var httpStatusCode = default(HttpStatusCode); + do + { + formData["start"] = Time.DateTimeToUnixTimeStampNanoseconds(start).ToStringInvariant(); + + using var content = new FormUrlEncodedContent(formData); + + using var requestMessage = new HttpRequestMessage(HttpMethod.Post, "/v0/timeseries.get_range") + { + Content = content + }; + + using var response = _httpClient.Send(requestMessage); + + if (response.Headers.TryGetValues("x-warning", out var warnings)) + { + foreach (var warning in warnings) + { + Log.Trace($"{nameof(HistoricalAPIClient)}.{nameof(GetRange)}: {warning}"); + } + } + + using var stream = response.Content.ReadAsStream(); + + if (stream.Length == 0) + { + continue; + } + + using var reader = new StreamReader(stream); + + var line = default(string); + if (response.StatusCode == HttpStatusCode.UnprocessableContent) + { + line = reader.ReadLine(); + Log.Trace($"{nameof(HistoricalAPIClient)}.{nameof(GetRange)}.Response: {line}. " + + $"Request: [{response.RequestMessage?.Method}]({response.RequestMessage?.RequestUri}), " + + $"Payload: {string.Join(", ", formData.Select(kvp => $"{kvp.Key}: {kvp.Value}"))}"); + yield break; + } + + httpStatusCode = response.EnsureSuccessStatusCode().StatusCode; + + var data = default(T); + while ((line = reader.ReadLine()) != null) + { + data = line.DeserializeKebabCase(); + yield return data; + } + start = data.Header.UtcTime.AddTicks(1); + } while (httpStatusCode == HttpStatusCode.PartialContent); + } + + public void Dispose() + { + _httpClient?.DisposeSafely(); + } +} diff --git a/QuantConnect.DataBento/DataBentoDataDownloader.cs b/QuantConnect.DataBento/DataBentoDataDownloader.cs index 841d070..9d78ce2 100644 --- a/QuantConnect.DataBento/DataBentoDataDownloader.cs +++ b/QuantConnect.DataBento/DataBentoDataDownloader.cs @@ -1,6 +1,6 @@ /* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,222 +14,84 @@ * */ -using System; -using System.Collections.Generic; -using System.Globalization; -using NodaTime; -using System.IO; -using System.Linq; -using System.Net.Http; -using System.Text; -using CsvHelper; -using QuantConnect.Configuration; using QuantConnect.Data; -using QuantConnect.Data.Market; -using QuantConnect.Lean.DataSource.DataBento.Models; using QuantConnect.Util; using QuantConnect.Securities; +using QuantConnect.Configuration; + +namespace QuantConnect.Lean.DataSource.DataBento; -namespace QuantConnect.Lean.DataSource.DataBento +/// +/// Data downloader class for pulling data from DataBento +/// +public class DataBentoDataDownloader : IDataDownloader, IDisposable { /// - /// Data downloader for historical data from DataBento's Raw HTTP API - /// Converts DataBento data to Lean data types + /// Provides access to historical market data via the DataBento service. /// - public class DataBentoDataDownloader : IDataDownloader, IDisposable - { - private readonly HttpClient _httpClient = new(); - private readonly string _apiKey; - private readonly DataBentoSymbolMapper _symbolMapper; - private readonly MarketHoursDatabase _marketHoursDatabase; - private readonly Dictionary _symbolExchangeTimeZones = new(); - - /// - /// Initializes a new instance of the - /// - /// The DataBento API key. - public DataBentoDataDownloader(string apiKey, MarketHoursDatabase marketHoursDatabase) - { - _marketHoursDatabase = marketHoursDatabase; - _apiKey = apiKey; - _httpClient.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.UTF8.GetBytes($"{_apiKey}:"))); - _symbolMapper = new DataBentoSymbolMapper(); - } - - /// - /// Get historical data enumerable for a single symbol, type and resolution given this start and end time (in UTC). - /// - /// Parameters for the historical data request - /// Enumerable of base data for this symbol - /// - public IEnumerable Get(DataDownloaderGetParameters parameters) - { - var symbol = parameters.Symbol; - var resolution = parameters.Resolution; - var tickType = parameters.TickType; - - /// - /// Dataset for CME Globex futures - /// https://databento.com/docs/venues-and-datasets has more information on datasets through DataBento - /// - const string dataset = "GLBX.MDP3"; // hard coded for now. Later on can add equities and options with different mapping - var schema = GetSchema(resolution, tickType); - var databentoSymbol = _symbolMapper.GetBrokerageSymbol(symbol); + private readonly DataBentoProvider _historyProvider; - // prepare body for Raw HTTP request - var body = new StringBuilder() - .Append($"dataset={dataset}") - .Append($"&symbols={databentoSymbol}") - .Append($"&schema={schema}") - .Append($"&start={parameters.StartUtc:yyyy-MM-ddTHH:mm}") - .Append($"&end={parameters.EndUtc:yyyy-MM-ddTHH:mm}") - .Append("&stype_in=parent") - .Append("&encoding=csv") - .ToString(); - - using var request = new HttpRequestMessage(HttpMethod.Post, - "https://hist.databento.com/v0/timeseries.get_range") - { - Content = new StringContent(body, Encoding.UTF8, "application/x-www-form-urlencoded") - }; - - // send the request with the get range url - var response = _httpClient.Send(request); - - // Add error handling to see the actual error message - if (!response.IsSuccessStatusCode) - { - var errorContent = response.Content.ReadAsStringAsync().SynchronouslyAwaitTaskResult(); - throw new HttpRequestException($"DataBento API error ({response.StatusCode}): {errorContent}"); - } + /// + /// Provides exchange trading hours and market-specific time zone information. + /// + private readonly MarketHoursDatabase _marketHoursDatabase; - using var csv = new CsvReader( - new StreamReader(response.Content.ReadAsStream()), - CultureInfo.InvariantCulture - ); + /// + /// Initializes a new instance of the + /// getting the DataBento API key from the configuration + /// + public DataBentoDataDownloader() + : this(Config.Get("databento-api-key")) + { - return (tickType, resolution) switch - { - (TickType.Trade, Resolution.Tick) => - csv.ForEach(dt => - new Tick( - GetTickTime(symbol, dt.Timestamp), - symbol, - string.Empty, - string.Empty, - dt.Size, - dt.Price - ) - ), + } - (TickType.Trade, _) => - csv.ForEach(bar => - new TradeBar( - GetTickTime(symbol, bar.Timestamp), - symbol, - bar.Open, - bar.High, - bar.Low, - bar.Close, - bar.Volume - ) - ), + /// + /// Initializes a new instance of the + /// + /// The DataBento API key. + public DataBentoDataDownloader(string apiKey) + { + _historyProvider = new DataBentoProvider(apiKey); + _marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); + } - (TickType.Quote, Resolution.Tick) => - csv.ForEach(q => - new Tick( - GetTickTime(symbol, q.Timestamp), - symbol, - bidPrice: q.BidPrice, - askPrice: q.AskPrice, - bidSize: q.BidSize, - askSize: q.AskSize - ) - { - TickType = TickType.Quote - } - ), + /// + /// Get historical data enumerable for a single symbol, type and resolution given this start and end time (in UTC). + /// + /// Parameters for the historical data request + /// Enumerable of base data for this symbol + public IEnumerable? Get(DataDownloaderGetParameters parameters) + { + var symbol = parameters.Symbol; + var resolution = parameters.Resolution; + var startUtc = parameters.StartUtc; + var endUtc = parameters.EndUtc; + var tickType = parameters.TickType; - (TickType.Quote, _) => - csv.ForEach(q => - new QuoteBar( - GetTickTime(symbol, q.Timestamp), - symbol, - new Bar(q.BidPrice, q.BidPrice, q.BidPrice, q.BidPrice), q.BidSize, - new Bar(q.AskPrice, q.AskPrice, q.AskPrice, q.AskPrice), q.AskSize - ) - ), + var dataType = LeanData.GetDataType(resolution, tickType); + var exchangeHours = _marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType); + var dataTimeZone = _marketHoursDatabase.GetDataTimeZone(symbol.ID.Market, symbol, symbol.SecurityType); - _ => throw new NotSupportedException( - $"Unsupported tickType={tickType} resolution={resolution}") - }; - } + var historyRequest = new HistoryRequest(startUtc, endUtc, dataType, symbol, resolution, exchangeHours, dataTimeZone, resolution, + true, false, DataNormalizationMode.Raw, tickType); - /// - /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources. - /// - public void Dispose() - { - _httpClient?.DisposeSafely(); - } + var historyData = _historyProvider.GetHistory(historyRequest); - /// - /// Pick Databento schema from Lean resolution/ticktype - /// - private static string GetSchema(Resolution resolution, TickType tickType) + if (historyData == null) { - return (tickType, resolution) switch - { - (TickType.Trade, Resolution.Tick) => "mbp-1", - (TickType.Trade, Resolution.Second) => "ohlcv-1s", - (TickType.Trade, Resolution.Minute) => "ohlcv-1m", - (TickType.Trade, Resolution.Hour) => "ohlcv-1h", - (TickType.Trade, Resolution.Daily) => "ohlcv-1d", - - (TickType.Quote, _) => "mbp-1", - - _ => throw new NotSupportedException( - $"Unsupported resolution {resolution} / {tickType}" - ) - }; + return null; } - /// - /// Converts the given UTC time into the symbol security exchange time zone - /// - private DateTime GetTickTime(Symbol symbol, DateTime utcTime) - { - DateTimeZone exchangeTimeZone; - lock (_symbolExchangeTimeZones) - { - if (!_symbolExchangeTimeZones.TryGetValue(symbol, out exchangeTimeZone)) - { - // read the exchange time zone from market-hours-database - if (_marketHoursDatabase.TryGetEntry(symbol.ID.Market, symbol, symbol.SecurityType, out var entry)) - { - exchangeTimeZone = entry.ExchangeHours.TimeZone; - } - // If there is no entry for the given Symbol, default to New York - else - { - exchangeTimeZone = TimeZones.NewYork; - } - - _symbolExchangeTimeZones.Add(symbol, exchangeTimeZone); - } - } - - return utcTime.ConvertFromUtc(exchangeTimeZone); - } + return historyData; } -} -public static class CsvReaderExtensions -{ - public static IEnumerable ForEach( - this CsvReader csv, - Func map) + + /// + /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources. + /// + public void Dispose() { - return csv.GetRecords().Select(map).ToList(); + _historyProvider.DisposeSafely(); } -} +} \ No newline at end of file diff --git a/QuantConnect.DataBento/DataBentoDataProvider.cs b/QuantConnect.DataBento/DataBentoDataProvider.cs index aa9dd32..c8d4eed 100644 --- a/QuantConnect.DataBento/DataBentoDataProvider.cs +++ b/QuantConnect.DataBento/DataBentoDataProvider.cs @@ -1,6 +1,6 @@ /* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,6 +24,7 @@ using QuantConnect.Packets; using QuantConnect.Securities; using System.Collections.Concurrent; +using QuantConnect.Lean.DataSource.DataBento.Api; namespace QuantConnect.Lean.DataSource.DataBento { @@ -34,18 +35,25 @@ namespace QuantConnect.Lean.DataSource.DataBento /// public partial class DataBentoProvider : IDataQueueHandler { + /// + /// Resolves map files to correctly handle current and historical ticker symbols. + /// + private readonly IMapFileProvider _mapFileProvider = Composer.Instance.GetPart(); + + private HistoricalAPIClient _historicalApiClient; + + private readonly DataBentoSymbolMapper _symbolMapper = new DataBentoSymbolMapper(); + private readonly IDataAggregator _dataAggregator = Composer.Instance.GetExportedValueByTypeName( Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager"), forceTypeNameOnExisting: false); private EventBasedDataQueueHandlerSubscriptionManager _subscriptionManager; private DataBentoRawLiveClient _client; - private readonly DataBentoDataDownloader _dataDownloader; private bool _potentialUnsupportedResolutionMessageLogged; private bool _sessionStarted = false; private readonly object _sessionLock = new(); private readonly MarketHoursDatabase _marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); private readonly ConcurrentDictionary _symbolExchangeTimeZones = new(); private bool _initialized; - private bool _unsupportedTickTypeMessagedLogged; /// /// Returns true if we're currently connected to the Data Provider @@ -56,9 +64,19 @@ public partial class DataBentoProvider : IDataQueueHandler /// Initializes a new instance of the DataBentoProvider /// public DataBentoProvider() + : this(Config.Get("databento-api-key")) { - var apiKey = Config.Get("databento-api-key"); - _dataDownloader = new DataBentoDataDownloader(apiKey, _marketHoursDatabase); + } + + public DataBentoProvider(string apiKey) + { + if (string.IsNullOrWhiteSpace(apiKey)) + { + // If the API key is not provided, we can't do anything. + // The handler might going to be initialized using a node packet job. + return; + } + Initialize(apiKey); } @@ -112,6 +130,8 @@ private void Initialize(string apiKey) cancellationTokenSource.Token, TaskCreationOptions.LongRunning, TaskScheduler.Default); + + _historicalApiClient = new(apiKey); _initialized = true; Log.Debug("DataBentoProvider.Initialize(): Initialization complete"); @@ -166,11 +186,11 @@ public bool SubscriptionLogic(IEnumerable symbols, TickType tickType) /// /// The symbol /// returns true if Data Provider supports the specified symbol; otherwise false - private bool CanSubscribe(Symbol symbol) + private static bool CanSubscribe(Symbol symbol) { return !symbol.Value.Contains("universe", StringComparison.InvariantCultureIgnoreCase) && !symbol.IsCanonical() && - IsSecurityTypeSupported(symbol.SecurityType); + symbol.SecurityType == SecurityType.Future; } /// @@ -232,18 +252,6 @@ public void Dispose() _dataAggregator?.DisposeSafely(); _subscriptionManager?.DisposeSafely(); _client?.DisposeSafely(); - _dataDownloader?.DisposeSafely(); - } - - /// - /// Checks if the security type is supported - /// - /// Security type to check - /// True if supported - private bool IsSecurityTypeSupported(SecurityType securityType) - { - // DataBento primarily supports futures, but also has equity and option coverage - return securityType == SecurityType.Future; } /// diff --git a/QuantConnect.DataBento/DataBentoHistoryProivder.cs b/QuantConnect.DataBento/DataBentoHistoryProivder.cs index eb23154..b92744e 100644 --- a/QuantConnect.DataBento/DataBentoHistoryProivder.cs +++ b/QuantConnect.DataBento/DataBentoHistoryProivder.cs @@ -1,6 +1,6 @@ /* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,30 +16,34 @@ using NodaTime; using QuantConnect.Data; -using QuantConnect.Data.Market; -using QuantConnect.Lean.Engine.DataFeeds; -using QuantConnect.Lean.Engine.HistoricalData; -using QuantConnect.Logging; using QuantConnect.Util; -using QuantConnect.Interfaces; +using QuantConnect.Logging; using QuantConnect.Securities; +using QuantConnect.Data.Market; using QuantConnect.Data.Consolidators; +using QuantConnect.Lean.Engine.DataFeeds; +using QuantConnect.Lean.Engine.HistoricalData; namespace QuantConnect.Lean.DataSource.DataBento { /// - /// Impleements a history provider for DataBento historical data. + /// Implements a history provider for DataBento historical data. /// Uses consolidators to produce the requested resolution when necessary. /// public partial class DataBentoProvider : SynchronizingHistoryProvider { - private int _dataPointCount; + private static int _dataPointCount; /// /// Indicates whether a error for an invalid start time has been fired, where the start time is greater than or equal to the end time in UTC. /// private volatile bool _invalidStartTimeErrorFired; + /// + /// Indicates whether the warning for invalid has been fired. + /// + private volatile bool _invalidSecurityTypeWarningFired; + /// /// Gets the total number of data points emitted by this history provider /// @@ -64,11 +68,7 @@ public override void Initialize(HistoryProviderInitializeParameters parameters) var subscriptions = new List(); foreach (var request in requests) { - var history = GetHistory(request); - if (history == null) - { - continue; - } + var history = request.SplitHistoryRequestWithUpdatedMappedSymbol(_mapFileProvider).SelectMany(x => GetHistory(x) ?? []); var subscription = CreateSubscription(request, history); if (!subscription.MoveNext()) @@ -89,54 +89,92 @@ public override void Initialize(HistoryProviderInitializeParameters parameters) /// /// Gets the history for the requested security /// - /// The historical data request + /// The historical data request /// An enumerable of BaseData points - public IEnumerable? GetHistory(HistoryRequest request) + public IEnumerable? GetHistory(HistoryRequest historyRequest) { - if (!CanSubscribe(request.Symbol)) + if (!CanSubscribe(historyRequest.Symbol)) { - // It is Logged in IsSupported(...) - return null; - } - - if (request.TickType == TickType.OpenInterest) - { - if (!_unsupportedTickTypeMessagedLogged) + if (!_invalidSecurityTypeWarningFired) { - _unsupportedTickTypeMessagedLogged = true; - Log.Trace($"DataBentoProvider.GetHistory(): Unsupported tick type: {TickType.OpenInterest}"); + _invalidSecurityTypeWarningFired = true; + LogTrace(nameof(GetHistory), $"Unsupported SecurityType '{historyRequest.Symbol.SecurityType}' for symbol '{historyRequest.Symbol}'."); } return null; } - if (request.EndTimeUtc < request.StartTimeUtc) + if (historyRequest.EndTimeUtc < historyRequest.StartTimeUtc) { if (!_invalidStartTimeErrorFired) { _invalidStartTimeErrorFired = true; - Log.Error($"{nameof(DataBentoProvider)}.{nameof(GetHistory)}:InvalidDateRange. The history request start date must precede the end date, no history returned"); + Log.Error($"{nameof(DataBentoProvider)}.{nameof(GetHistory)}: Invalid date range: the start date must be earlier than the end date."); } return null; } + var history = default(IEnumerable); + var brokerageSymbol = _symbolMapper.GetBrokerageSymbol(historyRequest.Symbol); + switch (historyRequest.TickType) + { + case TickType.Trade when historyRequest.Resolution == Resolution.Tick: + history = GetHistoryThroughDataConsolidator(historyRequest, brokerageSymbol); + break; + case TickType.Trade: + history = GetAggregatedTradeBars(historyRequest, brokerageSymbol); + break; + case TickType.Quote: + history = GetHistoryThroughDataConsolidator(historyRequest, brokerageSymbol); + break; + case TickType.OpenInterest: + history = GetOpenInterestBars(historyRequest, brokerageSymbol); + break; + default: + throw new ArgumentException(""); + } - // Use the trade aggregates API for resolutions above tick for fastest results - if (request.TickType == TickType.Trade && request.Resolution > Resolution.Tick) + if (history == null) { - var data = GetAggregates(request); + return null; + } + + return FilterHistory(history, historyRequest, historyRequest.StartTimeLocal, historyRequest.EndTimeLocal); + } - if (data == null) + private static IEnumerable FilterHistory(IEnumerable history, HistoryRequest request, DateTime startTimeLocal, DateTime endTimeLocal) + { + // cleaning the data before returning it back to user + foreach (var bar in history) + { + if (bar.Time >= startTimeLocal && bar.EndTime <= endTimeLocal) { - return null; + if (request.ExchangeHours.IsOpen(bar.Time, bar.EndTime, request.IncludeExtendedMarketHours)) + { + Interlocked.Increment(ref _dataPointCount); + yield return bar; + } } + } + } - return data; + private IEnumerable GetOpenInterestBars(HistoryRequest request, string brokerageSymbol) + { + foreach (var oi in _historicalApiClient.GetOpenInterest(brokerageSymbol, request.StartTimeUtc, request.EndTimeUtc)) + { + yield return new OpenInterest(oi.Header.UtcTime.ConvertFromUtc(request.DataTimeZone), request.Symbol, oi.Quantity); } + } - return GetHistoryThroughDataConsolidator(request); + private IEnumerable GetAggregatedTradeBars(HistoryRequest request, string brokerageSymbol) + { + var period = request.Resolution.ToTimeSpan(); + foreach (var b in _historicalApiClient.GetHistoricalOhlcvBars(brokerageSymbol, request.StartTimeUtc, request.EndTimeUtc, request.Resolution, request.TickType)) + { + yield return new TradeBar(b.Header.UtcTime.ConvertFromUtc(request.DataTimeZone), request.Symbol, b.Open, b.High, b.Low, b.Close, b.Volume, period); + } } - private IEnumerable? GetHistoryThroughDataConsolidator(HistoryRequest request) + private IEnumerable? GetHistoryThroughDataConsolidator(HistoryRequest request, string brokerageSymbol) { IDataConsolidator consolidator; IEnumerable history; @@ -146,14 +184,14 @@ public override void Initialize(HistoryProviderInitializeParameters parameters) consolidator = request.Resolution != Resolution.Tick ? new TickConsolidator(request.Resolution.ToTimeSpan()) : FilteredIdentityDataConsolidator.ForTickType(request.TickType); - history = GetTrades(request); + history = GetTrades(request, brokerageSymbol); } else { consolidator = request.Resolution != Resolution.Tick ? new TickQuoteBarConsolidator(request.Resolution.ToTimeSpan()) : FilteredIdentityDataConsolidator.ForTickType(request.TickType); - history = GetQuotes(request); + history = GetQuotes(request, brokerageSymbol); } BaseData? consolidatedData = null; @@ -168,7 +206,6 @@ public override void Initialize(HistoryProviderInitializeParameters parameters) consolidator.Update(data); if (consolidatedData != null) { - Interlocked.Increment(ref _dataPointCount); yield return consolidatedData; consolidatedData = null; } @@ -179,47 +216,34 @@ public override void Initialize(HistoryProviderInitializeParameters parameters) } /// - /// Gets the trade bars for the specified history request + /// Gets the trade ticks that will potentially be aggregated for the specified history request /// - private IEnumerable GetAggregates(HistoryRequest request) + private IEnumerable GetTrades(HistoryRequest request, string brokerageSymbol) { - var resolutionTimeSpan = request.Resolution.ToTimeSpan(); - foreach (var date in Time.EachDay(request.StartTimeUtc, request.EndTimeUtc)) + foreach (var t in _historicalApiClient.GetTickBars(brokerageSymbol, request.StartTimeUtc, request.EndTimeUtc)) { - var start = date; - var end = date + Time.OneDay; - - var parameters = new DataDownloaderGetParameters(request.Symbol, request.Resolution, start, end, request.TickType); - var data = _dataDownloader.Get(parameters); - if (data == null) continue; - - foreach (var bar in data) - { - var tradeBar = (TradeBar)bar; - if (tradeBar.Time >= request.StartTimeUtc && tradeBar.EndTime <= request.EndTimeUtc) - { - yield return tradeBar; - } - } + yield return new Tick(t.Header.UtcTime.ConvertFromUtc(request.DataTimeZone), request.Symbol, "", "", t.Size, t.Price); } } /// - /// Gets the trade ticks that will potentially be aggregated for the specified history request + /// Gets the quote ticks that will potentially be aggregated for the specified history request /// - private IEnumerable GetTrades(HistoryRequest request) + private IEnumerable GetQuotes(HistoryRequest request, string brokerageSymbol) { - var parameters = new DataDownloaderGetParameters(request.Symbol, Resolution.Tick, request.StartTimeUtc, request.EndTimeUtc, request.TickType); - return _dataDownloader.Get(parameters); + foreach (var quoteBar in _historicalApiClient.GetTickBars(brokerageSymbol, request.StartTimeUtc, request.EndTimeUtc)) + { + var time = quoteBar.Header.UtcTime.ConvertFromUtc(request.DataTimeZone); + foreach (var level in quoteBar.Levels) + { + yield return new Tick(time, request.Symbol, level.BidSz, level.BidPx, level.AskSz, level.AskPx); + } + } } - /// - /// Gets the quote ticks that will potentially be aggregated for the specified history request - /// - private IEnumerable GetQuotes(HistoryRequest request) + private static void LogTrace(string methodName, string message) { - var parameters = new DataDownloaderGetParameters(request.Symbol, Resolution.Tick, request.StartTimeUtc, request.EndTimeUtc, request.TickType); - return _dataDownloader.Get(parameters); + Log.Trace($"{nameof(DataBentoProvider)}.{methodName}: {message}"); } } } diff --git a/QuantConnect.DataBento/DataBentoRawLiveClient.cs b/QuantConnect.DataBento/DataBentoRawLiveClient.cs index 2dab1d2..8d8277c 100644 --- a/QuantConnect.DataBento/DataBentoRawLiveClient.cs +++ b/QuantConnect.DataBento/DataBentoRawLiveClient.cs @@ -1,6 +1,6 @@ /* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/QuantConnect.DataBento/DataBentoSymbolMapper.cs b/QuantConnect.DataBento/DataBentoSymbolMapper.cs index 3c6b8b3..5c90304 100644 --- a/QuantConnect.DataBento/DataBentoSymbolMapper.cs +++ b/QuantConnect.DataBento/DataBentoSymbolMapper.cs @@ -1,6 +1,6 @@ /* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,9 +13,7 @@ * limitations under the License. */ -using QuantConnect; using QuantConnect.Brokerages; -using System.Globalization; namespace QuantConnect.Lean.DataSource.DataBento { @@ -24,9 +22,6 @@ namespace QuantConnect.Lean.DataSource.DataBento /// public class DataBentoSymbolMapper : ISymbolMapper { - private readonly Dictionary _leanSymbolsCache = new(); - private readonly Dictionary _brokerageSymbolsCache = new(); - private readonly object _locker = new(); /// /// Converts a Lean symbol instance to a brokerage symbol @@ -35,47 +30,12 @@ public class DataBentoSymbolMapper : ISymbolMapper /// The brokerage symbol public string GetBrokerageSymbol(Symbol symbol) { - if (symbol == null || string.IsNullOrWhiteSpace(symbol.Value)) + switch (symbol.SecurityType) { - throw new ArgumentException($"Invalid symbol: {(symbol == null ? "null" : symbol.ToString())}"); - } - - return GetBrokerageSymbol(symbol, false); - } - - /// - /// Converts a Lean symbol instance to a brokerage symbol with updating of cached symbol collection - /// - /// - /// - /// - /// - public string GetBrokerageSymbol(Symbol symbol, bool isUpdateCachedSymbol) - { - lock (_locker) - { - if (!_brokerageSymbolsCache.TryGetValue(symbol, out var brokerageSymbol) || isUpdateCachedSymbol) - { - switch (symbol.SecurityType) - { - case SecurityType.Future: - brokerageSymbol = $"{symbol.ID.Symbol}.FUT"; - break; - - case SecurityType.Equity: - brokerageSymbol = symbol.Value; - break; - - default: - throw new Exception($"DataBentoSymbolMapper.GetBrokerageSymbol(): unsupported security type: {symbol.SecurityType}"); - } - - // Lean-to-DataBento symbol conversion is accurate, so we can cache it both ways - _brokerageSymbolsCache[symbol] = brokerageSymbol; - _leanSymbolsCache[brokerageSymbol] = symbol; - } - - return brokerageSymbol; + case SecurityType.Future: + return SymbolRepresentation.GenerateFutureTicker(symbol.ID.Symbol, symbol.ID.Date, doubleDigitsYear: false, includeExpirationDate: false); + default: + throw new Exception($"The unsupported security type: {symbol.SecurityType}"); } } @@ -90,48 +50,12 @@ public string GetBrokerageSymbol(Symbol symbol, bool isUpdateCachedSymbol) public Symbol GetLeanSymbol(string brokerageSymbol, SecurityType securityType, string market, DateTime expirationDate = new DateTime(), decimal strike = 0, OptionRight optionRight = 0) { - if (string.IsNullOrWhiteSpace(brokerageSymbol)) - { - throw new ArgumentException("Invalid symbol: " + brokerageSymbol); - } - - lock (_locker) + switch (securityType) { - if (!_leanSymbolsCache.TryGetValue(brokerageSymbol, out var leanSymbol)) - { - switch (securityType) - { - case SecurityType.Future: - leanSymbol = Symbol.CreateFuture(brokerageSymbol, market, expirationDate); - break; - - default: - throw new Exception($"DataBentoSymbolMapper.GetLeanSymbol(): unsupported security type: {securityType}"); - } - - _leanSymbolsCache[brokerageSymbol] = leanSymbol; - _brokerageSymbolsCache[leanSymbol] = brokerageSymbol; - } - - return leanSymbol; - } - } - - /// - /// Gets the Lean symbol for the specified DataBento symbol - /// - /// The databento symbol - /// The corresponding Lean symbol - public Symbol GetLeanSymbol(string databentoSymbol) - { - lock (_locker) - { - if (!_leanSymbolsCache.TryGetValue(databentoSymbol, out var symbol)) - { - symbol = GetLeanSymbol(databentoSymbol, SecurityType.Equity, Market.USA); - } - - return symbol; + case SecurityType.Future: + return Symbol.CreateFuture(brokerageSymbol, market, expirationDate); + default: + throw new Exception($"The unsupported security type: {securityType}"); } } @@ -142,33 +66,13 @@ public Symbol GetLeanSymbol(string databentoSymbol) /// A new Lean Symbol instance public Symbol GetLeanSymbolForFuture(string brokerageSymbol) { - if (string.IsNullOrWhiteSpace(brokerageSymbol)) - { - throw new ArgumentException("Invalid symbol: " + brokerageSymbol); - } - // ignore futures spreads if (brokerageSymbol.Contains("-")) { return null; } - lock (_locker) - { - if (!_leanSymbolsCache.TryGetValue(brokerageSymbol, out var leanSymbol)) - { - leanSymbol = SymbolRepresentation.ParseFutureSymbol(brokerageSymbol); - - if (leanSymbol == null) - { - throw new ArgumentException("Invalid future symbol: " + brokerageSymbol); - } - - _leanSymbolsCache[brokerageSymbol] = leanSymbol; - } - - return leanSymbol; - } + return SymbolRepresentation.ParseFutureSymbol(brokerageSymbol); } } } diff --git a/QuantConnect.DataBento/Extensions.cs b/QuantConnect.DataBento/Extensions.cs new file mode 100644 index 0000000..14e8834 --- /dev/null +++ b/QuantConnect.DataBento/Extensions.cs @@ -0,0 +1,35 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using Newtonsoft.Json; +using Newtonsoft.Json.Serialization; +using QuantConnect.Lean.DataSource.DataBento.Serialization; + +namespace QuantConnect.Lean.DataSource.DataBento; + +public static class Extensions +{ + /// + /// Deserializes the specified JSON string to an object of type + /// using snake-case property name resolution. + /// + /// The target type of the deserialized object. + /// The JSON string to deserialize. + /// The deserialized object of type . + public static T? DeserializeKebabCase(this string json) + { + return JsonConvert.DeserializeObject(json, JsonSettings.SnakeCase); + } +} diff --git a/QuantConnect.DataBento/Models/DataBentoTypes.cs b/QuantConnect.DataBento/Models/DataBentoTypes.cs deleted file mode 100644 index b7fa58c..0000000 --- a/QuantConnect.DataBento/Models/DataBentoTypes.cs +++ /dev/null @@ -1,112 +0,0 @@ -using System; -using CsvHelper.Configuration.Attributes; -using QuantConnect; - -namespace QuantConnect.Lean.DataSource.DataBento.Models -{ - /// - /// Provides a constant for scaling price values from DataBento. - /// - public static class PriceScaling - { - /// - /// price scale factor is needed to find the true price from the message - /// Due to compression each "1 unit corresponds to 1e-9, i.e. 1/1,000,000,000 or 0.000000001" - /// https://databento.com/docs/api-reference-live/basics/schemas-and-conventions?historical=raw&live=raw&reference=raw - /// - public const decimal PriceScaleFactor = 1e-9m; - } - - /// - /// Represents a single bar of historical data from DataBento. - /// This class is used to map CSV data from HTTP requests into a structured format. - /// - public class DatabentoBar - { - [Name("ts_event")] - public long TimestampNanos { get; set; } - - public DateTime Timestamp => Time.UnixNanosecondTimeStampToDateTime(TimestampNanos); - - [Name("open")] - public long RawOpen { get; set; } - - [Name("high")] - public long RawHigh { get; set; } - - [Name("low")] - public long RawLow { get; set; } - - - - [Name("close")] - public long RawClose { get; set; } - - [Ignore] - public decimal Open => RawOpen == long.MaxValue ? 0m : RawOpen * PriceScaling.PriceScaleFactor; - - [Ignore] - public decimal High => RawHigh == long.MaxValue ? 0m : RawHigh * PriceScaling.PriceScaleFactor; - - [Ignore] - public decimal Low => RawLow == long.MaxValue ? 0m : RawLow * PriceScaling.PriceScaleFactor; - - [Ignore] - public decimal Close => RawClose == long.MaxValue ? 0m : RawClose * PriceScaling.PriceScaleFactor; - - [Name("volume")] - public long RawVolume { get; set; } - - [Ignore] - public decimal Volume => RawVolume == long.MaxValue ? 0m : RawVolume; - } - - /// - /// Represents a single trade event from DataBento. - /// - public class DatabentoTrade - { - [Name("ts_event")] - public long TimestampNanos { get; set; } - - public DateTime Timestamp => Time.UnixNanosecondTimeStampToDateTime(TimestampNanos); - - [Name("price")] - public long RawPrice { get; set; } - - [Ignore] - public decimal Price => RawPrice == long.MaxValue ? 0m : RawPrice * PriceScaling.PriceScaleFactor; - - [Name("size")] - public int Size { get; set; } - } - - /// - /// Represents a single quote from DataBento. - /// - public class DatabentoQuote - { - [Name("ts_event")] - public long TimestampNanos { get; set; } - - public DateTime Timestamp => Time.UnixNanosecondTimeStampToDateTime(TimestampNanos); - - [Name("bid_px_00")] - public long RawBidPrice { get; set; } - - [Ignore] - public decimal BidPrice => RawBidPrice == long.MaxValue ? 0m : RawBidPrice * PriceScaling.PriceScaleFactor; - - [Name("bid_sz_00")] - public int BidSize { get; set; } - - [Name("ask_px_00")] - public long RawAskPrice { get; set; } - - [Ignore] - public decimal AskPrice => RawAskPrice == long.MaxValue ? 0m : RawAskPrice * PriceScaling.PriceScaleFactor; - - [Name("ask_sz_00")] - public int AskSize { get; set; } - } -} diff --git a/QuantConnect.DataBento/Models/Enums/StatisticType.cs b/QuantConnect.DataBento/Models/Enums/StatisticType.cs new file mode 100644 index 0000000..51be1c2 --- /dev/null +++ b/QuantConnect.DataBento/Models/Enums/StatisticType.cs @@ -0,0 +1,117 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +namespace QuantConnect.Lean.DataSource.DataBento.Models.Enums; + +/// +/// Identifies the type of statistical market data reported for an instrument. +/// +/// +/// The statistic type defines how price, quantity, stat_flags, +/// and ts_ref should be interpreted for a given statistics record. +/// +public enum StatisticType +{ + /// + /// The price of the first trade of an instrument. + /// + OpeningPrice = 1, + + /// + /// The probable price and quantity of the first trade of an instrument, + /// published during the pre-open phase. + /// + IndicativeOpeningPrice = 2, + + /// + /// The settlement price of an instrument. + /// + /// + /// stat_flags indicate whether the settlement price is final or preliminary, + /// and whether it is actual or theoretical. + /// + SettlementPrice = 3, + + /// + /// The lowest trade price of an instrument during the trading session. + /// + TradingSessionLowPrice = 4, + + /// + /// The highest trade price of an instrument during the trading session. + /// + TradingSessionHighPrice = 5, + + /// + /// The number of contracts cleared for an instrument on the previous trading date. + /// + ClearedVolume = 6, + + /// + /// The lowest offer price for an instrument during the trading session. + /// + LowestOffer = 7, + + /// + /// The highest bid price for an instrument during the trading session. + /// + HighestBid = 8, + + /// + /// The current number of outstanding contracts of an instrument. + /// + OpenInterest = 9, + + /// + /// The volume-weighted average price (VWAP) for a fixing period. + /// + FixingPrice = 10, + + /// + /// The last trade price and quantity during a trading session. + /// + ClosePrice = 11, + + /// + /// The change in price from the previous session's close price + /// to the most recent close price. + /// + NetChange = 12, + + /// + /// The volume-weighted average price (VWAP) during the trading session. + /// + VolumeWeightedAveragePrice = 13, + + /// + /// The implied volatility associated with the settlement price. + /// + Volatility = 14, + + /// + /// The options delta associated with the settlement price. + /// + Delta = 15, + + /// + /// The auction uncrossing price and quantity. + /// + /// + /// Used for auctions that are neither the official opening auction + /// nor the official closing auction. + /// + UncrossingPrice = 16 +} + diff --git a/QuantConnect.DataBento/Models/Header.cs b/QuantConnect.DataBento/Models/Header.cs new file mode 100644 index 0000000..2a2ead1 --- /dev/null +++ b/QuantConnect.DataBento/Models/Header.cs @@ -0,0 +1,48 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +namespace QuantConnect.Lean.DataSource.DataBento.Models; + +/// +/// Metadata header for a historical market data record. +/// Contains event timing, record type, data source, and instrument identifiers. +/// +public sealed class Header +{ + /// + /// Event timestamp in nanoseconds since Unix epoch (UTC). + /// + public long TsEvent { get; set; } + + /// + /// Record type identifier defining the data schema (e.g. trade, quote, bar). + /// + public int Rtype { get; set; } + + /// + /// DataBento publisher (exchange / data source) identifier. + /// + public int PublisherId { get; set; } + + /// + /// Internal instrument identifier for the symbol. + /// + public long InstrumentId { get; set; } + + /// + /// Event time converted to UTC . + /// + public DateTime UtcTime => Time.UnixNanosecondTimeStampToDateTime(TsEvent); +} diff --git a/QuantConnect.DataBento/Models/LevelOneBookLevel.cs b/QuantConnect.DataBento/Models/LevelOneBookLevel.cs new file mode 100644 index 0000000..3eb51f5 --- /dev/null +++ b/QuantConnect.DataBento/Models/LevelOneBookLevel.cs @@ -0,0 +1,49 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +namespace QuantConnect.Lean.DataSource.DataBento.Models; + +public sealed class LevelOneBookLevel +{ + /// + /// Bid price at this book level. + /// + public decimal BidPx { get; set; } + + /// + /// Ask price at this book level. + /// + public decimal AskPx { get; set; } + + /// + /// Total bid size at this level. + /// + public int BidSz { get; set; } + + /// + /// Total ask size at this level. + /// + public int AskSz { get; set; } + + /// + /// Number of bid orders at this level. + /// + public int BidCt { get; set; } + + /// + /// Number of ask orders at this level. + /// + public int AskCt { get; set; } +} diff --git a/QuantConnect.DataBento/Models/LevelOneData.cs b/QuantConnect.DataBento/Models/LevelOneData.cs new file mode 100644 index 0000000..a4d3454 --- /dev/null +++ b/QuantConnect.DataBento/Models/LevelOneData.cs @@ -0,0 +1,69 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +namespace QuantConnect.Lean.DataSource.DataBento.Models; + +/// +/// Represents a level-one market data update containing best bid and ask information. +/// +public sealed class LevelOneData : MarketDataRecord +{ + /// + /// Timestamp when the message was received by the gateway, + /// expressed as nanoseconds since the UNIX epoch. + /// + public long TsRecv { get; set; } + + /// + /// The event type or order book operation. Can be Add, Cancel, Modify, cleaR book, Trade, Fill, or None. + /// + public char Action { get; set; } + + /// + /// Side of the book affected by the update. + /// + public char Side { get; set; } + + /// + /// Book depth level affected by this update. + /// + public int Depth { get; set; } + + /// + /// Price associated with the update. + /// + public decimal Price { get; set; } + + /// + /// The side that initiates the event. + /// + /// + /// Can be: + /// - Ask for a sell order (or sell aggressor in a trade); + /// - Bid for a buy order (or buy aggressor in a trade); + /// - None where no side is specified by the original source. + /// + public int Size { get; set; } + + /// + /// A bit field indicating event end, message characteristics, and data quality. + /// + public int Flags { get; set; } + + /// + /// Snapshot of level-one bid and ask data. + /// + public IReadOnlyList Levels { get; set; } = []; +} \ No newline at end of file diff --git a/QuantConnect.DataBento/Models/MarketDataRecord.cs b/QuantConnect.DataBento/Models/MarketDataRecord.cs new file mode 100644 index 0000000..31501d2 --- /dev/null +++ b/QuantConnect.DataBento/Models/MarketDataRecord.cs @@ -0,0 +1,30 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using Newtonsoft.Json; + +namespace QuantConnect.Lean.DataSource.DataBento.Models; + +/// +/// Base class for all market data records containing a standard metadata header. +/// +public abstract class MarketDataRecord +{ + /// + /// Gets or sets the standard metadata header for this market data record. + /// + [JsonProperty("hd")] + public required Header Header { get; set; } +} diff --git a/QuantConnect.DataBento/Models/OhlcvBar.cs b/QuantConnect.DataBento/Models/OhlcvBar.cs new file mode 100644 index 0000000..a7ce501 --- /dev/null +++ b/QuantConnect.DataBento/Models/OhlcvBar.cs @@ -0,0 +1,48 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +namespace QuantConnect.Lean.DataSource.DataBento.Models; + +/// +/// Open-High-Low-Close-Volume (OHLCV) bar representing aggregated market data +/// for a specific instrument and time interval. +/// +public sealed class OhlcvBar : MarketDataRecord +{ + /// + /// Opening price of the bar. + /// + public decimal Open { get; set; } + + /// + /// Highest traded price during the bar interval. + /// + public decimal High { get; set; } + + /// + /// Lowest traded price during the bar interval. + /// + public decimal Low { get; set; } + + /// + /// Closing price of the bar. + /// + public decimal Close { get; set; } + + /// + /// Total traded volume during the bar interval. + /// + public decimal Volume { get; set; } +} diff --git a/QuantConnect.DataBento/Models/StatisticsData.cs b/QuantConnect.DataBento/Models/StatisticsData.cs new file mode 100644 index 0000000..749127e --- /dev/null +++ b/QuantConnect.DataBento/Models/StatisticsData.cs @@ -0,0 +1,31 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using QuantConnect.Lean.DataSource.DataBento.Models.Enums; + +namespace QuantConnect.Lean.DataSource.DataBento.Models; + +public sealed class StatisticsData : MarketDataRecord +{ + /// + /// Quantity or value associated with the statistic. + /// + public decimal Quantity { get; set; } + + /// + /// Type of statistic represented by this record. + /// + public StatisticType StatType { get; set; } +} diff --git a/QuantConnect.DataBento/Serialization/JsonSettings.cs b/QuantConnect.DataBento/Serialization/JsonSettings.cs new file mode 100644 index 0000000..5247d97 --- /dev/null +++ b/QuantConnect.DataBento/Serialization/JsonSettings.cs @@ -0,0 +1,34 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using Newtonsoft.Json; + +namespace QuantConnect.Lean.DataSource.DataBento.Serialization; + +/// +/// Provides globally accessible instances of +/// preconfigured with custom contract resolvers, such as snake-case formatting. +/// +public static class JsonSettings +{ + /// + /// Gets a reusable instance of that uses + /// for snake-case property name formatting. + /// + public static readonly JsonSerializerSettings SnakeCase = new() + { + ContractResolver = SnakeCaseContractResolver.Instance + }; +} diff --git a/QuantConnect.DataBento/Serialization/SnakeCaseContractResolver.cs b/QuantConnect.DataBento/Serialization/SnakeCaseContractResolver.cs new file mode 100644 index 0000000..cdc2319 --- /dev/null +++ b/QuantConnect.DataBento/Serialization/SnakeCaseContractResolver.cs @@ -0,0 +1,39 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using Newtonsoft.Json.Serialization; + +namespace QuantConnect.Lean.DataSource.DataBento.Serialization; + +/// +/// A singleton implementation that applies +/// a to JSON property names. +/// +public sealed class SnakeCaseContractResolver : DefaultContractResolver +{ + /// + /// Gets the singleton instance of the . + /// + public static readonly SnakeCaseContractResolver Instance = new(); + + /// + /// Initializes a new instance of the class + /// with a . + /// + private SnakeCaseContractResolver() + { + NamingStrategy = new SnakeCaseNamingStrategy(); + } +} From cbc4bfeb1e8798ac620c1d0b04ac4999dd7c23be Mon Sep 17 00:00:00 2001 From: Romazes Date: Thu, 22 Jan 2026 10:47:46 +0200 Subject: [PATCH 07/13] refactor: clean, style, spacing, missed license block --- .../DataBentoDataDownloaderTests.cs | 253 ++-- .../DataBentoDataProviderHistoryTests.cs | 4 +- .../DataBentoJsonConverterTests.cs | 16 +- .../DataBentoRawLiveClientTests.cs | 202 ++- .../DataBentoSymbolMapperTests.cs.cs | 4 +- QuantConnect.DataBento.Tests/TestSetup.cs | 68 +- .../DataBentoDataProvider.cs | 515 ++++---- .../DataBentoHistoryProivder.cs | 343 +++-- .../DataBentoRawLiveClient.cs | 1102 ++++++++--------- .../DataBentoSymbolMapper.cs | 93 +- QuantConnect.DataBento/Extensions.cs | 1 - 11 files changed, 1301 insertions(+), 1300 deletions(-) diff --git a/QuantConnect.DataBento.Tests/DataBentoDataDownloaderTests.cs b/QuantConnect.DataBento.Tests/DataBentoDataDownloaderTests.cs index 7525823..b84893c 100644 --- a/QuantConnect.DataBento.Tests/DataBentoDataDownloaderTests.cs +++ b/QuantConnect.DataBento.Tests/DataBentoDataDownloaderTests.cs @@ -22,165 +22,164 @@ using QuantConnect.Securities; using QuantConnect.Data.Market; -namespace QuantConnect.Lean.DataSource.DataBento.Tests +namespace QuantConnect.Lean.DataSource.DataBento.Tests; + +[TestFixture] +public class DataBentoDataDownloaderTests { - [TestFixture] - public class DataBentoDataDownloaderTests + private DataBentoDataDownloader _downloader; + + private readonly MarketHoursDatabase _marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); + + [SetUp] + public void SetUp() { - private DataBentoDataDownloader _downloader; + _downloader = new DataBentoDataDownloader(); + } - private readonly MarketHoursDatabase _marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); + [TearDown] + public void TearDown() + { + _downloader?.Dispose(); + } - [SetUp] - public void SetUp() - { - _downloader = new DataBentoDataDownloader(); - } + [TestCase(Resolution.Daily)] + [TestCase(Resolution.Hour)] + [TestCase(Resolution.Minute)] + [TestCase(Resolution.Second)] + [TestCase(Resolution.Tick)] + public void DownloadsTradeDataForLeanFuture(Resolution resolution) + { + var symbol = Symbol.CreateFuture("ES", Market.CME, new DateTime(2026, 3, 20)); + var exchangeTimeZone = _marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType).TimeZone; - [TearDown] - public void TearDown() - { - _downloader?.Dispose(); - } + var startUtc = new DateTime(2026, 1, 18, 0, 0, 0, DateTimeKind.Utc); + var endUtc = new DateTime(2026, 1, 20, 0, 0, 0, DateTimeKind.Utc); - [TestCase(Resolution.Daily)] - [TestCase(Resolution.Hour)] - [TestCase(Resolution.Minute)] - [TestCase(Resolution.Second)] - [TestCase(Resolution.Tick)] - public void DownloadsTradeDataForLeanFuture(Resolution resolution) + if (resolution == Resolution.Tick) { - var symbol = Symbol.CreateFuture("ES", Market.CME, new DateTime(2026, 3, 20)); - var exchangeTimeZone = _marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType).TimeZone; + startUtc = new DateTime(2026, 1, 21, 9, 30, 0, DateTimeKind.Utc); + endUtc = startUtc.AddMinutes(15); + } - var startUtc = new DateTime(2026, 1, 18, 0, 0, 0, DateTimeKind.Utc); - var endUtc = new DateTime(2026, 1, 20, 0, 0, 0, DateTimeKind.Utc); + var parameters = new DataDownloaderGetParameters( + symbol, + resolution, + startUtc, + endUtc, + TickType.Trade + ); - if (resolution == Resolution.Tick) - { - startUtc = new DateTime(2026, 1, 21, 9, 30, 0, DateTimeKind.Utc); - endUtc = startUtc.AddMinutes(15); - } + var data = _downloader.Get(parameters).ToList(); - var parameters = new DataDownloaderGetParameters( - symbol, - resolution, - startUtc, - endUtc, - TickType.Trade - ); + Log.Trace($"Downloaded {data.Count} trade points for {symbol} @ {resolution}"); - var data = _downloader.Get(parameters).ToList(); + Assert.IsNotEmpty(data); - Log.Trace($"Downloaded {data.Count} trade points for {symbol} @ {resolution}"); + var startExchange = startUtc.ConvertFromUtc(exchangeTimeZone); + var endExchange = endUtc.ConvertFromUtc(exchangeTimeZone); - Assert.IsNotEmpty(data); - - var startExchange = startUtc.ConvertFromUtc(exchangeTimeZone); - var endExchange = endUtc.ConvertFromUtc(exchangeTimeZone); + foreach (var point in data) + { + Assert.AreEqual(symbol, point.Symbol); + Assert.That(point.Time, Is.InRange(startExchange, endExchange)); - foreach (var point in data) + switch (point) { - Assert.AreEqual(symbol, point.Symbol); - Assert.That(point.Time, Is.InRange(startExchange, endExchange)); - - switch (point) - { - case TradeBar bar: - Assert.Greater(bar.Open, 0); - Assert.Greater(bar.High, 0); - Assert.Greater(bar.Low, 0); - Assert.Greater(bar.Close, 0); - Assert.GreaterOrEqual(bar.Volume, 0); - Assert.GreaterOrEqual(bar.High, bar.Low); - break; - - case Tick tick: - Assert.Greater(tick.Value, 0); - Assert.GreaterOrEqual(tick.Quantity, 0); - break; - - default: - Assert.Fail($"Unexpected data type {point.GetType()}"); - break; - } + case TradeBar bar: + Assert.Greater(bar.Open, 0); + Assert.Greater(bar.High, 0); + Assert.Greater(bar.Low, 0); + Assert.Greater(bar.Close, 0); + Assert.GreaterOrEqual(bar.Volume, 0); + Assert.GreaterOrEqual(bar.High, bar.Low); + break; + + case Tick tick: + Assert.Greater(tick.Value, 0); + Assert.GreaterOrEqual(tick.Quantity, 0); + break; + + default: + Assert.Fail($"Unexpected data type {point.GetType()}"); + break; } } + } - [Test] - public void DownloadsQuoteTicksForLeanFuture() - { - var symbol = Symbol.CreateFuture("ES", Market.CME, new DateTime(2026, 3, 20)); - var exchangeTimeZone = _marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType).TimeZone; - - var startUtc = new DateTime(2026, 1, 20, 9, 30, 0, DateTimeKind.Utc); - var endUtc = startUtc.AddMinutes(15); - - var parameters = new DataDownloaderGetParameters( - symbol, - Resolution.Tick, - startUtc, - endUtc, - TickType.Quote - ); + [Test] + public void DownloadsQuoteTicksForLeanFuture() + { + var symbol = Symbol.CreateFuture("ES", Market.CME, new DateTime(2026, 3, 20)); + var exchangeTimeZone = _marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType).TimeZone; + + var startUtc = new DateTime(2026, 1, 20, 9, 30, 0, DateTimeKind.Utc); + var endUtc = startUtc.AddMinutes(15); - var data = _downloader.Get(parameters).ToList(); + var parameters = new DataDownloaderGetParameters( + symbol, + Resolution.Tick, + startUtc, + endUtc, + TickType.Quote + ); - Log.Trace($"Downloaded {data.Count} quote ticks for {symbol}"); + var data = _downloader.Get(parameters).ToList(); - Assert.IsNotEmpty(data); + Log.Trace($"Downloaded {data.Count} quote ticks for {symbol}"); - var startExchange = startUtc.ConvertFromUtc(exchangeTimeZone); - var endExchange = endUtc.ConvertFromUtc(exchangeTimeZone); + Assert.IsNotEmpty(data); - foreach (var point in data) + var startExchange = startUtc.ConvertFromUtc(exchangeTimeZone); + var endExchange = endUtc.ConvertFromUtc(exchangeTimeZone); + + foreach (var point in data) + { + Assert.AreEqual(symbol, point.Symbol); + Assert.That(point.Time, Is.InRange(startExchange, endExchange)); + + if (point is Tick tick) + { + Assert.AreEqual(TickType.Quote, tick.TickType); + Assert.IsTrue( + tick.BidPrice > 0 || tick.AskPrice > 0, + "Quote tick must have bid or ask" + ); + } + else if (point is QuoteBar bar) { - Assert.AreEqual(symbol, point.Symbol); - Assert.That(point.Time, Is.InRange(startExchange, endExchange)); - - if (point is Tick tick) - { - Assert.AreEqual(TickType.Quote, tick.TickType); - Assert.IsTrue( - tick.BidPrice > 0 || tick.AskPrice > 0, - "Quote tick must have bid or ask" - ); - } - else if (point is QuoteBar bar) - { - Assert.IsTrue(bar.Bid != null || bar.Ask != null); - } + Assert.IsTrue(bar.Bid != null || bar.Ask != null); } } + } - [Test] - public void DataIsSortedByTime() - { - var symbol = Symbol.CreateFuture("ES", Market.CME, new DateTime(2026, 3, 20)); + [Test] + public void DataIsSortedByTime() + { + var symbol = Symbol.CreateFuture("ES", Market.CME, new DateTime(2026, 3, 20)); - var startUtc = new DateTime(2026, 1, 20, 0, 0, 0, DateTimeKind.Utc); - var endUtc = new DateTime(2024, 1, 21, 0, 0, 0, DateTimeKind.Utc); + var startUtc = new DateTime(2026, 1, 20, 0, 0, 0, DateTimeKind.Utc); + var endUtc = new DateTime(2024, 1, 21, 0, 0, 0, DateTimeKind.Utc); - var parameters = new DataDownloaderGetParameters( - symbol, - Resolution.Minute, - startUtc, - endUtc, - TickType.Trade - ); + var parameters = new DataDownloaderGetParameters( + symbol, + Resolution.Minute, + startUtc, + endUtc, + TickType.Trade + ); - var data = _downloader.Get(parameters).ToList(); + var data = _downloader.Get(parameters).ToList(); - Assert.IsNotEmpty(data); + Assert.IsNotEmpty(data); - for (int i = 1; i < data.Count; i++) - { - Assert.GreaterOrEqual( - data[i].Time, - data[i - 1].Time, - $"Data not sorted at index {i}" - ); - } + for (int i = 1; i < data.Count; i++) + { + Assert.GreaterOrEqual( + data[i].Time, + data[i - 1].Time, + $"Data not sorted at index {i}" + ); } } } diff --git a/QuantConnect.DataBento.Tests/DataBentoDataProviderHistoryTests.cs b/QuantConnect.DataBento.Tests/DataBentoDataProviderHistoryTests.cs index f0d2e2f..27a52d3 100644 --- a/QuantConnect.DataBento.Tests/DataBentoDataProviderHistoryTests.cs +++ b/QuantConnect.DataBento.Tests/DataBentoDataProviderHistoryTests.cs @@ -15,14 +15,12 @@ */ using System; -using System.Linq; using NUnit.Framework; using QuantConnect.Data; using QuantConnect.Util; using QuantConnect.Securities; -using System.Collections.Generic; -using QuantConnect.Logging; using QuantConnect.Data.Market; +using System.Collections.Generic; namespace QuantConnect.Lean.DataSource.DataBento.Tests; diff --git a/QuantConnect.DataBento.Tests/DataBentoJsonConverterTests.cs b/QuantConnect.DataBento.Tests/DataBentoJsonConverterTests.cs index 74d1126..9d43db8 100644 --- a/QuantConnect.DataBento.Tests/DataBentoJsonConverterTests.cs +++ b/QuantConnect.DataBento.Tests/DataBentoJsonConverterTests.cs @@ -1,4 +1,18 @@ -using Newtonsoft.Json; +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + using NUnit.Framework; using QuantConnect.Lean.DataSource.DataBento.Models; using QuantConnect.Lean.DataSource.DataBento.Models.Enums; diff --git a/QuantConnect.DataBento.Tests/DataBentoRawLiveClientTests.cs b/QuantConnect.DataBento.Tests/DataBentoRawLiveClientTests.cs index 10536a3..ec03477 100644 --- a/QuantConnect.DataBento.Tests/DataBentoRawLiveClientTests.cs +++ b/QuantConnect.DataBento.Tests/DataBentoRawLiveClientTests.cs @@ -15,134 +15,132 @@ */ using System; -using System.Threading; using NUnit.Framework; -using QuantConnect.Configuration; +using System.Threading; using QuantConnect.Data; -using QuantConnect.Data.Market; -using QuantConnect.Lean.DataSource.DataBento; using QuantConnect.Logging; +using QuantConnect.Data.Market; +using QuantConnect.Configuration; -namespace QuantConnect.Lean.DataSource.DataBento.Tests +namespace QuantConnect.Lean.DataSource.DataBento.Tests; + +[TestFixture] +public class DataBentoRawLiveClientSyncTests { - [TestFixture] - public class DataBentoRawLiveClientSyncTests + private DataBentoRawLiveClient _client; + protected readonly string ApiKey = Config.Get("databento-api-key"); + + private static Symbol CreateEsFuture() { - private DataBentoRawLiveClient _client; - protected readonly string ApiKey = Config.Get("databento-api-key"); + var expiration = new DateTime(2026, 3, 20); + return Symbol.CreateFuture("ES", Market.CME, expiration); + } - private static Symbol CreateEsFuture() - { - var expiration = new DateTime(2026, 3, 20); - return Symbol.CreateFuture("ES", Market.CME, expiration); - } + [SetUp] + public void SetUp() + { + Log.Trace("DataBentoLiveClientTests: Using API Key: " + ApiKey); + _client = new DataBentoRawLiveClient(ApiKey); + } - [SetUp] - public void SetUp() - { - Log.Trace("DataBentoLiveClientTests: Using API Key: " + ApiKey); - _client = new DataBentoRawLiveClient(ApiKey); - } + [TearDown] + public void TearDown() + { + _client?.Dispose(); + } - [TearDown] - public void TearDown() - { - _client?.Dispose(); - } + [Test] + public void Connects() + { + var connected = _client.Connect(); - [Test] - public void Connects() - { - var connected = _client.Connect(); + Assert.IsTrue(connected); + Assert.IsTrue(_client.IsConnected); - Assert.IsTrue(connected); - Assert.IsTrue(_client.IsConnected); + Log.Trace("Connected successfully"); + } - Log.Trace("Connected successfully"); - } + [Test] + public void SubscribesToLeanFutureSymbol() + { + Assert.IsTrue(_client.Connect()); + + var symbol = CreateEsFuture(); + + Assert.IsTrue(_client.Subscribe(symbol, TickType.Trade)); + Assert.IsTrue(_client.StartSession()); + + Thread.Sleep(1000); + + Assert.IsTrue(_client.Unsubscribe(symbol)); + } - [Test] - public void SubscribesToLeanFutureSymbol() + [Test] + public void ReceivesTradeOrQuoteTicks() + { + var receivedEvent = new ManualResetEventSlim(false); + BaseData received = null; + + _client.DataReceived += (_, data) => { - Assert.IsTrue(_client.Connect()); + received = data; + receivedEvent.Set(); + }; - var symbol = CreateEsFuture(); + Assert.IsTrue(_client.Connect()); - Assert.IsTrue(_client.Subscribe(symbol, TickType.Trade)); - Assert.IsTrue(_client.StartSession()); + var symbol = CreateEsFuture(); - Thread.Sleep(1000); + Assert.IsTrue(_client.Subscribe(symbol, TickType.Trade)); + Assert.IsTrue(_client.StartSession()); - Assert.IsTrue(_client.Unsubscribe(symbol)); - } + var gotData = receivedEvent.Wait(TimeSpan.FromMinutes(2)); - [Test] - public void ReceivesTradeOrQuoteTicks() + if (!gotData) { - var receivedEvent = new ManualResetEventSlim(false); - BaseData received = null; - - _client.DataReceived += (_, data) => - { - received = data; - receivedEvent.Set(); - }; - - Assert.IsTrue(_client.Connect()); - - var symbol = CreateEsFuture(); - - Assert.IsTrue(_client.Subscribe(symbol, TickType.Trade)); - Assert.IsTrue(_client.StartSession()); - - var gotData = receivedEvent.Wait(TimeSpan.FromMinutes(2)); - - if (!gotData) - { - Assert.Inconclusive("No data received (likely outside market hours)"); - return; - } - - Assert.NotNull(received); - Assert.AreEqual(symbol, received.Symbol); - - if (received is Tick tick) - { - Assert.Greater(tick.Time, DateTime.MinValue); - Assert.Greater(tick.Value, 0); - } - else if (received is TradeBar bar) - { - Assert.Greater(bar.Close, 0); - } - else - { - Assert.Fail($"Unexpected data type: {received.GetType()}"); - } + Assert.Inconclusive("No data received (likely outside market hours)"); + return; } - [Test] - public void DisposeIsIdempotent() + Assert.NotNull(received); + Assert.AreEqual(symbol, received.Symbol); + + if (received is Tick tick) { - var client = new DataBentoRawLiveClient(ApiKey); - Assert.DoesNotThrow(client.Dispose); - Assert.DoesNotThrow(client.Dispose); + Assert.Greater(tick.Time, DateTime.MinValue); + Assert.Greater(tick.Value, 0); } - - [Test] - public void SymbolMappingDoesNotThrow() + else if (received is TradeBar bar) { - Assert.IsTrue(_client.Connect()); + Assert.Greater(bar.Close, 0); + } + else + { + Assert.Fail($"Unexpected data type: {received.GetType()}"); + } + } - var symbol = CreateEsFuture(); + [Test] + public void DisposeIsIdempotent() + { + var client = new DataBentoRawLiveClient(ApiKey); + Assert.DoesNotThrow(client.Dispose); + Assert.DoesNotThrow(client.Dispose); + } - Assert.DoesNotThrow(() => - { - _client.Subscribe(symbol, TickType.Trade); - _client.StartSession(); - Thread.Sleep(500); - _client.Unsubscribe(symbol); - }); - } + [Test] + public void SymbolMappingDoesNotThrow() + { + Assert.IsTrue(_client.Connect()); + + var symbol = CreateEsFuture(); + + Assert.DoesNotThrow(() => + { + _client.Subscribe(symbol, TickType.Trade); + _client.StartSession(); + Thread.Sleep(500); + _client.Unsubscribe(symbol); + }); } } diff --git a/QuantConnect.DataBento.Tests/DataBentoSymbolMapperTests.cs.cs b/QuantConnect.DataBento.Tests/DataBentoSymbolMapperTests.cs.cs index 471519a..ad14aad 100644 --- a/QuantConnect.DataBento.Tests/DataBentoSymbolMapperTests.cs.cs +++ b/QuantConnect.DataBento.Tests/DataBentoSymbolMapperTests.cs.cs @@ -14,8 +14,8 @@ * */ -using NUnit.Framework; using System; +using NUnit.Framework; using System.Collections.Generic; namespace QuantConnect.Lean.DataSource.DataBento.Tests; @@ -40,7 +40,7 @@ private static IEnumerable LeanSymbolTestCases // TSLA - Equity var es = Symbol.CreateFuture(Securities.Futures.Indices.SP500EMini, Market.CME, new DateTime(2026, 3, 20)); yield return new TestCaseData(es, "ESH6"); - + } } diff --git a/QuantConnect.DataBento.Tests/TestSetup.cs b/QuantConnect.DataBento.Tests/TestSetup.cs index 023eae0..86d2f07 100644 --- a/QuantConnect.DataBento.Tests/TestSetup.cs +++ b/QuantConnect.DataBento.Tests/TestSetup.cs @@ -4,7 +4,6 @@ * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. - * * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software @@ -15,52 +14,51 @@ */ using System; -using System.Collections; using System.IO; using NUnit.Framework; -using QuantConnect.Configuration; +using System.Collections; using QuantConnect.Logging; +using QuantConnect.Configuration; + +namespace QuantConnect.Lean.DataSource.DataBento.Tests; -namespace QuantConnect.Lean.DataSource.DataBento.Tests +[SetUpFixture] +public class TestSetup { - [SetUpFixture] - public class TestSetup + [OneTimeSetUp] + public void GlobalSetup() { - [OneTimeSetUp] - public void GlobalSetup() - { - Log.DebuggingEnabled = true; - Log.LogHandler = new CompositeLogHandler(); - Log.Trace("TestSetup(): starting..."); - ReloadConfiguration(); - } + Log.DebuggingEnabled = true; + Log.LogHandler = new CompositeLogHandler(); + Log.Trace("TestSetup(): starting..."); + ReloadConfiguration(); + } + + private static void ReloadConfiguration() + { + // nunit 3 sets the current folder to a temp folder we need it to be the test bin output folder + var dir = TestContext.CurrentContext.TestDirectory; + Environment.CurrentDirectory = dir; + Directory.SetCurrentDirectory(dir); + // reload config from current path + Config.Reset(); - private static void ReloadConfiguration() + var environment = Environment.GetEnvironmentVariables(); + foreach (DictionaryEntry entry in environment) { - // nunit 3 sets the current folder to a temp folder we need it to be the test bin output folder - var dir = TestContext.CurrentContext.TestDirectory; - Environment.CurrentDirectory = dir; - Directory.SetCurrentDirectory(dir); - // reload config from current path - Config.Reset(); + var envKey = entry.Key.ToString(); + var value = entry.Value.ToString(); - var environment = Environment.GetEnvironmentVariables(); - foreach (DictionaryEntry entry in environment) + if (envKey.StartsWith("QC_")) { - var envKey = entry.Key.ToString(); - var value = entry.Value.ToString(); + var key = envKey.Substring(3).Replace("_", "-").ToLower(); - if (envKey.StartsWith("QC_")) - { - var key = envKey.Substring(3).Replace("_", "-").ToLower(); - - Log.Trace($"TestSetup(): Updating config setting '{key}' from environment var '{envKey}'"); - Config.Set(key, value); - } + Log.Trace($"TestSetup(): Updating config setting '{key}' from environment var '{envKey}'"); + Config.Set(key, value); } - - // resets the version among other things - Globals.Reset(); } + + // resets the version among other things + Globals.Reset(); } } diff --git a/QuantConnect.DataBento/DataBentoDataProvider.cs b/QuantConnect.DataBento/DataBentoDataProvider.cs index c8d4eed..40fc095 100644 --- a/QuantConnect.DataBento/DataBentoDataProvider.cs +++ b/QuantConnect.DataBento/DataBentoDataProvider.cs @@ -16,342 +16,341 @@ using NodaTime; using QuantConnect.Data; -using QuantConnect.Data.Market; using QuantConnect.Util; -using QuantConnect.Interfaces; -using QuantConnect.Configuration; using QuantConnect.Logging; using QuantConnect.Packets; +using QuantConnect.Interfaces; using QuantConnect.Securities; +using QuantConnect.Data.Market; +using QuantConnect.Configuration; using System.Collections.Concurrent; using QuantConnect.Lean.DataSource.DataBento.Api; -namespace QuantConnect.Lean.DataSource.DataBento +namespace QuantConnect.Lean.DataSource.DataBento; + +/// +/// A data Provider for DataBento that provides live market data and historical data. +/// Handles Subscribing, Unsubscribing, and fetching historical data from DataBento. +/// It will handle if a symbol is subscribable and will log errors if it is not. +/// +public partial class DataBentoProvider : IDataQueueHandler { /// - /// A data Provider for DataBento that provides live market data and historical data. - /// Handles Subscribing, Unsubscribing, and fetching historical data from DataBento. - /// It will handle if a symbol is subscribable and will log errors if it is not. + /// Resolves map files to correctly handle current and historical ticker symbols. /// - public partial class DataBentoProvider : IDataQueueHandler + private readonly IMapFileProvider _mapFileProvider = Composer.Instance.GetPart(); + + private HistoricalAPIClient _historicalApiClient; + + private readonly DataBentoSymbolMapper _symbolMapper = new(); + + private readonly IDataAggregator _dataAggregator = Composer.Instance.GetExportedValueByTypeName( + Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager"), forceTypeNameOnExisting: false); + private EventBasedDataQueueHandlerSubscriptionManager _subscriptionManager; + private DataBentoRawLiveClient _client; + private bool _potentialUnsupportedResolutionMessageLogged; + private bool _sessionStarted = false; + private readonly object _sessionLock = new(); + private readonly MarketHoursDatabase _marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); + private readonly ConcurrentDictionary _symbolExchangeTimeZones = new(); + private bool _initialized; + + /// + /// Returns true if we're currently connected to the Data Provider + /// + public bool IsConnected => _client?.IsConnected == true; + + /// + /// Initializes a new instance of the DataBentoProvider + /// + public DataBentoProvider() + : this(Config.Get("databento-api-key")) { - /// - /// Resolves map files to correctly handle current and historical ticker symbols. - /// - private readonly IMapFileProvider _mapFileProvider = Composer.Instance.GetPart(); - - private HistoricalAPIClient _historicalApiClient; - - private readonly DataBentoSymbolMapper _symbolMapper = new DataBentoSymbolMapper(); - - private readonly IDataAggregator _dataAggregator = Composer.Instance.GetExportedValueByTypeName( - Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager"), forceTypeNameOnExisting: false); - private EventBasedDataQueueHandlerSubscriptionManager _subscriptionManager; - private DataBentoRawLiveClient _client; - private bool _potentialUnsupportedResolutionMessageLogged; - private bool _sessionStarted = false; - private readonly object _sessionLock = new(); - private readonly MarketHoursDatabase _marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); - private readonly ConcurrentDictionary _symbolExchangeTimeZones = new(); - private bool _initialized; - - /// - /// Returns true if we're currently connected to the Data Provider - /// - public bool IsConnected => _client?.IsConnected == true; - - /// - /// Initializes a new instance of the DataBentoProvider - /// - public DataBentoProvider() - : this(Config.Get("databento-api-key")) + } + + public DataBentoProvider(string apiKey) + { + if (string.IsNullOrWhiteSpace(apiKey)) { + // If the API key is not provided, we can't do anything. + // The handler might going to be initialized using a node packet job. + return; } - public DataBentoProvider(string apiKey) + Initialize(apiKey); + } + + /// + /// Common initialization logic + /// DataBento API key from config file retrieved on constructor + /// + private void Initialize(string apiKey) + { + Log.Debug("DataBentoProvider.Initialize(): Starting initialization"); + _subscriptionManager = new EventBasedDataQueueHandlerSubscriptionManager() { - if (string.IsNullOrWhiteSpace(apiKey)) + SubscribeImpl = (symbols, tickType) => + { + return SubscriptionLogic(symbols, tickType); + }, + UnsubscribeImpl = (symbols, tickType) => { - // If the API key is not provided, we can't do anything. - // The handler might going to be initialized using a node packet job. - return; + return UnsubscribeLogic(symbols, tickType); } + }; - Initialize(apiKey); - } + // Initialize the live client + _client = new DataBentoRawLiveClient(apiKey); + _client.DataReceived += OnDataReceived; - /// - /// Common initialization logic - /// DataBento API key from config file retrieved on constructor - /// - private void Initialize(string apiKey) + // Connect to live gateway + Log.Debug("DataBentoProvider.Initialize(): Attempting connection to DataBento live gateway"); + var cancellationTokenSource = new CancellationTokenSource(); + Task.Factory.StartNew(() => { - Log.Debug("DataBentoProvider.Initialize(): Starting initialization"); - _subscriptionManager = new EventBasedDataQueueHandlerSubscriptionManager() + try { - SubscribeImpl = (symbols, tickType) => - { - return SubscriptionLogic(symbols, tickType); - }, - UnsubscribeImpl = (symbols, tickType) => - { - return UnsubscribeLogic(symbols, tickType); - } - }; + var connected = _client.Connect(); + Log.Debug($"DataBentoProvider.Initialize(): Connect() returned {connected}"); - // Initialize the live client - _client = new DataBentoRawLiveClient(apiKey); - _client.DataReceived += OnDataReceived; - - // Connect to live gateway - Log.Debug("DataBentoProvider.Initialize(): Attempting connection to DataBento live gateway"); - var cancellationTokenSource = new CancellationTokenSource(); - Task.Factory.StartNew(() => - { - try + if (connected) { - var connected = _client.Connect(); - Log.Debug($"DataBentoProvider.Initialize(): Connect() returned {connected}"); - - if (connected) - { - Log.Debug("DataBentoProvider.Initialize(): Successfully connected to DataBento live gateway"); - } - else - { - Log.Error("DataBentoProvider.Initialize(): Failed to connect to DataBento live gateway"); - } + Log.Debug("DataBentoProvider.Initialize(): Successfully connected to DataBento live gateway"); } - catch (Exception ex) + else { - Log.Error($"DataBentoProvider.Initialize(): Exception during Connect(): {ex.Message}\n{ex.StackTrace}"); + Log.Error("DataBentoProvider.Initialize(): Failed to connect to DataBento live gateway"); } - }, - cancellationTokenSource.Token, - TaskCreationOptions.LongRunning, - TaskScheduler.Default); + } + catch (Exception ex) + { + Log.Error($"DataBentoProvider.Initialize(): Exception during Connect(): {ex.Message}\n{ex.StackTrace}"); + } + }, + cancellationTokenSource.Token, + TaskCreationOptions.LongRunning, + TaskScheduler.Default); - _historicalApiClient = new(apiKey); - _initialized = true; + _historicalApiClient = new(apiKey); + _initialized = true; - Log.Debug("DataBentoProvider.Initialize(): Initialization complete"); - } + Log.Debug("DataBentoProvider.Initialize(): Initialization complete"); + } - /// - /// Logic to unsubscribe from the specified symbols - /// - public bool UnsubscribeLogic(IEnumerable symbols, TickType tickType) + /// + /// Logic to unsubscribe from the specified symbols + /// + public bool UnsubscribeLogic(IEnumerable symbols, TickType tickType) + { + foreach (var symbol in symbols) { - foreach (var symbol in symbols) + Log.Debug($"DataBentoProvider.UnsubscribeImpl(): Processing symbol {symbol}"); + if (_client?.IsConnected != true) { - Log.Debug($"DataBentoProvider.UnsubscribeImpl(): Processing symbol {symbol}"); - if (_client?.IsConnected != true) - { - throw new InvalidOperationException($"DataBentoProvider.UnsubscribeImpl(): Client is not connected. Cannot unsubscribe from {symbol}"); - } - - _client.Unsubscribe(symbol); + throw new InvalidOperationException($"DataBentoProvider.UnsubscribeImpl(): Client is not connected. Cannot unsubscribe from {symbol}"); } - return true; + _client.Unsubscribe(symbol); } - /// - /// Logic to subscribe to the specified symbols - /// - public bool SubscriptionLogic(IEnumerable symbols, TickType tickType) + return true; + } + + /// + /// Logic to subscribe to the specified symbols + /// + public bool SubscriptionLogic(IEnumerable symbols, TickType tickType) + { + if (_client?.IsConnected != true) { - if (_client?.IsConnected != true) + Log.Error("DataBentoProvider.SubscriptionLogic(): Client is not connected. Cannot subscribe to symbols"); + return false; + } + + foreach (var symbol in symbols) + { + if (!CanSubscribe(symbol)) { - Log.Error("DataBentoProvider.SubscriptionLogic(): Client is not connected. Cannot subscribe to symbols"); + Log.Error($"DataBentoProvider.SubscriptionLogic(): Unsupported subscription: {symbol}"); return false; } - foreach (var symbol in symbols) - { - if (!CanSubscribe(symbol)) - { - Log.Error($"DataBentoProvider.SubscriptionLogic(): Unsupported subscription: {symbol}"); - return false; - } + _client.Subscribe(symbol, tickType); + } - _client.Subscribe(symbol, tickType); - } + return true; + } - return true; - } + /// + /// Checks if this Data provider supports the specified symbol + /// + /// The symbol + /// returns true if Data Provider supports the specified symbol; otherwise false + private static bool CanSubscribe(Symbol symbol) + { + return !symbol.Value.Contains("universe", StringComparison.InvariantCultureIgnoreCase) && + !symbol.IsCanonical() && + symbol.SecurityType == SecurityType.Future; + } - /// - /// Checks if this Data provider supports the specified symbol - /// - /// The symbol - /// returns true if Data Provider supports the specified symbol; otherwise false - private static bool CanSubscribe(Symbol symbol) + /// + /// Subscribe to the specified configuration + /// + /// defines the parameters to subscribe to a data feed + /// handler to be fired on new data available + /// The new enumerator for this subscription request + public IEnumerator? Subscribe(SubscriptionDataConfig dataConfig, EventHandler newDataAvailableHandler) + { + if (!IsSupported(dataConfig.SecurityType, dataConfig.Type, dataConfig.TickType, dataConfig.Resolution)) { - return !symbol.Value.Contains("universe", StringComparison.InvariantCultureIgnoreCase) && - !symbol.IsCanonical() && - symbol.SecurityType == SecurityType.Future; + return null; } - /// - /// Subscribe to the specified configuration - /// - /// defines the parameters to subscribe to a data feed - /// handler to be fired on new data available - /// The new enumerator for this subscription request - public IEnumerator? Subscribe(SubscriptionDataConfig dataConfig, EventHandler newDataAvailableHandler) + lock (_sessionLock) { - if (!IsSupported(dataConfig.SecurityType, dataConfig.Type, dataConfig.TickType, dataConfig.Resolution)) + if (!_sessionStarted) { - return null; + Log.Debug("DataBentoProvider.SubscriptionLogic(): Starting session"); + _sessionStarted = _client.StartSession(); } + } - lock (_sessionLock) - { - if (!_sessionStarted) - { - Log.Debug("DataBentoProvider.SubscriptionLogic(): Starting session"); - _sessionStarted = _client.StartSession(); - } - } + var enumerator = _dataAggregator.Add(dataConfig, newDataAvailableHandler); + _subscriptionManager.Subscribe(dataConfig); - var enumerator = _dataAggregator.Add(dataConfig, newDataAvailableHandler); - _subscriptionManager.Subscribe(dataConfig); + return enumerator; + } - return enumerator; - } + /// + /// Removes the specified configuration + /// + /// Subscription config to be removed + public void Unsubscribe(SubscriptionDataConfig dataConfig) + { + Log.Debug($"DataBentoProvider.Unsubscribe(): Received unsubscription request for {dataConfig.Symbol}, Resolution={dataConfig.Resolution}, TickType={dataConfig.TickType}"); + _subscriptionManager.Unsubscribe(dataConfig); + _dataAggregator.Remove(dataConfig); + } - /// - /// Removes the specified configuration - /// - /// Subscription config to be removed - public void Unsubscribe(SubscriptionDataConfig dataConfig) + /// + /// Sets the job we're subscribing for + /// + /// Job we're subscribing for + public void SetJob(LiveNodePacket job) + { + if (_initialized) { - Log.Debug($"DataBentoProvider.Unsubscribe(): Received unsubscription request for {dataConfig.Symbol}, Resolution={dataConfig.Resolution}, TickType={dataConfig.TickType}"); - _subscriptionManager.Unsubscribe(dataConfig); - _dataAggregator.Remove(dataConfig); + return; } + } + + /// + /// Dispose of unmanaged resources. + /// + public void Dispose() + { + _dataAggregator?.DisposeSafely(); + _subscriptionManager?.DisposeSafely(); + _client?.DisposeSafely(); + } - /// - /// Sets the job we're subscribing for - /// - /// Job we're subscribing for - public void SetJob(LiveNodePacket job) + /// + /// Determines if the specified subscription is supported + /// + private bool IsSupported(SecurityType securityType, Type dataType, TickType tickType, Resolution resolution) + { + // Check supported data types + if (dataType != typeof(TradeBar) && + dataType != typeof(QuoteBar) && + dataType != typeof(Tick) && + dataType != typeof(OpenInterest)) { - if (_initialized) - { - return; - } + throw new NotSupportedException($"Unsupported data type: {dataType}"); } - /// - /// Dispose of unmanaged resources. - /// - public void Dispose() + // Warn about potential limitations for tick data + // I'm mimicing polygon implementation with this + if (!_potentialUnsupportedResolutionMessageLogged) { - _dataAggregator?.DisposeSafely(); - _subscriptionManager?.DisposeSafely(); - _client?.DisposeSafely(); + _potentialUnsupportedResolutionMessageLogged = true; + Log.Trace("DataBentoDataProvider.IsSupported(): " + + $"Subscription for {securityType}-{dataType}-{tickType}-{resolution} will be attempted. " + + $"An Advanced DataBento subscription plan is required to stream tick data."); } - /// - /// Determines if the specified subscription is supported - /// - private bool IsSupported(SecurityType securityType, Type dataType, TickType tickType, Resolution resolution) + return true; + } + + /// + /// Converts the given UTC time into the symbol security exchange time zone + /// + private DateTime GetTickTime(Symbol symbol, DateTime utcTime) + { + DateTimeZone exchangeTimeZone; + lock (_symbolExchangeTimeZones) { - // Check supported data types - if (dataType != typeof(TradeBar) && - dataType != typeof(QuoteBar) && - dataType != typeof(Tick) && - dataType != typeof(OpenInterest)) + if (!_symbolExchangeTimeZones.TryGetValue(symbol, out exchangeTimeZone)) { - throw new NotSupportedException($"Unsupported data type: {dataType}"); - } + // read the exchange time zone from market-hours-database + if (_marketHoursDatabase.TryGetEntry(symbol.ID.Market, symbol, symbol.SecurityType, out var entry)) + { + exchangeTimeZone = entry.ExchangeHours.TimeZone; + } + // If there is no entry for the given Symbol, default to New York + else + { + exchangeTimeZone = TimeZones.NewYork; + } - // Warn about potential limitations for tick data - // I'm mimicing polygon implementation with this - if (!_potentialUnsupportedResolutionMessageLogged) - { - _potentialUnsupportedResolutionMessageLogged = true; - Log.Trace("DataBentoDataProvider.IsSupported(): " + - $"Subscription for {securityType}-{dataType}-{tickType}-{resolution} will be attempted. " + - $"An Advanced DataBento subscription plan is required to stream tick data."); + _symbolExchangeTimeZones[symbol] = exchangeTimeZone; } - - return true; } - /// - /// Converts the given UTC time into the symbol security exchange time zone - /// - private DateTime GetTickTime(Symbol symbol, DateTime utcTime) + return utcTime.ConvertFromUtc(exchangeTimeZone); + } + + /// + /// Handles data received from the live client + /// + private void OnDataReceived(object _, BaseData data) + { + try { - DateTimeZone exchangeTimeZone; - lock (_symbolExchangeTimeZones) + switch (data) { - if (!_symbolExchangeTimeZones.TryGetValue(symbol, out exchangeTimeZone)) - { - // read the exchange time zone from market-hours-database - if (_marketHoursDatabase.TryGetEntry(symbol.ID.Market, symbol, symbol.SecurityType, out var entry)) + case Tick tick: + tick.Time = GetTickTime(tick.Symbol, tick.Time); + lock (_dataAggregator) { - exchangeTimeZone = entry.ExchangeHours.TimeZone; + _dataAggregator.Update(tick); } - // If there is no entry for the given Symbol, default to New York - else + // Log.Trace($"DataBentoProvider.OnDataReceived(): Updated tick - Symbol: {tick.Symbol}, " + + // $"TickType: {tick.TickType}, Price: {tick.Value}, Quantity: {tick.Quantity}"); + break; + + case TradeBar tradeBar: + tradeBar.Time = GetTickTime(tradeBar.Symbol, tradeBar.Time); + tradeBar.EndTime = GetTickTime(tradeBar.Symbol, tradeBar.EndTime); + lock (_dataAggregator) { - exchangeTimeZone = TimeZones.NewYork; + _dataAggregator.Update(tradeBar); } + // Log.Trace($"DataBentoProvider.OnDataReceived(): Updated TradeBar - Symbol: {tradeBar.Symbol}, " + + // $"O:{tradeBar.Open} H:{tradeBar.High} L:{tradeBar.Low} C:{tradeBar.Close} V:{tradeBar.Volume}"); + break; - _symbolExchangeTimeZones[symbol] = exchangeTimeZone; - } + default: + data.Time = GetTickTime(data.Symbol, data.Time); + lock (_dataAggregator) + { + _dataAggregator.Update(data); + } + break; } - - return utcTime.ConvertFromUtc(exchangeTimeZone); } - - /// - /// Handles data received from the live client - /// - private void OnDataReceived(object _, BaseData data) + catch (Exception ex) { - try - { - switch (data) - { - case Tick tick: - tick.Time = GetTickTime(tick.Symbol, tick.Time); - lock (_dataAggregator) - { - _dataAggregator.Update(tick); - } - // Log.Trace($"DataBentoProvider.OnDataReceived(): Updated tick - Symbol: {tick.Symbol}, " + - // $"TickType: {tick.TickType}, Price: {tick.Value}, Quantity: {tick.Quantity}"); - break; - - case TradeBar tradeBar: - tradeBar.Time = GetTickTime(tradeBar.Symbol, tradeBar.Time); - tradeBar.EndTime = GetTickTime(tradeBar.Symbol, tradeBar.EndTime); - lock (_dataAggregator) - { - _dataAggregator.Update(tradeBar); - } - // Log.Trace($"DataBentoProvider.OnDataReceived(): Updated TradeBar - Symbol: {tradeBar.Symbol}, " + - // $"O:{tradeBar.Open} H:{tradeBar.High} L:{tradeBar.Low} C:{tradeBar.Close} V:{tradeBar.Volume}"); - break; - - default: - data.Time = GetTickTime(data.Symbol, data.Time); - lock (_dataAggregator) - { - _dataAggregator.Update(data); - } - break; - } - } - catch (Exception ex) - { - Log.Error($"DataBentoProvider.OnDataReceived(): Error updating data aggregator: {ex.Message}\n{ex.StackTrace}"); - } + Log.Error($"DataBentoProvider.OnDataReceived(): Error updating data aggregator: {ex.Message}\n{ex.StackTrace}"); } } } diff --git a/QuantConnect.DataBento/DataBentoHistoryProivder.cs b/QuantConnect.DataBento/DataBentoHistoryProivder.cs index b92744e..2485ff5 100644 --- a/QuantConnect.DataBento/DataBentoHistoryProivder.cs +++ b/QuantConnect.DataBento/DataBentoHistoryProivder.cs @@ -24,226 +24,225 @@ using QuantConnect.Lean.Engine.DataFeeds; using QuantConnect.Lean.Engine.HistoricalData; -namespace QuantConnect.Lean.DataSource.DataBento +namespace QuantConnect.Lean.DataSource.DataBento; + +/// +/// Implements a history provider for DataBento historical data. +/// Uses consolidators to produce the requested resolution when necessary. +/// +public partial class DataBentoProvider : SynchronizingHistoryProvider { + private static int _dataPointCount; + /// - /// Implements a history provider for DataBento historical data. - /// Uses consolidators to produce the requested resolution when necessary. + /// Indicates whether a error for an invalid start time has been fired, where the start time is greater than or equal to the end time in UTC. /// - public partial class DataBentoProvider : SynchronizingHistoryProvider - { - private static int _dataPointCount; - - /// - /// Indicates whether a error for an invalid start time has been fired, where the start time is greater than or equal to the end time in UTC. - /// - private volatile bool _invalidStartTimeErrorFired; - - /// - /// Indicates whether the warning for invalid has been fired. - /// - private volatile bool _invalidSecurityTypeWarningFired; - - /// - /// Gets the total number of data points emitted by this history provider - /// - public override int DataPointCount => _dataPointCount; - - /// - /// Initializes this history provider to work for the specified job - /// - /// The initialization parameters - public override void Initialize(HistoryProviderInitializeParameters parameters) - { - } + private volatile bool _invalidStartTimeErrorFired; - /// - /// Gets the history for the requested securities - /// - /// The historical data requests - /// The time zone used when time stamping the slice instances - /// An enumerable of the slices of data covering the span specified in each request - public override IEnumerable? GetHistory(IEnumerable requests, DateTimeZone sliceTimeZone) - { - var subscriptions = new List(); - foreach (var request in requests) - { - var history = request.SplitHistoryRequestWithUpdatedMappedSymbol(_mapFileProvider).SelectMany(x => GetHistory(x) ?? []); + /// + /// Indicates whether the warning for invalid has been fired. + /// + private volatile bool _invalidSecurityTypeWarningFired; - var subscription = CreateSubscription(request, history); - if (!subscription.MoveNext()) - { - continue; - } + /// + /// Gets the total number of data points emitted by this history provider + /// + public override int DataPointCount => _dataPointCount; - subscriptions.Add(subscription); - } + /// + /// Initializes this history provider to work for the specified job + /// + /// The initialization parameters + public override void Initialize(HistoryProviderInitializeParameters parameters) + { + } + + /// + /// Gets the history for the requested securities + /// + /// The historical data requests + /// The time zone used when time stamping the slice instances + /// An enumerable of the slices of data covering the span specified in each request + public override IEnumerable? GetHistory(IEnumerable requests, DateTimeZone sliceTimeZone) + { + var subscriptions = new List(); + foreach (var request in requests) + { + var history = request.SplitHistoryRequestWithUpdatedMappedSymbol(_mapFileProvider).SelectMany(x => GetHistory(x) ?? []); - if (subscriptions.Count == 0) + var subscription = CreateSubscription(request, history); + if (!subscription.MoveNext()) { - return null; + continue; } - return CreateSliceEnumerableFromSubscriptions(subscriptions, sliceTimeZone); + + subscriptions.Add(subscription); } - /// - /// Gets the history for the requested security - /// - /// The historical data request - /// An enumerable of BaseData points - public IEnumerable? GetHistory(HistoryRequest historyRequest) + if (subscriptions.Count == 0) { - if (!CanSubscribe(historyRequest.Symbol)) - { - if (!_invalidSecurityTypeWarningFired) - { - _invalidSecurityTypeWarningFired = true; - LogTrace(nameof(GetHistory), $"Unsupported SecurityType '{historyRequest.Symbol.SecurityType}' for symbol '{historyRequest.Symbol}'."); - } - return null; - } + return null; + } + return CreateSliceEnumerableFromSubscriptions(subscriptions, sliceTimeZone); + } - if (historyRequest.EndTimeUtc < historyRequest.StartTimeUtc) + /// + /// Gets the history for the requested security + /// + /// The historical data request + /// An enumerable of BaseData points + public IEnumerable? GetHistory(HistoryRequest historyRequest) + { + if (!CanSubscribe(historyRequest.Symbol)) + { + if (!_invalidSecurityTypeWarningFired) { - if (!_invalidStartTimeErrorFired) - { - _invalidStartTimeErrorFired = true; - Log.Error($"{nameof(DataBentoProvider)}.{nameof(GetHistory)}: Invalid date range: the start date must be earlier than the end date."); - } - return null; + _invalidSecurityTypeWarningFired = true; + LogTrace(nameof(GetHistory), $"Unsupported SecurityType '{historyRequest.Symbol.SecurityType}' for symbol '{historyRequest.Symbol}'."); } + return null; + } - var history = default(IEnumerable); - var brokerageSymbol = _symbolMapper.GetBrokerageSymbol(historyRequest.Symbol); - switch (historyRequest.TickType) + if (historyRequest.EndTimeUtc < historyRequest.StartTimeUtc) + { + if (!_invalidStartTimeErrorFired) { - case TickType.Trade when historyRequest.Resolution == Resolution.Tick: - history = GetHistoryThroughDataConsolidator(historyRequest, brokerageSymbol); - break; - case TickType.Trade: - history = GetAggregatedTradeBars(historyRequest, brokerageSymbol); - break; - case TickType.Quote: - history = GetHistoryThroughDataConsolidator(historyRequest, brokerageSymbol); - break; - case TickType.OpenInterest: - history = GetOpenInterestBars(historyRequest, brokerageSymbol); - break; - default: - throw new ArgumentException(""); + _invalidStartTimeErrorFired = true; + Log.Error($"{nameof(DataBentoProvider)}.{nameof(GetHistory)}: Invalid date range: the start date must be earlier than the end date."); } + return null; + } - if (history == null) - { - return null; - } + var history = default(IEnumerable); + var brokerageSymbol = _symbolMapper.GetBrokerageSymbol(historyRequest.Symbol); + switch (historyRequest.TickType) + { + case TickType.Trade when historyRequest.Resolution == Resolution.Tick: + history = GetHistoryThroughDataConsolidator(historyRequest, brokerageSymbol); + break; + case TickType.Trade: + history = GetAggregatedTradeBars(historyRequest, brokerageSymbol); + break; + case TickType.Quote: + history = GetHistoryThroughDataConsolidator(historyRequest, brokerageSymbol); + break; + case TickType.OpenInterest: + history = GetOpenInterestBars(historyRequest, brokerageSymbol); + break; + default: + throw new ArgumentException(""); + } - return FilterHistory(history, historyRequest, historyRequest.StartTimeLocal, historyRequest.EndTimeLocal); + if (history == null) + { + return null; } - private static IEnumerable FilterHistory(IEnumerable history, HistoryRequest request, DateTime startTimeLocal, DateTime endTimeLocal) + return FilterHistory(history, historyRequest, historyRequest.StartTimeLocal, historyRequest.EndTimeLocal); + } + + private static IEnumerable FilterHistory(IEnumerable history, HistoryRequest request, DateTime startTimeLocal, DateTime endTimeLocal) + { + // cleaning the data before returning it back to user + foreach (var bar in history) { - // cleaning the data before returning it back to user - foreach (var bar in history) + if (bar.Time >= startTimeLocal && bar.EndTime <= endTimeLocal) { - if (bar.Time >= startTimeLocal && bar.EndTime <= endTimeLocal) + if (request.ExchangeHours.IsOpen(bar.Time, bar.EndTime, request.IncludeExtendedMarketHours)) { - if (request.ExchangeHours.IsOpen(bar.Time, bar.EndTime, request.IncludeExtendedMarketHours)) - { - Interlocked.Increment(ref _dataPointCount); - yield return bar; - } + Interlocked.Increment(ref _dataPointCount); + yield return bar; } } } + } - private IEnumerable GetOpenInterestBars(HistoryRequest request, string brokerageSymbol) - { - foreach (var oi in _historicalApiClient.GetOpenInterest(brokerageSymbol, request.StartTimeUtc, request.EndTimeUtc)) - { - yield return new OpenInterest(oi.Header.UtcTime.ConvertFromUtc(request.DataTimeZone), request.Symbol, oi.Quantity); - } + private IEnumerable GetOpenInterestBars(HistoryRequest request, string brokerageSymbol) + { + foreach (var oi in _historicalApiClient.GetOpenInterest(brokerageSymbol, request.StartTimeUtc, request.EndTimeUtc)) + { + yield return new OpenInterest(oi.Header.UtcTime.ConvertFromUtc(request.DataTimeZone), request.Symbol, oi.Quantity); } + } - private IEnumerable GetAggregatedTradeBars(HistoryRequest request, string brokerageSymbol) + private IEnumerable GetAggregatedTradeBars(HistoryRequest request, string brokerageSymbol) + { + var period = request.Resolution.ToTimeSpan(); + foreach (var b in _historicalApiClient.GetHistoricalOhlcvBars(brokerageSymbol, request.StartTimeUtc, request.EndTimeUtc, request.Resolution, request.TickType)) { - var period = request.Resolution.ToTimeSpan(); - foreach (var b in _historicalApiClient.GetHistoricalOhlcvBars(brokerageSymbol, request.StartTimeUtc, request.EndTimeUtc, request.Resolution, request.TickType)) - { - yield return new TradeBar(b.Header.UtcTime.ConvertFromUtc(request.DataTimeZone), request.Symbol, b.Open, b.High, b.Low, b.Close, b.Volume, period); - } + yield return new TradeBar(b.Header.UtcTime.ConvertFromUtc(request.DataTimeZone), request.Symbol, b.Open, b.High, b.Low, b.Close, b.Volume, period); } + } - private IEnumerable? GetHistoryThroughDataConsolidator(HistoryRequest request, string brokerageSymbol) - { - IDataConsolidator consolidator; - IEnumerable history; + private IEnumerable? GetHistoryThroughDataConsolidator(HistoryRequest request, string brokerageSymbol) + { + IDataConsolidator consolidator; + IEnumerable history; - if (request.TickType == TickType.Trade) - { - consolidator = request.Resolution != Resolution.Tick - ? new TickConsolidator(request.Resolution.ToTimeSpan()) - : FilteredIdentityDataConsolidator.ForTickType(request.TickType); - history = GetTrades(request, brokerageSymbol); - } - else - { - consolidator = request.Resolution != Resolution.Tick - ? new TickQuoteBarConsolidator(request.Resolution.ToTimeSpan()) - : FilteredIdentityDataConsolidator.ForTickType(request.TickType); - history = GetQuotes(request, brokerageSymbol); - } + if (request.TickType == TickType.Trade) + { + consolidator = request.Resolution != Resolution.Tick + ? new TickConsolidator(request.Resolution.ToTimeSpan()) + : FilteredIdentityDataConsolidator.ForTickType(request.TickType); + history = GetTrades(request, brokerageSymbol); + } + else + { + consolidator = request.Resolution != Resolution.Tick + ? new TickQuoteBarConsolidator(request.Resolution.ToTimeSpan()) + : FilteredIdentityDataConsolidator.ForTickType(request.TickType); + history = GetQuotes(request, brokerageSymbol); + } - BaseData? consolidatedData = null; - DataConsolidatedHandler onDataConsolidated = (s, e) => - { - consolidatedData = (BaseData)e; - }; - consolidator.DataConsolidated += onDataConsolidated; + BaseData? consolidatedData = null; + DataConsolidatedHandler onDataConsolidated = (s, e) => + { + consolidatedData = (BaseData)e; + }; + consolidator.DataConsolidated += onDataConsolidated; - foreach (var data in history) + foreach (var data in history) + { + consolidator.Update(data); + if (consolidatedData != null) { - consolidator.Update(data); - if (consolidatedData != null) - { - yield return consolidatedData; - consolidatedData = null; - } + yield return consolidatedData; + consolidatedData = null; } - - consolidator.DataConsolidated -= onDataConsolidated; - consolidator.DisposeSafely(); } - /// - /// Gets the trade ticks that will potentially be aggregated for the specified history request - /// - private IEnumerable GetTrades(HistoryRequest request, string brokerageSymbol) + consolidator.DataConsolidated -= onDataConsolidated; + consolidator.DisposeSafely(); + } + + /// + /// Gets the trade ticks that will potentially be aggregated for the specified history request + /// + private IEnumerable GetTrades(HistoryRequest request, string brokerageSymbol) + { + foreach (var t in _historicalApiClient.GetTickBars(brokerageSymbol, request.StartTimeUtc, request.EndTimeUtc)) { - foreach (var t in _historicalApiClient.GetTickBars(brokerageSymbol, request.StartTimeUtc, request.EndTimeUtc)) - { - yield return new Tick(t.Header.UtcTime.ConvertFromUtc(request.DataTimeZone), request.Symbol, "", "", t.Size, t.Price); - } + yield return new Tick(t.Header.UtcTime.ConvertFromUtc(request.DataTimeZone), request.Symbol, "", "", t.Size, t.Price); } + } - /// - /// Gets the quote ticks that will potentially be aggregated for the specified history request - /// - private IEnumerable GetQuotes(HistoryRequest request, string brokerageSymbol) + /// + /// Gets the quote ticks that will potentially be aggregated for the specified history request + /// + private IEnumerable GetQuotes(HistoryRequest request, string brokerageSymbol) + { + foreach (var quoteBar in _historicalApiClient.GetTickBars(brokerageSymbol, request.StartTimeUtc, request.EndTimeUtc)) { - foreach (var quoteBar in _historicalApiClient.GetTickBars(brokerageSymbol, request.StartTimeUtc, request.EndTimeUtc)) + var time = quoteBar.Header.UtcTime.ConvertFromUtc(request.DataTimeZone); + foreach (var level in quoteBar.Levels) { - var time = quoteBar.Header.UtcTime.ConvertFromUtc(request.DataTimeZone); - foreach (var level in quoteBar.Levels) - { - yield return new Tick(time, request.Symbol, level.BidSz, level.BidPx, level.AskSz, level.AskPx); - } + yield return new Tick(time, request.Symbol, level.BidSz, level.BidPx, level.AskSz, level.AskPx); } } + } - private static void LogTrace(string methodName, string message) - { - Log.Trace($"{nameof(DataBentoProvider)}.{methodName}: {message}"); - } + private static void LogTrace(string methodName, string message) + { + Log.Trace($"{nameof(DataBentoProvider)}.{methodName}: {message}"); } } diff --git a/QuantConnect.DataBento/DataBentoRawLiveClient.cs b/QuantConnect.DataBento/DataBentoRawLiveClient.cs index 8d8277c..9c65544 100644 --- a/QuantConnect.DataBento/DataBentoRawLiveClient.cs +++ b/QuantConnect.DataBento/DataBentoRawLiveClient.cs @@ -19,678 +19,676 @@ using System.Security.Cryptography; using System.Collections.Concurrent; using System.Text.Json; -using System.Threading.Tasks; using QuantConnect.Data; using QuantConnect.Data.Market; using QuantConnect.Logging; -namespace QuantConnect.Lean.DataSource.DataBento +namespace QuantConnect.Lean.DataSource.DataBento; + +/// +/// DataBento Raw TCP client for live streaming data +/// +public class DataBentoRawLiveClient : IDisposable { /// - /// DataBento Raw TCP client for live streaming data + /// The DataBento API key for authentication + /// + private readonly string _apiKey; + /// + /// The DataBento live gateway address to receive data from + /// + private const string _gateway = "glbx-mdp3.lsg.databento.com:13000"; + /// + /// The dataset to subscribe to + /// + private readonly string _dataset; + private readonly TcpClient? _tcpClient; + private readonly string _host; + private readonly int _port; + private NetworkStream? _stream; + private StreamReader _reader; + private StreamWriter _writer; + private readonly CancellationTokenSource _cancellationTokenSource; + private readonly ConcurrentDictionary _subscriptions; + private readonly object _connectionLock = new object(); + private bool _isConnected; + private bool _disposed; + private const decimal PriceScaleFactor = 1e-9m; + private readonly ConcurrentDictionary _instrumentIdToSymbol = new ConcurrentDictionary(); + private readonly DataBentoSymbolMapper _symbolMapper; + + /// + /// Event fired when new data is received + /// + public event EventHandler DataReceived; + + /// + /// Event fired when connection status changes /// - public class DataBentoRawLiveClient : IDisposable + public event EventHandler ConnectionStatusChanged; + + /// + /// Gets whether the client is currently connected + /// + public bool IsConnected => _isConnected && _tcpClient?.Connected == true; + + /// + /// Initializes a new instance of the DataBentoRawLiveClient + /// The DataBento API key. + /// + public DataBentoRawLiveClient(string apiKey, string dataset = "GLBX.MDP3") { - /// - /// The DataBento API key for authentication - /// - private readonly string _apiKey; - /// - /// The DataBento live gateway address to receive data from - /// - private const string _gateway = "glbx-mdp3.lsg.databento.com:13000"; - /// - /// The dataset to subscribe to - /// - private readonly string _dataset; - private readonly TcpClient? _tcpClient; - private readonly string _host; - private readonly int _port; - private NetworkStream? _stream; - private StreamReader _reader; - private StreamWriter _writer; - private readonly CancellationTokenSource _cancellationTokenSource; - private readonly ConcurrentDictionary _subscriptions; - private readonly object _connectionLock = new object(); - private bool _isConnected; - private bool _disposed; - private const decimal PriceScaleFactor = 1e-9m; - private readonly ConcurrentDictionary _instrumentIdToSymbol = new ConcurrentDictionary(); - private readonly DataBentoSymbolMapper _symbolMapper; - - /// - /// Event fired when new data is received - /// - public event EventHandler DataReceived; - - /// - /// Event fired when connection status changes - /// - public event EventHandler ConnectionStatusChanged; - - /// - /// Gets whether the client is currently connected - /// - public bool IsConnected => _isConnected && _tcpClient?.Connected == true; - - /// - /// Initializes a new instance of the DataBentoRawLiveClient - /// The DataBento API key. - /// - public DataBentoRawLiveClient(string apiKey, string dataset = "GLBX.MDP3") + _apiKey = apiKey ?? throw new ArgumentNullException(nameof(apiKey)); + _dataset = dataset; + _tcpClient = new TcpClient(); + _subscriptions = new ConcurrentDictionary(); + _cancellationTokenSource = new CancellationTokenSource(); + _symbolMapper = new DataBentoSymbolMapper(); + + var parts = _gateway.Split(':'); + _host = parts[0]; + _port = parts.Length > 1 ? int.Parse(parts[1]) : 13000; + } + + /// + /// Connects to the DataBento live gateway + /// + public bool Connect() + { + Log.Trace("DataBentoRawLiveClient.Connect(): Connecting to DataBento live gateway"); + if (_isConnected) { - _apiKey = apiKey ?? throw new ArgumentNullException(nameof(apiKey)); - _dataset = dataset; - _tcpClient = new TcpClient(); - _subscriptions = new ConcurrentDictionary(); - _cancellationTokenSource = new CancellationTokenSource(); - _symbolMapper = new DataBentoSymbolMapper(); - - var parts = _gateway.Split(':'); - _host = parts[0]; - _port = parts.Length > 1 ? int.Parse(parts[1]) : 13000; + return _isConnected; } - /// - /// Connects to the DataBento live gateway - /// - public bool Connect() + try { - Log.Trace("DataBentoRawLiveClient.Connect(): Connecting to DataBento live gateway"); - if (_isConnected) - { - return _isConnected; - } + _tcpClient.Connect(_host, _port); + _stream = _tcpClient.GetStream(); + _reader = new StreamReader(_stream, Encoding.ASCII); + _writer = new StreamWriter(_stream, Encoding.ASCII) { AutoFlush = true }; - try + // Perform authentication handshake + if (Authenticate()) { - _tcpClient.Connect(_host, _port); - _stream = _tcpClient.GetStream(); - _reader = new StreamReader(_stream, Encoding.ASCII); - _writer = new StreamWriter(_stream, Encoding.ASCII) { AutoFlush = true }; + _isConnected = true; + ConnectionStatusChanged?.Invoke(this, true); - // Perform authentication handshake - if (Authenticate()) - { - _isConnected = true; - ConnectionStatusChanged?.Invoke(this, true); - - // Start message processing - Task.Run(ProcessMessages, _cancellationTokenSource.Token); + // Start message processing + Task.Run(ProcessMessages, _cancellationTokenSource.Token); - Log.Trace("DataBentoRawLiveClient.Connect(): Connected and authenticated to DataBento live gateway"); - return true; - } - } - catch (Exception ex) - { - Log.Error($"DataBentoRawLiveClient.Connect(): Failed to connect: {ex.Message}"); - Disconnect(); + Log.Trace("DataBentoRawLiveClient.Connect(): Connected and authenticated to DataBento live gateway"); + return true; } - - return false; } - - /// - /// Authenticates with the DataBento gateway using CRAM-SHA256 - /// - private bool Authenticate() + catch (Exception ex) { - try - { - // Read greeting and challenge - var versionLine = _reader.ReadLine(); - var cramLine = _reader.ReadLine(); + Log.Error($"DataBentoRawLiveClient.Connect(): Failed to connect: {ex.Message}"); + Disconnect(); + } - if (string.IsNullOrEmpty(versionLine) || string.IsNullOrEmpty(cramLine)) - { - Log.Error("DataBentoRawLiveClient.Authenticate(): Failed to receive greeting or challenge"); - return false; - } + return false; + } - // Parse challenge - var cramParts = cramLine.Split('='); - if (cramParts.Length != 2 || cramParts[0] != "cram") - { - Log.Error("DataBentoRawLiveClient.Authenticate(): Invalid challenge format"); - return false; - } - var cram = cramParts[1].Trim(); + /// + /// Authenticates with the DataBento gateway using CRAM-SHA256 + /// + private bool Authenticate() + { + try + { + // Read greeting and challenge + var versionLine = _reader.ReadLine(); + var cramLine = _reader.ReadLine(); - // Auth - _writer.WriteLine($"auth={GetAuthStringFromCram(cram)}|dataset={_dataset}|encoding=json|ts_out=0"); - var authResp = _reader.ReadLine(); - if (!authResp.Contains("success=1")) - { - Log.Error($"DataBentoRawLiveClient.Authenticate(): Authentication failed: {authResp}"); - return false; - } + if (string.IsNullOrEmpty(versionLine) || string.IsNullOrEmpty(cramLine)) + { + Log.Error("DataBentoRawLiveClient.Authenticate(): Failed to receive greeting or challenge"); + return false; + } - Log.Trace("DataBentoRawLiveClient.Authenticate(): Authentication successful"); - return true; + // Parse challenge + var cramParts = cramLine.Split('='); + if (cramParts.Length != 2 || cramParts[0] != "cram") + { + Log.Error("DataBentoRawLiveClient.Authenticate(): Invalid challenge format"); + return false; } - catch (Exception ex) + var cram = cramParts[1].Trim(); + + // Auth + _writer.WriteLine($"auth={GetAuthStringFromCram(cram)}|dataset={_dataset}|encoding=json|ts_out=0"); + var authResp = _reader.ReadLine(); + if (!authResp.Contains("success=1")) { - Log.Error($"DataBentoRawLiveClient.Authenticate(): Authentication failed: {ex.Message}"); + Log.Error($"DataBentoRawLiveClient.Authenticate(): Authentication failed: {authResp}"); return false; } - } - /// - /// Handles the DataBento authentication string from a CRAM challenge - /// - /// The CRAM challenge string - /// The auth string to send to the server - private string GetAuthStringFromCram(string cram) + Log.Trace("DataBentoRawLiveClient.Authenticate(): Authentication successful"); + return true; + } + catch (Exception ex) { - if (string.IsNullOrWhiteSpace(cram)) - throw new ArgumentException("CRAM challenge cannot be null or empty", nameof(cram)); + Log.Error($"DataBentoRawLiveClient.Authenticate(): Authentication failed: {ex.Message}"); + return false; + } + } + + /// + /// Handles the DataBento authentication string from a CRAM challenge + /// + /// The CRAM challenge string + /// The auth string to send to the server + private string GetAuthStringFromCram(string cram) + { + if (string.IsNullOrWhiteSpace(cram)) + throw new ArgumentException("CRAM challenge cannot be null or empty", nameof(cram)); - string concat = $"{cram}|{_apiKey}"; - string hashHex = ComputeSHA256(concat); - string bucketId = _apiKey.Substring(_apiKey.Length - 5); + string concat = $"{cram}|{_apiKey}"; + string hashHex = ComputeSHA256(concat); + string bucketId = _apiKey.Substring(_apiKey.Length - 5); - return $"{hashHex}-{bucketId}"; - } + return $"{hashHex}-{bucketId}"; + } - /// - /// Subscribes to live data for a symbol - /// - public bool Subscribe(Symbol symbol, TickType tickType) + /// + /// Subscribes to live data for a symbol + /// + public bool Subscribe(Symbol symbol, TickType tickType) + { + if (!IsConnected) { - if (!IsConnected) - { - Log.Error("DataBentoRawLiveClient.Subscribe(): Not connected to gateway"); - return false; - } + Log.Error("DataBentoRawLiveClient.Subscribe(): Not connected to gateway"); + return false; + } - try - { - // Get the databento symbol form LEAN symbol - var databentoSymbol = _symbolMapper.GetBrokerageSymbol(symbol); - var schema = "mbp-1"; - var resolution = Resolution.Tick; + try + { + // Get the databento symbol form LEAN symbol + var databentoSymbol = _symbolMapper.GetBrokerageSymbol(symbol); + var schema = "mbp-1"; + var resolution = Resolution.Tick; - // subscribe - var subscribeMessage = $"schema={schema}|stype_in=parent|symbols={databentoSymbol}"; - Log.Debug($"DataBentoRawLiveClient.Subscribe(): Subscribing with message: {subscribeMessage}"); + // subscribe + var subscribeMessage = $"schema={schema}|stype_in=parent|symbols={databentoSymbol}"; + Log.Debug($"DataBentoRawLiveClient.Subscribe(): Subscribing with message: {subscribeMessage}"); - // Send subscribe message - _writer.WriteLine(subscribeMessage); + // Send subscribe message + _writer.WriteLine(subscribeMessage); - // Store subscription - _subscriptions.TryAdd(symbol, (resolution, tickType)); - Log.Debug($"DataBentoRawLiveClient.Subscribe(): Subscribed to {symbol} ({databentoSymbol}) at {resolution} resolution for {tickType}"); + // Store subscription + _subscriptions.TryAdd(symbol, (resolution, tickType)); + Log.Debug($"DataBentoRawLiveClient.Subscribe(): Subscribed to {symbol} ({databentoSymbol}) at {resolution} resolution for {tickType}"); - return true; - } - catch (Exception ex) - { - Log.Error($"DataBentoRawLiveClient.Subscribe(): Failed to subscribe to {symbol}: {ex.Message}"); - return false; - } + return true; + } + catch (Exception ex) + { + Log.Error($"DataBentoRawLiveClient.Subscribe(): Failed to subscribe to {symbol}: {ex.Message}"); + return false; } + } - /// - /// Starts the session to begin receiving data - /// - public bool StartSession() + /// + /// Starts the session to begin receiving data + /// + public bool StartSession() + { + if (!IsConnected) { - if (!IsConnected) - { - Log.Error("DataBentoRawLiveClient.StartSession(): Not connected"); - return false; - } + Log.Error("DataBentoRawLiveClient.StartSession(): Not connected"); + return false; + } - try - { - Log.Trace("DataBentoRawLiveClient.StartSession(): Starting session"); - _writer.WriteLine("start_session=1"); - return true; - } - catch (Exception ex) - { - Log.Error($"DataBentoRawLiveClient.StartSession(): Failed to start session: {ex.Message}"); - return false; - } + try + { + Log.Trace("DataBentoRawLiveClient.StartSession(): Starting session"); + _writer.WriteLine("start_session=1"); + return true; } + catch (Exception ex) + { + Log.Error($"DataBentoRawLiveClient.StartSession(): Failed to start session: {ex.Message}"); + return false; + } + } - /// - /// Unsubscribes from live data for a symbol - /// - public bool Unsubscribe(Symbol symbol) + /// + /// Unsubscribes from live data for a symbol + /// + public bool Unsubscribe(Symbol symbol) + { + try { - try - { - if (_subscriptions.TryRemove(symbol, out _)) - { - Log.Debug($"DataBentoRawLiveClient.Unsubscribe(): Unsubscribed from {symbol}"); - } - return true; - } - catch (Exception ex) + if (_subscriptions.TryRemove(symbol, out _)) { - Log.Error($"DataBentoRawLiveClient.Unsubscribe(): Failed to unsubscribe from {symbol}: {ex.Message}"); - return false; + Log.Debug($"DataBentoRawLiveClient.Unsubscribe(): Unsubscribed from {symbol}"); } + return true; } - - /// - /// Processes incoming messages from the DataBento gateway - /// - private void ProcessMessages() + catch (Exception ex) { - Log.Debug("DataBentoRawLiveClient.ProcessMessages(): Starting message processing"); + Log.Error($"DataBentoRawLiveClient.Unsubscribe(): Failed to unsubscribe from {symbol}: {ex.Message}"); + return false; + } + } - try + /// + /// Processes incoming messages from the DataBento gateway + /// + private void ProcessMessages() + { + Log.Debug("DataBentoRawLiveClient.ProcessMessages(): Starting message processing"); + + try + { + while (!_cancellationTokenSource.IsCancellationRequested && IsConnected) { - while (!_cancellationTokenSource.IsCancellationRequested && IsConnected) + var line = _reader.ReadLine(); + if (string.IsNullOrWhiteSpace(line)) { - var line = _reader.ReadLine(); - if (string.IsNullOrWhiteSpace(line)) - { - Log.Trace("DataBentoRawLiveClient.ProcessMessages(): Line is null or empty. Issue receiving data."); - break; - } - - ProcessSingleMessage(line); + Log.Trace("DataBentoRawLiveClient.ProcessMessages(): Line is null or empty. Issue receiving data."); + break; } + + ProcessSingleMessage(line); } - catch (OperationCanceledException) - { - Log.Trace("DataBentoRawLiveClient.ProcessMessages(): Message processing cancelled"); - } - catch (IOException ex) when (ex.InnerException is SocketException) - { - Log.Trace($"DataBentoRawLiveClient.ProcessMessages(): Socket exception: {ex.Message}"); - } - catch (Exception ex) - { - Log.Error($"DataBentoRawLiveClient.ProcessMessages(): Error processing messages: {ex.Message}\n{ex.StackTrace}"); - } - finally - { - Disconnect(); - } } + catch (OperationCanceledException) + { + Log.Trace("DataBentoRawLiveClient.ProcessMessages(): Message processing cancelled"); + } + catch (IOException ex) when (ex.InnerException is SocketException) + { + Log.Trace($"DataBentoRawLiveClient.ProcessMessages(): Socket exception: {ex.Message}"); + } + catch (Exception ex) + { + Log.Error($"DataBentoRawLiveClient.ProcessMessages(): Error processing messages: {ex.Message}\n{ex.StackTrace}"); + } + finally + { + Disconnect(); + } + } - /// - /// Processes a single message from DataBento - /// - private void ProcessSingleMessage(string message) + /// + /// Processes a single message from DataBento + /// + private void ProcessSingleMessage(string message) + { + try { - try - { - using var document = JsonDocument.Parse(message); - var root = document.RootElement; + using var document = JsonDocument.Parse(message); + var root = document.RootElement; - // Check for error messages - if (root.TryGetProperty("hd", out var headerElement)) + // Check for error messages + if (root.TryGetProperty("hd", out var headerElement)) + { + if (headerElement.TryGetProperty("rtype", out var rtypeElement)) { - if (headerElement.TryGetProperty("rtype", out var rtypeElement)) - { - var rtype = rtypeElement.GetInt32(); + var rtype = rtypeElement.GetInt32(); - switch (rtype) - { - case 23: - // System message - if (root.TryGetProperty("msg", out var msgElement)) - { - Log.Debug($"DataBentoRawLiveClient: System message: {msgElement.GetString()}"); - } - return; - - case 22: - // Symbol mapping message - if (root.TryGetProperty("stype_in_symbol", out var inSymbol) && - root.TryGetProperty("stype_out_symbol", out var outSymbol) && - headerElement.TryGetProperty("instrument_id", out var instId)) + switch (rtype) + { + case 23: + // System message + if (root.TryGetProperty("msg", out var msgElement)) + { + Log.Debug($"DataBentoRawLiveClient: System message: {msgElement.GetString()}"); + } + return; + + case 22: + // Symbol mapping message + if (root.TryGetProperty("stype_in_symbol", out var inSymbol) && + root.TryGetProperty("stype_out_symbol", out var outSymbol) && + headerElement.TryGetProperty("instrument_id", out var instId)) + { + var instrumentId = instId.GetInt64(); + var outSymbolStr = outSymbol.GetString(); + + Log.Debug($"DataBentoRawLiveClient: Symbol mapping: {inSymbol.GetString()} -> {outSymbolStr} (instrument_id: {instrumentId})"); + + if (outSymbolStr != null) { - var instrumentId = instId.GetInt64(); - var outSymbolStr = outSymbol.GetString(); - - Log.Debug($"DataBentoRawLiveClient: Symbol mapping: {inSymbol.GetString()} -> {outSymbolStr} (instrument_id: {instrumentId})"); - - if (outSymbolStr != null) + // Let's find the subscribed symbol to get the market and security type + var inSymbolStr = inSymbol.GetString(); + var subscription = _subscriptions.Keys.FirstOrDefault(s => _symbolMapper.GetBrokerageSymbol(s) == inSymbolStr); + if (subscription != null) { - // Let's find the subscribed symbol to get the market and security type - var inSymbolStr = inSymbol.GetString(); - var subscription = _subscriptions.Keys.FirstOrDefault(s => _symbolMapper.GetBrokerageSymbol(s) == inSymbolStr); - if (subscription != null) + if (subscription.SecurityType == SecurityType.Future) { - if (subscription.SecurityType == SecurityType.Future) + var leanSymbol = _symbolMapper.GetLeanSymbolForFuture(outSymbolStr); + if (leanSymbol == null) { - var leanSymbol = _symbolMapper.GetLeanSymbolForFuture(outSymbolStr); - if (leanSymbol == null) - { - Log.Trace($"DataBentoRawLiveClient: Future spreads are not supported: {outSymbolStr}. Skipping mapping."); - return; - } - _instrumentIdToSymbol[instrumentId] = leanSymbol; - Log.Debug($"DataBentoRawLiveClient: Mapped instrument_id {instrumentId} to {leanSymbol}"); + Log.Trace($"DataBentoRawLiveClient: Future spreads are not supported: {outSymbolStr}. Skipping mapping."); + return; } + _instrumentIdToSymbol[instrumentId] = leanSymbol; + Log.Debug($"DataBentoRawLiveClient: Mapped instrument_id {instrumentId} to {leanSymbol}"); } } } - return; - - case 1: - // MBP-1 (Market By Price) - HandleMBPMessage(root, headerElement); - return; - - case 0: - // Trade messages - HandleTradeTickMessage(root, headerElement); - return; - - case 32: - case 33: - case 34: - case 35: - // OHLCV bar messages - HandleOHLCVMessage(root, headerElement); - return; - - default: - Log.Error($"DataBentoRawLiveClient: Unknown rtype {rtype} in message"); - return; - } + } + return; + + case 1: + // MBP-1 (Market By Price) + HandleMBPMessage(root, headerElement); + return; + + case 0: + // Trade messages + HandleTradeTickMessage(root, headerElement); + return; + + case 32: + case 33: + case 34: + case 35: + // OHLCV bar messages + HandleOHLCVMessage(root, headerElement); + return; + + default: + Log.Error($"DataBentoRawLiveClient: Unknown rtype {rtype} in message"); + return; } } - - // Handle other message types if needed - if (root.TryGetProperty("error", out var errorElement)) - { - Log.Error($"DataBentoRawLiveClient: Server error: {errorElement.GetString()}"); - } } - catch (JsonException ex) - { - Log.Error($"DataBentoRawLiveClient.ProcessSingleMessage(): JSON parse error: {ex.Message}"); - } - catch (Exception ex) + + // Handle other message types if needed + if (root.TryGetProperty("error", out var errorElement)) { - Log.Error($"DataBentoRawLiveClient.ProcessSingleMessage(): Error: {ex.Message}"); + Log.Error($"DataBentoRawLiveClient: Server error: {errorElement.GetString()}"); } } - - /// - /// Handles OHLCV messages and converts to LEAN TradeBar data - /// - private void HandleOHLCVMessage(JsonElement root, JsonElement header) + catch (JsonException ex) { - try + Log.Error($"DataBentoRawLiveClient.ProcessSingleMessage(): JSON parse error: {ex.Message}"); + } + catch (Exception ex) + { + Log.Error($"DataBentoRawLiveClient.ProcessSingleMessage(): Error: {ex.Message}"); + } + } + + /// + /// Handles OHLCV messages and converts to LEAN TradeBar data + /// + private void HandleOHLCVMessage(JsonElement root, JsonElement header) + { + try + { + if (!header.TryGetProperty("ts_event", out var tsElement) || + !header.TryGetProperty("instrument_id", out var instIdElement)) { - if (!header.TryGetProperty("ts_event", out var tsElement) || - !header.TryGetProperty("instrument_id", out var instIdElement)) - { - return; - } + return; + } - // Convert timestamp from nanoseconds to DateTime - var timestampNs = long.Parse(tsElement.GetString()!); - var unixEpoch = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc); - var timestamp = unixEpoch.AddTicks(timestampNs / 100); + // Convert timestamp from nanoseconds to DateTime + var timestampNs = long.Parse(tsElement.GetString()!); + var unixEpoch = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc); + var timestamp = unixEpoch.AddTicks(timestampNs / 100); - var instrumentId = instIdElement.GetInt64(); + var instrumentId = instIdElement.GetInt64(); - if (!_instrumentIdToSymbol.TryGetValue(instrumentId, out var matchedSymbol)) - { - Log.Debug($"DataBentoRawLiveClient: No mapping for instrument_id {instrumentId} in OHLCV message."); - return; - } + if (!_instrumentIdToSymbol.TryGetValue(instrumentId, out var matchedSymbol)) + { + Log.Debug($"DataBentoRawLiveClient: No mapping for instrument_id {instrumentId} in OHLCV message."); + return; + } - // Get the resolution for this symbol - if (!_subscriptions.TryGetValue(matchedSymbol, out var subscription)) - { - return; - } + // Get the resolution for this symbol + if (!_subscriptions.TryGetValue(matchedSymbol, out var subscription)) + { + return; + } - var resolution = subscription.Item1; + var resolution = subscription.Item1; - // Extract OHLCV data - if (root.TryGetProperty("open", out var openElement) && - root.TryGetProperty("high", out var highElement) && - root.TryGetProperty("low", out var lowElement) && - root.TryGetProperty("close", out var closeElement) && - root.TryGetProperty("volume", out var volumeElement)) - { - // Parse prices - var openRaw = long.Parse(openElement.GetString()!); - var highRaw = long.Parse(highElement.GetString()!); - var lowRaw = long.Parse(lowElement.GetString()!); - var closeRaw = long.Parse(closeElement.GetString()!); - var volume = volumeElement.GetInt64(); - - var open = openRaw * PriceScaleFactor; - var high = highRaw * PriceScaleFactor; - var low = lowRaw * PriceScaleFactor; - var close = closeRaw * PriceScaleFactor; - - // Determine the period based on resolution - TimeSpan period = resolution switch - { - Resolution.Second => TimeSpan.FromSeconds(1), - Resolution.Minute => TimeSpan.FromMinutes(1), - Resolution.Hour => TimeSpan.FromHours(1), - Resolution.Daily => TimeSpan.FromDays(1), - _ => TimeSpan.FromMinutes(1) - }; - - // Create TradeBar - var tradeBar = new TradeBar( - timestamp, - matchedSymbol, - open, - high, - low, - close, - volume, - period - ); - - // Log.Trace($"DataBentoRawLiveClient: OHLCV bar: {matchedSymbol} O={open} H={high} L={low} C={close} V={volume} at {timestamp}"); - DataReceived?.Invoke(this, tradeBar); - } - } - catch (Exception ex) + // Extract OHLCV data + if (root.TryGetProperty("open", out var openElement) && + root.TryGetProperty("high", out var highElement) && + root.TryGetProperty("low", out var lowElement) && + root.TryGetProperty("close", out var closeElement) && + root.TryGetProperty("volume", out var volumeElement)) { - Log.Error($"DataBentoRawLiveClient.HandleOHLCVMessage(): Error: {ex.Message}"); + // Parse prices + var openRaw = long.Parse(openElement.GetString()!); + var highRaw = long.Parse(highElement.GetString()!); + var lowRaw = long.Parse(lowElement.GetString()!); + var closeRaw = long.Parse(closeElement.GetString()!); + var volume = volumeElement.GetInt64(); + + var open = openRaw * PriceScaleFactor; + var high = highRaw * PriceScaleFactor; + var low = lowRaw * PriceScaleFactor; + var close = closeRaw * PriceScaleFactor; + + // Determine the period based on resolution + TimeSpan period = resolution switch + { + Resolution.Second => TimeSpan.FromSeconds(1), + Resolution.Minute => TimeSpan.FromMinutes(1), + Resolution.Hour => TimeSpan.FromHours(1), + Resolution.Daily => TimeSpan.FromDays(1), + _ => TimeSpan.FromMinutes(1) + }; + + // Create TradeBar + var tradeBar = new TradeBar( + timestamp, + matchedSymbol, + open, + high, + low, + close, + volume, + period + ); + + // Log.Trace($"DataBentoRawLiveClient: OHLCV bar: {matchedSymbol} O={open} H={high} L={low} C={close} V={volume} at {timestamp}"); + DataReceived?.Invoke(this, tradeBar); } } + catch (Exception ex) + { + Log.Error($"DataBentoRawLiveClient.HandleOHLCVMessage(): Error: {ex.Message}"); + } + } - /// - /// Handles MBP messages for quote ticks - /// - private void HandleMBPMessage(JsonElement root, JsonElement header) + /// + /// Handles MBP messages for quote ticks + /// + private void HandleMBPMessage(JsonElement root, JsonElement header) + { + try { - try + if (!header.TryGetProperty("ts_event", out var tsElement) || + !header.TryGetProperty("instrument_id", out var instIdElement)) { - if (!header.TryGetProperty("ts_event", out var tsElement) || - !header.TryGetProperty("instrument_id", out var instIdElement)) - { - return; - } - - // Convert timestamp from nanoseconds to DateTime - var timestampNs = long.Parse(tsElement.GetString()!); - var unixEpoch = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc); - var timestamp = unixEpoch.AddTicks(timestampNs / 100); + return; + } - var instrumentId = instIdElement.GetInt64(); + // Convert timestamp from nanoseconds to DateTime + var timestampNs = long.Parse(tsElement.GetString()!); + var unixEpoch = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc); + var timestamp = unixEpoch.AddTicks(timestampNs / 100); - if (!_instrumentIdToSymbol.TryGetValue(instrumentId, out var matchedSymbol)) - { - Log.Trace($"DataBentoRawLiveClient: No mapping for instrument_id {instrumentId} in MBP message."); - return; - } + var instrumentId = instIdElement.GetInt64(); - // For MBP-1, bid/ask data is in the levels array at index 0 - if (root.TryGetProperty("levels", out var levelsElement) && - levelsElement.GetArrayLength() > 0) - { - var level0 = levelsElement[0]; - - var quoteTick = new Tick - { - Symbol = matchedSymbol, - Time = timestamp, - TickType = TickType.Quote - }; - - if (level0.TryGetProperty("ask_px", out var askPxElement) && - level0.TryGetProperty("ask_sz", out var askSzElement)) - { - var askPriceRaw = long.Parse(askPxElement.GetString()!); - quoteTick.AskPrice = askPriceRaw * PriceScaleFactor; - quoteTick.AskSize = askSzElement.GetInt32(); - } - - if (level0.TryGetProperty("bid_px", out var bidPxElement) && - level0.TryGetProperty("bid_sz", out var bidSzElement)) - { - var bidPriceRaw = long.Parse(bidPxElement.GetString()!); - quoteTick.BidPrice = bidPriceRaw * PriceScaleFactor; - quoteTick.BidSize = bidSzElement.GetInt32(); - } - - // Set the tick value to the mid price - quoteTick.Value = (quoteTick.BidPrice + quoteTick.AskPrice) / 2; - - // QuantConnect convention: Quote ticks should have zero Price and Quantity - quoteTick.Quantity = 0; - - // Log.Trace($"DataBentoRawLiveClient: Quote tick: {matchedSymbol} Bid={quoteTick.BidPrice}x{quoteTick.BidSize} Ask={quoteTick.AskPrice}x{quoteTick.AskSize}"); - DataReceived?.Invoke(this, quoteTick); - } - } - catch (Exception ex) + if (!_instrumentIdToSymbol.TryGetValue(instrumentId, out var matchedSymbol)) { - Log.Error($"DataBentoRawLiveClient.HandleMBPMessage(): Error: {ex.Message}"); + Log.Trace($"DataBentoRawLiveClient: No mapping for instrument_id {instrumentId} in MBP message."); + return; } - } - /// - /// Handles trade tick messages. Aggressor fills - /// - private void HandleTradeTickMessage(JsonElement root, JsonElement header) - { - try + // For MBP-1, bid/ask data is in the levels array at index 0 + if (root.TryGetProperty("levels", out var levelsElement) && + levelsElement.GetArrayLength() > 0) { - if (!header.TryGetProperty("ts_event", out var tsElement) || - !header.TryGetProperty("instrument_id", out var instIdElement)) - { - return; - } + var level0 = levelsElement[0]; - // Convert timestamp from nanoseconds to DateTime - var timestampNs = long.Parse(tsElement.GetString()!); - var unixEpoch = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc); - var timestamp = unixEpoch.AddTicks(timestampNs / 100); - - var instrumentId = instIdElement.GetInt64(); + var quoteTick = new Tick + { + Symbol = matchedSymbol, + Time = timestamp, + TickType = TickType.Quote + }; - if (!_instrumentIdToSymbol.TryGetValue(instrumentId, out var matchedSymbol)) + if (level0.TryGetProperty("ask_px", out var askPxElement) && + level0.TryGetProperty("ask_sz", out var askSzElement)) { - Log.Trace($"DataBentoRawLiveClient: No mapping for instrument_id {instrumentId} in trade message."); - return; + var askPriceRaw = long.Parse(askPxElement.GetString()!); + quoteTick.AskPrice = askPriceRaw * PriceScaleFactor; + quoteTick.AskSize = askSzElement.GetInt32(); } - if (root.TryGetProperty("price", out var priceElement) && - root.TryGetProperty("size", out var sizeElement)) + if (level0.TryGetProperty("bid_px", out var bidPxElement) && + level0.TryGetProperty("bid_sz", out var bidSzElement)) { - var priceRaw = long.Parse(priceElement.GetString()!); - var size = sizeElement.GetInt32(); - var price = priceRaw * PriceScaleFactor; - - var tradeTick = new Tick - { - Symbol = matchedSymbol, - Time = timestamp, - Value = price, - Quantity = size, - TickType = TickType.Trade, - // Trade ticks should have zero bid/ask values - BidPrice = 0, - BidSize = 0, - AskPrice = 0, - AskSize = 0 - }; - - // Log.Trace($"DataBentoRawLiveClient: Trade tick: {matchedSymbol} Price={price} Quantity={size}"); - DataReceived?.Invoke(this, tradeTick); + var bidPriceRaw = long.Parse(bidPxElement.GetString()!); + quoteTick.BidPrice = bidPriceRaw * PriceScaleFactor; + quoteTick.BidSize = bidSzElement.GetInt32(); } + + // Set the tick value to the mid price + quoteTick.Value = (quoteTick.BidPrice + quoteTick.AskPrice) / 2; + + // QuantConnect convention: Quote ticks should have zero Price and Quantity + quoteTick.Quantity = 0; + + // Log.Trace($"DataBentoRawLiveClient: Quote tick: {matchedSymbol} Bid={quoteTick.BidPrice}x{quoteTick.BidSize} Ask={quoteTick.AskPrice}x{quoteTick.AskSize}"); + DataReceived?.Invoke(this, quoteTick); } - catch (Exception ex) - { - Log.Error($"DataBentoRawLiveClient.HandleTradeTickMessage(): Error: {ex.Message}"); - } } + catch (Exception ex) + { + Log.Error($"DataBentoRawLiveClient.HandleMBPMessage(): Error: {ex.Message}"); + } + } - /// - /// Disconnects from the DataBento gateway - /// - public void Disconnect() + /// + /// Handles trade tick messages. Aggressor fills + /// + private void HandleTradeTickMessage(JsonElement root, JsonElement header) + { + try { - lock (_connectionLock) + if (!header.TryGetProperty("ts_event", out var tsElement) || + !header.TryGetProperty("instrument_id", out var instIdElement)) { - if (!_isConnected) - return; + return; + } - _isConnected = false; - _cancellationTokenSource?.Cancel(); + // Convert timestamp from nanoseconds to DateTime + var timestampNs = long.Parse(tsElement.GetString()!); + var unixEpoch = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc); + var timestamp = unixEpoch.AddTicks(timestampNs / 100); - try - { - _reader?.Dispose(); - _writer?.Dispose(); - _stream?.Close(); - _tcpClient?.Close(); - } - catch (Exception ex) - { - Log.Trace($"DataBentoRawLiveClient.Disconnect(): Error during disconnect: {ex.Message}"); - } + var instrumentId = instIdElement.GetInt64(); - ConnectionStatusChanged?.Invoke(this, false); - Log.Trace("DataBentoRawLiveClient.Disconnect(): Disconnected from DataBento gateway"); + if (!_instrumentIdToSymbol.TryGetValue(instrumentId, out var matchedSymbol)) + { + Log.Trace($"DataBentoRawLiveClient: No mapping for instrument_id {instrumentId} in trade message."); + return; } + + if (root.TryGetProperty("price", out var priceElement) && + root.TryGetProperty("size", out var sizeElement)) + { + var priceRaw = long.Parse(priceElement.GetString()!); + var size = sizeElement.GetInt32(); + var price = priceRaw * PriceScaleFactor; + + var tradeTick = new Tick + { + Symbol = matchedSymbol, + Time = timestamp, + Value = price, + Quantity = size, + TickType = TickType.Trade, + // Trade ticks should have zero bid/ask values + BidPrice = 0, + BidSize = 0, + AskPrice = 0, + AskSize = 0 + }; + + // Log.Trace($"DataBentoRawLiveClient: Trade tick: {matchedSymbol} Price={price} Quantity={size}"); + DataReceived?.Invoke(this, tradeTick); + } + } + catch (Exception ex) + { + Log.Error($"DataBentoRawLiveClient.HandleTradeTickMessage(): Error: {ex.Message}"); } + } - /// - /// Disposes of resources - /// - public void Dispose() + /// + /// Disconnects from the DataBento gateway + /// + public void Disconnect() + { + lock (_connectionLock) { - if (_disposed) + if (!_isConnected) return; - _disposed = true; - Disconnect(); + _isConnected = false; + _cancellationTokenSource?.Cancel(); - _cancellationTokenSource?.Dispose(); - _reader?.Dispose(); - _writer?.Dispose(); - _stream?.Dispose(); - _tcpClient?.Dispose(); - } - - /// - /// Computes the SHA-256 hash of the input string - /// - private static string ComputeSHA256(string input) - { - using var sha = SHA256.Create(); - var hash = sha.ComputeHash(Encoding.UTF8.GetBytes(input)); - var sb = new StringBuilder(); - foreach (byte b in hash) + try { - sb.Append(b.ToString("x2")); + _reader?.Dispose(); + _writer?.Dispose(); + _stream?.Close(); + _tcpClient?.Close(); } - return sb.ToString(); + catch (Exception ex) + { + Log.Trace($"DataBentoRawLiveClient.Disconnect(): Error during disconnect: {ex.Message}"); + } + + ConnectionStatusChanged?.Invoke(this, false); + Log.Trace("DataBentoRawLiveClient.Disconnect(): Disconnected from DataBento gateway"); } + } + + /// + /// Disposes of resources + /// + public void Dispose() + { + if (_disposed) + return; + _disposed = true; + Disconnect(); + + _cancellationTokenSource?.Dispose(); + _reader?.Dispose(); + _writer?.Dispose(); + _stream?.Dispose(); + _tcpClient?.Dispose(); } + + /// + /// Computes the SHA-256 hash of the input string + /// + private static string ComputeSHA256(string input) + { + using var sha = SHA256.Create(); + var hash = sha.ComputeHash(Encoding.UTF8.GetBytes(input)); + var sb = new StringBuilder(); + foreach (byte b in hash) + { + sb.Append(b.ToString("x2")); + } + return sb.ToString(); + } + } diff --git a/QuantConnect.DataBento/DataBentoSymbolMapper.cs b/QuantConnect.DataBento/DataBentoSymbolMapper.cs index 5c90304..7294922 100644 --- a/QuantConnect.DataBento/DataBentoSymbolMapper.cs +++ b/QuantConnect.DataBento/DataBentoSymbolMapper.cs @@ -15,64 +15,63 @@ using QuantConnect.Brokerages; -namespace QuantConnect.Lean.DataSource.DataBento +namespace QuantConnect.Lean.DataSource.DataBento; + +/// +/// Provides the mapping between Lean symbols and DataBento symbols. +/// +public class DataBentoSymbolMapper : ISymbolMapper { + /// - /// Provides the mapping between Lean symbols and DataBento symbols. + /// Converts a Lean symbol instance to a brokerage symbol /// - public class DataBentoSymbolMapper : ISymbolMapper + /// A Lean symbol instance + /// The brokerage symbol + public string GetBrokerageSymbol(Symbol symbol) { - - /// - /// Converts a Lean symbol instance to a brokerage symbol - /// - /// A Lean symbol instance - /// The brokerage symbol - public string GetBrokerageSymbol(Symbol symbol) + switch (symbol.SecurityType) { - switch (symbol.SecurityType) - { - case SecurityType.Future: - return SymbolRepresentation.GenerateFutureTicker(symbol.ID.Symbol, symbol.ID.Date, doubleDigitsYear: false, includeExpirationDate: false); - default: - throw new Exception($"The unsupported security type: {symbol.SecurityType}"); - } + case SecurityType.Future: + return SymbolRepresentation.GenerateFutureTicker(symbol.ID.Symbol, symbol.ID.Date, doubleDigitsYear: false, includeExpirationDate: false); + default: + throw new Exception($"The unsupported security type: {symbol.SecurityType}"); } + } - /// - /// Converts a brokerage symbol to a Lean symbol instance - /// - /// The brokerage symbol - /// The security type - /// The market - /// Expiration date of the security(if applicable) - /// A new Lean Symbol instance - public Symbol GetLeanSymbol(string brokerageSymbol, SecurityType securityType, string market, - DateTime expirationDate = new DateTime(), decimal strike = 0, OptionRight optionRight = 0) + /// + /// Converts a brokerage symbol to a Lean symbol instance + /// + /// The brokerage symbol + /// The security type + /// The market + /// Expiration date of the security(if applicable) + /// A new Lean Symbol instance + public Symbol GetLeanSymbol(string brokerageSymbol, SecurityType securityType, string market, + DateTime expirationDate = new DateTime(), decimal strike = 0, OptionRight optionRight = 0) + { + switch (securityType) { - switch (securityType) - { - case SecurityType.Future: - return Symbol.CreateFuture(brokerageSymbol, market, expirationDate); - default: - throw new Exception($"The unsupported security type: {securityType}"); - } + case SecurityType.Future: + return Symbol.CreateFuture(brokerageSymbol, market, expirationDate); + default: + throw new Exception($"The unsupported security type: {securityType}"); } + } - /// - /// Converts a brokerage future symbol to a Lean symbol instance - /// - /// The brokerage symbol - /// A new Lean Symbol instance - public Symbol GetLeanSymbolForFuture(string brokerageSymbol) + /// + /// Converts a brokerage future symbol to a Lean symbol instance + /// + /// The brokerage symbol + /// A new Lean Symbol instance + public Symbol GetLeanSymbolForFuture(string brokerageSymbol) + { + // ignore futures spreads + if (brokerageSymbol.Contains("-")) { - // ignore futures spreads - if (brokerageSymbol.Contains("-")) - { - return null; - } - - return SymbolRepresentation.ParseFutureSymbol(brokerageSymbol); + return null; } + + return SymbolRepresentation.ParseFutureSymbol(brokerageSymbol); } } diff --git a/QuantConnect.DataBento/Extensions.cs b/QuantConnect.DataBento/Extensions.cs index 14e8834..a4514db 100644 --- a/QuantConnect.DataBento/Extensions.cs +++ b/QuantConnect.DataBento/Extensions.cs @@ -14,7 +14,6 @@ */ using Newtonsoft.Json; -using Newtonsoft.Json.Serialization; using QuantConnect.Lean.DataSource.DataBento.Serialization; namespace QuantConnect.Lean.DataSource.DataBento; From 3513bcfbf78760a6fa71aa07dc1a23d02655228a Mon Sep 17 00:00:00 2001 From: Romazes Date: Mon, 26 Jan 2026 15:22:05 +0200 Subject: [PATCH 08/13] feat: debug mode config --- QuantConnect.DataBento.Tests/TestSetup.cs | 4 ++-- QuantConnect.DataBento.Tests/config.json | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/QuantConnect.DataBento.Tests/TestSetup.cs b/QuantConnect.DataBento.Tests/TestSetup.cs index 86d2f07..e56a03d 100644 --- a/QuantConnect.DataBento.Tests/TestSetup.cs +++ b/QuantConnect.DataBento.Tests/TestSetup.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. * @@ -28,7 +28,7 @@ public class TestSetup [OneTimeSetUp] public void GlobalSetup() { - Log.DebuggingEnabled = true; + Log.DebuggingEnabled = Config.GetBool("debug-mode"); Log.LogHandler = new CompositeLogHandler(); Log.Trace("TestSetup(): starting..."); ReloadConfiguration(); diff --git a/QuantConnect.DataBento.Tests/config.json b/QuantConnect.DataBento.Tests/config.json index 2464c3e..0d61d83 100644 --- a/QuantConnect.DataBento.Tests/config.json +++ b/QuantConnect.DataBento.Tests/config.json @@ -3,5 +3,6 @@ "job-user-id": "", "api-access-token": "", "job-organization-id": "", - "databento-api-key": "" + "databento-api-key": "", + "debug-mode": false } \ No newline at end of file From ee90b571aba7d97f2ded8aa437b5131a23253527 Mon Sep 17 00:00:00 2001 From: Romazes Date: Mon, 26 Jan 2026 15:24:05 +0200 Subject: [PATCH 09/13] remove: IsSupported() doesn't validate anything --- .../DataBentoDataProvider.cs | 32 ------------------- 1 file changed, 32 deletions(-) diff --git a/QuantConnect.DataBento/DataBentoDataProvider.cs b/QuantConnect.DataBento/DataBentoDataProvider.cs index 40fc095..9b74d9e 100644 --- a/QuantConnect.DataBento/DataBentoDataProvider.cs +++ b/QuantConnect.DataBento/DataBentoDataProvider.cs @@ -201,11 +201,6 @@ private static bool CanSubscribe(Symbol symbol) /// The new enumerator for this subscription request public IEnumerator? Subscribe(SubscriptionDataConfig dataConfig, EventHandler newDataAvailableHandler) { - if (!IsSupported(dataConfig.SecurityType, dataConfig.Type, dataConfig.TickType, dataConfig.Resolution)) - { - return null; - } - lock (_sessionLock) { if (!_sessionStarted) @@ -254,33 +249,6 @@ public void Dispose() _client?.DisposeSafely(); } - /// - /// Determines if the specified subscription is supported - /// - private bool IsSupported(SecurityType securityType, Type dataType, TickType tickType, Resolution resolution) - { - // Check supported data types - if (dataType != typeof(TradeBar) && - dataType != typeof(QuoteBar) && - dataType != typeof(Tick) && - dataType != typeof(OpenInterest)) - { - throw new NotSupportedException($"Unsupported data type: {dataType}"); - } - - // Warn about potential limitations for tick data - // I'm mimicing polygon implementation with this - if (!_potentialUnsupportedResolutionMessageLogged) - { - _potentialUnsupportedResolutionMessageLogged = true; - Log.Trace("DataBentoDataProvider.IsSupported(): " + - $"Subscription for {securityType}-{dataType}-{tickType}-{resolution} will be attempted. " + - $"An Advanced DataBento subscription plan is required to stream tick data."); - } - - return true; - } - /// /// Converts the given UTC time into the symbol security exchange time zone /// From 531056a75c254d323974d26727f8a355b7114897 Mon Sep 17 00:00:00 2001 From: Romazes Date: Mon, 26 Jan 2026 15:35:15 +0200 Subject: [PATCH 10/13] feat: map Lean.Market <-> DataBento.DataSet --- .../DataBentoHistoricalApiClientTests.cs | 13 ++++- .../DataBentoSymbolMapperTests.cs.cs | 10 ++++ .../Api/HistoricalAPIClient.cs | 29 ++++-------- .../DataBentoDataProvider.cs | 15 ++++++ .../DataBentoHistoryProivder.cs | 47 +++++++++++++------ .../DataBentoSymbolMapper.cs | 17 +++++++ 6 files changed, 94 insertions(+), 37 deletions(-) diff --git a/QuantConnect.DataBento.Tests/DataBentoHistoricalApiClientTests.cs b/QuantConnect.DataBento.Tests/DataBentoHistoricalApiClientTests.cs index 7edac6d..4623776 100644 --- a/QuantConnect.DataBento.Tests/DataBentoHistoricalApiClientTests.cs +++ b/QuantConnect.DataBento.Tests/DataBentoHistoricalApiClientTests.cs @@ -26,6 +26,15 @@ public class DataBentoHistoricalApiClientTests { private HistoricalAPIClient _client; + /// + /// Dataset for CME Globex futures + /// https://databento.com/docs/venues-and-datasets has more information on datasets through DataBento + /// + /// + /// TODO: Hard coded for now. Later on can add equities and options with different mapping + /// + private const string Dataset = "GLBX.MDP3"; + [OneTimeSetUp] public void OneTimeSetUp() { @@ -48,7 +57,7 @@ public void CanInitializeHistoricalApiClient(string ticker, DateTime startDate, { var dataCounter = 0; var previousEndTime = DateTime.MinValue; - foreach (var data in _client.GetHistoricalOhlcvBars(ticker, startDate, endDate, resolution, TickType.Trade)) + foreach (var data in _client.GetHistoricalOhlcvBars(ticker, startDate, endDate, resolution, Dataset)) { Assert.IsNotNull(data); @@ -75,7 +84,7 @@ public void ShouldFetchOpenInterest(string ticker, DateTime startDate, DateTime { var dataCounter = 0; var previousEndTime = DateTime.MinValue; - foreach (var data in _client.GetOpenInterest(ticker, startDate, endDate)) + foreach (var data in _client.GetOpenInterest(ticker, startDate, endDate, Dataset)) { Assert.IsNotNull(data); diff --git a/QuantConnect.DataBento.Tests/DataBentoSymbolMapperTests.cs.cs b/QuantConnect.DataBento.Tests/DataBentoSymbolMapperTests.cs.cs index ad14aad..feb393a 100644 --- a/QuantConnect.DataBento.Tests/DataBentoSymbolMapperTests.cs.cs +++ b/QuantConnect.DataBento.Tests/DataBentoSymbolMapperTests.cs.cs @@ -53,4 +53,14 @@ public void ReturnsCorrectBrokerageSymbol(Symbol symbol, string expectedBrokerag Assert.IsNotEmpty(brokerageSymbol); Assert.AreEqual(expectedBrokerageSymbol, brokerageSymbol); } + + [TestCase(Market.CME, "GLBX.MDP3", true)] + [TestCase(Market.EUREX, "XEUR.EOBI", true)] + [TestCase(Market.USA, null, false)] + public void ReturnsCorrectDataBentoDataSet(string market, string expectedDataSet, bool expectedExist) + { + var actualExist = _symbolMapper.DataBentoDataSetByLeanMarket.TryGetValue(market, out var actualDataSet); + Assert.AreEqual(expectedExist, actualExist); + Assert.AreEqual(expectedDataSet, actualDataSet); + } } diff --git a/QuantConnect.DataBento/Api/HistoricalAPIClient.cs b/QuantConnect.DataBento/Api/HistoricalAPIClient.cs index 92dc868..23c79ee 100644 --- a/QuantConnect.DataBento/Api/HistoricalAPIClient.cs +++ b/QuantConnect.DataBento/Api/HistoricalAPIClient.cs @@ -24,17 +24,6 @@ namespace QuantConnect.Lean.DataSource.DataBento.Api; public class HistoricalAPIClient : IDisposable { - //private const string - - /// - /// Dataset for CME Globex futures - /// https://databento.com/docs/venues-and-datasets has more information on datasets through DataBento - /// - /// - /// TODO: Hard coded for now. Later on can add equities and options with different mapping - /// - private const string Dataset = "GLBX.MDP3"; - private readonly HttpClient _httpClient = new() { BaseAddress = new Uri("https://hist.databento.com") @@ -45,11 +34,11 @@ public HistoricalAPIClient(string apiKey) _httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue( AuthenticationSchemes.Basic.ToString(), // Basic Auth expects "username:password". Using ":" means API key with an empty password. - Convert.ToBase64String(Encoding.UTF8.GetBytes($"{apiKey}:")) + Convert.ToBase64String(Encoding.UTF8.GetBytes($"{apiKey}:")) ); } - public IEnumerable GetHistoricalOhlcvBars(string symbol, DateTime startDateTimeUtc, DateTime endDateTimeUtc, Resolution resolution, TickType tickType) + public IEnumerable GetHistoricalOhlcvBars(string symbol, DateTime startDateTimeUtc, DateTime endDateTimeUtc, Resolution resolution, string dataSet) { string schema; switch (resolution) @@ -70,17 +59,17 @@ public IEnumerable GetHistoricalOhlcvBars(string symbol, DateTime star throw new ArgumentException($"Unsupported resolution {resolution} for OHLCV data."); } - return GetRange(symbol, startDateTimeUtc, endDateTimeUtc, schema); + return GetRange(symbol, startDateTimeUtc, endDateTimeUtc, schema, dataSet); } - public IEnumerable GetTickBars(string symbol, DateTime startDateTimeUtc, DateTime endDateTimeUtc) + public IEnumerable GetTickBars(string symbol, DateTime startDateTimeUtc, DateTime endDateTimeUtc, string dataSet) { - return GetRange(symbol, startDateTimeUtc, endDateTimeUtc, "mbp-1", useLimit: true); + return GetRange(symbol, startDateTimeUtc, endDateTimeUtc, "mbp-1", dataSet, useLimit: true); } - public IEnumerable GetOpenInterest(string symbol, DateTime startDateTimeUtc, DateTime endDateTimeUtc) + public IEnumerable GetOpenInterest(string symbol, DateTime startDateTimeUtc, DateTime endDateTimeUtc, string dataSet) { - foreach (var statistics in GetRange(symbol, startDateTimeUtc, endDateTimeUtc, "statistics")) + foreach (var statistics in GetRange(symbol, startDateTimeUtc, endDateTimeUtc, "statistics", dataSet)) { if (statistics.StatType == Models.Enums.StatisticType.OpenInterest) { @@ -89,11 +78,11 @@ public IEnumerable GetOpenInterest(string symbol, DateTime start } } - private IEnumerable GetRange(string symbol, DateTime startDateTimeUtc, DateTime endDateTimeUtc, string schema, bool useLimit = false) where T : MarketDataRecord + private IEnumerable GetRange(string symbol, DateTime startDateTimeUtc, DateTime endDateTimeUtc, string schema, string dataSet, bool useLimit = false) where T : MarketDataRecord { var formData = new Dictionary { - { "dataset", Dataset }, + { "dataset", dataSet }, { "end", Time.DateTimeToUnixTimeStampNanoseconds(endDateTimeUtc).ToStringInvariant() }, { "symbols", symbol }, { "schema", schema }, diff --git a/QuantConnect.DataBento/DataBentoDataProvider.cs b/QuantConnect.DataBento/DataBentoDataProvider.cs index 9b74d9e..ba228d1 100644 --- a/QuantConnect.DataBento/DataBentoDataProvider.cs +++ b/QuantConnect.DataBento/DataBentoDataProvider.cs @@ -156,6 +156,21 @@ public bool UnsubscribeLogic(IEnumerable symbols, TickType tickType) return true; } + /// + /// Attempts to resolve the DataBento dataset for the specified symbol based on its Lean market. + /// + /// The symbol whose market is used to determine the DataBento dataset. + /// + /// When this method returns true, contains the resolved DataBento dataset; otherwise, null. + /// + /// + /// true if a DataBento dataset mapping exists for the symbol's market; otherwise, false. + /// + private bool TryGetDataBentoDataSet(Symbol symbol, out string? dataSet) + { + return _symbolMapper.DataBentoDataSetByLeanMarket.TryGetValue(symbol.ID.Market, out dataSet); + } + /// /// Logic to subscribe to the specified symbols /// diff --git a/QuantConnect.DataBento/DataBentoHistoryProivder.cs b/QuantConnect.DataBento/DataBentoHistoryProivder.cs index 2485ff5..fabf42e 100644 --- a/QuantConnect.DataBento/DataBentoHistoryProivder.cs +++ b/QuantConnect.DataBento/DataBentoHistoryProivder.cs @@ -44,6 +44,11 @@ public partial class DataBentoProvider : SynchronizingHistoryProvider /// private volatile bool _invalidSecurityTypeWarningFired; + /// + /// Indicates whether a DataBento dataset error has already been logged. + /// + private bool _dataBentoDatasetErrorFired; + /// /// Gets the total number of data points emitted by this history provider /// @@ -113,21 +118,33 @@ public override void Initialize(HistoryProviderInitializeParameters parameters) return null; } + if (!TryGetDataBentoDataSet(historyRequest.Symbol, out var dataSet) || dataSet == null) + { + if (!_dataBentoDatasetErrorFired) + { + _dataBentoDatasetErrorFired = true; + Log.Error($"{nameof(DataBentoProvider)}.{nameof(GetHistory)}: " + + $"DataBento dataset not found for symbol '{historyRequest.Symbol.Value}, Market = {historyRequest.Symbol.ID.Market}." + ); + } + return null; + } + var history = default(IEnumerable); var brokerageSymbol = _symbolMapper.GetBrokerageSymbol(historyRequest.Symbol); switch (historyRequest.TickType) { case TickType.Trade when historyRequest.Resolution == Resolution.Tick: - history = GetHistoryThroughDataConsolidator(historyRequest, brokerageSymbol); + history = GetHistoryThroughDataConsolidator(historyRequest, brokerageSymbol, dataSet); break; case TickType.Trade: - history = GetAggregatedTradeBars(historyRequest, brokerageSymbol); + history = GetAggregatedTradeBars(historyRequest, brokerageSymbol, dataSet); break; case TickType.Quote: - history = GetHistoryThroughDataConsolidator(historyRequest, brokerageSymbol); + history = GetHistoryThroughDataConsolidator(historyRequest, brokerageSymbol, dataSet); break; case TickType.OpenInterest: - history = GetOpenInterestBars(historyRequest, brokerageSymbol); + history = GetOpenInterestBars(historyRequest, brokerageSymbol, dataSet); break; default: throw new ArgumentException(""); @@ -157,24 +174,24 @@ private static IEnumerable FilterHistory(IEnumerable history } } - private IEnumerable GetOpenInterestBars(HistoryRequest request, string brokerageSymbol) + private IEnumerable GetOpenInterestBars(HistoryRequest request, string brokerageSymbol, string dataBentoDataSet) { - foreach (var oi in _historicalApiClient.GetOpenInterest(brokerageSymbol, request.StartTimeUtc, request.EndTimeUtc)) + foreach (var oi in _historicalApiClient.GetOpenInterest(brokerageSymbol, request.StartTimeUtc, request.EndTimeUtc, dataBentoDataSet)) { yield return new OpenInterest(oi.Header.UtcTime.ConvertFromUtc(request.DataTimeZone), request.Symbol, oi.Quantity); } } - private IEnumerable GetAggregatedTradeBars(HistoryRequest request, string brokerageSymbol) + private IEnumerable GetAggregatedTradeBars(HistoryRequest request, string brokerageSymbol, string dataBentoDataSet) { var period = request.Resolution.ToTimeSpan(); - foreach (var b in _historicalApiClient.GetHistoricalOhlcvBars(brokerageSymbol, request.StartTimeUtc, request.EndTimeUtc, request.Resolution, request.TickType)) + foreach (var b in _historicalApiClient.GetHistoricalOhlcvBars(brokerageSymbol, request.StartTimeUtc, request.EndTimeUtc, request.Resolution, dataBentoDataSet)) { yield return new TradeBar(b.Header.UtcTime.ConvertFromUtc(request.DataTimeZone), request.Symbol, b.Open, b.High, b.Low, b.Close, b.Volume, period); } } - private IEnumerable? GetHistoryThroughDataConsolidator(HistoryRequest request, string brokerageSymbol) + private IEnumerable? GetHistoryThroughDataConsolidator(HistoryRequest request, string brokerageSymbol, string dataBentoDataSet) { IDataConsolidator consolidator; IEnumerable history; @@ -184,14 +201,14 @@ private IEnumerable GetAggregatedTradeBars(HistoryRequest request, str consolidator = request.Resolution != Resolution.Tick ? new TickConsolidator(request.Resolution.ToTimeSpan()) : FilteredIdentityDataConsolidator.ForTickType(request.TickType); - history = GetTrades(request, brokerageSymbol); + history = GetTrades(request, brokerageSymbol, dataBentoDataSet); } else { consolidator = request.Resolution != Resolution.Tick ? new TickQuoteBarConsolidator(request.Resolution.ToTimeSpan()) : FilteredIdentityDataConsolidator.ForTickType(request.TickType); - history = GetQuotes(request, brokerageSymbol); + history = GetQuotes(request, brokerageSymbol, dataBentoDataSet); } BaseData? consolidatedData = null; @@ -218,9 +235,9 @@ private IEnumerable GetAggregatedTradeBars(HistoryRequest request, str /// /// Gets the trade ticks that will potentially be aggregated for the specified history request /// - private IEnumerable GetTrades(HistoryRequest request, string brokerageSymbol) + private IEnumerable GetTrades(HistoryRequest request, string brokerageSymbol, string dataBentoDataSet) { - foreach (var t in _historicalApiClient.GetTickBars(brokerageSymbol, request.StartTimeUtc, request.EndTimeUtc)) + foreach (var t in _historicalApiClient.GetTickBars(brokerageSymbol, request.StartTimeUtc, request.EndTimeUtc, dataBentoDataSet)) { yield return new Tick(t.Header.UtcTime.ConvertFromUtc(request.DataTimeZone), request.Symbol, "", "", t.Size, t.Price); } @@ -229,9 +246,9 @@ private IEnumerable GetTrades(HistoryRequest request, string brokerage /// /// Gets the quote ticks that will potentially be aggregated for the specified history request /// - private IEnumerable GetQuotes(HistoryRequest request, string brokerageSymbol) + private IEnumerable GetQuotes(HistoryRequest request, string brokerageSymbol, string dataBentoDataSet) { - foreach (var quoteBar in _historicalApiClient.GetTickBars(brokerageSymbol, request.StartTimeUtc, request.EndTimeUtc)) + foreach (var quoteBar in _historicalApiClient.GetTickBars(brokerageSymbol, request.StartTimeUtc, request.EndTimeUtc, dataBentoDataSet)) { var time = quoteBar.Header.UtcTime.ConvertFromUtc(request.DataTimeZone); foreach (var level in quoteBar.Levels) diff --git a/QuantConnect.DataBento/DataBentoSymbolMapper.cs b/QuantConnect.DataBento/DataBentoSymbolMapper.cs index 7294922..1a824ee 100644 --- a/QuantConnect.DataBento/DataBentoSymbolMapper.cs +++ b/QuantConnect.DataBento/DataBentoSymbolMapper.cs @@ -14,6 +14,7 @@ */ using QuantConnect.Brokerages; +using System.Collections.Frozen; namespace QuantConnect.Lean.DataSource.DataBento; @@ -22,6 +23,22 @@ namespace QuantConnect.Lean.DataSource.DataBento; /// public class DataBentoSymbolMapper : ISymbolMapper { + /// + /// Dataset for CME Globex futures + /// https://databento.com/docs/venues-and-datasets has more information on datasets through DataBento + /// + public FrozenDictionary DataBentoDataSetByLeanMarket = new Dictionary + { + { Market.EUREX, "XEUR.EOBI" }, + + { Market.CBOT, "GLBX.MDP3" }, + { Market.CME, "GLBX.MDP3" }, + { Market.COMEX, "GLBX.MDP3" }, + { Market.NYMEX, "GLBX.MDP3" }, + + { Market.ICE, "IFUS.IMPACT" }, + { Market.NYSELIFFE, "IFUS.IMPACT" } + }.ToFrozenDictionary(); /// /// Converts a Lean symbol instance to a brokerage symbol From 4c322c0e8ba4af61498b394a9c8aab1b3acf1d84 Mon Sep 17 00:00:00 2001 From: Romazes Date: Tue, 27 Jan 2026 13:20:31 +0200 Subject: [PATCH 11/13] feat: Data Live TcpClient Wrapper feat: Record Type of different DataBento scheme feat: handle Auth-Request/Response msgs test:feat: deserialize Heartbeat json msg, parse tcp response/request test:feat: LiveApiClient quick connection and debug process --- .../DataBentoJsonConverterTests.cs | 46 ++++ .../DataBentoLiveAPIClientTests.cs | 65 +++++ QuantConnect.DataBento/Api/LiveAPIClient.cs | 83 +++++++ .../Api/LiveDataTcpClientWrapper.cs | 229 ++++++++++++++++++ .../DataBentoRawLiveClient.cs | 2 + .../Models/Enums/RecordType.cs | 86 +++++++ QuantConnect.DataBento/Models/Header.cs | 4 +- .../Live/AuthenticationMessageRequest.cs | 69 ++++++ .../Live/AuthenticationMessageResponse.cs | 54 +++++ .../Models/Live/HeartbeatMessage.cs | 22 ++ 10 files changed, 659 insertions(+), 1 deletion(-) create mode 100644 QuantConnect.DataBento.Tests/DataBentoLiveAPIClientTests.cs create mode 100644 QuantConnect.DataBento/Api/LiveAPIClient.cs create mode 100644 QuantConnect.DataBento/Api/LiveDataTcpClientWrapper.cs create mode 100644 QuantConnect.DataBento/Models/Enums/RecordType.cs create mode 100644 QuantConnect.DataBento/Models/Live/AuthenticationMessageRequest.cs create mode 100644 QuantConnect.DataBento/Models/Live/AuthenticationMessageResponse.cs create mode 100644 QuantConnect.DataBento/Models/Live/HeartbeatMessage.cs diff --git a/QuantConnect.DataBento.Tests/DataBentoJsonConverterTests.cs b/QuantConnect.DataBento.Tests/DataBentoJsonConverterTests.cs index 9d43db8..5b9de56 100644 --- a/QuantConnect.DataBento.Tests/DataBentoJsonConverterTests.cs +++ b/QuantConnect.DataBento.Tests/DataBentoJsonConverterTests.cs @@ -16,6 +16,7 @@ using NUnit.Framework; using QuantConnect.Lean.DataSource.DataBento.Models; using QuantConnect.Lean.DataSource.DataBento.Models.Enums; +using QuantConnect.Lean.DataSource.DataBento.Models.Live; namespace QuantConnect.Lean.DataSource.DataBento.Tests; @@ -146,4 +147,49 @@ public void DeserializeHistoricalStatisticsData() Assert.AreEqual(470m, res.Quantity); Assert.AreEqual(StatisticType.OpenInterest, res.StatType); } + + [Test] + public void DeserializeLiveHeartbeatMessage() + { + var json = @"{""hd"":{""ts_event"":""1769176693139629181"",""rtype"":23,""publisher_id"":0,""instrument_id"":0},""msg"":""Heartbeat""}"; + + var res = json.DeserializeKebabCase(); + + Assert.IsNotNull(res); + Assert.AreEqual(1769176693139629181, res.Header.TsEvent); + Assert.AreEqual(RecordType.System, res.Header.Rtype); + Assert.AreEqual(0, res.Header.PublisherId); + Assert.AreEqual(0, res.Header.InstrumentId); + Assert.AreEqual("Heartbeat", res.Msg); + } + + [TestCase("success=0|error=Unknown subscription param 'sssauth'", false)] + [TestCase("success=0|error=Authentication failed.", false)] + [TestCase("success=1|session_id=1769508116", true)] + public void ParsePotentialAuthenticationMessageResponses(string authenticationResponse, bool success) + { + var auth = new AuthenticationMessageResponse(authenticationResponse); + + if (success) + { + Assert.IsTrue(auth.Success); + Assert.AreNotEqual(0, auth.SessionId); + } + else + { + Assert.IsFalse(auth.Success); + Assert.That(auth.Error, Is.Not.Null.And.Not.Empty); + } + } + + [TestCase("cram=HCxTgxMcqglVMTMeaDZ2ICmcnrW8j92e", "auth=a6c5c23e06854dc0310e11ce6d3081509e415a5a37a323bb94bc90f64c9214d4-12345|dataset=GLBX.MDP3|pretty_px=1|encoding=json|heartbeat_interval_s=5")] + [TestCase("cram=HCxTgxMcqglVMTMeaDZ2ICmcnrW8j92e\n", "auth=a6c5c23e06854dc0310e11ce6d3081509e415a5a37a323bb94bc90f64c9214d4-12345|dataset=GLBX.MDP3|pretty_px=1|encoding=json|heartbeat_interval_s=5")] + public void ParsePotentialCramChallenges(string challenge, string expectedString) + { + var auth = new AuthenticationMessageRequest(challenge, "my-api-key-12345", "GLBX.MDP3"); + + var actualString = auth.ToString(); + + Assert.AreEqual(expectedString, actualString); + } } diff --git a/QuantConnect.DataBento.Tests/DataBentoLiveAPIClientTests.cs b/QuantConnect.DataBento.Tests/DataBentoLiveAPIClientTests.cs new file mode 100644 index 0000000..1124895 --- /dev/null +++ b/QuantConnect.DataBento.Tests/DataBentoLiveAPIClientTests.cs @@ -0,0 +1,65 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System; +using NUnit.Framework; +using System.Threading; +using QuantConnect.Configuration; +using QuantConnect.Lean.DataSource.DataBento.Api; + +namespace QuantConnect.Lean.DataSource.DataBento.Tests; + +[TestFixture] +public class DataBentoLiveAPIClientTests +{ + /// + /// Dataset for CME Globex futures + /// https://databento.com/docs/venues-and-datasets has more information on datasets through DataBento + /// + /// + /// TODO: Hard coded for now. Later on can add equities and options with different mapping + /// + private const string Dataset = "GLBX.MDP3"; + + private LiveAPIClient _live; + + [OneTimeSetUp] + public void OneTimeSetUp() + { + var apiKey = Config.Get("databento-api-key"); + if (string.IsNullOrEmpty(apiKey)) + { + Assert.Inconclusive("Please set the 'databento-api-key' in your configuration to enable these tests."); + } + + _live = new LiveAPIClient(apiKey); + } + + [OneTimeTearDown] + public void OneTimeTearDown() + { + _live.Dispose(); + } + + [Test] + public void TestExample() + { + var dataAvailableEvent = new AutoResetEvent(false); + + _live.Start(Dataset); + + dataAvailableEvent.WaitOne(TimeSpan.FromSeconds(60)); + } +} diff --git a/QuantConnect.DataBento/Api/LiveAPIClient.cs b/QuantConnect.DataBento/Api/LiveAPIClient.cs new file mode 100644 index 0000000..f8b60df --- /dev/null +++ b/QuantConnect.DataBento/Api/LiveAPIClient.cs @@ -0,0 +1,83 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +using QuantConnect.Util; +using QuantConnect.Logging; + +namespace QuantConnect.Lean.DataSource.DataBento.Api; + +public sealed class LiveAPIClient : IDisposable +{ + private readonly string _apiKey; + + private readonly Dictionary _tcpClientByDataSet = []; + + public LiveAPIClient(string apiKey) + { + _apiKey = apiKey; + } + + public void Dispose() + { + foreach (var tcpClient in _tcpClientByDataSet.Values) + { + tcpClient.DisposeSafely(); + } + _tcpClientByDataSet.Clear(); + } + + public bool Start(string dataSet) + { + LogTrace(nameof(Start), "Starting connection to DataBento live API"); + + if (_tcpClientByDataSet.TryGetValue(dataSet, out var existingClient) && existingClient.IsConnected) + { + LogTrace(nameof(Start), $"Already connected to DataBento live API (Dataset: {dataSet})"); + return true; + } + + var liveDataTcpClient = new LiveDataTcpClientWrapper(dataSet, _apiKey); + _tcpClientByDataSet[dataSet] = liveDataTcpClient; + liveDataTcpClient.Connect(); + + LogTrace(nameof(Start), $"Successfully connected to DataBento live API (Dataset: {dataSet})"); + + return true; + } + + public bool Subscribe(string dataSet, string symbol) + { + if (!_tcpClientByDataSet.TryGetValue(dataSet, out var tcpClient) || !tcpClient.IsConnected) + { + LogError(nameof(Subscribe), $"Not connected to DataBento live API (Dataset: {dataSet})"); + return false; + } + + tcpClient.SubscribeOnMarketBestPriceLevelOne(symbol); + + return true; + } + + private static void LogTrace(string method, string message) + { + Log.Trace($"LiveAPIClient.{method}: {message}"); + } + + private static void LogError(string method, string message) + { + Log.Error($"LiveAPIClient.{method}: {message}"); + } +} diff --git a/QuantConnect.DataBento/Api/LiveDataTcpClientWrapper.cs b/QuantConnect.DataBento/Api/LiveDataTcpClientWrapper.cs new file mode 100644 index 0000000..205ff78 --- /dev/null +++ b/QuantConnect.DataBento/Api/LiveDataTcpClientWrapper.cs @@ -0,0 +1,229 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +using System.Text; +using QuantConnect.Util; +using System.Net.Sockets; +using QuantConnect.Logging; +using QuantConnect.Lean.DataSource.DataBento.Models.Live; + +namespace QuantConnect.Lean.DataSource.DataBento.Api; + +public sealed class LiveDataTcpClientWrapper : IDisposable +{ + private const int DefaultPort = 13000; + private const int ReceiveBufferSize = 8192; + + private readonly string _gateway; + private readonly string _dataSet; + private readonly string _apiKey; + private readonly TimeSpan _heartBeatInterval = TimeSpan.FromSeconds(10); + + private readonly TcpClient _tcpClient = new(); + private readonly byte[] _receiveBuffer = new byte[ReceiveBufferSize]; + private readonly CancellationTokenSource _cancellationTokenSource = new(); + private readonly char[] _newLine = Environment.NewLine.ToCharArray(); + + private NetworkStream? _stream; + private Task? _dataReceiverTask; + private bool _isConnected; + + /// + /// Is client connected + /// + public bool IsConnected => _isConnected; + + public LiveDataTcpClientWrapper(string dataSet, string apiKey) + { + _apiKey = apiKey; + _dataSet = dataSet; + _gateway = DetermineGateway(dataSet); + } + + public void Connect() + { + _tcpClient.Connect(_gateway, DefaultPort); + _stream = _tcpClient.GetStream(); + + if (!Authenticate(_dataSet).SynchronouslyAwaitTask()) + throw new Exception("Authentication failed"); + + _dataReceiverTask = new Task(async () => await DataReceiverAsync(_cancellationTokenSource.Token), _cancellationTokenSource.Token, TaskCreationOptions.LongRunning); + _dataReceiverTask.Start(); + + _isConnected = true; + } + + public void Dispose() + { + _isConnected = false; + + _stream?.Close(); + _stream?.DisposeSafely(); + + _cancellationTokenSource?.Cancel(); + _cancellationTokenSource?.DisposeSafely(); + + _dataReceiverTask?.DisposeSafely(); + _tcpClient?.Close(); + _tcpClient?.DisposeSafely(); + } + + public void SubscribeOnMarketBestPriceLevelOne(string symbol) + { + var request = $"schema=mbp-1|stype_in=raw_symbol|symbols={symbol}"; + WriteData(request); + } + + private async Task DataReceiverAsync(CancellationToken ct) + { + var methodName = nameof(DataReceiverAsync); + + var readTimeoutCts = CancellationTokenSource.CreateLinkedTokenSource(ct); + + var readTimeout = _heartBeatInterval.Add(TimeSpan.FromSeconds(5)); + + LogTrace(methodName, "Receiver started"); + + try + { + while (!ct.IsCancellationRequested && IsConnected) + { + // Reset timeout + readTimeoutCts.CancelAfter(readTimeout); + + var line = await ReadDataAsync(readTimeoutCts.Token); + + if (line == null) + { + Log.Error("Remote closed connection"); + break; + } + + LogTrace(methodName, $"Received: {line}"); + } + } + catch (OperationCanceledException) + { + if (!_tcpClient.Connected) + { + LogError("DataReceiverAsync", "GG"); + } + + Log.Trace("DataBentoRawLiveClient.ProcessMessages(): Message processing cancelled"); + } + catch (IOException ex) when (ex.InnerException is SocketException) + { + Log.Trace($"DataBentoRawLiveClient.ProcessMessages(): Socket exception: {ex.Message}"); + } + catch (Exception ex) + { + Log.Error($"DataBentoRawLiveClient.ProcessMessages(): Error processing messages: {ex.Message}\n{ex.StackTrace}"); + } + finally + { + LogTrace(methodName, "Receiver stopped"); + readTimeoutCts.Dispose(); + } + } + + private async Task ReadDataAsync(CancellationToken cancellationToken) + { + var numberOfBytesToRead = await _stream.ReadAsync(_receiveBuffer.AsMemory(0, _receiveBuffer.Length), cancellationToken).ConfigureAwait(false); + + if (numberOfBytesToRead == 0) + { + return null; + } + + using var memoryStream = new MemoryStream(); + await memoryStream.WriteAsync(_receiveBuffer.AsMemory(0, numberOfBytesToRead), cancellationToken).ConfigureAwait(false); + return Encoding.ASCII.GetString(memoryStream.ToArray(), 0, numberOfBytesToRead).TrimEnd(_newLine); + } + + private void WriteData(string data) + { + if (!data.EndsWith('\n')) + { + data += '\n'; + } + var bytes = Encoding.ASCII.GetBytes(data); + _stream.Write(bytes, 0, bytes.Length); + } + + private async Task Authenticate(string dataSet) + { + var cts = CancellationTokenSource.CreateLinkedTokenSource(_cancellationTokenSource.Token); + + try + { + var versionLine = await ReadDataAsync(cts.Token); + var cramLine = await ReadDataAsync(cts.Token); + + if (Log.DebuggingEnabled) + { + LogDebug(nameof(Authenticate), $"Received initial message: {versionLine}, {cramLine}"); + } + + var request = new AuthenticationMessageRequest(cramLine, _apiKey, dataSet, _heartBeatInterval); + + LogTrace("Authenticate", $"Sending CRAM reply: {request}"); + + WriteData(request.ToString()); + + var authResponse = await ReadDataAsync(cts.Token); + + var authenticationResponse = new AuthenticationMessageResponse(authResponse); + + if (!authenticationResponse.Success) + { + LogError(nameof(Authenticate), $"Authentication response: {authResponse}"); + return false; + } + + LogTrace(nameof(Authenticate), $"Successfully authenticated with session ID: {authenticationResponse.SessionId}"); + + WriteData(request.GetStartSessionMessage()); // after start_session -> we get heartbeats and data + + return true; + } + finally + { + cts.DisposeSafely(); + } + } + + private static string DetermineGateway(string dataset) + { + dataset = dataset.Replace('.', '-').ToLowerInvariant(); + return dataset + ".lsg.databento.com"; + } + + private void LogTrace(string method, string message) + { + Log.Trace($"LiveDataTcpClientWrapper[{_dataSet}].{method}: {message}"); + } + + private void LogError(string method, string message) + { + Log.Error($"LiveDataTcpClientWrapper[{_dataSet}].{method}: {message}"); + } + + private void LogDebug(string method, string message) + { + Log.Debug($"LiveDataTcpClientWrapper[{_dataSet}].{method}: {message}"); + } +} diff --git a/QuantConnect.DataBento/DataBentoRawLiveClient.cs b/QuantConnect.DataBento/DataBentoRawLiveClient.cs index 9c65544..06d8e6a 100644 --- a/QuantConnect.DataBento/DataBentoRawLiveClient.cs +++ b/QuantConnect.DataBento/DataBentoRawLiveClient.cs @@ -261,6 +261,8 @@ public bool Unsubscribe(Symbol symbol) { try { + // Please note there is no unsubscribe method. Subscriptions end when the TCP connection closes. + if (_subscriptions.TryRemove(symbol, out _)) { Log.Debug($"DataBentoRawLiveClient.Unsubscribe(): Unsubscribed from {symbol}"); diff --git a/QuantConnect.DataBento/Models/Enums/RecordType.cs b/QuantConnect.DataBento/Models/Enums/RecordType.cs new file mode 100644 index 0000000..67226e1 --- /dev/null +++ b/QuantConnect.DataBento/Models/Enums/RecordType.cs @@ -0,0 +1,86 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +namespace QuantConnect.Lean.DataSource.DataBento.Models.Enums; + +/// +/// Record type identifier (rtype) used in market data messages. +/// +public enum RecordType : byte +{ + /// Market-by-price record with book depth 0 (trades). + MarketByPriceDepth0 = 0, + + /// Market-by-price record with book depth 1 (TBBO, MBP-1). + MarketByPriceDepth1 = 1, + + /// Market-by-price record with book depth 10. + MarketByPriceDepth10 = 10, + + /// Exchange status record. + Status = 18, + + /// Instrument definition record. + Definition = 19, + + /// Order imbalance record. + Imbalance = 20, + + /// Error record from the live gateway. + Error = 21, + + /// Symbol mapping record from the live gateway. + SymbolMapping = 22, + + /// System record from the live gateway (e.g. heartbeat). + System = 23, + + /// Statistics record from the publisher. + Statistics = 24, + + /// OHLCV record at 1-second cadence. + OpenHighLowCloseVolume1Second = 32, + + /// OHLCV record at 1-minute cadence. + OpenHighLowCloseVolume1Minute = 33, + + /// OHLCV record at hourly cadence. + OpenHighLowCloseVolume1Hour = 34, + + /// OHLCV record at daily cadence. + OpenHighLowCloseVolume1Day = 35, + + /// Market-by-order record. + MarketByOrder = 160, + + /// Consolidated market-by-price record with book depth 1. + ConsolidatedMarketByPriceDepth1 = 177, + + /// Consolidated BBO at 1-second cadence. + ConsolidatedBestBidAndOffer1Second = 192, + + /// Consolidated BBO at 1-minute cadence. + ConsolidatedBestBidAndOffer1Minute = 193, + + /// Consolidated BBO with trades only. + TradeWithConsolidatedBestBidAndOffer = 194, + + /// Market-by-price BBO at 1-second cadence. + BBO1Second = 195, + + /// Market-by-price BBO at 1-minute cadence. + BBO1Minute = 196 +} diff --git a/QuantConnect.DataBento/Models/Header.cs b/QuantConnect.DataBento/Models/Header.cs index 2a2ead1..36774a2 100644 --- a/QuantConnect.DataBento/Models/Header.cs +++ b/QuantConnect.DataBento/Models/Header.cs @@ -13,6 +13,8 @@ * limitations under the License. */ +using QuantConnect.Lean.DataSource.DataBento.Models.Enums; + namespace QuantConnect.Lean.DataSource.DataBento.Models; /// @@ -29,7 +31,7 @@ public sealed class Header /// /// Record type identifier defining the data schema (e.g. trade, quote, bar). /// - public int Rtype { get; set; } + public RecordType Rtype { get; set; } /// /// DataBento publisher (exchange / data source) identifier. diff --git a/QuantConnect.DataBento/Models/Live/AuthenticationMessageRequest.cs b/QuantConnect.DataBento/Models/Live/AuthenticationMessageRequest.cs new file mode 100644 index 0000000..79d0715 --- /dev/null +++ b/QuantConnect.DataBento/Models/Live/AuthenticationMessageRequest.cs @@ -0,0 +1,69 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +namespace QuantConnect.Lean.DataSource.DataBento.Models.Live; + +public readonly struct AuthenticationMessageRequest +{ + private const int BucketIdLength = 5; + + private readonly string _dataset; + + private readonly string _auth; + + private readonly TimeSpan _heartBeatInterval; + + + public AuthenticationMessageRequest(string cramLine, string apiKey, string dataSet, TimeSpan? heartBeatInterval = default) + { + _dataset = dataSet; + + var newLineIndex = cramLine.IndexOf('\n'); + if (newLineIndex >= 0) + { + cramLine = cramLine[..newLineIndex]; + } + + cramLine = cramLine[(cramLine.IndexOf('=') + 1)..]; + + var challengeKey = cramLine + '|' + apiKey; + var bucketId = apiKey[^BucketIdLength..]; + + _auth = $"{QuantConnect.Extensions.ToSHA256(challengeKey)}-{bucketId}"; + + switch (heartBeatInterval) + { + case null: + _heartBeatInterval = TimeSpan.FromSeconds(5); + break; + case { TotalSeconds: < 5 }: + throw new ArgumentOutOfRangeException(nameof(heartBeatInterval), "The Heartbeat interval must be not les 5 seconds."); + default: + _heartBeatInterval = heartBeatInterval.Value; + break; + } + } + + public override string ToString() + { + return $"auth={_auth}|dataset={_dataset}|pretty_px=1|encoding=json|heartbeat_interval_s={_heartBeatInterval.TotalSeconds}"; + } + + public string GetStartSessionMessage() + { + return "start_session"; + } +} diff --git a/QuantConnect.DataBento/Models/Live/AuthenticationMessageResponse.cs b/QuantConnect.DataBento/Models/Live/AuthenticationMessageResponse.cs new file mode 100644 index 0000000..51720a3 --- /dev/null +++ b/QuantConnect.DataBento/Models/Live/AuthenticationMessageResponse.cs @@ -0,0 +1,54 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +using QuantConnect.Logging; + +namespace QuantConnect.Lean.DataSource.DataBento.Models.Live; + +public readonly struct AuthenticationMessageResponse +{ + public bool Success { get; } + public string? Error { get; } + public int SessionId { get; } + + public AuthenticationMessageResponse(string input) + { + if (Log.DebuggingEnabled) + { + Log.Debug($"{nameof(AuthenticationMessageResponse)}.ctor: Authentication response: {input}"); + } + + var parts = input.Split('|', StringSplitOptions.RemoveEmptyEntries); + + foreach (var part in parts) + { + var kv = part.Split('=', 2); + + switch (kv[0]) + { + case "success": + Success = kv[1] == "1"; + break; + case "error": + Error = kv[1]; + break; + case "session_id": + SessionId = int.Parse(kv[1]); + break; + } + } + } +} diff --git a/QuantConnect.DataBento/Models/Live/HeartbeatMessage.cs b/QuantConnect.DataBento/Models/Live/HeartbeatMessage.cs new file mode 100644 index 0000000..4f1b5b7 --- /dev/null +++ b/QuantConnect.DataBento/Models/Live/HeartbeatMessage.cs @@ -0,0 +1,22 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +namespace QuantConnect.Lean.DataSource.DataBento.Models.Live; + +public sealed class HeartbeatMessage : MarketDataRecord +{ + public required string Msg { get; set; } +} From 2a2392f76a0d7317a20ca163c50a65eade1dba80 Mon Sep 17 00:00:00 2001 From: Romazes Date: Wed, 28 Jan 2026 01:50:56 +0200 Subject: [PATCH 12/13] feat: Data Queue Handler --- .../DataBentoJsonConverterTests.cs | 98 ++- .../DataBentoLiveAPIClientTests.cs | 37 +- .../DataBentoRawLiveClientTests.cs | 146 ---- QuantConnect.DataBento/Api/LiveAPIClient.cs | 57 +- .../Api/LiveDataTcpClientWrapper.cs | 27 +- .../Converters/LiveDataConverter.cs | 82 +++ .../DataBentoDataProvider.cs | 366 +++++---- .../DataBentoRawLiveClient.cs | 696 ------------------ .../DataBentoSymbolMapper.cs | 24 +- QuantConnect.DataBento/Extensions.cs | 15 + QuantConnect.DataBento/Models/Header.cs | 4 +- QuantConnect.DataBento/Models/LevelOneData.cs | 3 +- .../SymbolMappingConfirmationEventArgs.cs | 43 ++ .../Models/Live/SymbolMappingMessage.cs | 30 + .../Serialization/JsonSettings.cs | 11 + 15 files changed, 584 insertions(+), 1055 deletions(-) delete mode 100644 QuantConnect.DataBento.Tests/DataBentoRawLiveClientTests.cs create mode 100644 QuantConnect.DataBento/Converters/LiveDataConverter.cs delete mode 100644 QuantConnect.DataBento/DataBentoRawLiveClient.cs create mode 100644 QuantConnect.DataBento/Models/Live/SymbolMappingConfirmationEventArgs.cs create mode 100644 QuantConnect.DataBento/Models/Live/SymbolMappingMessage.cs diff --git a/QuantConnect.DataBento.Tests/DataBentoJsonConverterTests.cs b/QuantConnect.DataBento.Tests/DataBentoJsonConverterTests.cs index 5b9de56..d29d488 100644 --- a/QuantConnect.DataBento.Tests/DataBentoJsonConverterTests.cs +++ b/QuantConnect.DataBento.Tests/DataBentoJsonConverterTests.cs @@ -44,7 +44,7 @@ public void DeserializeHistoricalOhlcvBar() Assert.IsNotNull(res); Assert.AreEqual(1738281600000000000m, res.Header.TsEvent); - Assert.AreEqual(35, res.Header.Rtype); + Assert.AreEqual(RecordType.OpenHighLowCloseVolume1Day, res.Header.Rtype); Assert.AreEqual(1, res.Header.PublisherId); Assert.AreEqual(42140878, res.Header.InstrumentId); @@ -92,7 +92,7 @@ public void DeserializeHistoricalLevelOneData() Assert.AreEqual(1768137063449660443, res.TsRecv); Assert.AreEqual(1768137063107829777, res.Header.TsEvent); - Assert.AreEqual(1, res.Header.Rtype); + Assert.AreEqual(RecordType.MarketByPriceDepth1, res.Header.Rtype); Assert.AreEqual(1, res.Header.PublisherId); Assert.AreEqual(42140878, res.Header.InstrumentId); @@ -141,7 +141,7 @@ public void DeserializeHistoricalStatisticsData() Assert.AreEqual(1768156232522476283, res.Header.TsEvent); - Assert.AreEqual(24, res.Header.Rtype); + Assert.AreEqual(RecordType.Statistics, res.Header.Rtype); Assert.AreEqual(1, res.Header.PublisherId); Assert.AreEqual(42566722, res.Header.InstrumentId); Assert.AreEqual(470m, res.Quantity); @@ -192,4 +192,96 @@ public void ParsePotentialCramChallenges(string challenge, string expectedString Assert.AreEqual(expectedString, actualString); } + + [Test] + public void DeserializeSymbolMappingMessage() + { + var json = @"{ + ""hd"": { + ""ts_event"": ""1769546804979770503"", + ""rtype"": 22, + ""publisher_id"": 0, + ""instrument_id"": 42140878 + }, + ""stype_in_symbol"": ""ESH6"", + ""stype_out_symbol"": ""ESH6"", + ""start_ts"": ""18446744073709551615"", + ""end_ts"": ""18446744073709551615"" +}"; + + var marketData = json.DeserializeSnakeCaseLiveData(); + + Assert.IsNotNull(marketData); + Assert.AreEqual(1769546804979770503, marketData.Header.TsEvent); + Assert.AreEqual(RecordType.SymbolMapping, marketData.Header.Rtype); + Assert.AreEqual(0, marketData.Header.PublisherId); + Assert.AreEqual(42140878, marketData.Header.InstrumentId); + + Assert.IsInstanceOf(marketData); + + var sm = marketData as SymbolMappingMessage; + + Assert.AreEqual("ESH6", sm.StypeInSymbol); + Assert.AreEqual("ESH6", sm.StypeOutSymbol); + } + + + [Test] + public void DeserializeMarketByPriceMessage() + { + var json = @" +{ + ""ts_recv"": ""1769546804990938439"", + ""hd"": { + ""ts_event"": ""1769546804990833083"", + ""rtype"": 1, + ""publisher_id"": 1, + ""instrument_id"": 42005017 + }, + ""action"": ""A"", + ""side"": ""A"", + ""depth"": 0, + ""price"": ""2676.400000000"", + ""size"": 1, + ""flags"": 128, + ""ts_in_delta"": 17695, + ""sequence"": 126257483, + ""levels"": [ + { + ""bid_px"": ""2676.300000000"", + ""ask_px"": ""2676.400000000"", + ""bid_sz"": 14, + ""ask_sz"": 2, + ""bid_ct"": 8, + ""ask_ct"": 2 + } + ] +}"; + var marketData = json.DeserializeSnakeCaseLiveData(); + + Assert.IsNotNull(marketData); + Assert.AreEqual(1769546804990833083, marketData.Header.TsEvent); + Assert.AreEqual(RecordType.MarketByPriceDepth1, marketData.Header.Rtype); + Assert.AreEqual(1, marketData.Header.PublisherId); + Assert.AreEqual(42005017, marketData.Header.InstrumentId); + + Assert.IsInstanceOf(marketData); + + var mbp = marketData as LevelOneData; + Assert.AreEqual('A', mbp.Action); + Assert.AreEqual('A', mbp.Side); + Assert.AreEqual(0, mbp.Depth); + Assert.AreEqual(2676.4m, mbp.Price); + Assert.AreEqual(1, mbp.Size); + Assert.AreEqual(128, mbp.Flags); + Assert.IsNotNull(mbp.Levels); + Assert.AreEqual(1, mbp.Levels.Count); + var level = mbp.Levels[0]; + Assert.AreEqual(2676.3m, level.BidPx); + Assert.AreEqual(2676.4m, level.AskPx); + Assert.AreEqual(14, level.BidSz); + Assert.AreEqual(2, level.AskSz); + Assert.AreEqual(8, level.BidCt); + Assert.AreEqual(2, level.AskCt); + } } diff --git a/QuantConnect.DataBento.Tests/DataBentoLiveAPIClientTests.cs b/QuantConnect.DataBento.Tests/DataBentoLiveAPIClientTests.cs index 1124895..ea66096 100644 --- a/QuantConnect.DataBento.Tests/DataBentoLiveAPIClientTests.cs +++ b/QuantConnect.DataBento.Tests/DataBentoLiveAPIClientTests.cs @@ -17,6 +17,7 @@ using NUnit.Framework; using System.Threading; using QuantConnect.Configuration; +using System.Collections.Generic; using QuantConnect.Lean.DataSource.DataBento.Api; namespace QuantConnect.Lean.DataSource.DataBento.Tests; @@ -44,7 +45,7 @@ public void OneTimeSetUp() Assert.Inconclusive("Please set the 'databento-api-key' in your configuration to enable these tests."); } - _live = new LiveAPIClient(apiKey); + _live = new LiveAPIClient(apiKey, null); } [OneTimeTearDown] @@ -54,12 +55,40 @@ public void OneTimeTearDown() } [Test] - public void TestExample() + public void ShouldReceiveSymbolMappingConfirmation() { var dataAvailableEvent = new AutoResetEvent(false); - _live.Start(Dataset); + var subs = new Dictionary() + { + { Securities.Futures.Indices.SP500EMini + "H6", 0 }, + { Securities.Futures.Indices.Russell2000EMini + "H6", 0 } + }; + + void OnSymbolMappingConfirmation(object sender, Models.Live.SymbolMappingConfirmationEventArgs e) + { + if (subs.ContainsKey(e.Symbol)) + { + subs[e.Symbol] = e.InstrumentId; + dataAvailableEvent.Set(); + } + } + + _live.SymbolMappingConfirmation += OnSymbolMappingConfirmation; + + foreach (var s in subs.Keys) + { + _live.Subscribe(Dataset, s); + dataAvailableEvent.WaitOne(TimeSpan.FromSeconds(5)); + } + + dataAvailableEvent.WaitOne(TimeSpan.FromSeconds(1)); + + foreach (var instrumentId in subs.Values) + { + Assert.Greater(instrumentId, 0); + } - dataAvailableEvent.WaitOne(TimeSpan.FromSeconds(60)); + _live.SymbolMappingConfirmation -= OnSymbolMappingConfirmation; } } diff --git a/QuantConnect.DataBento.Tests/DataBentoRawLiveClientTests.cs b/QuantConnect.DataBento.Tests/DataBentoRawLiveClientTests.cs deleted file mode 100644 index ec03477..0000000 --- a/QuantConnect.DataBento.Tests/DataBentoRawLiveClientTests.cs +++ /dev/null @@ -1,146 +0,0 @@ -/* - * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * -*/ - -using System; -using NUnit.Framework; -using System.Threading; -using QuantConnect.Data; -using QuantConnect.Logging; -using QuantConnect.Data.Market; -using QuantConnect.Configuration; - -namespace QuantConnect.Lean.DataSource.DataBento.Tests; - -[TestFixture] -public class DataBentoRawLiveClientSyncTests -{ - private DataBentoRawLiveClient _client; - protected readonly string ApiKey = Config.Get("databento-api-key"); - - private static Symbol CreateEsFuture() - { - var expiration = new DateTime(2026, 3, 20); - return Symbol.CreateFuture("ES", Market.CME, expiration); - } - - [SetUp] - public void SetUp() - { - Log.Trace("DataBentoLiveClientTests: Using API Key: " + ApiKey); - _client = new DataBentoRawLiveClient(ApiKey); - } - - [TearDown] - public void TearDown() - { - _client?.Dispose(); - } - - [Test] - public void Connects() - { - var connected = _client.Connect(); - - Assert.IsTrue(connected); - Assert.IsTrue(_client.IsConnected); - - Log.Trace("Connected successfully"); - } - - [Test] - public void SubscribesToLeanFutureSymbol() - { - Assert.IsTrue(_client.Connect()); - - var symbol = CreateEsFuture(); - - Assert.IsTrue(_client.Subscribe(symbol, TickType.Trade)); - Assert.IsTrue(_client.StartSession()); - - Thread.Sleep(1000); - - Assert.IsTrue(_client.Unsubscribe(symbol)); - } - - [Test] - public void ReceivesTradeOrQuoteTicks() - { - var receivedEvent = new ManualResetEventSlim(false); - BaseData received = null; - - _client.DataReceived += (_, data) => - { - received = data; - receivedEvent.Set(); - }; - - Assert.IsTrue(_client.Connect()); - - var symbol = CreateEsFuture(); - - Assert.IsTrue(_client.Subscribe(symbol, TickType.Trade)); - Assert.IsTrue(_client.StartSession()); - - var gotData = receivedEvent.Wait(TimeSpan.FromMinutes(2)); - - if (!gotData) - { - Assert.Inconclusive("No data received (likely outside market hours)"); - return; - } - - Assert.NotNull(received); - Assert.AreEqual(symbol, received.Symbol); - - if (received is Tick tick) - { - Assert.Greater(tick.Time, DateTime.MinValue); - Assert.Greater(tick.Value, 0); - } - else if (received is TradeBar bar) - { - Assert.Greater(bar.Close, 0); - } - else - { - Assert.Fail($"Unexpected data type: {received.GetType()}"); - } - } - - [Test] - public void DisposeIsIdempotent() - { - var client = new DataBentoRawLiveClient(ApiKey); - Assert.DoesNotThrow(client.Dispose); - Assert.DoesNotThrow(client.Dispose); - } - - [Test] - public void SymbolMappingDoesNotThrow() - { - Assert.IsTrue(_client.Connect()); - - var symbol = CreateEsFuture(); - - Assert.DoesNotThrow(() => - { - _client.Subscribe(symbol, TickType.Trade); - _client.StartSession(); - Thread.Sleep(500); - _client.Unsubscribe(symbol); - }); - } -} diff --git a/QuantConnect.DataBento/Api/LiveAPIClient.cs b/QuantConnect.DataBento/Api/LiveAPIClient.cs index f8b60df..810dd9b 100644 --- a/QuantConnect.DataBento/Api/LiveAPIClient.cs +++ b/QuantConnect.DataBento/Api/LiveAPIClient.cs @@ -16,6 +16,8 @@ using QuantConnect.Util; using QuantConnect.Logging; +using QuantConnect.Lean.DataSource.DataBento.Models; +using QuantConnect.Lean.DataSource.DataBento.Models.Live; namespace QuantConnect.Lean.DataSource.DataBento.Api; @@ -25,9 +27,16 @@ public sealed class LiveAPIClient : IDisposable private readonly Dictionary _tcpClientByDataSet = []; - public LiveAPIClient(string apiKey) + private readonly Action _levelOneDataHandler; + + public event EventHandler? SymbolMappingConfirmation; + + public bool IsConnected => _tcpClientByDataSet.Values.All(c => c.IsConnected); + + public LiveAPIClient(string apiKey, Action levelOneDataHandler) { _apiKey = apiKey; + _levelOneDataHandler = levelOneDataHandler; } public void Dispose() @@ -39,36 +48,52 @@ public void Dispose() _tcpClientByDataSet.Clear(); } - public bool Start(string dataSet) + private LiveDataTcpClientWrapper EnsureDatasetConnection(string dataSet) { - LogTrace(nameof(Start), "Starting connection to DataBento live API"); - - if (_tcpClientByDataSet.TryGetValue(dataSet, out var existingClient) && existingClient.IsConnected) + if (_tcpClientByDataSet.TryGetValue(dataSet, out var liveDataTcpClient)) { - LogTrace(nameof(Start), $"Already connected to DataBento live API (Dataset: {dataSet})"); - return true; + return liveDataTcpClient; } - var liveDataTcpClient = new LiveDataTcpClientWrapper(dataSet, _apiKey); + LogTrace(nameof(EnsureDatasetConnection), "Starting connection to DataBento live API"); + + liveDataTcpClient = new LiveDataTcpClientWrapper(dataSet, _apiKey, MessageReceived); _tcpClientByDataSet[dataSet] = liveDataTcpClient; liveDataTcpClient.Connect(); - LogTrace(nameof(Start), $"Successfully connected to DataBento live API (Dataset: {dataSet})"); + LogTrace(nameof(EnsureDatasetConnection), $"Successfully connected to DataBento live API (Dataset: {dataSet})"); - return true; + return liveDataTcpClient; } public bool Subscribe(string dataSet, string symbol) { - if (!_tcpClientByDataSet.TryGetValue(dataSet, out var tcpClient) || !tcpClient.IsConnected) + EnsureDatasetConnection(dataSet).SubscribeOnMarketBestPriceLevelOne(symbol); + return true; + } + + private void MessageReceived(string message) + { + var data = message.DeserializeSnakeCaseLiveData(); + + if (data == null) { - LogError(nameof(Subscribe), $"Not connected to DataBento live API (Dataset: {dataSet})"); - return false; + LogError(nameof(MessageReceived), $"Failed to deserialize live data message: {message}"); + return; } - tcpClient.SubscribeOnMarketBestPriceLevelOne(symbol); - - return true; + switch (data) + { + case SymbolMappingMessage smm: + SymbolMappingConfirmation?.Invoke(this, new(smm.StypeInSymbol, smm.Header.InstrumentId)); + break; + case LevelOneData lod: + _levelOneDataHandler?.Invoke(lod); + break; + default: + LogError(nameof(MessageReceived), $"Received unsupported record type: {data.Header.Rtype}. Message: {message}"); + break; + } } private static void LogTrace(string method, string message) diff --git a/QuantConnect.DataBento/Api/LiveDataTcpClientWrapper.cs b/QuantConnect.DataBento/Api/LiveDataTcpClientWrapper.cs index 205ff78..b55e14a 100644 --- a/QuantConnect.DataBento/Api/LiveDataTcpClientWrapper.cs +++ b/QuantConnect.DataBento/Api/LiveDataTcpClientWrapper.cs @@ -25,7 +25,6 @@ namespace QuantConnect.Lean.DataSource.DataBento.Api; public sealed class LiveDataTcpClientWrapper : IDisposable { private const int DefaultPort = 13000; - private const int ReceiveBufferSize = 8192; private readonly string _gateway; private readonly string _dataSet; @@ -33,30 +32,33 @@ public sealed class LiveDataTcpClientWrapper : IDisposable private readonly TimeSpan _heartBeatInterval = TimeSpan.FromSeconds(10); private readonly TcpClient _tcpClient = new(); - private readonly byte[] _receiveBuffer = new byte[ReceiveBufferSize]; private readonly CancellationTokenSource _cancellationTokenSource = new(); - private readonly char[] _newLine = Environment.NewLine.ToCharArray(); private NetworkStream? _stream; + private StreamReader? _reader; private Task? _dataReceiverTask; private bool _isConnected; + private readonly Action MessageReceived; + /// /// Is client connected /// public bool IsConnected => _isConnected; - public LiveDataTcpClientWrapper(string dataSet, string apiKey) + public LiveDataTcpClientWrapper(string dataSet, string apiKey, Action messageReceived) { _apiKey = apiKey; _dataSet = dataSet; _gateway = DetermineGateway(dataSet); + MessageReceived = messageReceived; } public void Connect() { _tcpClient.Connect(_gateway, DefaultPort); _stream = _tcpClient.GetStream(); + _reader = new StreamReader(_stream, Encoding.ASCII); if (!Authenticate(_dataSet).SynchronouslyAwaitTask()) throw new Exception("Authentication failed"); @@ -71,8 +73,8 @@ public void Dispose() { _isConnected = false; - _stream?.Close(); - _stream?.DisposeSafely(); + _reader?.Close(); + _reader?.DisposeSafely(); _cancellationTokenSource?.Cancel(); _cancellationTokenSource?.DisposeSafely(); @@ -113,7 +115,7 @@ private async Task DataReceiverAsync(CancellationToken ct) break; } - LogTrace(methodName, $"Received: {line}"); + MessageReceived.Invoke(line); } } catch (OperationCanceledException) @@ -142,16 +144,7 @@ private async Task DataReceiverAsync(CancellationToken ct) private async Task ReadDataAsync(CancellationToken cancellationToken) { - var numberOfBytesToRead = await _stream.ReadAsync(_receiveBuffer.AsMemory(0, _receiveBuffer.Length), cancellationToken).ConfigureAwait(false); - - if (numberOfBytesToRead == 0) - { - return null; - } - - using var memoryStream = new MemoryStream(); - await memoryStream.WriteAsync(_receiveBuffer.AsMemory(0, numberOfBytesToRead), cancellationToken).ConfigureAwait(false); - return Encoding.ASCII.GetString(memoryStream.ToArray(), 0, numberOfBytesToRead).TrimEnd(_newLine); + return await _reader.ReadLineAsync(cancellationToken); } private void WriteData(string data) diff --git a/QuantConnect.DataBento/Converters/LiveDataConverter.cs b/QuantConnect.DataBento/Converters/LiveDataConverter.cs new file mode 100644 index 0000000..395706f --- /dev/null +++ b/QuantConnect.DataBento/Converters/LiveDataConverter.cs @@ -0,0 +1,82 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using QuantConnect.Lean.DataSource.DataBento.Models; +using QuantConnect.Lean.DataSource.DataBento.Models.Enums; +using QuantConnect.Lean.DataSource.DataBento.Models.Live; + +namespace QuantConnect.Lean.DataSource.DataBento.Converters; + +public class LiveDataConverter : JsonConverter +{ + private const string headerIdentifier = "hd"; + + private const string recordTypeIdentifier = "rtype"; + + private static JsonSerializer _snakeSerializer = new JsonSerializer + { + ContractResolver = Serialization.SnakeCaseContractResolver.Instance + }; + + /// + /// Gets a value indicating whether this can write JSON. + /// + /// true if this can write JSON; otherwise, false. + public override bool CanWrite => false; + + /// + /// Gets a value indicating whether this can read JSON. + /// + /// true if this can read JSON; otherwise, false. + public override bool CanRead => true; + + /// + /// Reads the JSON representation of the object. + /// + /// The to read from. + /// Type of the object. + /// The existing property value of the JSON that is being converted. + /// The calling serializer. + /// The object value. + public override MarketDataRecord ReadJson(JsonReader reader, Type objectType, MarketDataRecord? existingValue, bool hasExistingValue, JsonSerializer serializer) + { + var jObject = JObject.Load(reader); + + var recordType = jObject[headerIdentifier]?[recordTypeIdentifier]?.ToObject(); + + switch (recordType) + { + case RecordType.SymbolMapping: + return jObject.ToObject(_snakeSerializer); + case RecordType.MarketByPriceDepth1: + return jObject.ToObject(_snakeSerializer); + default: + return null; + } + } + + /// + /// Writes the JSON representation of the object. + /// + /// The to write to. + /// The value. + /// The calling serializer. + public override void WriteJson(JsonWriter writer, MarketDataRecord? value, JsonSerializer serializer) + { + throw new NotImplementedException(); + } +} \ No newline at end of file diff --git a/QuantConnect.DataBento/DataBentoDataProvider.cs b/QuantConnect.DataBento/DataBentoDataProvider.cs index ba228d1..5127a45 100644 --- a/QuantConnect.DataBento/DataBentoDataProvider.cs +++ b/QuantConnect.DataBento/DataBentoDataProvider.cs @@ -14,17 +14,24 @@ * */ -using NodaTime; -using QuantConnect.Data; +using System.Net; +using System.Text; +using Newtonsoft.Json; +using QuantConnect.Api; using QuantConnect.Util; +using QuantConnect.Data; +using Newtonsoft.Json.Linq; using QuantConnect.Logging; using QuantConnect.Packets; using QuantConnect.Interfaces; -using QuantConnect.Securities; -using QuantConnect.Data.Market; using QuantConnect.Configuration; +using System.Security.Cryptography; +using System.Net.NetworkInformation; using System.Collections.Concurrent; +using QuantConnect.Brokerages.LevelOneOrderBook; using QuantConnect.Lean.DataSource.DataBento.Api; +using QuantConnect.Lean.DataSource.DataBento.Models; +using QuantConnect.Lean.DataSource.DataBento.Models.Live; namespace QuantConnect.Lean.DataSource.DataBento; @@ -44,21 +51,27 @@ public partial class DataBentoProvider : IDataQueueHandler private readonly DataBentoSymbolMapper _symbolMapper = new(); - private readonly IDataAggregator _dataAggregator = Composer.Instance.GetExportedValueByTypeName( - Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager"), forceTypeNameOnExisting: false); - private EventBasedDataQueueHandlerSubscriptionManager _subscriptionManager; - private DataBentoRawLiveClient _client; - private bool _potentialUnsupportedResolutionMessageLogged; - private bool _sessionStarted = false; - private readonly object _sessionLock = new(); - private readonly MarketHoursDatabase _marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); - private readonly ConcurrentDictionary _symbolExchangeTimeZones = new(); + private readonly ConcurrentDictionary _pendingSubscriptions = []; + + private readonly Dictionary _subscribedSymbolsByDataBentoInstrumentId = []; + + /// + /// Manages Level 1 market data subscriptions and routing of updates to the shared . + /// Responsible for tracking and updating individual instances per symbol. + /// + private LevelOneServiceManager _levelOneServiceManager; + + private IDataAggregator _aggregator; + + private LiveAPIClient _liveApiClient; + private bool _initialized; /// /// Returns true if we're currently connected to the Data Provider /// - public bool IsConnected => _client?.IsConnected == true; + public bool IsConnected => _liveApiClient.IsConnected; + /// /// Initializes a new instance of the DataBentoProvider @@ -86,74 +99,50 @@ public DataBentoProvider(string apiKey) /// private void Initialize(string apiKey) { - Log.Debug("DataBentoProvider.Initialize(): Starting initialization"); - _subscriptionManager = new EventBasedDataQueueHandlerSubscriptionManager() - { - SubscribeImpl = (symbols, tickType) => - { - return SubscriptionLogic(symbols, tickType); - }, - UnsubscribeImpl = (symbols, tickType) => - { - return UnsubscribeLogic(symbols, tickType); - } - }; - - // Initialize the live client - _client = new DataBentoRawLiveClient(apiKey); - _client.DataReceived += OnDataReceived; + ValidateSubscription(); - // Connect to live gateway - Log.Debug("DataBentoProvider.Initialize(): Attempting connection to DataBento live gateway"); - var cancellationTokenSource = new CancellationTokenSource(); - Task.Factory.StartNew(() => + _aggregator = Composer.Instance.GetPart(); + if (_aggregator == null) { - try - { - var connected = _client.Connect(); - Log.Debug($"DataBentoProvider.Initialize(): Connect() returned {connected}"); + var aggregatorName = Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager"); + Log.Trace($"{nameof(DataBentoProvider)}.{nameof(Initialize)}: found no data aggregator instance, creating {aggregatorName}"); + _aggregator = Composer.Instance.GetExportedValueByTypeName(aggregatorName); + } - if (connected) - { - Log.Debug("DataBentoProvider.Initialize(): Successfully connected to DataBento live gateway"); - } - else - { - Log.Error("DataBentoProvider.Initialize(): Failed to connect to DataBento live gateway"); - } - } - catch (Exception ex) - { - Log.Error($"DataBentoProvider.Initialize(): Exception during Connect(): {ex.Message}\n{ex.StackTrace}"); - } - }, - cancellationTokenSource.Token, - TaskCreationOptions.LongRunning, - TaskScheduler.Default); + _liveApiClient = new LiveAPIClient(apiKey, HandleLevelOneData); + _liveApiClient.SymbolMappingConfirmation += OnSymbolMappingConfirmation; _historicalApiClient = new(apiKey); + + _levelOneServiceManager = new LevelOneServiceManager( + _aggregator, + (symbols, _) => Subscribe(symbols), + (symbols, _) => Unsubscribe(symbols)); + _initialized = true; + } - Log.Debug("DataBentoProvider.Initialize(): Initialization complete"); + private void OnSymbolMappingConfirmation(object? _, SymbolMappingConfirmationEventArgs smce) + { + if (_pendingSubscriptions.TryRemove(smce.Symbol, out var symbol)) + { + _subscribedSymbolsByDataBentoInstrumentId[smce.InstrumentId] = symbol; + } } - /// - /// Logic to unsubscribe from the specified symbols - /// - public bool UnsubscribeLogic(IEnumerable symbols, TickType tickType) + private void HandleLevelOneData(LevelOneData levelOneData) { - foreach (var symbol in symbols) + if (_subscribedSymbolsByDataBentoInstrumentId.TryGetValue(levelOneData.Header.InstrumentId, out var symbol)) { - Log.Debug($"DataBentoProvider.UnsubscribeImpl(): Processing symbol {symbol}"); - if (_client?.IsConnected != true) + var time = levelOneData.Header.UtcTime; + + _levelOneServiceManager.HandleLastTrade(symbol, time, levelOneData.Size, levelOneData.Price); + + foreach (var l in levelOneData.Levels) { - throw new InvalidOperationException($"DataBentoProvider.UnsubscribeImpl(): Client is not connected. Cannot unsubscribe from {symbol}"); + _levelOneServiceManager.HandleQuote(symbol, time, l.BidPx, l.BidSz, l.AskPx, l.AskSz); } - - _client.Unsubscribe(symbol); } - - return true; } /// @@ -174,23 +163,37 @@ private bool TryGetDataBentoDataSet(Symbol symbol, out string? dataSet) /// /// Logic to subscribe to the specified symbols /// - public bool SubscriptionLogic(IEnumerable symbols, TickType tickType) + public bool Subscribe(IEnumerable symbols) { - if (_client?.IsConnected != true) + foreach (var symbol in symbols) { - Log.Error("DataBentoProvider.SubscriptionLogic(): Client is not connected. Cannot subscribe to symbols"); - return false; + if (!TryGetDataBentoDataSet(symbol, out var dataSet)) + { + throw new ArgumentException($"No DataBento dataset mapping found for symbol {symbol} in market {symbol.ID.Market}. Cannot subscribe."); + } + + var brokerageSymbol = _symbolMapper.GetBrokerageSymbol(symbol); + + _pendingSubscriptions[brokerageSymbol] = symbol; + + _liveApiClient.Subscribe(dataSet, brokerageSymbol); } - foreach (var symbol in symbols) + return true; + } + + public bool Unsubscribe(IEnumerable symbols) + { + // Please note there is no unsubscribe method. Subscriptions end when the TCP connection closes. + + var symbolsToRemove = symbols.ToHashSet(); + + foreach (var (instrumentId, symbol) in _subscribedSymbolsByDataBentoInstrumentId) { - if (!CanSubscribe(symbol)) + if (symbolsToRemove.Contains(symbol)) { - Log.Error($"DataBentoProvider.SubscriptionLogic(): Unsupported subscription: {symbol}"); - return false; + _subscribedSymbolsByDataBentoInstrumentId.Remove(instrumentId); } - - _client.Subscribe(symbol, tickType); } return true; @@ -216,17 +219,13 @@ private static bool CanSubscribe(Symbol symbol) /// The new enumerator for this subscription request public IEnumerator? Subscribe(SubscriptionDataConfig dataConfig, EventHandler newDataAvailableHandler) { - lock (_sessionLock) + if (!CanSubscribe(dataConfig.Symbol)) { - if (!_sessionStarted) - { - Log.Debug("DataBentoProvider.SubscriptionLogic(): Starting session"); - _sessionStarted = _client.StartSession(); - } + return null; } - var enumerator = _dataAggregator.Add(dataConfig, newDataAvailableHandler); - _subscriptionManager.Subscribe(dataConfig); + var enumerator = _aggregator.Add(dataConfig, newDataAvailableHandler); + _levelOneServiceManager.Subscribe(dataConfig); return enumerator; } @@ -237,9 +236,8 @@ private static bool CanSubscribe(Symbol symbol) /// Subscription config to be removed public void Unsubscribe(SubscriptionDataConfig dataConfig) { - Log.Debug($"DataBentoProvider.Unsubscribe(): Received unsubscription request for {dataConfig.Symbol}, Resolution={dataConfig.Resolution}, TickType={dataConfig.TickType}"); - _subscriptionManager.Unsubscribe(dataConfig); - _dataAggregator.Remove(dataConfig); + _levelOneServiceManager.Unsubscribe(dataConfig); + _aggregator.Remove(dataConfig); } /// @@ -252,6 +250,13 @@ public void SetJob(LiveNodePacket job) { return; } + + if (!job.BrokerageData.TryGetValue("databento-api-key", out var apiKey) || string.IsNullOrWhiteSpace(apiKey)) + { + throw new ArgumentException("The DataBento API key is missing from the brokerage data."); + } + + Initialize(apiKey); } /// @@ -259,81 +264,150 @@ public void SetJob(LiveNodePacket job) /// public void Dispose() { - _dataAggregator?.DisposeSafely(); - _subscriptionManager?.DisposeSafely(); - _client?.DisposeSafely(); + _levelOneServiceManager?.DisposeSafely(); + _aggregator?.DisposeSafely(); + _liveApiClient?.DisposeSafely(); + _historicalApiClient?.DisposeSafely(); } - /// - /// Converts the given UTC time into the symbol security exchange time zone - /// - private DateTime GetTickTime(Symbol symbol, DateTime utcTime) + private class ModulesReadLicenseRead : RestResponse { - DateTimeZone exchangeTimeZone; - lock (_symbolExchangeTimeZones) - { - if (!_symbolExchangeTimeZones.TryGetValue(symbol, out exchangeTimeZone)) - { - // read the exchange time zone from market-hours-database - if (_marketHoursDatabase.TryGetEntry(symbol.ID.Market, symbol, symbol.SecurityType, out var entry)) - { - exchangeTimeZone = entry.ExchangeHours.TimeZone; - } - // If there is no entry for the given Symbol, default to New York - else - { - exchangeTimeZone = TimeZones.NewYork; - } + [JsonProperty(PropertyName = "license")] + public string License; - _symbolExchangeTimeZones[symbol] = exchangeTimeZone; - } - } - - return utcTime.ConvertFromUtc(exchangeTimeZone); + [JsonProperty(PropertyName = "organizationId")] + public string OrganizationId; } /// - /// Handles data received from the live client + /// Validate the user of this project has permission to be using it via our web API. /// - private void OnDataReceived(object _, BaseData data) + private static void ValidateSubscription() { try { - switch (data) + const int productId = 306; + var userId = Globals.UserId; + var token = Globals.UserToken; + var organizationId = Globals.OrganizationID; + // Verify we can authenticate with this user and token + var api = new ApiConnection(userId, token); + if (!api.Connected) { - case Tick tick: - tick.Time = GetTickTime(tick.Symbol, tick.Time); - lock (_dataAggregator) - { - _dataAggregator.Update(tick); - } - // Log.Trace($"DataBentoProvider.OnDataReceived(): Updated tick - Symbol: {tick.Symbol}, " + - // $"TickType: {tick.TickType}, Price: {tick.Value}, Quantity: {tick.Quantity}"); - break; - - case TradeBar tradeBar: - tradeBar.Time = GetTickTime(tradeBar.Symbol, tradeBar.Time); - tradeBar.EndTime = GetTickTime(tradeBar.Symbol, tradeBar.EndTime); - lock (_dataAggregator) + throw new ArgumentException("Invalid api user id or token, cannot authenticate subscription."); + } + // Compile the information we want to send when validating + var information = new Dictionary() + { + {"productId", productId}, + {"machineName", Environment.MachineName}, + {"userName", Environment.UserName}, + {"domainName", Environment.UserDomainName}, + {"os", Environment.OSVersion} + }; + // IP and Mac Address Information + try + { + var interfaceDictionary = new List>(); + foreach (var nic in NetworkInterface.GetAllNetworkInterfaces().Where(nic => nic.OperationalStatus == OperationalStatus.Up)) + { + var interfaceInformation = new Dictionary(); + // Get UnicastAddresses + var addresses = nic.GetIPProperties().UnicastAddresses + .Select(uniAddress => uniAddress.Address) + .Where(address => !IPAddress.IsLoopback(address)).Select(x => x.ToString()); + // If this interface has non-loopback addresses, we will include it + if (!addresses.IsNullOrEmpty()) { - _dataAggregator.Update(tradeBar); + interfaceInformation.Add("unicastAddresses", addresses); + // Get MAC address + interfaceInformation.Add("MAC", nic.GetPhysicalAddress().ToString()); + // Add Interface name + interfaceInformation.Add("name", nic.Name); + // Add these to our dictionary + interfaceDictionary.Add(interfaceInformation); } - // Log.Trace($"DataBentoProvider.OnDataReceived(): Updated TradeBar - Symbol: {tradeBar.Symbol}, " + - // $"O:{tradeBar.Open} H:{tradeBar.High} L:{tradeBar.Low} C:{tradeBar.Close} V:{tradeBar.Volume}"); - break; + } + information.Add("networkInterfaces", interfaceDictionary); + } + catch (Exception) + { + // NOP, not necessary to crash if fails to extract and add this information + } + // Include our OrganizationId if specified + if (!string.IsNullOrEmpty(organizationId)) + { + information.Add("organizationId", organizationId); + } - default: - data.Time = GetTickTime(data.Symbol, data.Time); - lock (_dataAggregator) - { - _dataAggregator.Update(data); - } - break; + // Create HTTP request + using var request = ApiUtils.CreateJsonPostRequest("modules/license/read", information); + + api.TryRequest(request, out ModulesReadLicenseRead result); + if (!result.Success) + { + throw new InvalidOperationException($"Request for subscriptions from web failed, Response Errors : {string.Join(',', result.Errors)}"); + } + + var encryptedData = result.License; + // Decrypt the data we received + DateTime? expirationDate = null; + long? stamp = null; + bool? isValid = null; + if (encryptedData != null) + { + // Fetch the org id from the response if it was not set, we need it to generate our validation key + if (string.IsNullOrEmpty(organizationId)) + { + organizationId = result.OrganizationId; + } + // Create our combination key + var password = $"{token}-{organizationId}"; + var key = SHA256.HashData(Encoding.UTF8.GetBytes(password)); + // Split the data + var info = encryptedData.Split("::"); + var buffer = Convert.FromBase64String(info[0]); + var iv = Convert.FromBase64String(info[1]); + // Decrypt our information + using var aes = new AesManaged(); + var decryptor = aes.CreateDecryptor(key, iv); + using var memoryStream = new MemoryStream(buffer); + using var cryptoStream = new CryptoStream(memoryStream, decryptor, CryptoStreamMode.Read); + using var streamReader = new StreamReader(cryptoStream); + var decryptedData = streamReader.ReadToEnd(); + if (!decryptedData.IsNullOrEmpty()) + { + var jsonInfo = JsonConvert.DeserializeObject(decryptedData); + expirationDate = jsonInfo["expiration"]?.Value(); + isValid = jsonInfo["isValid"]?.Value(); + stamp = jsonInfo["stamped"]?.Value(); + } + } + // Validate our conditions + if (!expirationDate.HasValue || !isValid.HasValue || !stamp.HasValue) + { + throw new InvalidOperationException("Failed to validate subscription."); + } + + var nowUtc = DateTime.UtcNow; + var timeSpan = nowUtc - Time.UnixTimeStampToDateTime(stamp.Value); + if (timeSpan > TimeSpan.FromHours(12)) + { + throw new InvalidOperationException("Invalid API response."); + } + if (!isValid.Value) + { + throw new ArgumentException($"Your subscription is not valid, please check your product subscriptions on our website."); + } + if (expirationDate < nowUtc) + { + throw new ArgumentException($"Your subscription expired {expirationDate}, please renew in order to use this product."); } } - catch (Exception ex) + catch (Exception e) { - Log.Error($"DataBentoProvider.OnDataReceived(): Error updating data aggregator: {ex.Message}\n{ex.StackTrace}"); + Log.Error($"PolygonDataProvider.ValidateSubscription(): Failed during validation, shutting down. Error : {e.Message}"); + throw; } } } diff --git a/QuantConnect.DataBento/DataBentoRawLiveClient.cs b/QuantConnect.DataBento/DataBentoRawLiveClient.cs deleted file mode 100644 index 06d8e6a..0000000 --- a/QuantConnect.DataBento/DataBentoRawLiveClient.cs +++ /dev/null @@ -1,696 +0,0 @@ -/* - * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * -*/ - -using System.Text; -using System.Net.Sockets; -using System.Security.Cryptography; -using System.Collections.Concurrent; -using System.Text.Json; -using QuantConnect.Data; -using QuantConnect.Data.Market; -using QuantConnect.Logging; - -namespace QuantConnect.Lean.DataSource.DataBento; - -/// -/// DataBento Raw TCP client for live streaming data -/// -public class DataBentoRawLiveClient : IDisposable -{ - /// - /// The DataBento API key for authentication - /// - private readonly string _apiKey; - /// - /// The DataBento live gateway address to receive data from - /// - private const string _gateway = "glbx-mdp3.lsg.databento.com:13000"; - /// - /// The dataset to subscribe to - /// - private readonly string _dataset; - private readonly TcpClient? _tcpClient; - private readonly string _host; - private readonly int _port; - private NetworkStream? _stream; - private StreamReader _reader; - private StreamWriter _writer; - private readonly CancellationTokenSource _cancellationTokenSource; - private readonly ConcurrentDictionary _subscriptions; - private readonly object _connectionLock = new object(); - private bool _isConnected; - private bool _disposed; - private const decimal PriceScaleFactor = 1e-9m; - private readonly ConcurrentDictionary _instrumentIdToSymbol = new ConcurrentDictionary(); - private readonly DataBentoSymbolMapper _symbolMapper; - - /// - /// Event fired when new data is received - /// - public event EventHandler DataReceived; - - /// - /// Event fired when connection status changes - /// - public event EventHandler ConnectionStatusChanged; - - /// - /// Gets whether the client is currently connected - /// - public bool IsConnected => _isConnected && _tcpClient?.Connected == true; - - /// - /// Initializes a new instance of the DataBentoRawLiveClient - /// The DataBento API key. - /// - public DataBentoRawLiveClient(string apiKey, string dataset = "GLBX.MDP3") - { - _apiKey = apiKey ?? throw new ArgumentNullException(nameof(apiKey)); - _dataset = dataset; - _tcpClient = new TcpClient(); - _subscriptions = new ConcurrentDictionary(); - _cancellationTokenSource = new CancellationTokenSource(); - _symbolMapper = new DataBentoSymbolMapper(); - - var parts = _gateway.Split(':'); - _host = parts[0]; - _port = parts.Length > 1 ? int.Parse(parts[1]) : 13000; - } - - /// - /// Connects to the DataBento live gateway - /// - public bool Connect() - { - Log.Trace("DataBentoRawLiveClient.Connect(): Connecting to DataBento live gateway"); - if (_isConnected) - { - return _isConnected; - } - - try - { - _tcpClient.Connect(_host, _port); - _stream = _tcpClient.GetStream(); - _reader = new StreamReader(_stream, Encoding.ASCII); - _writer = new StreamWriter(_stream, Encoding.ASCII) { AutoFlush = true }; - - // Perform authentication handshake - if (Authenticate()) - { - _isConnected = true; - ConnectionStatusChanged?.Invoke(this, true); - - // Start message processing - Task.Run(ProcessMessages, _cancellationTokenSource.Token); - - Log.Trace("DataBentoRawLiveClient.Connect(): Connected and authenticated to DataBento live gateway"); - return true; - } - } - catch (Exception ex) - { - Log.Error($"DataBentoRawLiveClient.Connect(): Failed to connect: {ex.Message}"); - Disconnect(); - } - - return false; - } - - /// - /// Authenticates with the DataBento gateway using CRAM-SHA256 - /// - private bool Authenticate() - { - try - { - // Read greeting and challenge - var versionLine = _reader.ReadLine(); - var cramLine = _reader.ReadLine(); - - if (string.IsNullOrEmpty(versionLine) || string.IsNullOrEmpty(cramLine)) - { - Log.Error("DataBentoRawLiveClient.Authenticate(): Failed to receive greeting or challenge"); - return false; - } - - // Parse challenge - var cramParts = cramLine.Split('='); - if (cramParts.Length != 2 || cramParts[0] != "cram") - { - Log.Error("DataBentoRawLiveClient.Authenticate(): Invalid challenge format"); - return false; - } - var cram = cramParts[1].Trim(); - - // Auth - _writer.WriteLine($"auth={GetAuthStringFromCram(cram)}|dataset={_dataset}|encoding=json|ts_out=0"); - var authResp = _reader.ReadLine(); - if (!authResp.Contains("success=1")) - { - Log.Error($"DataBentoRawLiveClient.Authenticate(): Authentication failed: {authResp}"); - return false; - } - - Log.Trace("DataBentoRawLiveClient.Authenticate(): Authentication successful"); - return true; - } - catch (Exception ex) - { - Log.Error($"DataBentoRawLiveClient.Authenticate(): Authentication failed: {ex.Message}"); - return false; - } - } - - /// - /// Handles the DataBento authentication string from a CRAM challenge - /// - /// The CRAM challenge string - /// The auth string to send to the server - private string GetAuthStringFromCram(string cram) - { - if (string.IsNullOrWhiteSpace(cram)) - throw new ArgumentException("CRAM challenge cannot be null or empty", nameof(cram)); - - string concat = $"{cram}|{_apiKey}"; - string hashHex = ComputeSHA256(concat); - string bucketId = _apiKey.Substring(_apiKey.Length - 5); - - return $"{hashHex}-{bucketId}"; - } - - /// - /// Subscribes to live data for a symbol - /// - public bool Subscribe(Symbol symbol, TickType tickType) - { - if (!IsConnected) - { - Log.Error("DataBentoRawLiveClient.Subscribe(): Not connected to gateway"); - return false; - } - - try - { - // Get the databento symbol form LEAN symbol - var databentoSymbol = _symbolMapper.GetBrokerageSymbol(symbol); - var schema = "mbp-1"; - var resolution = Resolution.Tick; - - // subscribe - var subscribeMessage = $"schema={schema}|stype_in=parent|symbols={databentoSymbol}"; - Log.Debug($"DataBentoRawLiveClient.Subscribe(): Subscribing with message: {subscribeMessage}"); - - // Send subscribe message - _writer.WriteLine(subscribeMessage); - - // Store subscription - _subscriptions.TryAdd(symbol, (resolution, tickType)); - Log.Debug($"DataBentoRawLiveClient.Subscribe(): Subscribed to {symbol} ({databentoSymbol}) at {resolution} resolution for {tickType}"); - - return true; - } - catch (Exception ex) - { - Log.Error($"DataBentoRawLiveClient.Subscribe(): Failed to subscribe to {symbol}: {ex.Message}"); - return false; - } - } - - /// - /// Starts the session to begin receiving data - /// - public bool StartSession() - { - if (!IsConnected) - { - Log.Error("DataBentoRawLiveClient.StartSession(): Not connected"); - return false; - } - - try - { - Log.Trace("DataBentoRawLiveClient.StartSession(): Starting session"); - _writer.WriteLine("start_session=1"); - return true; - } - catch (Exception ex) - { - Log.Error($"DataBentoRawLiveClient.StartSession(): Failed to start session: {ex.Message}"); - return false; - } - } - - /// - /// Unsubscribes from live data for a symbol - /// - public bool Unsubscribe(Symbol symbol) - { - try - { - // Please note there is no unsubscribe method. Subscriptions end when the TCP connection closes. - - if (_subscriptions.TryRemove(symbol, out _)) - { - Log.Debug($"DataBentoRawLiveClient.Unsubscribe(): Unsubscribed from {symbol}"); - } - return true; - } - catch (Exception ex) - { - Log.Error($"DataBentoRawLiveClient.Unsubscribe(): Failed to unsubscribe from {symbol}: {ex.Message}"); - return false; - } - } - - /// - /// Processes incoming messages from the DataBento gateway - /// - private void ProcessMessages() - { - Log.Debug("DataBentoRawLiveClient.ProcessMessages(): Starting message processing"); - - try - { - while (!_cancellationTokenSource.IsCancellationRequested && IsConnected) - { - var line = _reader.ReadLine(); - if (string.IsNullOrWhiteSpace(line)) - { - Log.Trace("DataBentoRawLiveClient.ProcessMessages(): Line is null or empty. Issue receiving data."); - break; - } - - ProcessSingleMessage(line); - } - } - catch (OperationCanceledException) - { - Log.Trace("DataBentoRawLiveClient.ProcessMessages(): Message processing cancelled"); - } - catch (IOException ex) when (ex.InnerException is SocketException) - { - Log.Trace($"DataBentoRawLiveClient.ProcessMessages(): Socket exception: {ex.Message}"); - } - catch (Exception ex) - { - Log.Error($"DataBentoRawLiveClient.ProcessMessages(): Error processing messages: {ex.Message}\n{ex.StackTrace}"); - } - finally - { - Disconnect(); - } - } - - /// - /// Processes a single message from DataBento - /// - private void ProcessSingleMessage(string message) - { - try - { - using var document = JsonDocument.Parse(message); - var root = document.RootElement; - - // Check for error messages - if (root.TryGetProperty("hd", out var headerElement)) - { - if (headerElement.TryGetProperty("rtype", out var rtypeElement)) - { - var rtype = rtypeElement.GetInt32(); - - switch (rtype) - { - case 23: - // System message - if (root.TryGetProperty("msg", out var msgElement)) - { - Log.Debug($"DataBentoRawLiveClient: System message: {msgElement.GetString()}"); - } - return; - - case 22: - // Symbol mapping message - if (root.TryGetProperty("stype_in_symbol", out var inSymbol) && - root.TryGetProperty("stype_out_symbol", out var outSymbol) && - headerElement.TryGetProperty("instrument_id", out var instId)) - { - var instrumentId = instId.GetInt64(); - var outSymbolStr = outSymbol.GetString(); - - Log.Debug($"DataBentoRawLiveClient: Symbol mapping: {inSymbol.GetString()} -> {outSymbolStr} (instrument_id: {instrumentId})"); - - if (outSymbolStr != null) - { - // Let's find the subscribed symbol to get the market and security type - var inSymbolStr = inSymbol.GetString(); - var subscription = _subscriptions.Keys.FirstOrDefault(s => _symbolMapper.GetBrokerageSymbol(s) == inSymbolStr); - if (subscription != null) - { - if (subscription.SecurityType == SecurityType.Future) - { - var leanSymbol = _symbolMapper.GetLeanSymbolForFuture(outSymbolStr); - if (leanSymbol == null) - { - Log.Trace($"DataBentoRawLiveClient: Future spreads are not supported: {outSymbolStr}. Skipping mapping."); - return; - } - _instrumentIdToSymbol[instrumentId] = leanSymbol; - Log.Debug($"DataBentoRawLiveClient: Mapped instrument_id {instrumentId} to {leanSymbol}"); - } - } - } - } - return; - - case 1: - // MBP-1 (Market By Price) - HandleMBPMessage(root, headerElement); - return; - - case 0: - // Trade messages - HandleTradeTickMessage(root, headerElement); - return; - - case 32: - case 33: - case 34: - case 35: - // OHLCV bar messages - HandleOHLCVMessage(root, headerElement); - return; - - default: - Log.Error($"DataBentoRawLiveClient: Unknown rtype {rtype} in message"); - return; - } - } - } - - // Handle other message types if needed - if (root.TryGetProperty("error", out var errorElement)) - { - Log.Error($"DataBentoRawLiveClient: Server error: {errorElement.GetString()}"); - } - } - catch (JsonException ex) - { - Log.Error($"DataBentoRawLiveClient.ProcessSingleMessage(): JSON parse error: {ex.Message}"); - } - catch (Exception ex) - { - Log.Error($"DataBentoRawLiveClient.ProcessSingleMessage(): Error: {ex.Message}"); - } - } - - /// - /// Handles OHLCV messages and converts to LEAN TradeBar data - /// - private void HandleOHLCVMessage(JsonElement root, JsonElement header) - { - try - { - if (!header.TryGetProperty("ts_event", out var tsElement) || - !header.TryGetProperty("instrument_id", out var instIdElement)) - { - return; - } - - // Convert timestamp from nanoseconds to DateTime - var timestampNs = long.Parse(tsElement.GetString()!); - var unixEpoch = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc); - var timestamp = unixEpoch.AddTicks(timestampNs / 100); - - var instrumentId = instIdElement.GetInt64(); - - if (!_instrumentIdToSymbol.TryGetValue(instrumentId, out var matchedSymbol)) - { - Log.Debug($"DataBentoRawLiveClient: No mapping for instrument_id {instrumentId} in OHLCV message."); - return; - } - - // Get the resolution for this symbol - if (!_subscriptions.TryGetValue(matchedSymbol, out var subscription)) - { - return; - } - - var resolution = subscription.Item1; - - // Extract OHLCV data - if (root.TryGetProperty("open", out var openElement) && - root.TryGetProperty("high", out var highElement) && - root.TryGetProperty("low", out var lowElement) && - root.TryGetProperty("close", out var closeElement) && - root.TryGetProperty("volume", out var volumeElement)) - { - // Parse prices - var openRaw = long.Parse(openElement.GetString()!); - var highRaw = long.Parse(highElement.GetString()!); - var lowRaw = long.Parse(lowElement.GetString()!); - var closeRaw = long.Parse(closeElement.GetString()!); - var volume = volumeElement.GetInt64(); - - var open = openRaw * PriceScaleFactor; - var high = highRaw * PriceScaleFactor; - var low = lowRaw * PriceScaleFactor; - var close = closeRaw * PriceScaleFactor; - - // Determine the period based on resolution - TimeSpan period = resolution switch - { - Resolution.Second => TimeSpan.FromSeconds(1), - Resolution.Minute => TimeSpan.FromMinutes(1), - Resolution.Hour => TimeSpan.FromHours(1), - Resolution.Daily => TimeSpan.FromDays(1), - _ => TimeSpan.FromMinutes(1) - }; - - // Create TradeBar - var tradeBar = new TradeBar( - timestamp, - matchedSymbol, - open, - high, - low, - close, - volume, - period - ); - - // Log.Trace($"DataBentoRawLiveClient: OHLCV bar: {matchedSymbol} O={open} H={high} L={low} C={close} V={volume} at {timestamp}"); - DataReceived?.Invoke(this, tradeBar); - } - } - catch (Exception ex) - { - Log.Error($"DataBentoRawLiveClient.HandleOHLCVMessage(): Error: {ex.Message}"); - } - } - - /// - /// Handles MBP messages for quote ticks - /// - private void HandleMBPMessage(JsonElement root, JsonElement header) - { - try - { - if (!header.TryGetProperty("ts_event", out var tsElement) || - !header.TryGetProperty("instrument_id", out var instIdElement)) - { - return; - } - - // Convert timestamp from nanoseconds to DateTime - var timestampNs = long.Parse(tsElement.GetString()!); - var unixEpoch = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc); - var timestamp = unixEpoch.AddTicks(timestampNs / 100); - - var instrumentId = instIdElement.GetInt64(); - - if (!_instrumentIdToSymbol.TryGetValue(instrumentId, out var matchedSymbol)) - { - Log.Trace($"DataBentoRawLiveClient: No mapping for instrument_id {instrumentId} in MBP message."); - return; - } - - // For MBP-1, bid/ask data is in the levels array at index 0 - if (root.TryGetProperty("levels", out var levelsElement) && - levelsElement.GetArrayLength() > 0) - { - var level0 = levelsElement[0]; - - var quoteTick = new Tick - { - Symbol = matchedSymbol, - Time = timestamp, - TickType = TickType.Quote - }; - - if (level0.TryGetProperty("ask_px", out var askPxElement) && - level0.TryGetProperty("ask_sz", out var askSzElement)) - { - var askPriceRaw = long.Parse(askPxElement.GetString()!); - quoteTick.AskPrice = askPriceRaw * PriceScaleFactor; - quoteTick.AskSize = askSzElement.GetInt32(); - } - - if (level0.TryGetProperty("bid_px", out var bidPxElement) && - level0.TryGetProperty("bid_sz", out var bidSzElement)) - { - var bidPriceRaw = long.Parse(bidPxElement.GetString()!); - quoteTick.BidPrice = bidPriceRaw * PriceScaleFactor; - quoteTick.BidSize = bidSzElement.GetInt32(); - } - - // Set the tick value to the mid price - quoteTick.Value = (quoteTick.BidPrice + quoteTick.AskPrice) / 2; - - // QuantConnect convention: Quote ticks should have zero Price and Quantity - quoteTick.Quantity = 0; - - // Log.Trace($"DataBentoRawLiveClient: Quote tick: {matchedSymbol} Bid={quoteTick.BidPrice}x{quoteTick.BidSize} Ask={quoteTick.AskPrice}x{quoteTick.AskSize}"); - DataReceived?.Invoke(this, quoteTick); - } - } - catch (Exception ex) - { - Log.Error($"DataBentoRawLiveClient.HandleMBPMessage(): Error: {ex.Message}"); - } - } - - /// - /// Handles trade tick messages. Aggressor fills - /// - private void HandleTradeTickMessage(JsonElement root, JsonElement header) - { - try - { - if (!header.TryGetProperty("ts_event", out var tsElement) || - !header.TryGetProperty("instrument_id", out var instIdElement)) - { - return; - } - - // Convert timestamp from nanoseconds to DateTime - var timestampNs = long.Parse(tsElement.GetString()!); - var unixEpoch = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc); - var timestamp = unixEpoch.AddTicks(timestampNs / 100); - - var instrumentId = instIdElement.GetInt64(); - - if (!_instrumentIdToSymbol.TryGetValue(instrumentId, out var matchedSymbol)) - { - Log.Trace($"DataBentoRawLiveClient: No mapping for instrument_id {instrumentId} in trade message."); - return; - } - - if (root.TryGetProperty("price", out var priceElement) && - root.TryGetProperty("size", out var sizeElement)) - { - var priceRaw = long.Parse(priceElement.GetString()!); - var size = sizeElement.GetInt32(); - var price = priceRaw * PriceScaleFactor; - - var tradeTick = new Tick - { - Symbol = matchedSymbol, - Time = timestamp, - Value = price, - Quantity = size, - TickType = TickType.Trade, - // Trade ticks should have zero bid/ask values - BidPrice = 0, - BidSize = 0, - AskPrice = 0, - AskSize = 0 - }; - - // Log.Trace($"DataBentoRawLiveClient: Trade tick: {matchedSymbol} Price={price} Quantity={size}"); - DataReceived?.Invoke(this, tradeTick); - } - } - catch (Exception ex) - { - Log.Error($"DataBentoRawLiveClient.HandleTradeTickMessage(): Error: {ex.Message}"); - } - } - - /// - /// Disconnects from the DataBento gateway - /// - public void Disconnect() - { - lock (_connectionLock) - { - if (!_isConnected) - return; - - _isConnected = false; - _cancellationTokenSource?.Cancel(); - - try - { - _reader?.Dispose(); - _writer?.Dispose(); - _stream?.Close(); - _tcpClient?.Close(); - } - catch (Exception ex) - { - Log.Trace($"DataBentoRawLiveClient.Disconnect(): Error during disconnect: {ex.Message}"); - } - - ConnectionStatusChanged?.Invoke(this, false); - Log.Trace("DataBentoRawLiveClient.Disconnect(): Disconnected from DataBento gateway"); - } - } - - /// - /// Disposes of resources - /// - public void Dispose() - { - if (_disposed) - return; - - _disposed = true; - Disconnect(); - - _cancellationTokenSource?.Dispose(); - _reader?.Dispose(); - _writer?.Dispose(); - _stream?.Dispose(); - _tcpClient?.Dispose(); - } - - /// - /// Computes the SHA-256 hash of the input string - /// - private static string ComputeSHA256(string input) - { - using var sha = SHA256.Create(); - var hash = sha.ComputeHash(Encoding.UTF8.GetBytes(input)); - var sb = new StringBuilder(); - foreach (byte b in hash) - { - sb.Append(b.ToString("x2")); - } - return sb.ToString(); - } - -} diff --git a/QuantConnect.DataBento/DataBentoSymbolMapper.cs b/QuantConnect.DataBento/DataBentoSymbolMapper.cs index 1a824ee..a0b8913 100644 --- a/QuantConnect.DataBento/DataBentoSymbolMapper.cs +++ b/QuantConnect.DataBento/DataBentoSymbolMapper.cs @@ -67,28 +67,6 @@ public string GetBrokerageSymbol(Symbol symbol) public Symbol GetLeanSymbol(string brokerageSymbol, SecurityType securityType, string market, DateTime expirationDate = new DateTime(), decimal strike = 0, OptionRight optionRight = 0) { - switch (securityType) - { - case SecurityType.Future: - return Symbol.CreateFuture(brokerageSymbol, market, expirationDate); - default: - throw new Exception($"The unsupported security type: {securityType}"); - } - } - - /// - /// Converts a brokerage future symbol to a Lean symbol instance - /// - /// The brokerage symbol - /// A new Lean Symbol instance - public Symbol GetLeanSymbolForFuture(string brokerageSymbol) - { - // ignore futures spreads - if (brokerageSymbol.Contains("-")) - { - return null; - } - - return SymbolRepresentation.ParseFutureSymbol(brokerageSymbol); + throw new NotImplementedException("This method is not used in the current implementation."); } } diff --git a/QuantConnect.DataBento/Extensions.cs b/QuantConnect.DataBento/Extensions.cs index a4514db..7240d6a 100644 --- a/QuantConnect.DataBento/Extensions.cs +++ b/QuantConnect.DataBento/Extensions.cs @@ -14,6 +14,7 @@ */ using Newtonsoft.Json; +using QuantConnect.Lean.DataSource.DataBento.Models; using QuantConnect.Lean.DataSource.DataBento.Serialization; namespace QuantConnect.Lean.DataSource.DataBento; @@ -31,4 +32,18 @@ public static class Extensions { return JsonConvert.DeserializeObject(json, JsonSettings.SnakeCase); } + + /// + /// Deserializes the specified JSON string to a + /// using snake-case property name resolution. + /// + /// The JSON string to deserialize. + /// + /// The deserialized object, + /// or null if deserialization fails or the input is null or empty. + /// + public static MarketDataRecord? DeserializeSnakeCaseLiveData(this string json) + { + return JsonConvert.DeserializeObject(json, JsonSettings.LiveDataSnakeCase); + } } diff --git a/QuantConnect.DataBento/Models/Header.cs b/QuantConnect.DataBento/Models/Header.cs index 36774a2..08b585d 100644 --- a/QuantConnect.DataBento/Models/Header.cs +++ b/QuantConnect.DataBento/Models/Header.cs @@ -24,7 +24,7 @@ namespace QuantConnect.Lean.DataSource.DataBento.Models; public sealed class Header { /// - /// Event timestamp in nanoseconds since Unix epoch (UTC). + /// The matching-engine-received timestamp expressed as the number of nanoseconds since the UNIX epoch. /// public long TsEvent { get; set; } @@ -41,7 +41,7 @@ public sealed class Header /// /// Internal instrument identifier for the symbol. /// - public long InstrumentId { get; set; } + public uint InstrumentId { get; set; } /// /// Event time converted to UTC . diff --git a/QuantConnect.DataBento/Models/LevelOneData.cs b/QuantConnect.DataBento/Models/LevelOneData.cs index a4d3454..cc68dd4 100644 --- a/QuantConnect.DataBento/Models/LevelOneData.cs +++ b/QuantConnect.DataBento/Models/LevelOneData.cs @@ -21,8 +21,7 @@ namespace QuantConnect.Lean.DataSource.DataBento.Models; public sealed class LevelOneData : MarketDataRecord { /// - /// Timestamp when the message was received by the gateway, - /// expressed as nanoseconds since the UNIX epoch. + /// The capture-server-received timestamp expressed as the number of nanoseconds since the UNIX epoch. /// public long TsRecv { get; set; } diff --git a/QuantConnect.DataBento/Models/Live/SymbolMappingConfirmationEventArgs.cs b/QuantConnect.DataBento/Models/Live/SymbolMappingConfirmationEventArgs.cs new file mode 100644 index 0000000..461d0a2 --- /dev/null +++ b/QuantConnect.DataBento/Models/Live/SymbolMappingConfirmationEventArgs.cs @@ -0,0 +1,43 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +namespace QuantConnect.Lean.DataSource.DataBento.Models.Live; + +/// +/// Provides data for the symbol-mapping confirmation event. +/// +public sealed class SymbolMappingConfirmationEventArgs : EventArgs +{ + /// + /// Gets the original symbol that was mapped. + /// + public string Symbol { get; } + + /// + /// Gets the internal instrument identifier associated with the symbol. + /// + public uint InstrumentId { get; } + + /// + /// Initializes a new instance of the class. + /// + /// The symbol that was mapped. + /// The internal instrument identifier. + public SymbolMappingConfirmationEventArgs(string symbol, uint instrumentId) + { + Symbol = symbol; + InstrumentId = instrumentId; + } +} diff --git a/QuantConnect.DataBento/Models/Live/SymbolMappingMessage.cs b/QuantConnect.DataBento/Models/Live/SymbolMappingMessage.cs new file mode 100644 index 0000000..f0ffdfc --- /dev/null +++ b/QuantConnect.DataBento/Models/Live/SymbolMappingMessage.cs @@ -0,0 +1,30 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +namespace QuantConnect.Lean.DataSource.DataBento.Models.Live; + +public class SymbolMappingMessage : MarketDataRecord +{ + /// + /// The input symbol from the subscription + /// + public string? StypeInSymbol { get; set; } + + /// + /// The output symbol from the subscription + /// + public string? StypeOutSymbol { get; set; } +} diff --git a/QuantConnect.DataBento/Serialization/JsonSettings.cs b/QuantConnect.DataBento/Serialization/JsonSettings.cs index 5247d97..4badd80 100644 --- a/QuantConnect.DataBento/Serialization/JsonSettings.cs +++ b/QuantConnect.DataBento/Serialization/JsonSettings.cs @@ -31,4 +31,15 @@ public static class JsonSettings { ContractResolver = SnakeCaseContractResolver.Instance }; + + /// + /// Gets a reusable instance of that uses + /// for snake-case property name formatting + /// and custom converter. + /// + public static readonly JsonSerializerSettings LiveDataSnakeCase = new() + { + ContractResolver = SnakeCaseContractResolver.Instance, + Converters = { new Converters.LiveDataConverter() } + }; } From a80bdccd702c09afede9599c48de5b7c0b4ad58b Mon Sep 17 00:00:00 2001 From: Romazes Date: Wed, 28 Jan 2026 16:12:13 +0200 Subject: [PATCH 13/13] feat: reconnection process and resubscription --- .../DataBentoDataDownloaderTests.cs | 2 +- .../DataBentoDataProviderHistoryTests.cs | 2 +- .../DataBentoDataQueueHandlerTests.cs | 234 ++++++++++++++++++ .../DataBentoLiveAPIClientTests.cs | 3 +- QuantConnect.DataBento/Api/LiveAPIClient.cs | 26 +- .../Api/LiveDataTcpClientWrapper.cs | 74 ++++-- .../DataBentoDataProvider.cs | 14 +- .../Models/Live/ConnectionLostEventArgs.cs | 34 +++ 8 files changed, 359 insertions(+), 30 deletions(-) create mode 100644 QuantConnect.DataBento.Tests/DataBentoDataQueueHandlerTests.cs create mode 100644 QuantConnect.DataBento/Models/Live/ConnectionLostEventArgs.cs diff --git a/QuantConnect.DataBento.Tests/DataBentoDataDownloaderTests.cs b/QuantConnect.DataBento.Tests/DataBentoDataDownloaderTests.cs index b84893c..940813c 100644 --- a/QuantConnect.DataBento.Tests/DataBentoDataDownloaderTests.cs +++ b/QuantConnect.DataBento.Tests/DataBentoDataDownloaderTests.cs @@ -40,7 +40,7 @@ public void SetUp() [TearDown] public void TearDown() { - _downloader?.Dispose(); + _downloader?.DisposeSafely(); } [TestCase(Resolution.Daily)] diff --git a/QuantConnect.DataBento.Tests/DataBentoDataProviderHistoryTests.cs b/QuantConnect.DataBento.Tests/DataBentoDataProviderHistoryTests.cs index 27a52d3..be19cbd 100644 --- a/QuantConnect.DataBento.Tests/DataBentoDataProviderHistoryTests.cs +++ b/QuantConnect.DataBento.Tests/DataBentoDataProviderHistoryTests.cs @@ -38,7 +38,7 @@ public void SetUp() [TearDown] public void TearDown() { - _historyDataProvider?.Dispose(); + _historyDataProvider?.DisposeSafely(); } internal static IEnumerable TestParameters diff --git a/QuantConnect.DataBento.Tests/DataBentoDataQueueHandlerTests.cs b/QuantConnect.DataBento.Tests/DataBentoDataQueueHandlerTests.cs new file mode 100644 index 0000000..dd57c25 --- /dev/null +++ b/QuantConnect.DataBento.Tests/DataBentoDataQueueHandlerTests.cs @@ -0,0 +1,234 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + + +using System; +using System.Text; +using NUnit.Framework; +using System.Threading; +using QuantConnect.Util; +using QuantConnect.Data; +using QuantConnect.Logging; +using System.Threading.Tasks; +using QuantConnect.Securities; +using QuantConnect.Data.Market; +using System.Collections.Generic; +using QuantConnect.Lean.Engine.DataFeeds.Enumerators; + +namespace QuantConnect.Lean.DataSource.DataBento.Tests; + +[TestFixture] +public class DataBentoDataQueueHandlerTests +{ + private DataBentoProvider _dataProvider; + private CancellationTokenSource _cancellationTokenSource; + + [SetUp] + public void SetUp() + { + _cancellationTokenSource = new(); + _dataProvider = new(); + } + + [TearDown] + public void TearDown() + { + _cancellationTokenSource?.Cancel(); + _cancellationTokenSource.DisposeSafely(); + _dataProvider?.DisposeSafely(); + } + + private static IEnumerable TestParameters + { + get + { + var sp500EMiniMarch = Symbol.CreateFuture(Futures.Indices.SP500EMini, Market.CME, new(2026, 03, 20)); + + yield return new TestCaseData(new Symbol[] { sp500EMiniMarch }, Resolution.Second); + } + } + + [Test, TestCaseSource(nameof(TestParameters))] + public void CanSubscribeAndUnsubscribeOnDifferentResolution(Symbol[] symbols, Resolution resolution) + { + + var configs = new List(); + + var dataFromEnumerator = new Dictionary>(); + + foreach (var symbol in symbols) + { + dataFromEnumerator[symbol] = new Dictionary(); + foreach (var config in GetSubscriptionDataConfigs(symbol, resolution)) + { + configs.Add(config); + + var tickType = config.TickType switch + { + TickType.Quote => typeof(QuoteBar), + TickType.Trade => typeof(TradeBar), + _ => throw new NotImplementedException() + }; + + dataFromEnumerator[symbol][tickType] = 0; + } + } + + Assert.That(configs, Is.Not.Empty); + + Action callback = (dataPoint) => + { + if (dataPoint == null) + { + return; + } + + switch (dataPoint) + { + case TradeBar tb: + dataFromEnumerator[tb.Symbol][typeof(TradeBar)] += 1; + break; + case QuoteBar qb: + Assert.GreaterOrEqual(qb.Ask.Open, qb.Bid.Open, $"QuoteBar validation failed for {qb.Symbol}: Ask.Open ({qb.Ask.Open}) <= Bid.Open ({qb.Bid.Open}). Full data: {DisplayBaseData(qb)}"); + Assert.GreaterOrEqual(qb.Ask.High, qb.Bid.High, $"QuoteBar validation failed for {qb.Symbol}: Ask.High ({qb.Ask.High}) <= Bid.High ({qb.Bid.High}). Full data: {DisplayBaseData(qb)}"); + Assert.GreaterOrEqual(qb.Ask.Low, qb.Bid.Low, $"QuoteBar validation failed for {qb.Symbol}: Ask.Low ({qb.Ask.Low}) <= Bid.Low ({qb.Bid.Low}). Full data: {DisplayBaseData(qb)}"); + Assert.GreaterOrEqual(qb.Ask.Close, qb.Bid.Close, $"QuoteBar validation failed for {qb.Symbol}: Ask.Close ({qb.Ask.Close}) <= Bid.Close ({qb.Bid.Close}). Full data: {DisplayBaseData(qb)}"); + dataFromEnumerator[qb.Symbol][typeof(QuoteBar)] += 1; + break; + } + ; + }; + + foreach (var config in configs) + { + ProcessFeed(_dataProvider.Subscribe(config, (sender, args) => + { + var dataPoint = ((NewDataAvailableEventArgs)args).DataPoint; + Log.Trace($"{dataPoint}. Time span: {dataPoint.Time} - {dataPoint.EndTime}"); + }), _cancellationTokenSource.Token, callback: callback); + } + + Thread.Sleep(TimeSpan.FromSeconds(120)); + + Log.Trace("Unsubscribing symbols"); + foreach (var config in configs) + { + _dataProvider.Unsubscribe(config); + } + + Thread.Sleep(TimeSpan.FromSeconds(5)); + + _cancellationTokenSource.Cancel(); + + var str = new StringBuilder(); + + str.AppendLine($"{nameof(DataBentoDataQueueHandlerTests)}.{nameof(CanSubscribeAndUnsubscribeOnDifferentResolution)}: ***** Summary *****"); + + foreach (var symbol in symbols) + { + str.AppendLine($"Input parameters: ticker:{symbol} | securityType:{symbol.SecurityType} | resolution:{resolution}"); + + foreach (var tickType in dataFromEnumerator[symbol]) + { + str.AppendLine($"[{tickType.Key}] = {tickType.Value}"); + + if (symbol.SecurityType != SecurityType.Index) + { + Assert.Greater(tickType.Value, 0); + } + // The ThetaData returns TradeBar seldom. Perhaps should find more relevant ticker. + Assert.GreaterOrEqual(tickType.Value, 0); + } + str.AppendLine(new string('-', 30)); + } + + Log.Trace(str.ToString()); + } + + private static string DisplayBaseData(BaseData item) + { + switch (item) + { + case TradeBar tradeBar: + return $"Data Type: {item.DataType} | " + tradeBar.ToString() + $" Time: {tradeBar.Time}, EndTime: {tradeBar.EndTime}"; + default: + return $"DEFAULT: Data Type: {item.DataType} | Time: {item.Time} | End Time: {item.EndTime} | Symbol: {item.Symbol} | Price: {item.Price} | IsFillForward: {item.IsFillForward}"; + } + } + + private static IEnumerable GetSubscriptionDataConfigs(Symbol symbol, Resolution resolution) + { + yield return GetSubscriptionDataConfig(symbol, resolution); + yield return GetSubscriptionDataConfig(symbol, resolution); + } + + public static IEnumerable GetSubscriptionTickDataConfigs(Symbol symbol) + { + yield return new SubscriptionDataConfig(GetSubscriptionDataConfig(symbol, Resolution.Tick), tickType: TickType.Trade); + yield return new SubscriptionDataConfig(GetSubscriptionDataConfig(symbol, Resolution.Tick), tickType: TickType.Quote); + } + + private static SubscriptionDataConfig GetSubscriptionDataConfig(Symbol symbol, Resolution resolution) + { + return new SubscriptionDataConfig( + typeof(T), + symbol, + resolution, + TimeZones.Utc, + TimeZones.Utc, + true, + extendedHours: false, + false); + } + + private Task ProcessFeed( + IEnumerator enumerator, + CancellationToken cancellationToken, + int cancellationTokenDelayMilliseconds = 100, + Action callback = null, + Action throwExceptionCallback = null) + { + return Task.Factory.StartNew(() => + { + try + { + while (enumerator.MoveNext() && !cancellationToken.IsCancellationRequested) + { + BaseData tick = enumerator.Current; + + if (tick != null) + { + callback?.Invoke(tick); + } + + cancellationToken.WaitHandle.WaitOne(TimeSpan.FromMilliseconds(cancellationTokenDelayMilliseconds)); + } + } + catch (Exception ex) + { + Log.Debug($"{nameof(DataBentoDataQueueHandlerTests)}.{nameof(ProcessFeed)}.Exception: {ex.Message}"); + throw; + } + }, cancellationToken).ContinueWith(task => + { + if (throwExceptionCallback != null) + { + throwExceptionCallback(); + } + Log.Debug("The throwExceptionCallback is null."); + }, TaskContinuationOptions.OnlyOnFaulted); + } +} diff --git a/QuantConnect.DataBento.Tests/DataBentoLiveAPIClientTests.cs b/QuantConnect.DataBento.Tests/DataBentoLiveAPIClientTests.cs index ea66096..69d5f1d 100644 --- a/QuantConnect.DataBento.Tests/DataBentoLiveAPIClientTests.cs +++ b/QuantConnect.DataBento.Tests/DataBentoLiveAPIClientTests.cs @@ -16,6 +16,7 @@ using System; using NUnit.Framework; using System.Threading; +using QuantConnect.Util; using QuantConnect.Configuration; using System.Collections.Generic; using QuantConnect.Lean.DataSource.DataBento.Api; @@ -51,7 +52,7 @@ public void OneTimeSetUp() [OneTimeTearDown] public void OneTimeTearDown() { - _live.Dispose(); + _live?.DisposeSafely(); } [Test] diff --git a/QuantConnect.DataBento/Api/LiveAPIClient.cs b/QuantConnect.DataBento/Api/LiveAPIClient.cs index 810dd9b..3629b95 100644 --- a/QuantConnect.DataBento/Api/LiveAPIClient.cs +++ b/QuantConnect.DataBento/Api/LiveAPIClient.cs @@ -31,6 +31,8 @@ public sealed class LiveAPIClient : IDisposable public event EventHandler? SymbolMappingConfirmation; + public event EventHandler? ConnectionLost; + public bool IsConnected => _tcpClientByDataSet.Values.All(c => c.IsConnected); public LiveAPIClient(string apiKey, Action levelOneDataHandler) @@ -50,17 +52,35 @@ public void Dispose() private LiveDataTcpClientWrapper EnsureDatasetConnection(string dataSet) { - if (_tcpClientByDataSet.TryGetValue(dataSet, out var liveDataTcpClient)) + if (_tcpClientByDataSet.TryGetValue(dataSet, out var liveDataTcpClient) && liveDataTcpClient.IsConnected) { return liveDataTcpClient; } LogTrace(nameof(EnsureDatasetConnection), "Starting connection to DataBento live API"); - liveDataTcpClient = new LiveDataTcpClientWrapper(dataSet, _apiKey, MessageReceived); - _tcpClientByDataSet[dataSet] = liveDataTcpClient; + if (liveDataTcpClient == null) + { + liveDataTcpClient = new LiveDataTcpClientWrapper(dataSet, _apiKey, MessageReceived); + + liveDataTcpClient.ConnectionLost += (sender, message) => + { + LogError(nameof(EnsureDatasetConnection), $"Connection lost to DataBento live API (Dataset: {dataSet}). Reason: {message}"); + ConnectionLost?.Invoke(this, new ConnectionLostEventArgs(dataSet, message)); + }; + + _tcpClientByDataSet[dataSet] = liveDataTcpClient; + } + liveDataTcpClient.Connect(); + if (!liveDataTcpClient.IsConnected) + { + var msg = $"Unable to establish a connection to the DataBento Live API (Dataset: {dataSet})."; + LogError(nameof(EnsureDatasetConnection), msg); + throw new Exception(msg); + } + LogTrace(nameof(EnsureDatasetConnection), $"Successfully connected to DataBento live API (Dataset: {dataSet})"); return liveDataTcpClient; diff --git a/QuantConnect.DataBento/Api/LiveDataTcpClientWrapper.cs b/QuantConnect.DataBento/Api/LiveDataTcpClientWrapper.cs index b55e14a..87f1581 100644 --- a/QuantConnect.DataBento/Api/LiveDataTcpClientWrapper.cs +++ b/QuantConnect.DataBento/Api/LiveDataTcpClientWrapper.cs @@ -31,7 +31,7 @@ public sealed class LiveDataTcpClientWrapper : IDisposable private readonly string _apiKey; private readonly TimeSpan _heartBeatInterval = TimeSpan.FromSeconds(10); - private readonly TcpClient _tcpClient = new(); + private TcpClient _tcpClient; private readonly CancellationTokenSource _cancellationTokenSource = new(); private NetworkStream? _stream; @@ -41,6 +41,8 @@ public sealed class LiveDataTcpClientWrapper : IDisposable private readonly Action MessageReceived; + public event EventHandler? ConnectionLost; + /// /// Is client connected /// @@ -56,17 +58,46 @@ public LiveDataTcpClientWrapper(string dataSet, string apiKey, Action me public void Connect() { - _tcpClient.Connect(_gateway, DefaultPort); - _stream = _tcpClient.GetStream(); - _reader = new StreamReader(_stream, Encoding.ASCII); + var attemptToConnect = 1; + var error = default(string); + do + { + try + { + _tcpClient = new(_gateway, DefaultPort); + _stream = _tcpClient.GetStream(); + _reader = new StreamReader(_stream, Encoding.ASCII); + + if (!Authenticate(_dataSet).SynchronouslyAwaitTask()) + throw new Exception("Authentication failed"); - if (!Authenticate(_dataSet).SynchronouslyAwaitTask()) - throw new Exception("Authentication failed"); + _dataReceiverTask = new Task(async () => await DataReceiverAsync(_cancellationTokenSource.Token), _cancellationTokenSource.Token, TaskCreationOptions.LongRunning); + _dataReceiverTask.Start(); - _dataReceiverTask = new Task(async () => await DataReceiverAsync(_cancellationTokenSource.Token), _cancellationTokenSource.Token, TaskCreationOptions.LongRunning); - _dataReceiverTask.Start(); + _isConnected = true; + } + catch (Exception ex) + { + error = ex.Message; + } - _isConnected = true; + var retryDelayMs = attemptToConnect * 2 * 1000; + LogError(nameof(Connect), $"Connection attempt #{attemptToConnect} failed. Retrying in {retryDelayMs} ms. Error: {error}"); + _cancellationTokenSource.Token.WaitHandle.WaitOne(attemptToConnect * 2 * 1000); + + } while (attemptToConnect++ < 5 && !_isConnected); + } + + private void Close() + { + _isConnected = false; + + _reader?.Close(); + _reader?.DisposeSafely(); + + _dataReceiverTask?.DisposeSafely(); + _tcpClient?.Close(); + _tcpClient?.DisposeSafely(); } public void Dispose() @@ -98,7 +129,9 @@ private async Task DataReceiverAsync(CancellationToken ct) var readTimeout = _heartBeatInterval.Add(TimeSpan.FromSeconds(5)); - LogTrace(methodName, "Receiver started"); + LogTrace(methodName, "Task Receiver started"); + + var errorMessage = string.Empty; try { @@ -118,27 +151,22 @@ private async Task DataReceiverAsync(CancellationToken ct) MessageReceived.Invoke(line); } } - catch (OperationCanceledException) - { - if (!_tcpClient.Connected) - { - LogError("DataReceiverAsync", "GG"); - } - - Log.Trace("DataBentoRawLiveClient.ProcessMessages(): Message processing cancelled"); - } - catch (IOException ex) when (ex.InnerException is SocketException) + catch (OperationCanceledException oce) { - Log.Trace($"DataBentoRawLiveClient.ProcessMessages(): Socket exception: {ex.Message}"); + errorMessage = $"Read timeout exceeded: Outer CancellationToken: {ct.IsCancellationRequested}, Read Timeout: {readTimeoutCts.IsCancellationRequested}"; + LogTrace(methodName, errorMessage); } catch (Exception ex) { - Log.Error($"DataBentoRawLiveClient.ProcessMessages(): Error processing messages: {ex.Message}\n{ex.StackTrace}"); + errorMessage += ex.Message; + LogError(methodName, $"Error processing messages: {ex.Message}\n{ex.StackTrace}"); } finally { - LogTrace(methodName, "Receiver stopped"); + LogTrace(methodName, "Task Receiver stopped"); + Close(); readTimeoutCts.Dispose(); + ConnectionLost?.Invoke(this, new($"{errorMessage}. TcpConnected: {_tcpClient.Connected}")); } } diff --git a/QuantConnect.DataBento/DataBentoDataProvider.cs b/QuantConnect.DataBento/DataBentoDataProvider.cs index 5127a45..23fd465 100644 --- a/QuantConnect.DataBento/DataBentoDataProvider.cs +++ b/QuantConnect.DataBento/DataBentoDataProvider.cs @@ -106,11 +106,12 @@ private void Initialize(string apiKey) { var aggregatorName = Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager"); Log.Trace($"{nameof(DataBentoProvider)}.{nameof(Initialize)}: found no data aggregator instance, creating {aggregatorName}"); - _aggregator = Composer.Instance.GetExportedValueByTypeName(aggregatorName); + _aggregator = Composer.Instance.GetExportedValueByTypeName(aggregatorName, forceTypeNameOnExisting: false); } _liveApiClient = new LiveAPIClient(apiKey, HandleLevelOneData); _liveApiClient.SymbolMappingConfirmation += OnSymbolMappingConfirmation; + _liveApiClient.ConnectionLost += OnConnectionLost; _historicalApiClient = new(apiKey); @@ -122,6 +123,17 @@ private void Initialize(string apiKey) _initialized = true; } + private void OnConnectionLost(object? _, ConnectionLostEventArgs cle) + { + LogTrace(nameof(OnConnectionLost), "The connection was lost. Starting ReSubscription process"); + + var symbols = _levelOneServiceManager.GetSubscribedSymbols(); + + Subscribe(symbols); + + LogTrace(nameof(OnConnectionLost), $"Re-subscription completed successfully for {_levelOneServiceManager.Count} symbol(s)."); + } + private void OnSymbolMappingConfirmation(object? _, SymbolMappingConfirmationEventArgs smce) { if (_pendingSubscriptions.TryRemove(smce.Symbol, out var symbol)) diff --git a/QuantConnect.DataBento/Models/Live/ConnectionLostEventArgs.cs b/QuantConnect.DataBento/Models/Live/ConnectionLostEventArgs.cs new file mode 100644 index 0000000..6994e98 --- /dev/null +++ b/QuantConnect.DataBento/Models/Live/ConnectionLostEventArgs.cs @@ -0,0 +1,34 @@ +namespace QuantConnect.Lean.DataSource.DataBento.Models.Live; + +/// +/// Provides data for the event that is raised when a connection is lost. +/// +public class ConnectionLostEventArgs : EventArgs +{ + /// + /// Gets the identifier of the data set or logical stream + /// associated with the lost connection. + /// + public string DataSet { get; } + + /// + /// Gets a human-readable description of the reason + /// why the connection was lost. + /// + public string Reason { get; } + + /// + /// Initializes a new instance of the class. + /// + /// + /// The identifier of the data set or logical stream related to the connection. + /// + /// + /// A human-readable explanation describing why the connection was lost. + /// + public ConnectionLostEventArgs(string message, string reason) + { + DataSet = message; + Reason = reason; + } +}