Skip to content

Addressing comments from previous PR. #8

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Joseph-Matteo-Scorsone wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Addressing comments from previous PR. #8

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
221 changes: 122 additions & 99 deletions QuantConnect.DataBento.Tests/DataBentoDataDownloaderTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,24 +17,34 @@
using System;
using System.Linq;
using NUnit.Framework;
using QuantConnect.Configuration;
using QuantConnect.Data;
using QuantConnect.Data.Market;
using QuantConnect.Lean.DataSource.DataBento;
using QuantConnect.Logging;
using QuantConnect.Configuration;
using QuantConnect.Securities;
using QuantConnect.Util;

namespace QuantConnect.Lean.DataSource.DataBento.Tests
{
[TestFixture]
public class DataBentoDataDownloaderTests
{
private DataBentoDataDownloader _downloader;
private readonly string _apiKey = Config.Get("databento-api-key");
private MarketHoursDatabase _marketHoursDatabase;
protected readonly string ApiKey = Config.Get("databento-api-key");

private static Symbol CreateEsFuture()
Copy link

@Romazes Romazes Jan 2, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You can use the property (looking simpler):

private static Symbol ESH20 => Symbol.CreateFuture("ES", Market.CME, new DateTime(2026, 3, 20));

{
var expiration = new DateTime(2026, 3, 20);
return Symbol.CreateFuture("ES", Market.CME, expiration);
}

[SetUp]
public void SetUp()
{
_downloader = new DataBentoDataDownloader(_apiKey);
_marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
_downloader = new DataBentoDataDownloader(ApiKey, _marketHoursDatabase);
}

[TearDown]
Expand All @@ -43,141 +53,154 @@ public void TearDown()
_downloader?.Dispose();
}

[Test]
[TestCase("ESM3", SecurityType.Future, Market.CME, Resolution.Daily, TickType.Trade)]
[TestCase("ESM3", SecurityType.Future, Market.CME, Resolution.Hour, TickType.Trade)]
[TestCase("ESM3", SecurityType.Future, Market.CME, Resolution.Minute, TickType.Trade)]
[TestCase("ESM3", SecurityType.Future, Market.CME, Resolution.Second, TickType.Trade)]
[TestCase("ESM3", SecurityType.Future, Market.CME, Resolution.Tick, TickType.Trade)]
[Explicit("This test requires a configured DataBento API key")]
public void DownloadsHistoricalData(string ticker, SecurityType securityType, string market, Resolution resolution, TickType tickType)
[TestCase(Resolution.Daily)]
[TestCase(Resolution.Hour)]
[TestCase(Resolution.Minute)]
[TestCase(Resolution.Second)]
[TestCase(Resolution.Tick)]
public void DownloadsTradeDataForLeanFuture(Resolution resolution)
{
var symbol = Symbol.Create(ticker, securityType, market);
var startTime = new DateTime(2024, 1, 15);
var endTime = new DateTime(2024, 1, 16);
var param = new DataDownloaderGetParameters(symbol, resolution, startTime, endTime, tickType);
var symbol = CreateEsFuture();
var exchangeTimeZone = _marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType).TimeZone;

var downloadResponse = _downloader.Get(param).ToList();
var startUtc = new DateTime(2024, 5, 1, 0, 0, 0, DateTimeKind.Utc);
var endUtc = new DateTime(2024, 5, 2, 0, 0, 0, DateTimeKind.Utc);

Log.Trace($"Downloaded {downloadResponse.Count} data points for {symbol} at {resolution} resolution");
if (resolution == Resolution.Tick)
{
startUtc = new DateTime(2024, 5, 1, 9, 30, 0, DateTimeKind.Utc);
endUtc = startUtc.AddMinutes(15);
}

Assert.IsTrue(downloadResponse.Any(), "Expected to download at least one data point");
var parameters = new DataDownloaderGetParameters(
symbol,
resolution,
startUtc,
endUtc,
TickType.Trade
);

foreach (var data in downloadResponse)
var data = _downloader.Get(parameters).ToList();

Log.Trace($"Downloaded {data.Count} trade points for {symbol} @ {resolution}");

Assert.IsNotEmpty(data);

var startExchange = startUtc.ConvertFromUtc(exchangeTimeZone);
var endExchange = endUtc.ConvertFromUtc(exchangeTimeZone);

foreach (var point in data)
{
Assert.IsNotNull(data, "Data point should not be null");
Assert.AreEqual(symbol, data.Symbol, "Symbol should match requested symbol");
Assert.IsTrue(data.Time >= startTime && data.Time <= endTime, "Data time should be within requested range");
Assert.AreEqual(symbol, point.Symbol);
Assert.That(point.Time, Is.InRange(startExchange, endExchange));

if (data is TradeBar tradeBar)
switch (point)
Copy link

@Romazes Romazes Jan 2, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Add validation of Time, example:
https://github.com/QuantConnect/Lean.Brokerages.InteractiveBrokers/blob/master/QuantConnect.InteractiveBrokersBrokerage.Tests/InteractiveBrokersBrokerageAdditionalTests.cs#L1349

{
Assert.Greater(tradeBar.Close, 0, "Close price should be positive");
Assert.GreaterOrEqual(tradeBar.Volume, 0, "Volume should be non-negative");
Assert.Greater(tradeBar.High, 0, "High price should be positive");
Assert.Greater(tradeBar.Low, 0, "Low price should be positive");
Assert.Greater(tradeBar.Open, 0, "Open price should be positive");
Assert.GreaterOrEqual(tradeBar.High, tradeBar.Low, "High should be >= Low");
Assert.GreaterOrEqual(tradeBar.High, tradeBar.Open, "High should be >= Open");
Assert.GreaterOrEqual(tradeBar.High, tradeBar.Close, "High should be >= Close");
Assert.LessOrEqual(tradeBar.Low, tradeBar.Open, "Low should be <= Open");
Assert.LessOrEqual(tradeBar.Low, tradeBar.Close, "Low should be <= Close");
}
else if (data is QuoteBar quoteBar)
{
Assert.Greater(quoteBar.Close, 0, "Quote close price should be positive");
if (quoteBar.Bid != null)
{
Assert.Greater(quoteBar.Bid.Close, 0, "Bid price should be positive");
}
if (quoteBar.Ask != null)
{
Assert.Greater(quoteBar.Ask.Close, 0, "Ask price should be positive");
}
}
else if (data is Tick tick)
{
Assert.Greater(tick.Value, 0, "Tick value should be positive");
Assert.GreaterOrEqual(tick.Quantity, 0, "Tick quantity should be non-negative");
case TradeBar bar:
Assert.Greater(bar.Open, 0);
Assert.Greater(bar.High, 0);
Assert.Greater(bar.Low, 0);
Assert.Greater(bar.Close, 0);
Assert.GreaterOrEqual(bar.Volume, 0);
Assert.GreaterOrEqual(bar.High, bar.Low);
break;

case Tick tick:
Assert.Greater(tick.Value, 0);
Assert.GreaterOrEqual(tick.Quantity, 0);
break;

default:
Assert.Fail($"Unexpected data type {point.GetType()}");
break;
}
}
}

[Test]
[TestCase("ZNM3", SecurityType.Future, Market.CME, Resolution.Daily, TickType.Trade)]
[TestCase("ZNM3", SecurityType.Future, Market.CME, Resolution.Hour, TickType.Trade)]
[Explicit("This test requires a configured DataBento API key")]
public void DownloadsFuturesHistoricalData(string ticker, SecurityType securityType, string market, Resolution resolution, TickType tickType)
public void DownloadsQuoteTicksForLeanFuture()
{
var symbol = Symbol.Create(ticker, securityType, market);
var startTime = new DateTime(2024, 1, 15);
var endTime = new DateTime(2024, 1, 16);
var param = new DataDownloaderGetParameters(symbol, resolution, startTime, endTime, tickType);

var downloadResponse = _downloader.Get(param).ToList();
var symbol = CreateEsFuture();
var exchangeTimeZone = _marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType).TimeZone;

Log.Trace($"Downloaded {downloadResponse.Count} data points for futures {symbol}");
var startUtc = new DateTime(2024, 5, 1, 9, 30, 0, DateTimeKind.Utc);
var endUtc = startUtc.AddMinutes(15);

Assert.IsTrue(downloadResponse.Any(), "Expected to download futures data");
var parameters = new DataDownloaderGetParameters(
symbol,
Resolution.Tick,
startUtc,
endUtc,
TickType.Quote
);

foreach (var data in downloadResponse)
{
Assert.AreEqual(symbol, data.Symbol, "Symbol should match requested futures symbol");
Assert.Greater(data.Value, 0, "Data value should be positive");
}
}

[Test]
[TestCase("ESM3", SecurityType.Future, Market.CME, Resolution.Tick, TickType.Quote)]
[Explicit("This test requires a configured DataBento API key and advanced subscription")]
public void DownloadsQuoteData(string ticker, SecurityType securityType, string market, Resolution resolution, TickType tickType)
{
var symbol = Symbol.Create(ticker, securityType, market);
var startTime = new DateTime(2024, 1, 15, 9, 30, 0);
var endTime = new DateTime(2024, 1, 15, 9, 45, 0);
var param = new DataDownloaderGetParameters(symbol, resolution, startTime, endTime, tickType);
var data = _downloader.Get(parameters).ToList();

var downloadResponse = _downloader.Get(param).ToList();
Log.Trace($"Downloaded {data.Count} quote ticks for {symbol}");

Log.Trace($"Downloaded {downloadResponse.Count} quote data points for {symbol}");
Assert.IsNotEmpty(data);

Assert.IsTrue(downloadResponse.Any(), "Expected to download quote data");
var startExchange = startUtc.ConvertFromUtc(exchangeTimeZone);
var endExchange = endUtc.ConvertFromUtc(exchangeTimeZone);

foreach (var data in downloadResponse)
foreach (var point in data)
Copy link

@Romazes Romazes Jan 2, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Add validation of Time, example:
https://github.com/QuantConnect/Lean.Brokerages.InteractiveBrokers/blob/master/QuantConnect.InteractiveBrokersBrokerage.Tests/InteractiveBrokersBrokerageAdditionalTests.cs#L1349

{
Assert.AreEqual(symbol, data.Symbol, "Symbol should match requested symbol");
if (data is QuoteBar quoteBar)
Assert.AreEqual(symbol, point.Symbol);
Assert.That(point.Time, Is.InRange(startExchange, endExchange));

if (point is Tick tick)
{
Assert.AreEqual(TickType.Quote, tick.TickType);
Assert.IsTrue(
tick.BidPrice > 0 || tick.AskPrice > 0,
"Quote tick must have bid or ask"
);
}
else if (point is QuoteBar bar)
{
Assert.IsTrue(quoteBar.Bid != null || quoteBar.Ask != null, "Quote should have bid or ask data");
Assert.IsTrue(bar.Bid != null || bar.Ask != null);
}
}
}

[Test]
[Explicit("This test requires a configured DataBento API key")]
public void DataIsSortedByTime()
{
var symbol = Symbol.Create("ESM3", SecurityType.Future, Market.CME);
var startTime = new DateTime(2024, 1, 15);
var endTime = new DateTime(2024, 1, 16);
var param = new DataDownloaderGetParameters(symbol, Resolution.Minute, startTime, endTime, TickType.Trade);
var symbol = CreateEsFuture();

var downloadResponse = _downloader.Get(param).ToList();
var startUtc = new DateTime(2024, 5, 1, 0, 0, 0, DateTimeKind.Utc);
var endUtc = new DateTime(2024, 5, 2, 0, 0, 0, DateTimeKind.Utc);

Assert.IsTrue(downloadResponse.Any(), "Expected to download data for time sorting test");
var parameters = new DataDownloaderGetParameters(
symbol,
Resolution.Minute,
startUtc,
endUtc,
TickType.Trade
);

for (int i = 1; i < downloadResponse.Count; i++)
var data = _downloader.Get(parameters).ToList();

Assert.IsNotEmpty(data);

for (int i = 1; i < data.Count; i++)
{
Assert.GreaterOrEqual(downloadResponse[i].Time, downloadResponse[i - 1].Time,
$"Data should be sorted by time. Item {i} time {downloadResponse[i].Time} should be >= item {i - 1} time {downloadResponse[i - 1].Time}");
Assert.GreaterOrEqual(
data[i].Time,
data[i - 1].Time,
$"Data not sorted at index {i}"
);
}
}

[Test]
public void DisposesCorrectly()
public void DisposeIsIdempotent()
{
var downloader = new DataBentoDataDownloader();
Assert.DoesNotThrow(() => downloader.Dispose(), "Dispose should not throw");
Assert.DoesNotThrow(() => downloader.Dispose(), "Multiple dispose calls should not throw");
var downloader = new DataBentoDataDownloader(ApiKey,
MarketHoursDatabase.FromDataFolder());

Assert.DoesNotThrow(downloader.Dispose);
Assert.DoesNotThrow(downloader.Dispose);
}
}
}
Loading
Loading