From 583b8c91b5f5806b31ec88ba8fd57aa9bead73b6 Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Thu, 16 Jul 2026 11:54:34 +0100 Subject: [PATCH 1/4] docs: revamp README with feature showcase and auto-updating benchmarks Rewrites the README around what TUnit can actually do: hero snippet with real failure output, focused object-diff example, benchmark table vs xUnit/NUnit/MSTest, and sections for mocking, shared fixtures, custom assertions, and the ASP.NET Core/Aspire/Playwright/FsCheck integrations. The benchmark table lives between markers and is regenerated weekly: a new update-readme-benchmarks.js step in the speed-comparison workflow rewrites it from latest.json inside the existing automated benchmark PR. --- .github/scripts/update-readme-benchmarks.js | 81 ++++++ .github/workflows/speed-comparison.yml | 6 +- README.md | 270 ++++++++++++++++---- 3 files changed, 309 insertions(+), 48 deletions(-) create mode 100644 .github/scripts/update-readme-benchmarks.js diff --git a/.github/scripts/update-readme-benchmarks.js b/.github/scripts/update-readme-benchmarks.js new file mode 100644 index 00000000000..38088dd1fe7 --- /dev/null +++ b/.github/scripts/update-readme-benchmarks.js @@ -0,0 +1,81 @@ +const fs = require('fs'); + +const LATEST_JSON = 'docs/static/benchmarks/latest.json'; +const README = 'README.md'; +const START_MARKER = ''; +const END_MARKER = ''; + +const SCENARIOS = { + DataDrivenTests: 'Data-driven tests', + AsyncTests: 'Async-heavy tests', + MatrixTests: 'Matrix combinations', + ScaleTests: 'Large suites (scale)', + MassiveParallelTests: 'Massive parallelism', + SetupTeardownTests: 'Setup/teardown lifecycle', +}; + +const COLUMNS = ['TUnit_AOT', 'TUnit', 'xUnit3', 'NUnit', 'MSTest']; +const COLUMN_LABELS = { TUnit_AOT: 'TUnit (AOT)', TUnit: 'TUnit', xUnit3: 'xUnit v3', NUnit: 'NUnit', MSTest: 'MSTest' }; + +console.log('๐Ÿ“ Updating README benchmark section...'); + +const latest = JSON.parse(fs.readFileSync(LATEST_JSON, 'utf8')); + +function findResult(category, method) { + return (latest.categories[category] || []).find(r => r.Method === method); +} + +const rows = Object.entries(SCENARIOS) + .filter(([key]) => latest.categories[key]) + .map(([key, label]) => { + const cells = COLUMNS.map(col => { + const mean = findResult(key, col)?.Mean; + return mean && mean !== 'NA' ? mean : 'โ€”'; + }); + return `| ${label} | ${cells.join(' | ')} |`; + }); + +if (rows.length === 0) { + console.error('โŒ No runtime benchmark categories found in latest.json'); + process.exit(1); +} + +const versions = COLUMNS.map(col => { + for (const category of Object.keys(latest.categories)) { + const result = findResult(category, col); + if (result?.Version) return `${COLUMN_LABELS[col]} ${result.Version}`; + } + return null; +}).filter(Boolean); + +const date = (latest.timestamp || '').slice(0, 10); +const environment = [latest.environment?.sdk, latest.environment?.host].filter(Boolean).join(', '); + +const section = [ + `| Scenario | ${COLUMNS.map(c => COLUMN_LABELS[c]).join(' | ')} |`, + `|----------|${COLUMNS.map(() => '---').join('|')}|`, + ...rows, + '', + `Mean wall-clock time to run the same test suite. ${versions.join(' ยท ')}. ${environment}. Updated ${date} โ€” regenerated weekly by the [Speed Comparison workflow](https://github.com/thomhurst/TUnit/actions/workflows/speed-comparison.yml). Full results and methodology: [tunit.dev/docs/benchmarks](https://tunit.dev/docs/benchmarks/).`, +].join('\n'); + +const readme = fs.readFileSync(README, 'utf8'); +const startIndex = readme.indexOf(START_MARKER); +const endIndex = readme.indexOf(END_MARKER); + +if (startIndex === -1 || endIndex === -1 || endIndex < startIndex) { + console.error(`โŒ Could not find ${START_MARKER} / ${END_MARKER} markers in README.md`); + process.exit(1); +} + +const updated = + readme.slice(0, startIndex + START_MARKER.length) + + '\n' + section + '\n' + + readme.slice(endIndex); + +if (updated === readme) { + console.log('โœ… README benchmark section already up to date'); +} else { + fs.writeFileSync(README, updated); + console.log(`โœ… README benchmark section updated (${rows.length} scenarios, data from ${date})`); +} diff --git a/.github/workflows/speed-comparison.yml b/.github/workflows/speed-comparison.yml index 972880734f8..64d0232bc31 100644 --- a/.github/workflows/speed-comparison.yml +++ b/.github/workflows/speed-comparison.yml @@ -160,6 +160,10 @@ jobs: run: | node .github/scripts/process-benchmarks.js + - name: Update README Benchmark Section + run: | + node .github/scripts/update-readme-benchmarks.js + - name: Upload Individual Runtime Benchmarks uses: actions/upload-artifact@v7 if: always() @@ -266,7 +270,7 @@ jobs: - name: Check for Changes id: check_changes run: | - git add docs/docs/benchmarks/ docs/static/benchmarks/ + git add docs/docs/benchmarks/ docs/static/benchmarks/ README.md if git diff --quiet --staged; then echo "has_changes=false" >> $GITHUB_OUTPUT echo "No benchmark changes detected" diff --git a/README.md b/README.md index e7e974cc6eb..a26956dd89b 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ # TUnit -A modern .NET testing framework. Tests are source-generated at compile time, run in parallel by default, and support Native AOT โ€” all built on [Microsoft.Testing.Platform](https://learn.microsoft.com/en-us/dotnet/core/testing/microsoft-testing-platform-intro). +A modern .NET testing framework. Tests are discovered at compile time via source generators, run in parallel by default, and work under Native AOT โ€” all built on [Microsoft.Testing.Platform](https://learn.microsoft.com/en-us/dotnet/core/testing/microsoft-testing-platform-intro).
@@ -13,16 +13,63 @@ A modern .NET testing framework. Tests are source-generated at compile time, run
-## Features +## What it looks like -- **Compile-time test discovery** โ€” tests are generated at build time rather than discovered via reflection at runtime, which means faster startup and better IDE integration -- **Parallel by default** โ€” tests run concurrently; use `[DependsOn]` to express ordering and `[ParallelLimiter]` to cap concurrency -- **Data-driven testing** โ€” `[Arguments]`, `[Matrix]`, `[ClassData]`, and custom `DataSourceGenerator` sources -- **Async assertions** with detailed failure messages -- **Built-in Roslyn analyzers** โ€” catch mistakes at compile time, such as missing `async`, incorrect method signatures, and invalid attribute combinations -- **Extensible** โ€” write your own skip conditions, retry logic, and attributes -- **Native AOT & trimming support** -- **Lifecycle hooks** โ€” `[Before]` / `[After]` at method, class, assembly, or test session scope +```csharp +[Test] +[Arguments("GOLD", 100.00, 80.00)] +[Arguments("SILVER", 100.00, 90.00)] +public async Task Discount_Is_Applied(string tier, double subtotal, double expected) +{ + var checkout = new CheckoutService(); + + var total = await checkout.ApplyDiscountAsync(tier, subtotal); + + await Assert.That(total).IsEqualTo(expected); +} +``` + +When a test fails, TUnit tells you what happened โ€” including the actual expression you wrote: + +``` +Expected to be 80 +but found 100 + +at Assert.That(total).IsEqualTo(expected) +``` + +Comparing objects? Instead of dumping two object graphs at you, TUnit pinpoints the difference: + +``` +Expected to be equal to Employee { FirstName = "Victoria", LastName = "Apanii", Age = 30 } +but differs at member FirstName: expected "Victoria" but found "ictoria" + +at Assert.That(actualEmployee).IsEqualTo(expectedEmployee) +``` + +## Why TUnit? + +- **Compile-time test discovery** โ€” tests are wired up by a source generator at build time, not found via reflection at runtime. Faster startup, better IDE integration, and full Native AOT / trimming support. +- **Compile-time safety** โ€” a suite of Roslyn analyzers ships in the box, so mistakes like invalid hook signatures, broken data sources, and misused assertions fail your *build*, not your CI run. +- **Parallel by default, with real control** โ€” tests run concurrently out of the box; `[DependsOn]`, `[NotInParallel]`, and `[ParallelLimiter]` give you precise ordering and throttling when you need it. +- **Batteries included** โ€” rich async assertions, shared fixtures with dependency injection, lifecycle hooks at every scope, and a source-generated mocking library โ€” with first-class integrations for ASP.NET Core, Aspire, and Playwright. + +## Performance + +Source generation shifts work from run time to build time: builds are a fraction of a second slower, and every test run after that starts faster โ€” dramatically so under Native AOT. The same test suites, run on every framework: + + +| Scenario | TUnit (AOT) | TUnit | xUnit v3 | NUnit | MSTest | +|----------|---|---|---|---|---| +| Data-driven tests | 16.65 ms | 265.70 ms | 523.90 ms | 523.01 ms | 529.13 ms | +| Async-heavy tests | 113.1 ms | 360.0 ms | 605.6 ms | 608.6 ms | 676.0 ms | +| Matrix combinations | 119.6 ms | 474.1 ms | 1,469.2 ms | 1,468.9 ms | 1,515.3 ms | +| Large suites (scale) | 29.38 ms | 263.55 ms | 504.14 ms | 488.30 ms | 495.94 ms | +| Massive parallelism | 217.6 ms | 467.8 ms | 2,945.7 ms | 1,107.5 ms | 2,993.9 ms | +| Setup/teardown lifecycle | โ€” | 335.8 ms | 1,050.2 ms | 1,008.8 ms | 1,099.9 ms | + +Mean wall-clock time to run the same test suite. TUnit (AOT) 1.58.0 ยท TUnit 1.58.0 ยท xUnit v3 3.2.2 ยท NUnit 4.6.1 ยท MSTest 4.3.0. .NET SDK 10.0.301, .NET 10.0.9 (10.0.9, 10.0.926.27113), X64 RyuJIT x86-64-v4. Updated 2026-07-12 โ€” regenerated weekly by the [Speed Comparison workflow](https://github.com/thomhurst/TUnit/actions/workflows/speed-comparison.yml). Full results and methodology: [tunit.dev/docs/benchmarks](https://tunit.dev/docs/benchmarks/). + ## Getting Started @@ -43,65 +90,123 @@ dotnet add package TUnit [Getting Started Guide](https://tunit.dev/docs/getting-started/installation) ยท [Migration Guides](https://tunit.dev/docs/migration/xunit) -## Examples - -### Basic test with assertions - -```csharp -[Test] -public async Task Parsing_A_Valid_Date_Succeeds() -{ - var date = DateTime.Parse("2025-01-01"); - - await Assert.That(date.Year).IsEqualTo(2025); - await Assert.That(date.Month).IsEqualTo(1); -} -``` +## A tour of the good parts ### Data-driven tests ```csharp [Test] [Arguments("user1@test.com", "ValidPassword123")] -[Arguments("user2@test.com", "AnotherPassword456")] [Arguments("admin@test.com", "AdminPass789")] -public async Task User_Login_Should_Succeed(string email, string password) -{ - var result = await authService.LoginAsync(email, password); - await Assert.That(result.IsSuccess).IsTrue(); -} +public async Task User_Login_Succeeds(string email, string password) { ... } // Matrix โ€” generates a test for every combination (9 total here) [Test] [MatrixDataSource] public async Task Database_Operations_Work( [Matrix("Create", "Update", "Delete")] string operation, - [Matrix("User", "Product", "Order")] string entity) -{ - await Assert.That(await ExecuteOperation(operation, entity)) - .IsTrue(); -} + [Matrix("User", "Product", "Order")] string entity) { ... } ``` -### Hooks, dependencies, and retry +Need more? `[MethodDataSource]` pulls rows from a method, and custom `DataSourceGenerator` attributes let you build your own sources. + +### Assertions that explain themselves + +Assertions are async, chainable, and produce the focused failure messages shown above: ```csharp -[Before(Class)] -public static async Task SetupDatabase(ClassHookContext context) +await Assert.That(response.StatusCode).IsEqualTo(HttpStatusCode.OK) + .Because("the health endpoint should always be up"); + +await Assert.That(order.Items) + .HasCount(3) + .And.Contains(item => item.Sku == "ABC-123"); +``` + +Defining your own assertion is one attribute on a plain method โ€” TUnit generates the fluent extension for you: + +```csharp +[GenerateAssertion] +public static bool IsPositive(this int value) => value > 0; + +// Now available on Assert.That: +await Assert.That(account.Balance).IsPositive(); +``` + +### Shared fixtures without the ceremony + +Inject anything into your test classes with `[ClassDataSource]`. Implement `IAsyncInitializer` for async setup, `IAsyncDisposable` for teardown, and pick a sharing scope โ€” `None`, `PerClass`, `PerAssembly`, `PerTestSession`, or `Keyed`: + +```csharp +public class PostgresContainer : IAsyncInitializer, IAsyncDisposable { - await DatabaseHelper.InitializeAsync(); + public Task InitializeAsync() { /* start container */ } + public ValueTask DisposeAsync() { /* stop container */ } } +[ClassDataSource(Shared = SharedType.PerTestSession)] +public class OrderRepositoryTests(PostgresContainer postgres) +{ + [Test] + public async Task Saves_Order() { /* postgres is initialized and shared across the whole run */ } +} +``` + +Property injection works too, and disposal is reference-counted โ€” shared fixtures are torn down exactly when the last test using them finishes. + +### Parallelism you control + +Everything runs in parallel by default. Opt out or sequence tests where it matters: + +```csharp [Test] -[MethodDataSource(nameof(GetTestUsers))] -public async Task Register_User(string username, string password) { ... } +public async Task Register_User() { ... } [Test, DependsOn(nameof(Register_User))] [Retry(3)] -public async Task Login_With_Registered_User(string username, string password) -{ - // Guaranteed to run after Register_User passes -} +public async Task Login_With_Registered_User() { ... } // runs after Register_User passes + +[Test, NotInParallel("checkout-db")] // tests sharing a key never overlap +public async Task Migrates_Schema() { ... } +``` + +`[Repeat(n)]`, `[Timeout(ms)]`, and `[ParallelLimiter]` round out the set. + +### Lifecycle hooks at every scope + +```csharp +[Before(Test)] // also: Class, Assembly, TestSession +public async Task SetUp() { ... } + +[After(Class)] +public static async Task TearDownDatabase(ClassHookContext context) { ... } +``` + +### Mocking built in + +`TUnit.Mocks` is a source-generated, Native AOT-compatible mocking library โ€” no runtime proxies, no `Castle.Core`. It works with any test framework: + +```csharp +var gateway = IPaymentGateway.Mock(); // or Mock.Of() + +gateway.ChargeAsync(Any()).Returns(new ChargeResult(Success: true)); + +var checkout = new CheckoutService(gateway.Object); +await checkout.CompleteAsync(cart); + +gateway.ChargeAsync(99.99m).WasCalled(Times.Once); +``` + +Companion packages mock the annoying stuff for you: + +```csharp +// TUnit.Mocks.Http โ€” a real HttpClient backed by a scriptable handler +using var client = Mock.HttpClient("https://api.example.com"); +client.Handler.OnGet("/users/1").RespondWithJson("""{ "id": 1 }"""); + +// TUnit.Mocks.Logging โ€” capture and verify ILogger output +var logger = Mock.Logger(); +logger.VerifyLog().AtLevel(LogLevel.Warning).ContainingMessage("retrying").WasCalled(Times.Once); ``` ### Custom attributes @@ -121,7 +226,76 @@ public class WindowsOnlyAttribute : SkipAttribute public async Task Windows_Specific_Feature() { ... } ``` -See the [documentation](https://tunit.dev/docs/getting-started/attributes) for more examples, including custom retry logic and data sources. +## Integrations + +### ASP.NET Core + +```csharp +public class ApiFactory : TestWebApplicationFactory; + +[ClassDataSource(Shared = SharedType.PerTestSession)] +public class HealthCheckTests(ApiFactory factory) +{ + [Test] + public async Task Health_Endpoint_Responds() + { + using var client = factory.CreateClient(); + var response = await client.GetAsync("/health"); + + await Assert.That(response.StatusCode).IsEqualTo(HttpStatusCode.OK); + } +} +``` + +### Aspire + +Spin up your whole distributed app once per test session, with resource log forwarding and OpenTelemetry capture built in: + +```csharp +public class AppFixture : AspireFixture; + +[ClassDataSource(Shared = SharedType.PerTestSession)] +public class ApiServiceTests(AppFixture app) +{ + [Test] + public async Task Api_Returns_Data() + { + var client = app.CreateHttpClient("apiservice"); + var response = await client.GetAsync("/weather"); + + await Assert.That(response.IsSuccessStatusCode).IsTrue(); + } +} +``` + +### Playwright + +Inherit from `PageTest` and a browser page is waiting for you โ€” lifecycle fully managed: + +```csharp +public class HomePageTests : PageTest +{ + [Test] + public async Task Homepage_Loads() + { + await Page.GotoAsync("https://example.com"); + + await Assert.That(await Page.TitleAsync()).Contains("Example"); + } +} +``` + +### Property-based testing (FsCheck) + +```csharp +[Test, FsCheckProperty] +public bool Reversing_Twice_Returns_Original(int[] array) => + array.SequenceEqual(array.AsEnumerable().Reverse().Reverse()); +``` + +## More than C# + +TUnit runs F# and VB.NET test projects too, and `TUnit.Assertions.FSharp` provides idiomatic F# assertion helpers. ## IDE Support @@ -142,12 +316,14 @@ See the [documentation](https://tunit.dev/docs/getting-started/attributes) for m | `TUnit.Engine` | Execution engine for test projects | | `TUnit.Assertions` | Standalone assertions โ€” works with other test frameworks too | | `TUnit.Assertions.Should` | Optional FluentAssertions-style `value.Should().BeEqualTo(...)` syntax over `TUnit.Assertions` (beta) | -| `TUnit.Mocks` | Source-generated, AOT-compatible mocking framework โ€” works with any test runner | +| `TUnit.Mocks` | Source-generated, AOT-compatible mocking โ€” works with any test runner | | `TUnit.Mocks.Http` | `HttpClient` mocking helpers built on `TUnit.Mocks` | | `TUnit.Mocks.Logging` | `ILogger` capture/verification helpers built on `TUnit.Mocks` | | `TUnit.AspNetCore` | ASP.NET Core integration โ€” `WebApplicationFactory`-based test fixtures | | `TUnit.Aspire` | Aspire integration โ€” distributed app host fixtures with OpenTelemetry capture | | `TUnit.Playwright` | Playwright integration with automatic browser lifecycle management | +| `TUnit.FsCheck` | Property-based testing via FsCheck | +| `TUnit.OpenTelemetry` | OpenTelemetry instrumentation for test runs | ## Migrating from xUnit, NUnit, or MSTest? From e56c5a5d330e458e10f2ed51de8363cba25f859c Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Thu, 16 Jul 2026 13:04:12 +0100 Subject: [PATCH 2/4] docs: use non-obsolete Count().IsEqualTo in README assertion example HasCount(int) is [Obsolete]; showcase the recommended Count().IsEqualTo(3) path instead. Chains identically via CollectionCountEqualsAssertion. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a26956dd89b..80bd7f43e6b 100644 --- a/README.md +++ b/README.md @@ -119,7 +119,7 @@ await Assert.That(response.StatusCode).IsEqualTo(HttpStatusCode.OK) .Because("the health endpoint should always be up"); await Assert.That(order.Items) - .HasCount(3) + .Count().IsEqualTo(3) .And.Contains(item => item.Sku == "ABC-123"); ``` From 6605d48fe331dbcb6b278abb9408f877845adc36 Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Thu, 16 Jul 2026 13:09:11 +0100 Subject: [PATCH 3/4] docs: soften build-time claim in README performance section 'a fraction of a second slower' understates source-gen cost on very large suites (scales with test count). Reworded to 'pay a little up front'. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 80bd7f43e6b..3ff31b56642 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ at Assert.That(actualEmployee).IsEqualTo(expectedEmployee) ## Performance -Source generation shifts work from run time to build time: builds are a fraction of a second slower, and every test run after that starts faster โ€” dramatically so under Native AOT. The same test suites, run on every framework: +Source generation shifts work from run time to build time: you pay a little up front at build, and every test run after that starts faster โ€” dramatically so under Native AOT. The same test suites, run on every framework: | Scenario | TUnit (AOT) | TUnit | xUnit v3 | NUnit | MSTest | From 6e4ed6d0c3ae11f899d583d089f44de4cfbd9f68 Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Thu, 16 Jul 2026 13:11:50 +0100 Subject: [PATCH 4/4] docs: lead DI example with property injection Property injection lets subclasses inherit shared fixtures without re-threading constructor parameters; note constructor injection still works. --- README.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 3ff31b56642..70ed1f0eb8b 100644 --- a/README.md +++ b/README.md @@ -144,15 +144,17 @@ public class PostgresContainer : IAsyncInitializer, IAsyncDisposable public ValueTask DisposeAsync() { /* stop container */ } } -[ClassDataSource(Shared = SharedType.PerTestSession)] -public class OrderRepositoryTests(PostgresContainer postgres) +public class OrderRepositoryTests { + [ClassDataSource(Shared = SharedType.PerTestSession)] + public required PostgresContainer Postgres { get; init; } + [Test] - public async Task Saves_Order() { /* postgres is initialized and shared across the whole run */ } + public async Task Saves_Order() { /* Postgres is initialized and shared across the whole run */ } } ``` -Property injection works too, and disposal is reference-counted โ€” shared fixtures are torn down exactly when the last test using them finishes. +Property injection keeps base test classes clean โ€” subclasses inherit the fixture without re-threading constructor parameters. Prefer a constructor param? That works too. Disposal is reference-counted, so shared fixtures are torn down exactly when the last test using them finishes. ### Parallelism you control