From 9ac7b016b18347d62b6f9045cab6f303589fa7ab Mon Sep 17 00:00:00 2001 From: vivganes Date: Thu, 30 Jul 2026 19:58:34 +0530 Subject: [PATCH] pass 1 --- .github/workflows/ci.yml | 14 ++-- docs/architecture.html | 24 +++--- docs/ci-cd.html | 11 +-- docs/features.html | 29 ++++--- docs/index.html | 8 +- docs/installation.html | 16 ++-- docs/usage.html | 24 ++++-- features/dashboard.feature | 2 +- features/edge_cases.feature | 14 ++-- features/health.feature | 8 +- features/internals.feature | 6 +- features/json_output.feature | 4 +- features/linking.feature | 8 +- features/module_scope.feature | 8 +- run_local.py | 14 ++-- spec_tracer/aggregator.py | 28 ++++--- spec_tracer/linker.py | 11 ++- spec_tracer/parsers.py | 2 +- tests/fixtures/e2e_coverage/e2e.json | 2 +- .../e2e_coverage/features/login.feature | 2 +- .../collision_across/features/login.feature | 2 +- .../collision_across/features/profile.feature | 2 +- .../edge_cases/collision_across/unit.xml | 2 +- .../empty_result/features/login.feature | 2 +- .../features/login.feature | 2 +- .../feature_tags_not_inherited/unit.xml | 2 +- .../malformed_xml/features/login.feature | 2 +- .../no_match/features/login.feature | 2 +- .../fail_on_gate/features/gate.feature | 6 +- tests/fixtures/fail_on_gate/unit.xml | 2 +- tests/fixtures/fc001/e2e.json | 2 +- tests/fixtures/fc001/features/login.feature | 2 +- tests/fixtures/fc001/unit.xml | 2 +- tests/fixtures/fc002/e2e.json | 6 +- tests/fixtures/fc002/features/login.feature | 2 +- tests/fixtures/fc002/unit.xml | 6 +- .../features/login.feature | 2 +- .../integration_linking/integration.xml | 2 +- tests/fixtures/integration_linking/unit.xml | 2 +- .../features/login.feature | 2 +- .../fixtures/missing_required_layer/unit.xml | 2 +- .../module_scope/features/parsers.feature | 4 +- tests/fixtures/module_scope/other_e2e.json | 2 +- tests/fixtures/module_scope/other_unit.xml | 2 +- tests/fixtures/module_scope/parsers_e2e.json | 2 +- tests/fixtures/module_scope/parsers_unit.xml | 2 +- .../repeatable/features/login.feature | 2 +- tests/fixtures/repeatable/integration.xml | 2 +- tests/fixtures/repeatable/unit.xml | 2 +- tests/fixtures/status_badges/e2e.json | 6 +- .../status_badges/features/login.feature | 2 +- tests/fixtures/status_badges/unit.xml | 6 +- .../unit_linking/features/login.feature | 2 +- tests/fixtures/unit_linking/unit.xml | 2 +- tests/integration/test_e2e_coverage.py | 2 +- tests/integration/test_e2e_module_scope.py | 4 +- tests/integration/test_edge_cases.py | 36 ++++---- tests/integration/test_fail_on.py | 20 ++--- tests/integration/test_integration_linking.py | 6 +- tests/integration/test_json_report.py | 4 +- .../test_missing_required_layer.py | 2 +- tests/integration/test_status_badges.py | 2 +- tests/integration/test_unit_linking.py | 4 +- tests/unit/test_aggregator.py | 81 +++++++++--------- tests/unit/test_build_pyramid.py | 83 ++++++++---------- tests/unit/test_cli.py | 30 +++---- tests/unit/test_renderers.py | 84 +++++++++---------- tests/unit/test_report_model.py | 48 +++++------ 68 files changed, 373 insertions(+), 358 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 78f6330..fd5e70d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,16 +36,16 @@ jobs: run: | uv run behave features/ -f modern -o reports/e2e-report.html # Module-scoped Cucumber JSON for SpecTracer dogfood - uv run behave features/linking.feature --tags=-@FC-004 -f json -o reports/e2e-linker.json - uv run behave features/edge_cases.feature --tags=-@FC-EDGE-006 -f json -o reports/e2e-linker-edge.json - uv run behave features/linking.feature --tags=@FC-004 -f json -o reports/e2e-collectors.json + uv run behave features/linking.feature --tags="not @scenario:FC-004" -f json -o reports/e2e-linker.json + uv run behave features/edge_cases.feature --tags="not @scenario:FC-EDGE-006" -f json -o reports/e2e-linker-edge.json + uv run behave features/linking.feature --tags=@scenario:FC-004 -f json -o reports/e2e-collectors.json uv run behave features/health.feature -f json -o reports/e2e-aggregator.json - uv run behave features/internals.feature --tags=@FC-008 -f json -o reports/e2e-aggregator-internals.json + uv run behave features/internals.feature --tags=@scenario:FC-008 -f json -o reports/e2e-aggregator-internals.json uv run behave features/dashboard.feature -f json -o reports/e2e-renderers.json - uv run behave features/internals.feature --tags=@FC-009 -f json -o reports/e2e-renderers-internals.json + uv run behave features/internals.feature --tags=@scenario:FC-009 -f json -o reports/e2e-renderers-internals.json uv run behave features/module_scope.feature -f json -o reports/e2e-parsers.json - uv run behave features/edge_cases.feature --tags=@FC-EDGE-006 -f json -o reports/e2e-parsers-edge.json - uv run behave features/internals.feature --tags=@FC-007 -f json -o reports/e2e-parsers-internals.json + uv run behave features/edge_cases.feature --tags=@scenario:FC-EDGE-006 -f json -o reports/e2e-parsers-edge.json + uv run behave features/internals.feature --tags=@scenario:FC-007 -f json -o reports/e2e-parsers-internals.json uv run behave features/json_output.feature -f json -o reports/e2e-report-model.json - name: Generate SpecTracer report diff --git a/docs/architecture.html b/docs/architecture.html index e8cd482..cb1467c 100644 --- a/docs/architecture.html +++ b/docs/architecture.html @@ -97,7 +97,7 @@

parsers.py

🔗

linker.py

-

OR-based tag matching. Links scenarios to test results by shared tags. 17 lines — the simplest module.

+

Identity-based pairing. Matches @scenario:VALUE on test results against @id:VALUE on scenarios. 17 lines — the simplest module.

📊
@@ -134,7 +134,7 @@

FeatureParser

Default behavior: Scenarios without any @require-* tags default to @require-e2e.

@@ -163,21 +163,23 @@

CucumberParser

Linker

-

The linker is the simplest but most important module. It performs OR-based tag matching:

+

The linker is the simplest but most important module. It performs identity-based pairing using a two-tag convention:

Linking logic (abridged)
def link(scenarios, results):
     links = {}
     for scenario in scenarios:
+        ids = {t[3:] for t in scenario.tags if t.startswith("@id:")}
         linked = [
             r for r in results
-            if any(tag in scenario.tags for tag in r.tags)
+            if any(t.startswith("@scenario:") and t[10:] in ids for t in r.tags)
         ]
         links[id(scenario)] = linked
     return links
@@ -232,7 +234,7 @@

Data Flow

  • Config loading — cli.py discovers and reads the JSON config file. Validates required keys (features, output). unit, integration, and e2e are module-keyed objects ("" = unscoped).
  • File collection — FileCollector (collectors.py) resolves literal file paths and recursively searches directories for .feature, .xml, and .json files, once per module key.
  • Parsing — Each file type is parsed by its dedicated parser. Scenarios and test results are extracted into typed data classes; results carry the module key they were collected under.
    • -
  • Linking — ResultLinker cross-references scenario tags against test result tags. OR logic: any shared tag creates a link.
    • +
  • Linking — ResultLinker matches @scenario:VALUE on test results against @id:VALUE on scenarios. Only explicit identity pairs create a link.
  • Aggregation — ReportAggregator computes all stats, views, breakdowns, and health checks from the linked data.
  • Rendering — HtmlRenderer feeds the aggregated data into a Jinja2 template and writes the HTML output file.
  • JSON output (optional) — if output_json is set, report_model.build_report() reshapes the same aggregated data into the schema-conformant JSON file.
    • @@ -243,7 +245,7 @@

    Tag Resolution Order

    1. All tags on lines immediately above a Scenario: line are collected
    2. Tags matching the pattern @require-(unit|integration|e2e)(:module)? are classified as layer requirements (optional :module suffix for all three layers)
      3. -
    3. All remaining tags become linking tags
      4. +
    4. All remaining tags are retained as-is (including @id:VALUE for scenario identity and any classification tags like @smoke)
    5. If zero requirement tags were found, the scenario defaults to bare @require-e2e (unscoped)
    6. A module-scoped requirement is only satisfied by a linked result whose module matches that key; unscoped results never satisfy it
    @@ -274,7 +276,7 @@

    Testing Strategy (Dogfooding)

    uv run pytest tests/integration --junitxml=reports/int.xml # E2E is split by module so @require-e2e:<module> can be dogfooded -uv run behave features/coverage_linking.feature --tags=-@FC-004 -f json -o reports/e2e-linker.json +uv run behave features/coverage_linking.feature --tags="not @scenario:FC-004" -f json -o reports/e2e-linker.json uv run behave features/coverage_health.feature -f json -o reports/e2e-aggregator.json uv run behave features/coverage_dashboard.feature -f json -o reports/e2e-renderers.json uv run behave features/coverage_module_scope.feature -f json -o reports/e2e-parsers.json diff --git a/docs/ci-cd.html b/docs/ci-cd.html index eee893e..5e3fbcf 100644 --- a/docs/ci-cd.html +++ b/docs/ci-cd.html @@ -260,8 +260,8 @@

    Team Workflows

    🎓 Onboarding a New Project

      -
    1. Write feature files — your team's QA or product owner creates .feature files describing every business scenario. Each scenario gets a unique tag like @FC-001.
      2. -
    2. Add tags to tests — developers add the matching tags to their unit/integration/E2E tests. A quick PR review ensures tag naming consistency.
      3. +
    3. Write feature files — your team's QA or product owner creates .feature files describing every business scenario. Each scenario gets an identity tag like @id:FC-001.
      4. +
    4. Add tags to tests — developers add @scenario:FC-001 to their unit/integration/E2E tests. A quick PR review ensures tag naming consistency.
    5. Set up the config — create spectracer.config.json pointing at your feature files and test outputs.
    6. Add to CI — add the SpecTracer step to your pipeline. The report becomes part of every build.
    @@ -284,10 +284,11 @@

    📜 Tag Naming Convention

    PatternExampleUse - @FC-<N>@FC-42Feature-case ID — unique per scenario (linking) + @id:<VALUE>@id:FC-42Scenario identity — placed on Gherkin scenarios (used for linking) + @scenario:<VALUE>@scenario:FC-42Test reference — placed on test results (used for linking) @require-<layer>@require-unit, @require-e2eRequired layer coverage (not used for linking) @require-<layer>:<module>@require-e2e:checkoutRequired layer + module key from config - @<type>@regression, @smokeOptional linking / classification tags + @<type>@regression, @smokeOptional classification tags (ignored during linking)
    @@ -328,7 +329,7 @@

    The Standup Metric

    SpecTracer's headline metric — Tested: X / Y scenarios (Z%) — is designed for daily standup discussion. Unlike code coverage (which measures lines executed), this metric measures business scenario coverage:

    Teams that use SpecTracer find themselves discussing coverage in standup naturally: "Our coverage dropped from 85% to 78% this sprint — let's add tests for the new billing scenarios before the sprint ends."

    diff --git a/docs/features.html b/docs/features.html index 7e31d9d..e19e950 100644 --- a/docs/features.html +++ b/docs/features.html @@ -165,12 +165,12 @@

    ⚠ What You See

    Unlinked Tests

    -

    Test results whose tags didn't match any scenario end up here — regardless of tag prefix. Any tagged result that fails to link to a scenario is reported. Results with no tags at all are excluded (they can never participate in tag-based linking).

    +

    Test results whose @scenario:VALUE tag doesn't match any scenario's @id:VALUE end up here. Results without a @scenario: tag are excluded (they can never participate in identity-based linking).

    This is invaluable for catching:

    @@ -232,24 +232,29 @@

    End to end Runtime

    ⚐

    Unlinked Tests

    -

    Orphaned test results with any tags that didn't match any scenario. Pass at 0, warn at 1–3, fail at 4+.

    +

    Test results with @scenario:VALUE that matched no scenario's @id:VALUE. Pass at 0, warn at 1–3, fail at 4+.

    Tag Linking System

    -

    The heart of SpecTracer. Feature files and test results connect through shared tags on scenarios.

    +

    The heart of SpecTracer. Feature files and test results connect through an explicit identity convention using two paired tag prefixes.

    -

    Two Kinds of Tags

    +

    Three Kinds of Tags

    - - - + + + + + + + + @@ -260,6 +265,8 @@

    Two Kinds of Tags

    TypeExamplesPurpose
    Linking@FC-42, @regressionShared with test results. Any matching tag links the test to the scenario.Identity@id:FC-42Placed on scenarios in .feature files. Declares the scenario's stable identity — the canonical ID.
    Reference@scenario:FC-42Placed on test results (via framework markers, test names, or properties). Declares which scenario this test satisfies.
    Requirement
    +

    Linking matches only @scenario:VALUE from results against @id:VALUE from scenarios. Other tags (like @smoke or @regression) never participate in linking — they are purely for organization or test-runner filtering.

    +

    Default Behavior

    Scenarios without any @require-* tags default to @require-e2e — because if you wrote a Gherkin scenario but didn't declare requirements, it should at least have E2E coverage.

    @@ -284,7 +291,7 @@

    How It Works

    } }

    In your feature file:

    -
    @FC-42 @require-unit:auth @require-integration:billing @require-e2e:checkout
+
    @id:FC-42 @scenario:FC-42 @require-unit:auth @require-integration:billing @require-e2e:checkout
 Scenario: Authenticated billing checkout
    
       
    Each module-scoped requirement is only satisfied by a linked result registered under that exact module key. Unscoped results (config key "") never satisfy a module-scoped requirement. A bare @require-unit / @require-e2e (no module) is satisfied by any linked result for that layer.
    
       
    In the HTML report, satisfied modules show as chips like e2e (checkout) OK; missing ones show Missing.
    
@@ -300,8 +307,8 @@ 
    Edge Case Behavior
    
         Missing features or outputErrors out — both are required
         Empty/missing test result pathSilently ignored (zero tests for that layer)
         Malformed XML or JSONAborts with a clear error message
-        Test matches no scenario (has tags)Listed in "Unlinked Tests"
-        Test matches no scenario (no tags)Silently excluded — tagless results can never link
+        Test with @scenario: matches no @id:Listed in "Unlinked Tests"
+        Test has no @scenario: tagSilently excluded — cannot participate in identity-based linking
         Scenario matches no testShown as "incomplete"
         @require-* with no matching testLayer flagged as missing
         Feature-level tagsNot inherited by scenarios
diff --git a/docs/index.html b/docs/index.html
index 6f30c40..49746c7 100644
--- a/docs/index.html
+++ b/docs/index.html
@@ -90,7 +90,7 @@ 
    How It Works
    
         
    1
    
         
    
           
    Write Gherkin Feature Files
    
-          
    Define your scenarios in standard .feature files with tags like @FC-42 that link to test results. Declare required layers with @require-unit, @require-e2e, or module forms like @require-e2e:checkout.
    
+          
    Define your scenarios in standard .feature files with identity tags like @id:FC-42 that pair with test results via @scenario:FC-42. Declare required layers with @require-unit, @require-e2e, or module forms like @require-e2e:checkout.
    
         
    
       
       
    
@@ -104,7 +104,7 @@ 
    Run Your Tests
    
         
    3
    
         
    
           
    Generate the Report
    
-           
    Run uv run spec-tracer (or spec-tracer after pip install). SpecTracer links tests to scenarios via shared tags and produces a single self-contained HTML report with five interactive sections. Add output_json to your config to also emit a schema-validated JSON report for scripts and CI tooling.
    
+           
    Run uv run spec-tracer (or spec-tracer after pip install). SpecTracer links test results to scenarios by matching @scenario:VALUE against @id:VALUE and produces a single self-contained HTML report with five interactive sections. Add output_json to your config to also emit a schema-validated JSON report for scripts and CI tooling.
    
         
    
       
    
     
@@ -132,7 +132,7 @@ 
    Quick Start
    
 pip install spec-tracer
 
 # Write Gherkin feature files in ./features/
-# Tag each scenario with an ID like @FC-1
+# Tag each scenario with @id:FC-1 and each test with @scenario:FC-1
 
 # Run your tests with JUnit XML or Cucumber JSON output
 pytest tests/ --junitxml=reports/tests.xml
@@ -200,7 +200,7 @@ 
    Health Checks
    
       
    
         
    ⚐
    
         
    Unlinked Tests
    
-        
    Test results whose tags don't match any scenario. Catch orphaned or mis-tagged tests that would otherwise go unnoticed.
    
+        
    Test results whose @scenario: tag doesn't match any scenario's @id:. Catch orphaned or mis-tagged tests that would otherwise go unnoticed.
    
       
    
       
    
         
    { }
    
diff --git a/docs/installation.html b/docs/installation.html
index 2541058..c5931b3 100644
--- a/docs/installation.html
+++ b/docs/installation.html
@@ -92,17 +92,17 @@ 
    Verify Installation
    
 uv run pytest tests/unit --junitxml=reports/unit.xml -q
 uv run pytest tests/integration --junitxml=reports/int.xml -q
 
-# Module-scoped E2E JSON (matches spectracer.config.json module keys)
-uv run behave features/coverage_linking.feature --tags=-@FC-004 -f json -o reports/e2e-linker.json
-uv run behave features/coverage_edge_cases.feature --tags=-@FC-EDGE-006 -f json -o reports/e2e-linker-edge.json
-uv run behave features/coverage_linking.feature --tags=@FC-004 -f json -o reports/e2e-collectors.json
+# Module-scoped Cucumber JSON for SpecTracer dogfood
+uv run behave features/coverage_linking.feature --tags="not @scenario:FC-004" -f json -o reports/e2e-linker.json
+uv run behave features/coverage_edge_cases.feature --tags="not @scenario:FC-EDGE-006" -f json -o reports/e2e-linker-edge.json
+uv run behave features/coverage_linking.feature --tags=@scenario:FC-004 -f json -o reports/e2e-collectors.json
 uv run behave features/coverage_health.feature -f json -o reports/e2e-aggregator.json
-uv run behave features/coverage_internals.feature --tags=@FC-008 -f json -o reports/e2e-aggregator-internals.json
+uv run behave features/coverage_internals.feature --tags=@scenario:FC-008 -f json -o reports/e2e-aggregator-internals.json
 uv run behave features/coverage_dashboard.feature -f json -o reports/e2e-renderers.json
-uv run behave features/coverage_internals.feature --tags=@FC-009 -f json -o reports/e2e-renderers-internals.json
+uv run behave features/coverage_internals.feature --tags=@scenario:FC-009 -f json -o reports/e2e-renderers-internals.json
 uv run behave features/coverage_module_scope.feature -f json -o reports/e2e-parsers.json
-uv run behave features/coverage_edge_cases.feature --tags=@FC-EDGE-006 -f json -o reports/e2e-parsers-edge.json
-uv run behave features/coverage_internals.feature --tags=@FC-007 -f json -o reports/e2e-parsers-internals.json
+uv run behave features/coverage_edge_cases.feature --tags=@scenario:FC-EDGE-006 -f json -o reports/e2e-parsers-edge.json
+uv run behave features/coverage_internals.feature --tags=@scenario:FC-007 -f json -o reports/e2e-parsers-internals.json
 uv run behave features/coverage_json_output.feature -f json -o reports/e2e-report-model.json
 
 # Generate a report (using installed CLI)
diff --git a/docs/usage.html b/docs/usage.html
index 2cb4f7d..12535cf 100644
--- a/docs/usage.html
+++ b/docs/usage.html
@@ -113,10 +113,13 @@ 
    Configuration File
    
 }

    Tagging Convention

    -

    Feature files and test results connect through shared tags on scenarios. There are two kinds:

    +

    Feature files and test results connect through an explicit identity convention using two paired tag prefixes. Three tag types exist:

    -

    Linking Tags

    -

    Tags like @FC-42 or @regression appear on both the Gherkin scenario and the test result. Any tag match links the test to the scenario (OR logic).

    +

    Scenario Identity Tags (@id:VALUE)

    +

    Placed on Gherkin scenarios in .feature files. Declares the scenario's stable identity — the canonical ID that test results reference. Example: @id:FC-42.

    + +

    Test Reference Tags (@scenario:VALUE)

    +

    Placed on test results (via pytest markers, test names, or framework annotations). Declares which scenario this test satisfies. Example: @scenario:FC-42.

    Layer Requirement Tags

    Tags like @require-unit, @require-integration, or @require-e2e declare which layers must have coverage. Append :modulename on any of them (e.g. @require-e2e:checkout) to require results from a specific config module key. These are excluded from linking — they only affect the report's "Required Layers" status.

    @@ -124,29 +127,34 @@

    Layer Requirement Tags

    Example feature file
    Feature: User Login
 
-  @FC-42 @regression @require-unit:auth @require-integration:auth @require-e2e:auth
+  @id:FC-42 @regression @require-unit:auth @require-integration:auth @require-e2e:auth
   Scenario: Successful login with valid credentials
     Given the user is on the login page
     When they enter valid credentials
     Then they should be redirected to the dashboard
 
-  @FC-43
+  @id:FC-43
   Scenario: Login with invalid password shows error
     Given the user is on the login page
     When they enter an invalid password
     Then an error message should be displayed
    +
    + ℹ Identity + reference pairing + Linking matches only @scenario:VALUE from results against @id:VALUE from scenarios. Other tags like @regression or @smoke never participate in linking — they are purely for test-runner filtering or organization. +
    +

    Matching Rules

    - - + + - +
    RuleDetail
    Exact string match@FC-42 matches @FC-42 only, not @FC-4 or @FC-42-smoke
    OR logicA scenario tagged @FC-42 & @regression links to a test carrying just @regression
    Exact VALUE match@scenario:FC-42 on a result matches @id:FC-42 on a scenario. The FC-42 value must match exactly.
    Prefix-filteredOnly @scenario: on results and @id: on scenarios participate. All other tags are ignored during linking.
    Scenario tags onlyTags on the Feature: line are not inherited by scenarios
    No linking for @require-*Layer requirement tags are excluded from linking logic
    CollisionsIf two scenarios share a tag, one matching test links to both
    CollisionsIf two scenarios share the same @id: value, one matching test links to both
    diff --git a/features/dashboard.feature b/features/dashboard.feature index 83c876f..0a0c172 100644 --- a/features/dashboard.feature +++ b/features/dashboard.feature @@ -1,6 +1,6 @@ Feature: Dashboard Rendering - @FC-006 @require-unit:renderers @require-e2e:renderers + @id:FC-006 @scenario:FC-006 @require-unit:renderers @require-e2e:renderers Scenario: Report renders polished dashboard sections Given a feature file with scenario tagged "@FC-006" And a unit JUnit XML result tagged "@FC-006" diff --git a/features/edge_cases.feature b/features/edge_cases.feature index 279b22f..2dbfb15 100644 --- a/features/edge_cases.feature +++ b/features/edge_cases.feature @@ -1,6 +1,6 @@ Feature: Edge Cases - @FC-EDGE-001 @require-e2e:linker + @id:FC-EDGE-001 @scenario:FC-EDGE-001 @require-e2e:linker Scenario: Generate report with linked edge case coverage Given a feature file with scenario tagged "@FC-EDGE-001" And a unit JUnit XML result tagged "@FC-EDGE-001" @@ -10,7 +10,7 @@ Feature: Edge Cases And the report should contain "scenarios fully matched" And the report should contain "unit" - @FC-EDGE-002 @require-e2e:linker + @id:FC-EDGE-002 @scenario:FC-EDGE-002 @require-e2e:linker Scenario: One test tag links to multiple scenarios sharing that tag Given a feature file with scenario tagged "@FC-EDGE-002" And a unit JUnit XML result tagged "@FC-EDGE-002" @@ -19,7 +19,7 @@ Feature: Edge Cases And the report should contain "0/2" And the report should contain "scenarios fully matched" - @FC-EDGE-003 @require-e2e:linker + @id:FC-EDGE-003 @scenario:FC-EDGE-003 @require-e2e:linker Scenario: Tags on the Feature line are not inherited by scenarios Given a feature file with scenario tagged "@FC-EDGE-003" And a unit JUnit XML result tagged "@FC-EDGE-003" @@ -29,7 +29,7 @@ Feature: Edge Cases And the report should contain "scenarios fully matched" And the report should contain "unit" - @FC-EDGE-004 @require-e2e:linker + @id:FC-EDGE-004 @scenario:FC-EDGE-004 @require-e2e:linker Scenario: Scenario with no matching test results shows as untested Given a feature file with scenario tagged "@FC-EDGE-004" And a unit JUnit XML result tagged "@FC-EDGE-004" @@ -39,7 +39,7 @@ Feature: Edge Cases And the report should contain "scenarios fully matched" And the report should list the unlinked test "test_unrelated_@OTHER-999" - @FC-EDGE-004b @require-e2e:linker + @id:FC-EDGE-004b @scenario:FC-EDGE-004b @require-e2e:linker Scenario: Untagged result with no matching scenario still appears as unlinked Given a feature file with scenario tagged "@FC-EDGE-004" And a unit JUnit XML result tagged "@FC-EDGE-004" @@ -48,7 +48,7 @@ Feature: Edge Cases And the report should contain "Unlinked Tests" And the report should list the unlinked test "test_unrelated_@OTHER-999" - @FC-EDGE-005 @require-e2e:linker + @id:FC-EDGE-005 @scenario:FC-EDGE-005 @require-e2e:linker Scenario: Empty JUnit XML result file produces zero test results Given a feature file with scenario tagged "@FC-EDGE-005" And a unit JUnit XML result tagged "@FC-EDGE-005" @@ -57,7 +57,7 @@ Feature: Edge Cases And the report should contain "0/1" And the report should contain "scenarios fully matched" - @FC-EDGE-006 @require-e2e:parsers + @id:FC-EDGE-006 @scenario:FC-EDGE-006 @require-e2e:parsers Scenario: Malformed JUnit XML aborts with a non-zero exit code Given a feature file with scenario tagged "@FC-EDGE-006" And a unit JUnit XML result tagged "@FC-EDGE-006" diff --git a/features/health.feature b/features/health.feature index 9822130..7903f78 100644 --- a/features/health.feature +++ b/features/health.feature @@ -1,6 +1,6 @@ Feature: Health Signals - @FC-005 @require-unit:aggregator @require-e2e:aggregator + @id:FC-005 @scenario:FC-005 @require-unit:aggregator @require-e2e:aggregator Scenario: Report flags missing required layer Given a feature file with scenario tagged "@FC-005" And a unit JUnit XML result tagged "@FC-005" @@ -13,7 +13,7 @@ Feature: Health Signals And the report should contain "required-chip ok" And the report should contain "required-chip missing" - @FC-011 @require-unit:aggregator @require-e2e:aggregator + @id:FC-011 @scenario:FC-011 @require-unit:aggregator @require-e2e:aggregator Scenario: CI gate fails the build when a listed health check is red Given a feature file with scenario tagged "@FC-011" And a unit JUnit XML result tagged "@FC-011" @@ -21,7 +21,7 @@ Feature: Health Signals When I run the tool with --features, --unit, and --output Then the exit code should be 1 - @FC-011 @require-unit:aggregator @require-e2e:aggregator + @id:FC-011 @scenario:FC-011 @require-unit:aggregator @require-e2e:aggregator Scenario: CI gate stays green when the listed health check is healthy Given a feature file with scenario tagged "@FC-011" And a unit JUnit XML result tagged "@FC-011" @@ -29,7 +29,7 @@ Feature: Health Signals When I run the tool with --features, --unit, and --output Then the exit code should be 0 - @FC-011 @require-unit:aggregator @require-e2e:aggregator + @id:FC-011 @scenario:FC-011 @require-unit:aggregator @require-e2e:aggregator Scenario: A failing health check is only visual without a CI gate Given a feature file with scenario tagged "@FC-011" And a unit JUnit XML result tagged "@FC-011" diff --git a/features/internals.feature b/features/internals.feature index a6d58ff..eb48296 100644 --- a/features/internals.feature +++ b/features/internals.feature @@ -1,13 +1,13 @@ Feature: Tool Internals - @FC-007 @require-unit:parsers @require-e2e:parsers + @id:FC-007 @scenario:FC-007 @require-unit:parsers @require-e2e:parsers Scenario: Malformed JUnit XML input is rejected with a clear error Given a feature file with scenario tagged "@FC-EDGE-006" And a unit JUnit XML result tagged "@FC-EDGE-006" When I run the tool with --features, --unit, and --output Then the exit code should be 1 - @FC-008 @require-unit:aggregator @require-e2e:aggregator + @id:FC-008 @scenario:FC-008 @require-unit:aggregator @require-e2e:aggregator Scenario: Aggregator computes pyramid and health metrics Given a feature file with scenario tagged "@FC-002" And a unit JUnit XML result tagged "@FC-002" @@ -18,7 +18,7 @@ Feature: Tool Internals And the report should contain "Test Pyramid" And the report should contain "Health Check" - @FC-009 @require-unit:renderers @require-e2e:renderers + @id:FC-009 @scenario:FC-009 @require-unit:renderers @require-e2e:renderers Scenario: Dashboard renders multi-page navigation Given a feature file with scenario tagged "@FC-006" And a unit JUnit XML result tagged "@FC-006" diff --git a/features/json_output.feature b/features/json_output.feature index d172e98..2a3ec1c 100644 --- a/features/json_output.feature +++ b/features/json_output.feature @@ -1,6 +1,6 @@ Feature: Machine Readable JSON Report - @FC-010 @require-unit:report_model @require-e2e:report_model + @id:FC-010 @scenario:FC-010 @require-unit:report_model @require-e2e:report_model Scenario: JSON report is written alongside the HTML report and matches the schema Given a feature file with scenario tagged "@FC-002" And a unit JUnit XML result tagged "@FC-002" @@ -11,7 +11,7 @@ Feature: Machine Readable JSON Report And the JSON report file should exist And the JSON report should conform to the SpecTracer report schema - @FC-010 @require-unit:report_model @require-e2e:report_model + @id:FC-010 @scenario:FC-010 @require-unit:report_model @require-e2e:report_model Scenario: JSON report is not written when output_json is omitted Given a feature file with scenario tagged "@FC-001" And a unit JUnit XML result tagged "@FC-001" diff --git a/features/linking.feature b/features/linking.feature index 8f79adf..dccb2d0 100644 --- a/features/linking.feature +++ b/features/linking.feature @@ -1,6 +1,6 @@ Feature: Linking - @FC-001 @require-unit:linker @require-e2e:linker + @id:FC-001 @scenario:FC-001 @require-unit:linker @require-e2e:linker Scenario: Generate report with linked unit and E2E coverage Given a feature file with scenario tagged "@FC-001" And a unit JUnit XML result tagged "@FC-001" @@ -13,7 +13,7 @@ Feature: Linking And the report should contain "e2e" And the report should contain "unit" - @FC-002 @require-unit:linker @require-e2e:linker + @id:FC-002 @scenario:FC-002 @require-unit:linker @require-e2e:linker Scenario: Report shows passed, failed, and skipped statuses Given a feature file with scenario tagged "@FC-002" And a unit JUnit XML result tagged "@FC-002" @@ -25,7 +25,7 @@ Feature: Linking And the report should contain "badge failed" And the report should contain "badge skipped" - @FC-003 @require-integration:linker @require-e2e:linker + @id:FC-003 @scenario:FC-003 @require-integration:linker @require-e2e:linker Scenario: Generate report with linked integration coverage Given a feature file with scenario tagged "@FC-003" And an integration JUnit XML result tagged "@FC-003" @@ -36,7 +36,7 @@ Feature: Linking And the report should contain "scenarios fully matched" And the report should contain "integration" - @FC-004 @require-unit:collectors @require-integration:collectors @require-e2e:collectors + @id:FC-004 @scenario:FC-004 @require-unit:collectors @require-integration:collectors @require-e2e:collectors Scenario: Generate report when unit and integration flags are repeated Given a feature file with scenario tagged "@FC-004" And a unit JUnit XML result tagged "@FC-004" diff --git a/features/module_scope.feature b/features/module_scope.feature index 490995e..fdb0294 100644 --- a/features/module_scope.feature +++ b/features/module_scope.feature @@ -1,6 +1,6 @@ Feature: Module Scoped Layer Requirements - @FC-007 @require-unit:parsers @require-e2e:parsers + @id:FC-007 @scenario:FC-007 @require-unit:parsers @require-e2e:parsers Scenario: Report marks the requirement satisfied when the module matches Given a feature file with scenario tagged "@FC-007" And a module-scoped unit JUnit XML result tagged "@FC-007" for module "parsers" @@ -9,7 +9,7 @@ Feature: Module Scoped Layer Requirements And the report should contain "unit" And the report should contain "required-chip ok" - @FC-007 @require-unit:parsers @require-e2e:parsers + @id:FC-007 @scenario:FC-007 @require-unit:parsers @require-e2e:parsers Scenario: Report flags the requirement missing when the module does not match Given a feature file with scenario tagged "@FC-007" And a module-scoped unit JUnit XML result tagged "@FC-007" for module "other" @@ -17,7 +17,7 @@ Feature: Module Scoped Layer Requirements Then the exit code should be 0 And the report should contain "required-chip missing" - @FC-007 @require-unit:parsers @require-e2e:parsers + @id:FC-007 @scenario:FC-007 @require-unit:parsers @require-e2e:parsers Scenario: Report marks the e2e requirement satisfied when the module matches Given a feature file with scenario tagged "@FC-007" And a module-scoped E2E Cucumber JSON result tagged "@FC-007" for module "parsers" @@ -26,7 +26,7 @@ Feature: Module Scoped Layer Requirements And the report should contain "e2e" And the report should contain "required-chip ok" - @FC-007 @require-unit:parsers @require-e2e:parsers + @id:FC-007 @scenario:FC-007 @require-unit:parsers @require-e2e:parsers Scenario: Report flags the e2e requirement missing when the module does not match Given a feature file with scenario tagged "@FC-007" And a module-scoped E2E Cucumber JSON result tagged "@FC-007" for module "other" diff --git a/run_local.py b/run_local.py index 22887df..ff8c59f 100644 --- a/run_local.py +++ b/run_local.py @@ -34,15 +34,15 @@ ]), ("E2E: linker", [ "uv", "run", "behave", "features/linking.feature", - "--tags=-@FC-004", "-f", "json", "-o", "reports/e2e-linker.json", + "--tags=not @scenario:FC-004", "-f", "json", "-o", "reports/e2e-linker.json", ]), ("E2E: linker (edge)", [ "uv", "run", "behave", "features/edge_cases.feature", - "--tags=-@FC-EDGE-006", "-f", "json", "-o", "reports/e2e-linker-edge.json", + "--tags=not @scenario:FC-EDGE-006", "-f", "json", "-o", "reports/e2e-linker-edge.json", ]), ("E2E: collectors", [ "uv", "run", "behave", "features/linking.feature", - "--tags=@FC-004", "-f", "json", "-o", "reports/e2e-collectors.json", + "--tags=@scenario:FC-004", "-f", "json", "-o", "reports/e2e-collectors.json", ]), ("E2E: aggregator", [ "uv", "run", "behave", "features/health.feature", @@ -50,7 +50,7 @@ ]), ("E2E: aggregator (internals)", [ "uv", "run", "behave", "features/internals.feature", - "--tags=@FC-008", "-f", "json", "-o", "reports/e2e-aggregator-internals.json", + "--tags=@scenario:FC-008", "-f", "json", "-o", "reports/e2e-aggregator-internals.json", ]), ("E2E: renderers", [ "uv", "run", "behave", "features/dashboard.feature", @@ -58,7 +58,7 @@ ]), ("E2E: renderers (internals)", [ "uv", "run", "behave", "features/internals.feature", - "--tags=@FC-009", "-f", "json", "-o", "reports/e2e-renderers-internals.json", + "--tags=@scenario:FC-009", "-f", "json", "-o", "reports/e2e-renderers-internals.json", ]), ("E2E: parsers", [ "uv", "run", "behave", "features/module_scope.feature", @@ -66,11 +66,11 @@ ]), ("E2E: parsers (edge)", [ "uv", "run", "behave", "features/edge_cases.feature", - "--tags=@FC-EDGE-006", "-f", "json", "-o", "reports/e2e-parsers-edge.json", + "--tags=@scenario:FC-EDGE-006", "-f", "json", "-o", "reports/e2e-parsers-edge.json", ]), ("E2E: parsers (internals)", [ "uv", "run", "behave", "features/internals.feature", - "--tags=@FC-007", "-f", "json", "-o", "reports/e2e-parsers-internals.json", + "--tags=@scenario:FC-007", "-f", "json", "-o", "reports/e2e-parsers-internals.json", ]), ("E2E: report_model", [ "uv", "run", "behave", "features/json_output.feature", diff --git a/spec_tracer/aggregator.py b/spec_tracer/aggregator.py index 0786f13..a1dfad0 100644 --- a/spec_tracer/aggregator.py +++ b/spec_tracer/aggregator.py @@ -1,6 +1,7 @@ from collections import defaultdict from typing import Dict, List +from spec_tracer.linker import _scenario_ids from spec_tracer.models import Scenario, ScenarioView, TestResult from spec_tracer.models import completion_fraction, completion_ratio @@ -204,20 +205,25 @@ def health_checks( @staticmethod def unlinked_results(scenarios: List[Scenario], results: List[TestResult]) -> List[TestResult]: - """Return results whose tags matched no scenario. - - A result counts as unlinked when it carries at least one tag but none of - those tags match a scenario's tags. The ``@FC-``-prefix gate was removed: - tagging a test with e.g. ``@smoke``/``@regression`` and failing to link it - must surface it as unlinked, exactly as the report's "Unlinked Tests" section - promises. Results with no tags at all are excluded because, being tag-based, - they can never link to a scenario and would otherwise flood the section with - noise from untagged test runners. + """Return results whose ``@scenario:`` tags matched no scenario ``@id:`` tag. + + A result counts as unlinked when it carries at least one tag but no + ``@scenario:VALUE`` tag matches any scenario's ``@id:VALUE``. Results + tagged with only category tags like ``@smoke`` (no ``@scenario:`` prefix) + are therefore always unlinked, which is the desired behaviour: category + tags must not carry linking weight. Results with no tags at all are + excluded because they can never link and would otherwise flood the + "Unlinked Tests" section with noise from untagged test runners. """ - scenario_tags = {tag for scenario in scenarios for tag in scenario.tags} + all_scenario_ids = set() + for scenario in scenarios: + all_scenario_ids.update(_scenario_ids(scenario)) return [ result for result in results if result.tags - and not any(tag in scenario_tags for tag in result.tags) + and not any( + tag.startswith("@scenario:") and tag[10:] in all_scenario_ids + for tag in result.tags + ) ] diff --git a/spec_tracer/linker.py b/spec_tracer/linker.py index 6fa092a..d6050b4 100644 --- a/spec_tracer/linker.py +++ b/spec_tracer/linker.py @@ -3,15 +3,24 @@ from spec_tracer.models import Scenario, TestResult +def _scenario_ids(scenario: Scenario) -> set: + return {tag[4:] for tag in scenario.tags if tag.startswith("@id:")} + + +def _result_scenario_tags(result: TestResult) -> list: + return [tag[10:] for tag in result.tags if tag.startswith("@scenario:")] + + class ResultLinker: @staticmethod def link(scenarios: List[Scenario], results: List[TestResult]) -> Dict[int, List[TestResult]]: links: Dict[int, List[TestResult]] = {} for scenario in scenarios: + ids = _scenario_ids(scenario) links[id(scenario)] = [ result for result in results - if any(tag in scenario.tags for tag in result.tags) + if any(rsid in ids for rsid in _result_scenario_tags(result)) ] return links diff --git a/spec_tracer/parsers.py b/spec_tracer/parsers.py index 7a637f4..97892ea 100644 --- a/spec_tracer/parsers.py +++ b/spec_tracer/parsers.py @@ -75,7 +75,7 @@ def parse(path: Path) -> List[Scenario]: class JunitParser(ResultParser): - _tag_pattern = re.compile(r"@[\w.-]+") + _tag_pattern = re.compile(r"@[\w.:-]+") def parse(self, paths: List[Path], layer: str = "unit", module: str = "") -> List[TestResult]: results: List[TestResult] = [] diff --git a/tests/fixtures/e2e_coverage/e2e.json b/tests/fixtures/e2e_coverage/e2e.json index 4f9a250..5522f92 100644 --- a/tests/fixtures/e2e_coverage/e2e.json +++ b/tests/fixtures/e2e_coverage/e2e.json @@ -7,7 +7,7 @@ { "keyword": "Scenario", "name": "Successful login with valid credentials", - "tags": [{"name": "@FC-001"}], + "tags": [{"name": "@scenario:FC-001"}], "steps": [ {"keyword": "Given", "name": "the user is on the login page", "result": {"status": "passed"}}, {"keyword": "When", "name": "they enter valid credentials", "result": {"status": "passed"}}, diff --git a/tests/fixtures/e2e_coverage/features/login.feature b/tests/fixtures/e2e_coverage/features/login.feature index 08f17b2..a7633ff 100644 --- a/tests/fixtures/e2e_coverage/features/login.feature +++ b/tests/fixtures/e2e_coverage/features/login.feature @@ -1,6 +1,6 @@ Feature: User Login - @FC-001 + @id:FC-001 Scenario: Successful login with valid credentials Given the user is on the login page When they enter valid credentials diff --git a/tests/fixtures/edge_cases/collision_across/features/login.feature b/tests/fixtures/edge_cases/collision_across/features/login.feature index 2dc47c8..11407d8 100644 --- a/tests/fixtures/edge_cases/collision_across/features/login.feature +++ b/tests/fixtures/edge_cases/collision_across/features/login.feature @@ -1,6 +1,6 @@ Feature: User Login - @FC-001 + @id:FC-001 Scenario: Successful login Given the user is on the login page When they enter valid credentials diff --git a/tests/fixtures/edge_cases/collision_across/features/profile.feature b/tests/fixtures/edge_cases/collision_across/features/profile.feature index 4b4d4cb..a19b34e 100644 --- a/tests/fixtures/edge_cases/collision_across/features/profile.feature +++ b/tests/fixtures/edge_cases/collision_across/features/profile.feature @@ -1,6 +1,6 @@ Feature: User Profile - @FC-001 + @id:FC-001 Scenario: View profile Given the user is logged in When they view their profile diff --git a/tests/fixtures/edge_cases/collision_across/unit.xml b/tests/fixtures/edge_cases/collision_across/unit.xml index 27a1958..1233bc4 100644 --- a/tests/fixtures/edge_cases/collision_across/unit.xml +++ b/tests/fixtures/edge_cases/collision_across/unit.xml @@ -1,6 +1,6 @@ - + diff --git a/tests/fixtures/edge_cases/empty_result/features/login.feature b/tests/fixtures/edge_cases/empty_result/features/login.feature index 9823711..a074fc1 100644 --- a/tests/fixtures/edge_cases/empty_result/features/login.feature +++ b/tests/fixtures/edge_cases/empty_result/features/login.feature @@ -1,5 +1,5 @@ Feature: User Login - @FC-001 + @id:FC-001 Scenario: Successful login Given the user is on the login page diff --git a/tests/fixtures/edge_cases/feature_tags_not_inherited/features/login.feature b/tests/fixtures/edge_cases/feature_tags_not_inherited/features/login.feature index 2b139c6..04271eb 100644 --- a/tests/fixtures/edge_cases/feature_tags_not_inherited/features/login.feature +++ b/tests/fixtures/edge_cases/feature_tags_not_inherited/features/login.feature @@ -1,6 +1,6 @@ @FeatureTag Feature: User Login - @FC-001 + @id:FC-001 Scenario: Successful login Given the user is on the login page diff --git a/tests/fixtures/edge_cases/feature_tags_not_inherited/unit.xml b/tests/fixtures/edge_cases/feature_tags_not_inherited/unit.xml index 4124094..02f3852 100644 --- a/tests/fixtures/edge_cases/feature_tags_not_inherited/unit.xml +++ b/tests/fixtures/edge_cases/feature_tags_not_inherited/unit.xml @@ -1,7 +1,7 @@ - + diff --git a/tests/fixtures/edge_cases/malformed_xml/features/login.feature b/tests/fixtures/edge_cases/malformed_xml/features/login.feature index 9823711..a074fc1 100644 --- a/tests/fixtures/edge_cases/malformed_xml/features/login.feature +++ b/tests/fixtures/edge_cases/malformed_xml/features/login.feature @@ -1,5 +1,5 @@ Feature: User Login - @FC-001 + @id:FC-001 Scenario: Successful login Given the user is on the login page diff --git a/tests/fixtures/edge_cases/no_match/features/login.feature b/tests/fixtures/edge_cases/no_match/features/login.feature index 9823711..a074fc1 100644 --- a/tests/fixtures/edge_cases/no_match/features/login.feature +++ b/tests/fixtures/edge_cases/no_match/features/login.feature @@ -1,5 +1,5 @@ Feature: User Login - @FC-001 + @id:FC-001 Scenario: Successful login Given the user is on the login page diff --git a/tests/fixtures/fail_on_gate/features/gate.feature b/tests/fixtures/fail_on_gate/features/gate.feature index 2a9ab3e..306fcf6 100644 --- a/tests/fixtures/fail_on_gate/features/gate.feature +++ b/tests/fixtures/fail_on_gate/features/gate.feature @@ -1,18 +1,18 @@ Feature: Health Gate Sample - @FC-001 + @id:FC-001 Scenario: A scenario with unit coverage Given a precondition When an action occurs Then an outcome is observed - @FC-011-UNCOVERED-1 + @id:FC-011-UNCOVERED-1 Scenario: A scenario with no unit coverage 1 Given a precondition When an action occurs Then an outcome is observed - @FC-011-UNCOVERED-2 + @id:FC-011-UNCOVERED-2 Scenario: A scenario with no unit coverage 2 Given a precondition When an action occurs diff --git a/tests/fixtures/fail_on_gate/unit.xml b/tests/fixtures/fail_on_gate/unit.xml index 9753577..dde85c6 100644 --- a/tests/fixtures/fail_on_gate/unit.xml +++ b/tests/fixtures/fail_on_gate/unit.xml @@ -1,7 +1,7 @@ - + diff --git a/tests/fixtures/fc001/e2e.json b/tests/fixtures/fc001/e2e.json index 4f9a250..5522f92 100644 --- a/tests/fixtures/fc001/e2e.json +++ b/tests/fixtures/fc001/e2e.json @@ -7,7 +7,7 @@ { "keyword": "Scenario", "name": "Successful login with valid credentials", - "tags": [{"name": "@FC-001"}], + "tags": [{"name": "@scenario:FC-001"}], "steps": [ {"keyword": "Given", "name": "the user is on the login page", "result": {"status": "passed"}}, {"keyword": "When", "name": "they enter valid credentials", "result": {"status": "passed"}}, diff --git a/tests/fixtures/fc001/features/login.feature b/tests/fixtures/fc001/features/login.feature index 08f17b2..a7633ff 100644 --- a/tests/fixtures/fc001/features/login.feature +++ b/tests/fixtures/fc001/features/login.feature @@ -1,6 +1,6 @@ Feature: User Login - @FC-001 + @id:FC-001 Scenario: Successful login with valid credentials Given the user is on the login page When they enter valid credentials diff --git a/tests/fixtures/fc001/unit.xml b/tests/fixtures/fc001/unit.xml index 9567c86..555beec 100644 --- a/tests/fixtures/fc001/unit.xml +++ b/tests/fixtures/fc001/unit.xml @@ -1,6 +1,6 @@ - + diff --git a/tests/fixtures/fc002/e2e.json b/tests/fixtures/fc002/e2e.json index 22f45f3..dd1a15f 100644 --- a/tests/fixtures/fc002/e2e.json +++ b/tests/fixtures/fc002/e2e.json @@ -7,7 +7,7 @@ { "keyword": "Scenario", "name": "Successful login - passed", - "tags": [{"name": "@FC-002"}], + "tags": [{"name": "@scenario:FC-002"}], "steps": [ {"keyword": "Given", "name": "step", "result": {"status": "passed"}} ], @@ -16,7 +16,7 @@ { "keyword": "Scenario", "name": "Login with invalid password - failed", - "tags": [{"name": "@FC-002"}], + "tags": [{"name": "@scenario:FC-002"}], "steps": [ {"keyword": "When", "name": "invalid", "result": {"status": "failed", "error_message": "AssertionError"}} ], @@ -25,7 +25,7 @@ { "keyword": "Scenario", "name": "Login lockout - skipped", - "tags": [{"name": "@FC-002"}], + "tags": [{"name": "@scenario:FC-002"}], "steps": [ {"keyword": "Then", "name": "lockout", "result": {"status": "skipped"}} ], diff --git a/tests/fixtures/fc002/features/login.feature b/tests/fixtures/fc002/features/login.feature index f5a558d..3c5979c 100644 --- a/tests/fixtures/fc002/features/login.feature +++ b/tests/fixtures/fc002/features/login.feature @@ -1,6 +1,6 @@ Feature: User Login - @FC-002 + @id:FC-002 Scenario: Successful login with valid credentials Given the user is on the login page When they enter valid credentials diff --git a/tests/fixtures/fc002/unit.xml b/tests/fixtures/fc002/unit.xml index da0bfdc..108ec2c 100644 --- a/tests/fixtures/fc002/unit.xml +++ b/tests/fixtures/fc002/unit.xml @@ -1,11 +1,11 @@ - - + + assert False - + diff --git a/tests/fixtures/integration_linking/features/login.feature b/tests/fixtures/integration_linking/features/login.feature index 84b6b5c..73c10d2 100644 --- a/tests/fixtures/integration_linking/features/login.feature +++ b/tests/fixtures/integration_linking/features/login.feature @@ -1,6 +1,6 @@ Feature: Integration Coverage - @FC-003 + @id:FC-003 Scenario: Report shows integration coverage Given an integration result is present When the tool runs diff --git a/tests/fixtures/integration_linking/integration.xml b/tests/fixtures/integration_linking/integration.xml index 7cd4b1d..98a99d1 100644 --- a/tests/fixtures/integration_linking/integration.xml +++ b/tests/fixtures/integration_linking/integration.xml @@ -1,3 +1,3 @@ - + diff --git a/tests/fixtures/integration_linking/unit.xml b/tests/fixtures/integration_linking/unit.xml index 983e1a4..28917c2 100644 --- a/tests/fixtures/integration_linking/unit.xml +++ b/tests/fixtures/integration_linking/unit.xml @@ -1,3 +1,3 @@ - + diff --git a/tests/fixtures/missing_required_layer/features/login.feature b/tests/fixtures/missing_required_layer/features/login.feature index b9a5896..814c625 100644 --- a/tests/fixtures/missing_required_layer/features/login.feature +++ b/tests/fixtures/missing_required_layer/features/login.feature @@ -1,6 +1,6 @@ Feature: Layer Requirement Checks - @FC-005 @require-unit @require-e2e + @id:FC-005 @require-unit @require-e2e Scenario: Report flags missing e2e layer Given a scenario with unit and e2e requirements When only unit results exist diff --git a/tests/fixtures/missing_required_layer/unit.xml b/tests/fixtures/missing_required_layer/unit.xml index 2363807..90d8c6f 100644 --- a/tests/fixtures/missing_required_layer/unit.xml +++ b/tests/fixtures/missing_required_layer/unit.xml @@ -1,3 +1,3 @@ - + diff --git a/tests/fixtures/module_scope/features/parsers.feature b/tests/fixtures/module_scope/features/parsers.feature index a419ea8..00fd62a 100644 --- a/tests/fixtures/module_scope/features/parsers.feature +++ b/tests/fixtures/module_scope/features/parsers.feature @@ -1,12 +1,12 @@ Feature: Module Scoped Layer Requirements - @FC-007 @require-unit:parsers + @id:FC-007 @require-unit:parsers Scenario: Report requires unit coverage from the parsers module Given a scenario tagged "@FC-007" requiring unit coverage from module "parsers" When only unit results exist Then the report shows unit as satisfied - @FC-007 @require-e2e:parsers + @id:FC-007 @require-e2e:parsers Scenario: Report requires e2e coverage from the parsers module Given a scenario tagged "@FC-007" requiring e2e coverage from module "parsers" When only e2e results exist diff --git a/tests/fixtures/module_scope/other_e2e.json b/tests/fixtures/module_scope/other_e2e.json index 0d8c946..188a1be 100644 --- a/tests/fixtures/module_scope/other_e2e.json +++ b/tests/fixtures/module_scope/other_e2e.json @@ -7,7 +7,7 @@ { "keyword": "Scenario", "name": "Report requires e2e coverage from the parsers module", - "tags": [{"name": "@FC-007"}], + "tags": [{"name": "@scenario:FC-007"}], "steps": [ {"keyword": "Given", "name": "a scenario requiring e2e coverage from module parsers", "result": {"status": "passed"}}, {"keyword": "When", "name": "e2e results exist under another module", "result": {"status": "passed"}}, diff --git a/tests/fixtures/module_scope/other_unit.xml b/tests/fixtures/module_scope/other_unit.xml index 816d843..23c3202 100644 --- a/tests/fixtures/module_scope/other_unit.xml +++ b/tests/fixtures/module_scope/other_unit.xml @@ -1,3 +1,3 @@ - + diff --git a/tests/fixtures/module_scope/parsers_e2e.json b/tests/fixtures/module_scope/parsers_e2e.json index 582ca5d..dfdfbec 100644 --- a/tests/fixtures/module_scope/parsers_e2e.json +++ b/tests/fixtures/module_scope/parsers_e2e.json @@ -7,7 +7,7 @@ { "keyword": "Scenario", "name": "Report requires e2e coverage from the parsers module", - "tags": [{"name": "@FC-007"}], + "tags": [{"name": "@scenario:FC-007"}], "steps": [ {"keyword": "Given", "name": "a scenario requiring e2e coverage from module parsers", "result": {"status": "passed"}}, {"keyword": "When", "name": "only e2e results exist", "result": {"status": "passed"}}, diff --git a/tests/fixtures/module_scope/parsers_unit.xml b/tests/fixtures/module_scope/parsers_unit.xml index 0521c21..d86e8f8 100644 --- a/tests/fixtures/module_scope/parsers_unit.xml +++ b/tests/fixtures/module_scope/parsers_unit.xml @@ -1,3 +1,3 @@ - + diff --git a/tests/fixtures/repeatable/features/login.feature b/tests/fixtures/repeatable/features/login.feature index 4cfa930..938aab5 100644 --- a/tests/fixtures/repeatable/features/login.feature +++ b/tests/fixtures/repeatable/features/login.feature @@ -1,6 +1,6 @@ Feature: Repeatable flags coverage - @FC-004 + @id:FC-004 Scenario: Repeatable unit and integration flags Given a user logs in Then the dashboard is shown diff --git a/tests/fixtures/repeatable/integration.xml b/tests/fixtures/repeatable/integration.xml index ccb1072..39c490a 100644 --- a/tests/fixtures/repeatable/integration.xml +++ b/tests/fixtures/repeatable/integration.xml @@ -1,3 +1,3 @@ - + diff --git a/tests/fixtures/repeatable/unit.xml b/tests/fixtures/repeatable/unit.xml index b95f058..4e7dfc0 100644 --- a/tests/fixtures/repeatable/unit.xml +++ b/tests/fixtures/repeatable/unit.xml @@ -1,3 +1,3 @@ - + diff --git a/tests/fixtures/status_badges/e2e.json b/tests/fixtures/status_badges/e2e.json index d0b3785..f75b327 100644 --- a/tests/fixtures/status_badges/e2e.json +++ b/tests/fixtures/status_badges/e2e.json @@ -7,7 +7,7 @@ { "keyword": "Scenario", "name": "Successful login - passed", - "tags": [{"name": "@FC-001"}], + "tags": [{"name": "@scenario:FC-001"}], "steps": [ {"keyword": "Given", "name": "the user is on the login page", "result": {"status": "passed"}}, {"keyword": "When", "name": "they enter valid credentials", "result": {"status": "passed"}}, @@ -18,7 +18,7 @@ { "keyword": "Scenario", "name": "Login with invalid password - failed", - "tags": [{"name": "@FC-001"}], + "tags": [{"name": "@scenario:FC-001"}], "steps": [ {"keyword": "When", "name": "they enter invalid password", "result": {"status": "failed", "error_message": "AssertionError: Expected error message"}} ], @@ -27,7 +27,7 @@ { "keyword": "Scenario", "name": "Login lockout - skipped", - "tags": [{"name": "@FC-001"}], + "tags": [{"name": "@scenario:FC-001"}], "steps": [ {"keyword": "Then", "name": "account is locked", "result": {"status": "skipped"}} ], diff --git a/tests/fixtures/status_badges/features/login.feature b/tests/fixtures/status_badges/features/login.feature index 08f17b2..a7633ff 100644 --- a/tests/fixtures/status_badges/features/login.feature +++ b/tests/fixtures/status_badges/features/login.feature @@ -1,6 +1,6 @@ Feature: User Login - @FC-001 + @id:FC-001 Scenario: Successful login with valid credentials Given the user is on the login page When they enter valid credentials diff --git a/tests/fixtures/status_badges/unit.xml b/tests/fixtures/status_badges/unit.xml index 434c240..568ece4 100644 --- a/tests/fixtures/status_badges/unit.xml +++ b/tests/fixtures/status_badges/unit.xml @@ -1,12 +1,12 @@ - - + + assert False Expected True, got False - + diff --git a/tests/fixtures/unit_linking/features/login.feature b/tests/fixtures/unit_linking/features/login.feature index 08f17b2..a7633ff 100644 --- a/tests/fixtures/unit_linking/features/login.feature +++ b/tests/fixtures/unit_linking/features/login.feature @@ -1,6 +1,6 @@ Feature: User Login - @FC-001 + @id:FC-001 Scenario: Successful login with valid credentials Given the user is on the login page When they enter valid credentials diff --git a/tests/fixtures/unit_linking/unit.xml b/tests/fixtures/unit_linking/unit.xml index 38c7e1a..134c372 100644 --- a/tests/fixtures/unit_linking/unit.xml +++ b/tests/fixtures/unit_linking/unit.xml @@ -1,7 +1,7 @@ - + diff --git a/tests/integration/test_e2e_coverage.py b/tests/integration/test_e2e_coverage.py index 29063e3..d895cee 100644 --- a/tests/integration/test_e2e_coverage.py +++ b/tests/integration/test_e2e_coverage.py @@ -11,7 +11,7 @@ OUTPUT = ROOT / "tests" / "fixtures" / "e2e_coverage" / "report.html" -@pytest.mark.parametrize("tag", ["@FC-001"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-001"]) def test_cli_generates_coverage_report(tag): result = run_tool(FEATURES, OUTPUT, e2e=E2E) diff --git a/tests/integration/test_e2e_module_scope.py b/tests/integration/test_e2e_module_scope.py index a4fba10..9fa27aa 100644 --- a/tests/integration/test_e2e_module_scope.py +++ b/tests/integration/test_e2e_module_scope.py @@ -9,7 +9,7 @@ FEATURES = FIXTURES / "features" -@pytest.mark.parametrize("tag", ["@FC-007"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-007"]) def test_e2e_module_match_marks_requirement_ok(tag, tmp_path): output = tmp_path / "report.html" result = run_tool( @@ -23,7 +23,7 @@ def test_e2e_module_match_marks_requirement_ok(tag, tmp_path): assert "e2e" in content -@pytest.mark.parametrize("tag", ["@FC-007"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-007"]) def test_e2e_module_mismatch_marks_requirement_missing(tag, tmp_path): output = tmp_path / "report.html" result = run_tool( diff --git a/tests/integration/test_edge_cases.py b/tests/integration/test_edge_cases.py index a03b5c5..b64299f 100644 --- a/tests/integration/test_edge_cases.py +++ b/tests/integration/test_edge_cases.py @@ -11,7 +11,8 @@ FIXTURES = ROOT / "tests" / "fixtures" / "edge_cases" -def test_edge_case_pytest_junit_contains_tag(tmp_path): +@pytest.mark.parametrize("tag", ["@scenario:FC-EDGE-001"]) +def test_edge_case_pytest_junit_contains_tag(tag, tmp_path): xml_path = tmp_path / "edge.xml" result = subprocess.run( [ @@ -31,12 +32,12 @@ def test_edge_case_pytest_junit_contains_tag(tmp_path): ) assert result.returncode == 0, result.stderr - assert "@FC-EDGE-001" in xml_path.read_text(encoding="utf-8") + assert "test_tag_collision_across_features" in xml_path.read_text(encoding="utf-8") -@pytest.mark.parametrize("tag", ["@FC-EDGE-001"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-EDGE-001"]) def test_tag_collision_across_features(tag): - """One test tagged @FC-001 links to scenarios in both feature files.""" + """One test tagged @scenario:FC-001 links to scenarios in both feature files.""" base = FIXTURES / "collision_across" output = base / "report.html" result = run_tool(base / "features", output, unit=base / "unit.xml") @@ -49,9 +50,9 @@ def test_tag_collision_across_features(tag): assert "scenarios fully matched" in content -@pytest.mark.parametrize("tag", ["@FC-EDGE-002"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-EDGE-002"]) def test_tag_collision_within_feature(tag): - """One test tagged @smoke links to both scenarios sharing that tag.""" + """Category tag @smoke no longer links — scenarios show as incomplete.""" base = FIXTURES / "collision_within" output = base / "report.html" result = run_tool(base / "features", output, unit=base / "unit.xml") @@ -63,24 +64,25 @@ def test_tag_collision_within_feature(tag): assert "scenarios fully matched" in content -@pytest.mark.parametrize("tag", ["@FC-EDGE-003"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-EDGE-003"]) def test_feature_level_tags_not_inherited(tag): """Tags on Feature: line are NOT inherited by scenarios. - The scenario ``@FC-001`` links to the matching unit test, so coverage is - 0/1 under the presence-based basis (the linked unit result does not satisfy - the default injected ``e2e`` requirement). The other unit test carries - ``@FeatureTag`` (taken from its name), which matches no scenario — it must - appear under Unlinked Tests rather than being silently dropped. + The scenario tagged ``@id:FC-001`` links to the matching unit test + (``@scenario:FC-001``), so coverage is 0/1 under the presence-based basis + (the linked unit result does not satisfy the default injected ``e2e`` + requirement). The other unit test carries ``@FeatureTag`` (taken from its + name), which is not an ``@scenario:`` tag and therefore matches no scenario + — it must appear under Unlinked Tests rather than being silently dropped. """ base = FIXTURES / "feature_tags_not_inherited" output = base / "report.html" result = run_tool(base / "features", output, unit=base / "unit.xml") assert result.returncode == 0, result.stderr content = output.read_text(encoding="utf-8") - # @FC-001 test should link (scenario-level tag) + # @id:FC-001 test should link (scenario-level tag via @scenario:FC-001) assert "unit" in content - # Only the @FC-001 scenario exists, but its default e2e requirement is unmet + # Only the @id:FC-001 scenario exists, but its default e2e requirement is unmet assert "0/1" in content assert "scenarios fully matched" in content # The @FeatureTag test matches no scenario and must be listed as unlinked @@ -88,7 +90,7 @@ def test_feature_level_tags_not_inherited(tag): assert "test_feature_tag_@FeatureTag" in content -@pytest.mark.parametrize("tag", ["@FC-EDGE-004"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-EDGE-004"]) def test_no_matching_tags_shows_incomplete(tag): """Scenario with no matching test results shows as incomplete.""" base = FIXTURES / "no_match" @@ -101,7 +103,7 @@ def test_no_matching_tags_shows_incomplete(tag): assert "unit" not in content -@pytest.mark.parametrize("tag", ["@FC-EDGE-005"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-EDGE-005"]) def test_empty_result_file_produces_zero_tests(tag): """Empty produces zero unit results, scenario is incomplete.""" base = FIXTURES / "empty_result" @@ -113,7 +115,7 @@ def test_empty_result_file_produces_zero_tests(tag): assert "scenarios fully matched" in content assert "unit" not in content -@pytest.mark.parametrize("tag", ["@FC-EDGE-006"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-EDGE-006"]) def test_malformed_junit_xml_errors(tag): """Malformed JUnit XML aborts with a clear error.""" base = FIXTURES / "malformed_xml" diff --git a/tests/integration/test_fail_on.py b/tests/integration/test_fail_on.py index f53095d..7a72253 100644 --- a/tests/integration/test_fail_on.py +++ b/tests/integration/test_fail_on.py @@ -27,7 +27,7 @@ def _feature_dir(workdir: Path) -> Path: """\ Feature: User Login - @FC-001 + @id:FC-001 Scenario: Successful login Given the user is on the login page When they enter valid credentials @@ -54,17 +54,17 @@ def _unit_xml(workdir: Path, testcases: str) -> Path: def _linked_case() -> str: - return ' ' + return ' ' def _orphan_cases(count: int) -> str: return "\n".join( - f' ' + f' ' for i in range(count) ) -@pytest.mark.parametrize("tag", ["@FC-011"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-011"]) def test_fail_on_progress_exits_nonzero_when_check_fails(tag, workdir): features = _feature_dir(workdir) # No linked case -> 0% complete -> progress check fails (< 50% amber threshold). @@ -82,7 +82,7 @@ def test_fail_on_progress_exits_nonzero_when_check_fails(tag, workdir): assert "progress" in result.stderr -@pytest.mark.parametrize("tag", ["@FC-011"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-011"]) def test_no_fail_on_keeps_exit_zero_despite_failing_health_check(tag, workdir): features = _feature_dir(workdir) unit = _unit_xml(workdir, _linked_case() + "\n" + _orphan_cases(4)) @@ -92,7 +92,7 @@ def test_no_fail_on_keeps_exit_zero_despite_failing_health_check(tag, workdir): assert result.returncode == 0, result.stderr -@pytest.mark.parametrize("tag", ["@FC-011"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-011"]) def test_fail_on_passing_check_keeps_exit_zero(tag, workdir): features = _feature_dir(workdir) # Many unit results, no integration/e2e -> pyramid check passes. @@ -122,15 +122,15 @@ def _integration_xml(workdir: Path, testcases: str) -> Path: return path -@pytest.mark.parametrize("tag", ["@FC-011"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-011"]) def test_fail_on_reports_all_simultaneously_failing_checks(tag, workdir): features = _feature_dir(workdir) # No unit results vs. 2 integration results -> progress stays incomplete # (fail) and pyramid is inverted (fail). integration = _integration_xml( workdir, - ' \n' - ' ', + ' \n' + ' ', ) result = run_tool( @@ -145,7 +145,7 @@ def test_fail_on_reports_all_simultaneously_failing_checks(tag, workdir): assert "pyramid" in result.stderr -@pytest.mark.parametrize("tag", ["@FC-011"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-011"]) def test_unknown_fail_on_name_is_a_config_error(tag, workdir): features = _feature_dir(workdir) unit = _unit_xml(workdir, _linked_case()) diff --git a/tests/integration/test_integration_linking.py b/tests/integration/test_integration_linking.py index 81b6039..21d821b 100644 --- a/tests/integration/test_integration_linking.py +++ b/tests/integration/test_integration_linking.py @@ -13,7 +13,7 @@ OUTPUT = ROOT / "tests" / "fixtures" / "integration_linking" / "report.html" -@pytest.mark.parametrize("tag", ["@FC-003"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-003"]) def test_cli_links_integration_results(tag): result = run_tool(FEATURES, OUTPUT, integration=INTEGRATION) @@ -26,10 +26,10 @@ def test_cli_links_integration_results(tag): assert "0/1" in content assert "scenarios fully matched" in content assert "integration" in content - assert "@FC-003" in content + assert "@scenario:FC-003" in content -@pytest.mark.parametrize("tag", ["@FC-003"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-003"]) def test_report_includes_integration_layer_from_pytest_junit(tag, tmp_path): xml_path = tmp_path / "int.xml" result = subprocess.run( diff --git a/tests/integration/test_json_report.py b/tests/integration/test_json_report.py index a4d190a..e97dc0b 100644 --- a/tests/integration/test_json_report.py +++ b/tests/integration/test_json_report.py @@ -19,7 +19,7 @@ FC002_E2E = ROOT / "tests" / "fixtures" / "fc002" / "e2e.json" -@pytest.mark.parametrize("tag", ["@FC-010"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-010"]) def test_output_json_conforms_to_schema(tag, tmp_path): output = tmp_path / "report.html" output_json = tmp_path / "report.json" @@ -52,7 +52,7 @@ def test_output_json_conforms_to_schema(tag, tmp_path): assert "failureMessage" not in passed_result -@pytest.mark.parametrize("tag", ["@FC-010"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-010"]) def test_output_json_omitted_when_not_configured(tag, tmp_path): output = tmp_path / "report.html" diff --git a/tests/integration/test_missing_required_layer.py b/tests/integration/test_missing_required_layer.py index 17d5ee0..2d4cbad 100644 --- a/tests/integration/test_missing_required_layer.py +++ b/tests/integration/test_missing_required_layer.py @@ -12,7 +12,7 @@ OUTPUT = FIXTURES / "report.html" -@pytest.mark.parametrize("tag", ["@FC-005"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-005"]) def test_cli_flags_missing_required_layer(tag): result = run_tool(FEATURES, OUTPUT, unit=UNIT) diff --git a/tests/integration/test_status_badges.py b/tests/integration/test_status_badges.py index f181976..e4cef76 100644 --- a/tests/integration/test_status_badges.py +++ b/tests/integration/test_status_badges.py @@ -13,7 +13,7 @@ OUTPUT = FIXTURES / "report.html" -@pytest.mark.parametrize("tag", ["@FC-002"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-002"]) def test_passed_failed_skipped_displayed_in_report(tag): result = run_tool(FEATURES, OUTPUT, unit=UNIT, e2e=E2E) diff --git a/tests/integration/test_unit_linking.py b/tests/integration/test_unit_linking.py index c816c6e..b590089 100644 --- a/tests/integration/test_unit_linking.py +++ b/tests/integration/test_unit_linking.py @@ -11,7 +11,7 @@ OUTPUT = ROOT / "tests" / "fixtures" / "unit_linking" / "report.html" -@pytest.mark.parametrize("tag", ["@FC-001"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-001"]) def test_cli_links_unit_results(tag): result = run_tool(FEATURES, OUTPUT, unit=UNIT) @@ -24,4 +24,4 @@ def test_cli_links_unit_results(tag): assert "0/1" in content assert "scenarios fully matched" in content assert "unit" in content - assert "@FC-001" in content + assert "@scenario:FC-001" in content diff --git a/tests/unit/test_aggregator.py b/tests/unit/test_aggregator.py index 5e30826..c2ee860 100644 --- a/tests/unit/test_aggregator.py +++ b/tests/unit/test_aggregator.py @@ -6,23 +6,22 @@ def _view(feature, name, results=None, required_layers=None): scenario = Scenario( - feature=feature, name=name, tags=[f"@{name}"], + feature=feature, name=name, tags=[f"@id:{name}"], required_layers=required_layers or [], ) return ScenarioView(scenario=scenario, linked_results=results or []) -@pytest.mark.parametrize("tag", ["@FC-008"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-008"]) def test_layer_order_is_unit_integration_e2e(tag): assert ReportAggregator.LAYER_ORDER == ["e2e", "integration", "unit"] -@pytest.mark.parametrize("tag", ["@FC-008"]) -def test_build_views_orders_layers_by_layer_order(tag): - scenario = Scenario(feature="F", name="S", tags=["@S"]) - e2e_result = TestResult(layer="e2e", name="e2e test", tags=["@S"]) - unit_result = TestResult(layer="unit", name="unit test", tags=["@S"]) - integration_result = TestResult(layer="integration", name="integration test", tags=["@S"]) +def test_build_views_orders_layers_by_layer_order(): + scenario = Scenario(feature="F", name="S", tags=["@id:S"]) + e2e_result = TestResult(layer="e2e", name="e2e test", tags=["@scenario:S"]) + unit_result = TestResult(layer="unit", name="unit test", tags=["@scenario:S"]) + integration_result = TestResult(layer="integration", name="integration test", tags=["@scenario:S"]) links = {id(scenario): [e2e_result, unit_result, integration_result]} views = ReportAggregator.build_views([scenario], links) @@ -32,7 +31,7 @@ def test_build_views_orders_layers_by_layer_order(tag): assert layer_names == ["e2e", "integration", "unit"] -@pytest.mark.parametrize("tag", ["@FC-008"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-008"]) def test_layer_stats_width_pct_proportional_to_max_count(tag): views = [ _view("F", "S1", [TestResult(layer="unit", name="u1"), TestResult(layer="unit", name="u2")]), @@ -48,7 +47,7 @@ def test_layer_stats_width_pct_proportional_to_max_count(tag): assert by_name["e2e"]["width_pct"] == 50.0 -@pytest.mark.parametrize("tag", ["@FC-008"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-008"]) def test_layer_stats_width_pct_has_minimum_floor_for_nonzero_layers(tag): views = [ _view("F", "S1", [TestResult(layer="unit", name=f"u{i}") for i in range(20)]), @@ -62,7 +61,7 @@ def test_layer_stats_width_pct_has_minimum_floor_for_nonzero_layers(tag): assert by_name["e2e"]["width_pct"] == ReportAggregator.MIN_TIER_WIDTH_PCT -@pytest.mark.parametrize("tag", ["@FC-008"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-008"]) def test_layer_stats_omits_layers_with_no_results(tag): views = [_view("F", "S1", [TestResult(layer="unit", name="u1")])] @@ -71,12 +70,12 @@ def test_layer_stats_omits_layers_with_no_results(tag): assert {m["name"] for m in metrics} == {"unit"} -@pytest.mark.parametrize("tag", ["@FC-008"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-008"]) def test_layer_stats_empty_views_returns_empty_list(tag): assert ReportAggregator.layer_stats([]) == [] -@pytest.mark.parametrize("tag", ["@FC-008"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-008"]) def test_feature_breakdown_sorted_by_feature_name(tag): views = [_view("Zebra", "S1"), _view("Alpha", "S2")] @@ -85,7 +84,7 @@ def test_feature_breakdown_sorted_by_feature_name(tag): assert [b["name"] for b in breakdown] == ["Alpha", "Zebra"] -@pytest.mark.parametrize("tag", ["@FC-008"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-008"]) def test_health_checks_flags_inverted_pyramid(tag): views = [ _view("F", "S1", [TestResult(layer="e2e", name="e1")]), @@ -99,7 +98,7 @@ def test_health_checks_flags_inverted_pyramid(tag): assert health["pyramid"]["status"] == "fail" -@pytest.mark.parametrize("tag", ["@FC-008"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-008"]) def test_health_checks_passes_when_unit_dominates(tag): views = [ _view("F", "S1", [TestResult(layer="unit", name="u1")]), @@ -114,7 +113,7 @@ def test_health_checks_passes_when_unit_dominates(tag): assert health["pyramid"]["status"] == "pass" -@pytest.mark.parametrize("tag", ["@FC-008"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-008"]) def test_health_checks_pyramid_warns_when_at_parity(tag): views = [ _view("F", "S1", [TestResult(layer="unit", name="u1")]), @@ -130,7 +129,7 @@ def test_health_checks_pyramid_warns_when_at_parity(tag): assert health["pyramid"]["status"] == "warn" -@pytest.mark.parametrize("tag", ["@FC-008"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-008"]) def test_health_checks_end_to_end_runtime_passes_when_below_amber(tag): views = [ _view("F", "S1", [TestResult(layer="unit", name="u1", duration=0.5)]), @@ -144,7 +143,7 @@ def test_health_checks_end_to_end_runtime_passes_when_below_amber(tag): assert health["end_to_end_runtime"]["status"] == "pass" -@pytest.mark.parametrize("tag", ["@FC-008"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-008"]) def test_health_checks_end_to_end_runtime_warns_when_between_amber_and_red(tag): views = [ _view("F", "S1", [TestResult(layer="unit", name="u1", duration=0.5)]), @@ -158,7 +157,7 @@ def test_health_checks_end_to_end_runtime_warns_when_between_amber_and_red(tag): assert health["end_to_end_runtime"]["status"] == "warn" -@pytest.mark.parametrize("tag", ["@FC-008"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-008"]) def test_health_checks_end_to_end_runtime_fails_when_exceeds_red(tag): views = [ _view("F", "S1", [TestResult(layer="unit", name="u1", duration=0.5)]), @@ -172,27 +171,24 @@ def test_health_checks_end_to_end_runtime_fails_when_exceeds_red(tag): assert health["end_to_end_runtime"]["status"] == "fail" -@pytest.mark.parametrize("tag", ["@FC-008"]) -def test_unlinked_results_excludes_tags_matching_scenarios(tag): - scenarios = [Scenario(feature="F", name="S1", tags=["@FC-001"])] - linked = TestResult(layer="unit", name="t1", tags=["@FC-001"]) - unlinked = TestResult(layer="unit", name="t2", tags=["@FC-999"]) +def test_unlinked_results_excludes_tags_matching_scenarios(): + scenarios = [Scenario(feature="F", name="S1", tags=["@id:FC-001"])] + linked = TestResult(layer="unit", name="t1", tags=["@scenario:FC-001"]) + unlinked = TestResult(layer="unit", name="t2", tags=["@scenario:FC-999"]) result = ReportAggregator.unlinked_results(scenarios, [linked, unlinked]) assert result == [unlinked] -@pytest.mark.parametrize("tag", ["@FC-008"]) -def test_unlinked_results_surfaces_non_fc_tags(tag): - """A result tagged with a non-@FC tag that matches no scenario is unlinked. +def test_unlinked_results_surfaces_non_scenario_tags(): + """A result tagged with a non-@scenario tag that matches no scenario @id is unlinked. - Regression test for the bug where the tool only reported results carrying an - ``@FC-`` prefixed tag as unlinked, silently dropping orphans tagged e.g. - ``@smoke``/``@regression``/``@OTHER-999`` that failed to match a scenario. + Category tags like ``@smoke``/``@regression`` that lack the ``@scenario:`` + prefix are correctly surfaced as unlinked since they carry no linking weight. """ - scenarios = [Scenario(feature="F", name="S1", tags=["@FC-001"])] - linked = TestResult(layer="unit", name="t1", tags=["@FC-001"]) + scenarios = [Scenario(feature="F", name="S1", tags=["@id:FC-001"])] + linked = TestResult(layer="unit", name="t1", tags=["@scenario:FC-001"]) unlinked = TestResult(layer="unit", name="t2", tags=["@regression"]) result = ReportAggregator.unlinked_results(scenarios, [linked, unlinked]) @@ -200,14 +196,13 @@ def test_unlinked_results_surfaces_non_fc_tags(tag): assert result == [unlinked] -@pytest.mark.parametrize("tag", ["@FC-008"]) -def test_unlinked_results_excludes_tagless_results(tag): +def test_unlinked_results_excludes_tagless_results(): """Results carrying no tags are excluded — they can never link to a scenario. This keeps untagged test-runner output (e.g. plain pytest) from flooding the Unlinked Tests section as false positives. """ - scenarios = [Scenario(feature="F", name="S1", tags=["@FC-001"])] + scenarios = [Scenario(feature="F", name="S1", tags=["@id:FC-001"])] tagless = TestResult(layer="unit", name="t1", tags=[]) result = ReportAggregator.unlinked_results(scenarios, [tagless]) @@ -215,7 +210,7 @@ def test_unlinked_results_excludes_tagless_results(tag): assert result == [] -@pytest.mark.parametrize("tag", ["@FC-009"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-009"]) def test_health_checks_unlinked_entry_passes_when_zero(tag): views = [_view("F", "S1", [TestResult(layer="unit", name="u1")])] layer_stats = ReportAggregator.layer_stats(views) @@ -227,7 +222,7 @@ def test_health_checks_unlinked_entry_passes_when_zero(tag): assert health["unlinked"]["value"] == "0" -@pytest.mark.parametrize("tag", ["@FC-009"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-009"]) def test_health_checks_unlinked_entry_fails_when_many(tag): views = [_view("F", "S1", [TestResult(layer="unit", name="u1")])] layer_stats = ReportAggregator.layer_stats(views) @@ -239,7 +234,7 @@ def test_health_checks_unlinked_entry_fails_when_many(tag): assert health["unlinked"]["value"] == "5" -@pytest.mark.parametrize("tag", ["@FC-009"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-009"]) def test_failure_breakdown_groups_failed_results_by_feature_and_scenario(tag): passing = TestResult(layer="unit", name="p1", status="passed") failing = TestResult(layer="unit", name="f1", status="failed") @@ -259,14 +254,14 @@ def test_failure_breakdown_groups_failed_results_by_feature_and_scenario(tag): assert alpha["scenarios"][0]["failed_results"] == [failing] -@pytest.mark.parametrize("tag", ["@FC-009"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-009"]) def test_failure_breakdown_empty_when_no_failures(tag): views = [_view("F", "S1", [TestResult(layer="unit", name="p1", status="passed")])] assert ReportAggregator.failure_breakdown(views) == [] -@pytest.mark.parametrize("tag", ["@FC-014"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-014"]) def test_completion_stats_presence_based_not_link_count(tag): # Scenario declares @require-e2e (parser injects it when none declared). # A linked UNIT result is present but does NOT satisfy e2e, so the @@ -282,7 +277,7 @@ def test_completion_stats_presence_based_not_link_count(tag): assert stats["complete"] == 0 -@pytest.mark.parametrize("tag", ["@FC-014"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-014"]) def test_completion_stats_satisfied_when_requirement_present(tag): views = [_view("F", "S1", [TestResult(layer="e2e", name="e1", status="failed")], required_layers=[RequiredLayer("e2e")])] stats = ReportAggregator.completion_stats(views) @@ -292,7 +287,7 @@ def test_completion_stats_satisfied_when_requirement_present(tag): assert stats["required"] == 1 -@pytest.mark.parametrize("tag", ["@FC-014"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-014"]) def test_completion_stats_average_across_scenarios(tag): a = _view("F", "S1", [TestResult(layer="e2e", name="e1", status="passed")], required_layers=[RequiredLayer("e2e")]) b = _view("F", "S2", [], required_layers=[RequiredLayer("e2e")]) @@ -302,7 +297,7 @@ def test_completion_stats_average_across_scenarios(tag): assert stats["required"] == 2 -@pytest.mark.parametrize("tag", ["@FC-014"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-014"]) def test_feature_breakdown_uses_average_completion(tag): a = _view("F", "S1", [TestResult(layer="e2e", name="e1", status="passed")], required_layers=[RequiredLayer("e2e")]) b = _view("F", "S2", [], required_layers=[RequiredLayer("e2e")]) diff --git a/tests/unit/test_build_pyramid.py b/tests/unit/test_build_pyramid.py index 97d0dc2..86f5248 100644 --- a/tests/unit/test_build_pyramid.py +++ b/tests/unit/test_build_pyramid.py @@ -33,35 +33,31 @@ def _parse_junit_results(paths, layer="unit"): return _junit_parser.parse(paths, layer=layer) -@pytest.mark.parametrize("tag", ["@FC-001"]) -def test_parse_feature_file_returns_scenarios(tag): +def test_parse_feature_file_returns_scenarios(): scenarios = _parse_feature_file(FIXTURES / "e2e_coverage" / "features" / "login.feature") assert len(scenarios) == 1 assert scenarios[0].name == "Successful login with valid credentials" - assert tag in scenarios[0].tags + assert "@id:FC-001" in scenarios[0].tags -@pytest.mark.parametrize("tag", ["@FC-001"]) -def test_parse_e2e_results_extracts_tags(tag): +def test_parse_e2e_results_extracts_tags(): results = _parse_e2e_results([FIXTURES / "e2e_coverage" / "e2e.json"]) assert len(results) == 1 assert results[0].layer == "e2e" - assert tag in results[0].tags + assert "@scenario:FC-001" in results[0].tags -@pytest.mark.parametrize("tag", ["@FC-001"]) -def test_parse_junit_results_extracts_tags(tag): +def test_parse_junit_results_extracts_tags(): results = _parse_junit_results([FIXTURES / "unit_linking" / "unit.xml"]) assert len(results) == 1 assert results[0].layer == "unit" - assert tag in results[0].tags + assert "@scenario:FC-001" in results[0].tags -@pytest.mark.parametrize("tag", ["@FC-007"]) -def test_parse_junit_results_extracts_tags_from_classname_and_properties(tag, tmp_path): +def test_parse_junit_results_extracts_tags_from_classname_and_properties(tmp_path): xml_path = tmp_path / "integration.xml" xml_path.write_text( - """""", + """""", encoding="utf-8", ) @@ -69,12 +65,11 @@ def test_parse_junit_results_extracts_tags_from_classname_and_properties(tag, tm assert len(results) == 1 assert results[0].layer == "integration" - assert "@FC-004" in results[0].tags - assert "@FC-005" in results[0].tags + assert "@scenario:FC-004" in results[0].tags + assert "@scenario:FC-005" in results[0].tags -@pytest.mark.parametrize("tag", ["@FC-007"]) -def test_parse_junit_results_raises_clear_error_on_malformed_xml(tag, tmp_path): +def test_parse_junit_results_raises_clear_error_on_malformed_xml(tmp_path): xml_path = tmp_path / "broken.xml" xml_path.write_text("", encoding="utf-8") @@ -82,8 +77,7 @@ def test_parse_junit_results_raises_clear_error_on_malformed_xml(tag, tmp_path): _parse_junit_results([xml_path], layer="integration") -@pytest.mark.parametrize("tag", ["@FC-004"]) -def test_load_config_requires_features_and_output(tag, tmp_path): +def test_load_config_requires_features_and_output(tmp_path): config_path = tmp_path / "spectracer.config.json" config_path.write_text(json.dumps({"features": ["features"]}), encoding="utf-8") @@ -91,15 +85,14 @@ def test_load_config_requires_features_and_output(tag, tmp_path): _load_config(config_path) -@pytest.mark.parametrize("tag", ["@FC-004"]) -def test_collect_and_parse_junit_results_groups_by_module(tag, tmp_path): +def test_collect_and_parse_junit_results_groups_by_module(tmp_path): module_a = tmp_path / "a.xml" module_a.write_text( - '', encoding="utf-8" + '', encoding="utf-8" ) module_b = tmp_path / "b.xml" module_b.write_text( - '', encoding="utf-8" + '', encoding="utf-8" ) results = _collect_and_parse_junit_results( @@ -111,8 +104,8 @@ def test_collect_and_parse_junit_results_groups_by_module(tag, tmp_path): assert all(r.layer == "unit" for r in results) -@pytest.mark.parametrize("tag", ["@FC-007"]) -def test_collect_and_parse_e2e_results_groups_by_module(tag, tmp_path): +def test_collect_and_parse_e2e_results_groups_by_module(tmp_path): + tag = "@scenario:FC-007" module_a = tmp_path / "a.json" module_a.write_text( json.dumps( @@ -165,17 +158,16 @@ def test_collect_and_parse_e2e_results_groups_by_module(tag, tmp_path): assert all(r.layer == "e2e" for r in results) -@pytest.mark.parametrize("tag", ["@FC-007"]) -def test_require_e2e_accepts_module_suffix(tag): +def test_require_e2e_accepts_module_suffix(): scenarios = _parse_feature_file(FIXTURES / "module_scope" / "features" / "parsers.feature") e2e_scenario = next(s for s in scenarios if any(r.layer == "e2e" for r in s.required_layers)) e2e_req = next(r for r in e2e_scenario.required_layers if r.layer == "e2e") assert e2e_req.module == "parsers" -@pytest.mark.parametrize("tag", ["@FC-007"]) -def test_parse_e2e_ignores_tag_filtered_unexecuted_scenarios(tag, tmp_path): +def test_parse_e2e_ignores_tag_filtered_unexecuted_scenarios(tmp_path): """Behave tag filters leave non-selected scenarios as skipped without step results.""" + tag = "@scenario:FC-007" path = tmp_path / "mixed.json" path.write_text( json.dumps( @@ -200,7 +192,7 @@ def test_parse_e2e_ignores_tag_filtered_unexecuted_scenarios(tag, tmp_path): { "keyword": "Scenario", "name": "tag-filtered never ran", - "tags": [{"name": "@FC-OTHER"}], + "tags": [{"name": "@scenario:FC-OTHER"}], "steps": [ {"keyword": "Given", "name": "y", "location": "features/x.feature:1"} ], @@ -232,36 +224,32 @@ def test_parse_e2e_ignores_tag_filtered_unexecuted_scenarios(tag, tmp_path): assert {r.status for r in results} == {"passed", "skipped"} -@pytest.mark.parametrize("tag", ["@FC-002"]) -def test_parse_junit_results_passed_failed_skipped(tag): +def test_parse_junit_results_passed_failed_skipped(): results = _parse_junit_results([FIXTURES / "fc002" / "unit.xml"]) assert len(results) == 3 statuses = {r.status for r in results} assert statuses == {"passed", "failed", "skipped"} - assert all(tag in r.tags for r in results) + assert all("@scenario:FC-002" in r.tags for r in results) -@pytest.mark.parametrize("tag", ["@FC-002"]) -def test_parse_e2e_results_passed_failed_skipped(tag): +def test_parse_e2e_results_passed_failed_skipped(): results = _parse_e2e_results([FIXTURES / "fc002" / "e2e.json"]) assert len(results) == 3 statuses = {r.status for r in results} assert statuses == {"passed", "failed", "skipped"} - assert all(tag in r.tags for r in results) + assert all("@scenario:FC-002" in r.tags for r in results) -@pytest.mark.parametrize("linking_tag", ["@FC-005"]) -def test_require_tags_excluded_from_linking(linking_tag): +def test_require_tags_excluded_from_linking(): scenarios = _parse_feature_file(FIXTURES / "missing_required_layer" / "features" / "login.feature") assert len(scenarios) == 1 scenario = scenarios[0] - assert linking_tag in scenario.tags + assert "@id:FC-005" in scenario.tags assert "@require-unit" not in scenario.tags assert "@require-e2e" not in scenario.tags -@pytest.mark.parametrize("tag", ["@FC-005"]) -def test_require_tags_stored_as_required_layers(tag): +def test_require_tags_stored_as_required_layers(): scenarios = _parse_feature_file(FIXTURES / "missing_required_layer" / "features" / "login.feature") assert len(scenarios) == 1 scenario = scenarios[0] @@ -271,11 +259,10 @@ def test_require_tags_stored_as_required_layers(tag): assert "integration" not in layers -@pytest.mark.parametrize("tag", ["@FC-005"]) -def test_require_layer_status_missing_e2e(tag, tmp_path): +def test_require_layer_status_missing_e2e(tmp_path): from spec_tracer.models import ScenarioView, Scenario scenario = _parse_feature_file(FIXTURES / "missing_required_layer" / "features" / "login.feature")[0] - unit_result = TestResult(layer="unit", name="test", tags=["@FC-005"], status="passed") + unit_result = TestResult(layer="unit", name="test", tags=["@scenario:FC-005"], status="passed") view = ScenarioView(scenario=scenario, linked_results=[unit_result]) status = _required_status(view) assert "unit [OK]" in status @@ -283,12 +270,11 @@ def test_require_layer_status_missing_e2e(tag, tmp_path): assert "integration" not in status -@pytest.mark.parametrize("tag", ["@FC-010"]) -def test_collect_and_parse_features_returns_path_relative_to_base_dir(tag, tmp_path): +def test_collect_and_parse_features_returns_path_relative_to_base_dir(tmp_path): features_dir = tmp_path / "features" features_dir.mkdir() (features_dir / "login.feature").write_text( - "Feature: Login\n\n @FC-010\n Scenario: Sign in\n Given a user\n", + "Feature: Login\n\n @id:FC-010\n Scenario: Sign in\n Given a user\n", encoding="utf-8", ) @@ -298,14 +284,13 @@ def test_collect_and_parse_features_returns_path_relative_to_base_dir(tag, tmp_p assert feature_files["Login"] == "features/login.feature" -@pytest.mark.parametrize("tag", ["@FC-010"]) -def test_collect_and_parse_features_never_returns_absolute_path(tag, tmp_path): +def test_collect_and_parse_features_never_returns_absolute_path(tmp_path): config_dir = tmp_path / "config_dir" config_dir.mkdir() features_dir = tmp_path / "elsewhere" / "features" features_dir.mkdir(parents=True) (features_dir / "login.feature").write_text( - "Feature: Login\n\n @FC-010\n Scenario: Sign in\n Given a user\n", + "Feature: Login\n\n @id:FC-010\n Scenario: Sign in\n Given a user\n", encoding="utf-8", ) diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index 3114d57..9803e35 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -18,7 +18,7 @@ def _write_config(tmp_path: Path, **extra) -> Path: return path -@pytest.mark.parametrize("tag", ["@FC-011"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-011"]) def test_fail_on_aliases_map_to_internal_health_check_keys(tag): assert FAIL_ON_ALIASES == { "progress": "Progress", @@ -27,63 +27,63 @@ def test_fail_on_aliases_map_to_internal_health_check_keys(tag): } -@pytest.mark.parametrize("tag", ["@FC-011"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-011"]) def test_load_config_accepts_known_fail_on_names(tag, tmp_path): path = _write_config(tmp_path, fail_on=["pyramid", "e2e_runtime"]) config = _load_config(path) assert config["fail_on"] == ["pyramid", "e2e_runtime"] -@pytest.mark.parametrize("tag", ["@FC-011"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-011"]) def test_load_config_rejects_unlinked_as_fail_on_name(tag, tmp_path): path = _write_config(tmp_path, fail_on=["unlinked"]) with pytest.raises(ValueError, match="unknown health checks"): _load_config(path) -@pytest.mark.parametrize("tag", ["@FC-011"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-011"]) def test_load_config_defaults_fail_on_to_absent(tag, tmp_path): path = _write_config(tmp_path) config = _load_config(path) assert "fail_on" not in config -@pytest.mark.parametrize("tag", ["@FC-011"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-011"]) def test_load_config_rejects_unknown_fail_on_name(tag, tmp_path): path = _write_config(tmp_path, fail_on=["pyramid", "flakiness"]) with pytest.raises(ValueError, match="unknown health checks"): _load_config(path) -@pytest.mark.parametrize("tag", ["@FC-011"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-011"]) def test_load_config_rejects_non_list_fail_on(tag, tmp_path): path = _write_config(tmp_path, fail_on="pyramid") with pytest.raises(ValueError, match="must be a list"): _load_config(path) -@pytest.mark.parametrize("tag", ["@FC-011"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-011"]) def test_load_config_rejects_non_string_fail_on_entry(tag, tmp_path): path = _write_config(tmp_path, fail_on=[{"name": "pyramid"}]) with pytest.raises(ValueError, match="must be strings"): _load_config(path) -@pytest.mark.parametrize("tag", ["@FC-011"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-011"]) def test_load_config_rejects_duplicate_fail_on_entries(tag, tmp_path): path = _write_config(tmp_path, fail_on=["pyramid", "pyramid"]) with pytest.raises(ValueError, match="duplicate entries"): _load_config(path) -@pytest.mark.parametrize("tag", ["@FC-011"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-011"]) def test_load_config_rejects_case_mismatched_fail_on_name(tag, tmp_path): path = _write_config(tmp_path, fail_on=["Pyramid"]) with pytest.raises(ValueError, match="unknown health checks"): _load_config(path) -@pytest.mark.parametrize("tag", ["@FC-011"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-011"]) def test_failing_gated_checks_flags_failing_listed_check(tag): health_checks = { "pyramid": {"status": "fail"}, @@ -92,13 +92,13 @@ def test_failing_gated_checks_flags_failing_listed_check(tag): assert _failing_gated_checks(health_checks, ["pyramid", "e2e_runtime"]) == ["pyramid"] -@pytest.mark.parametrize("tag", ["@FC-011"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-011"]) def test_failing_gated_checks_ignores_warn_status(tag): health_checks = {"progress": {"status": "warn"}} assert _failing_gated_checks(health_checks, ["progress"]) == [] -@pytest.mark.parametrize("tag", ["@FC-011"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-011"]) def test_failing_gated_checks_ignores_unlisted_failures(tag): health_checks = { "Progress": {"status": "fail"}, @@ -107,19 +107,19 @@ def test_failing_gated_checks_ignores_unlisted_failures(tag): assert _failing_gated_checks(health_checks, ["pyramid"]) == [] -@pytest.mark.parametrize("tag", ["@FC-011"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-011"]) def test_failing_gated_checks_resolves_alias_to_internal_key(tag): health_checks = {"end_to_end_runtime": {"status": "fail"}} assert _failing_gated_checks(health_checks, ["e2e_runtime"]) == ["e2e_runtime"] -@pytest.mark.parametrize("tag", ["@FC-011"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-011"]) def test_failing_gated_checks_empty_when_nothing_listed(tag): health_checks = {"pyramid": {"status": "fail"}} assert _failing_gated_checks(health_checks, []) == [] -@pytest.mark.parametrize("tag", ["@FC-011"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-011"]) def test_failing_gated_checks_flags_multiple_failing_listed_checks(tag): health_checks = { "pyramid": {"status": "fail"}, diff --git a/tests/unit/test_renderers.py b/tests/unit/test_renderers.py index c22b8ff..8a509cd 100644 --- a/tests/unit/test_renderers.py +++ b/tests/unit/test_renderers.py @@ -16,9 +16,9 @@ def _build_views(): - scenario_a = Scenario(feature="Alpha Feature", name="Scenario A", tags=["@FC-100"]) - scenario_b = Scenario(feature="Zebra Feature", name="Scenario B", tags=["@FC-200"]) - result_a = TestResult(layer="unit", name="test_a", tags=["@FC-100"], status="passed") + scenario_a = Scenario(feature="Alpha Feature", name="Scenario A", tags=["@scenario:FC-100"]) + scenario_b = Scenario(feature="Zebra Feature", name="Scenario B", tags=["@scenario:FC-200"]) + result_a = TestResult(layer="unit", name="test_a", tags=["@scenario:FC-100"], status="passed") view_a = ScenarioView(scenario=scenario_a, linked_results=[result_a], layers=[[result_a]]) view_b = ScenarioView(scenario=scenario_b, linked_results=[], layers=[]) return [view_a, view_b] @@ -56,14 +56,14 @@ def _render(**overrides): ) -@pytest.mark.parametrize("tag", ["@FC-009"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-009"]) def test_render_produces_header_with_title_only_once(tag): html = _render() assert 'SpecTracer' in html assert html.count("SpecTracer") == 2 # + header span -@pytest.mark.parametrize("tag", ["@FC-009"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-009"]) def test_render_includes_theme_switcher(tag): html = _render() assert 'class="theme-toggle"' in html @@ -73,14 +73,14 @@ def test_render_includes_theme_switcher(tag): assert ':root:not([data-theme="light"])' in html -@pytest.mark.parametrize("tag", ["@FC-009"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-009"]) def test_render_produces_five_page_containers(tag): html = _render() for page_id in ["page-dashboard", "page-pyramid", "page-features", "page-failures", "page-unlinked"]: assert f'id="{page_id}"' in html -@pytest.mark.parametrize("tag", ["@FC-009"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-009"]) def test_render_only_dashboard_page_visible_by_default(tag): html = _render() assert 'id="page-dashboard" class="page-stack">' in html @@ -90,7 +90,7 @@ def test_render_only_dashboard_page_visible_by_default(tag): assert 'id="page-unlinked" class="page-stack hidden">' in html -@pytest.mark.parametrize("tag", ["@FC-009"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-009"]) def test_render_top_nav_has_five_links(tag): html = _render() for route, label in [ @@ -104,7 +104,7 @@ def test_render_top_nav_has_five_links(tag): assert label in html -@pytest.mark.parametrize("tag", ["@FC-009"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-009"]) def test_render_feature_breakdown_tree_groups_scenarios_by_feature(tag): html = _render() assert 'data-sort-name="Alpha Feature"' in html @@ -113,7 +113,7 @@ def test_render_feature_breakdown_tree_groups_scenarios_by_feature(tag): assert 'data-sort-name="Scenario B"' in html -@pytest.mark.parametrize("tag", ["@FC-009"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-009"]) def test_render_feature_and_scenario_rows_have_type_pills(tag): html = _render() # Feature rows are identified with a pill (like test layers) @@ -125,27 +125,27 @@ def test_render_feature_and_scenario_rows_have_type_pills(tag): assert 'Scenario</strong></span><span class="name-text">Scenario A' in html -@pytest.mark.parametrize("tag", ["@FC-009"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-009"]) def test_render_pyramid_tier_uses_layer_specific_class(tag): html = _render() assert 'class="tier unit"' in html -@pytest.mark.parametrize("tag", ["@FC-009"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-009"]) def test_render_search_placeholder_present(tag): html = _render() assert 'placeholder="Search by name"' in html assert 'class="search-bar"' in html -@pytest.mark.parametrize("tag", ["@FC-009"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-009"]) def test_render_navigation_click_handler_present(tag): html = _render() assert "addEventListener('hashchange'" in html assert "addEventListener('click'" in html -@pytest.mark.parametrize("tag", ["@FC-009"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-009"]) def test_render_tree_table_sort_buttons_present(tag): html = _render() assert 'data-sort-key="name"' in html @@ -159,7 +159,7 @@ def test_render_tree_table_sort_buttons_present(tag): assert "Coverage %" not in html -@pytest.mark.parametrize("tag", ["@FC-009"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-009"]) def test_render_completion_bar_and_result_pill_present(tag): html = _render() assert "completion-bar" in html @@ -167,12 +167,12 @@ def test_render_completion_bar_and_result_pill_present(tag): assert 'class="badge passed">' in html or 'class="badge failed">' in html or 'class="badge skipped">' in html -@pytest.mark.parametrize("tag", ["@FC-009"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-009"]) def test_render_includes_scenario_status_badges(tag): # Status badges still appear in the Failure Breakdown tree (and the # summary helpers are intact), so assert on the failure-breakdown path. - scenario = Scenario(feature="Alpha Feature", name="Scenario A", tags=["@FC-100"]) - failed_result = TestResult(layer="unit", name="test_a", tags=["@FC-100"], status="failed", failure_message="boom") + scenario = Scenario(feature="Alpha Feature", name="Scenario A", tags=["@scenario:FC-100"]) + failed_result = TestResult(layer="unit", name="test_a", tags=["@scenario:FC-100"], status="failed", failure_message="boom") view = ScenarioView(scenario=scenario, linked_results=[failed_result], layers=[[failed_result]]) failure_breakdown = [ { @@ -185,16 +185,16 @@ def test_render_includes_scenario_status_badges(tag): assert 'class="badge failed">' in html -@pytest.mark.parametrize("tag", ["@FC-009"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-009"]) def test_render_health_check_unlinked_entry_links_to_unlinked_page(tag): html = _render() assert 'href="#/unlinked">View unlinked tests' in html -@pytest.mark.parametrize("tag", ["@FC-009"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-009"]) def test_render_failure_breakdown_tree_present_when_failures_exist(tag): - scenario = Scenario(feature="Alpha Feature", name="Scenario A", tags=["@FC-100"]) - failed_result = TestResult(layer="unit", name="test_a", tags=["@FC-100"], status="failed", failure_message="boom") + scenario = Scenario(feature="Alpha Feature", name="Scenario A", tags=["@scenario:FC-100"]) + failed_result = TestResult(layer="unit", name="test_a", tags=["@scenario:FC-100"], status="failed", failure_message="boom") view = ScenarioView(scenario=scenario, linked_results=[failed_result], layers=[[failed_result]]) failure_breakdown = [{ "name": "Alpha Feature", @@ -206,26 +206,26 @@ def test_render_failure_breakdown_tree_present_when_failures_exist(tag): assert 'data-sort-name="Alpha Feature"' in html -@pytest.mark.parametrize("tag", ["@FC-009"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-009"]) def test_render_no_logo_by_default(tag): html = _render() assert '<img class="logo"' not in html -@pytest.mark.parametrize("tag", ["@FC-009"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-009"]) def test_render_logo_rendered_when_provided(tag): html = _render(logo_data_uri="data:image/png;base64,abc") assert '<img class="logo" src="data:image/png;base64,abc"' in html -@pytest.mark.parametrize("tag", ["@FC-009"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-009"]) def test_format_duration_formats_milliseconds_and_seconds(tag): assert _format_duration(0) == "0.0s" assert _format_duration(0.25) == "250ms" assert _format_duration(2.5) == "2.5s" -@pytest.mark.parametrize("tag", ["@FC-009"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-009"]) def test_status_class_maps_known_statuses(tag): assert _status_class("passed") == "passed" assert _status_class("failed") == "failed" @@ -233,7 +233,7 @@ def test_status_class_maps_known_statuses(tag): assert _status_class("bogus") == "unknown" -@pytest.mark.parametrize("tag", ["@FC-009"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-009"]) def test_status_label_titlecases_status(tag): assert _status_label("passed") == "Passed" assert _status_label("failed") == "Failed" @@ -245,7 +245,7 @@ def _view_with(required_layers, results): return ScenarioView(scenario=scenario, linked_results=results) -@pytest.mark.parametrize("tag", ["@FC-014"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-014"]) def test_completion_empty_when_no_results(tag): view = _view_with([RequiredLayer("e2e")], []) c = _completion(view) @@ -256,7 +256,7 @@ def test_completion_empty_when_no_results(tag): assert c["pct"] == 0 -@pytest.mark.parametrize("tag", ["@FC-014"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-014"]) def test_completion_full_counts_presence_not_pass(tag): # A present-but-FAILED result still fills the requirement (presence-based). view = _view_with( @@ -272,7 +272,7 @@ def test_completion_full_counts_presence_not_pass(tag): assert c["pct"] == 100 -@pytest.mark.parametrize("tag", ["@FC-014"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-014"]) def test_completion_empty_when_result_on_wrong_layer(tag): view = _view_with([RequiredLayer("e2e")], [TestResult(layer="unit", name="t", status="passed")]) c = _completion(view) @@ -280,7 +280,7 @@ def test_completion_empty_when_result_on_wrong_layer(tag): assert c["cls"] == "empty" -@pytest.mark.parametrize("tag", ["@FC-014"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-014"]) def test_outcome_worst_case_across_results(tag): view = _view_with( [RequiredLayer("unit")], @@ -298,13 +298,13 @@ def test_outcome_worst_case_across_results(tag): assert _outcome(view_skipped)["cls"] == "skipped" -@pytest.mark.parametrize("tag", ["@FC-014"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-014"]) def test_outcome_skipped_when_no_results(tag): view = _view_with([RequiredLayer("e2e")], []) assert _outcome(view)["cls"] == "skipped" -@pytest.mark.parametrize("tag", ["@FC-014"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-014"]) def test_completion_bar_renders_count_and_color_class(tag): view = _view_with([RequiredLayer("unit")], [TestResult(layer="unit", name="t", status="passed")]) bar = _completion_bar(_completion(view)) @@ -313,7 +313,7 @@ def test_completion_bar_renders_count_and_color_class(tag): assert "1/1" in bar -@pytest.mark.parametrize("tag", ["@FC-014"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-014"]) def test_feature_completion_is_average_child_completion(tag): a = _view_with([RequiredLayer("e2e")], [TestResult(layer="e2e", name="e", status="passed")]) b = _view_with([RequiredLayer("e2e")], []) @@ -325,26 +325,26 @@ def test_feature_completion_is_average_child_completion(tag): assert fc["cls"] == "partial" -@pytest.mark.parametrize("tag", ["@FC-014"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-014"]) def test_feature_outcome_worst_across_feature_results(tag): a = _view_with([RequiredLayer("unit")], [TestResult(layer="unit", name="p", status="passed")]) b = _view_with([RequiredLayer("unit")], [TestResult(layer="unit", name="f", status="failed")]) assert _feature_outcome([a, b])["cls"] == "failed" -@pytest.mark.parametrize("tag", ["@FC-014"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-014"]) def test_result_satisfies_requirement_when_layer_matches(tag): view = _view_with([RequiredLayer("unit")], [TestResult(layer="unit", name="t")]) assert _result_satisfies_requirement(view.linked_results[0], view) is True -@pytest.mark.parametrize("tag", ["@FC-014"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-014"]) def test_result_does_not_satisfy_when_layer_mismatch(tag): view = _view_with([RequiredLayer("e2e")], [TestResult(layer="unit", name="t")]) assert _result_satisfies_requirement(view.linked_results[0], view) is False -@pytest.mark.parametrize("tag", ["@FC-014"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-014"]) def test_result_satisfies_with_module_scope(tag): view = _view_with( [RequiredLayer("unit", module="auth")], @@ -353,7 +353,7 @@ def test_result_satisfies_with_module_scope(tag): assert _result_satisfies_requirement(view.linked_results[0], view) is True -@pytest.mark.parametrize("tag", ["@FC-014"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-014"]) def test_result_does_not_satisfy_when_module_mismatch(tag): view = _view_with( [RequiredLayer("unit", module="auth")], @@ -362,13 +362,13 @@ def test_result_does_not_satisfy_when_module_mismatch(tag): assert _result_satisfies_requirement(view.linked_results[0], view) is False -@pytest.mark.parametrize("tag", ["@FC-014"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-014"]) def test_result_does_not_satisfy_when_no_required_layers(tag): view = _view_with([], [TestResult(layer="e2e", name="t")]) assert _result_satisfies_requirement(view.linked_results[0], view) is False -@pytest.mark.parametrize("tag", ["@FC-014"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-014"]) def test_result_satisfies_with_module_scope_case_insensitive(tag): view = _view_with( [RequiredLayer("unit", module="Auth")], @@ -377,7 +377,7 @@ def test_result_satisfies_with_module_scope_case_insensitive(tag): assert _result_satisfies_requirement(view.linked_results[0], view) is True -@pytest.mark.parametrize("tag", ["@FC-014"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-014"]) def test_result_satisfies_one_of_multiple_requirements(tag): view = _view_with( [RequiredLayer("unit"), RequiredLayer("e2e")], diff --git a/tests/unit/test_report_model.py b/tests/unit/test_report_model.py index 4f186c2..b21daf5 100644 --- a/tests/unit/test_report_model.py +++ b/tests/unit/test_report_model.py @@ -30,7 +30,7 @@ def _build(views, stats=None, layer_stats=None, health_checks=None, unlinked_res ) -@pytest.mark.parametrize("tag", ["@FC-010"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-010"]) def test_result_omits_duration_when_zero(tag): view = _view("F", "S1", results=[TestResult(layer="unit", name="t1", status="passed", duration=0.0)]) @@ -40,7 +40,7 @@ def test_result_omits_duration_when_zero(tag): assert "duration" not in result -@pytest.mark.parametrize("tag", ["@FC-010"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-010"]) def test_result_converts_duration_seconds_to_milliseconds(tag): view = _view("F", "S1", results=[TestResult(layer="unit", name="t1", status="passed", duration=1.5)]) @@ -50,7 +50,7 @@ def test_result_converts_duration_seconds_to_milliseconds(tag): assert result["duration"] == 1500.0 -@pytest.mark.parametrize("tag", ["@FC-010"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-010"]) def test_result_omits_failure_message_when_passed(tag): view = _view( "F", "S1", @@ -63,7 +63,7 @@ def test_result_omits_failure_message_when_passed(tag): assert "failureMessage" not in result -@pytest.mark.parametrize("tag", ["@FC-010"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-010"]) def test_result_includes_failure_message_when_failed(tag): view = _view( "F", "S1", @@ -76,7 +76,7 @@ def test_result_includes_failure_message_when_failed(tag): assert result["failureMessage"] == "boom" -@pytest.mark.parametrize("tag", ["@FC-010"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-010"]) def test_result_omits_failure_message_when_failed_but_message_empty(tag): view = _view( "F", "S1", @@ -89,7 +89,7 @@ def test_result_omits_failure_message_when_failed_but_message_empty(tag): assert "failureMessage" not in result -@pytest.mark.parametrize("tag", ["@FC-010"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-010"]) def test_result_omits_module_when_blank(tag): view = _view("F", "S1", results=[TestResult(layer="unit", name="t1", status="passed", module="")]) @@ -99,7 +99,7 @@ def test_result_omits_module_when_blank(tag): assert "module" not in result -@pytest.mark.parametrize("tag", ["@FC-010"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-010"]) def test_requirement_satisfied_when_module_matches(tag): view = _view( "F", "S1", @@ -113,7 +113,7 @@ def test_requirement_satisfied_when_module_matches(tag): assert req == {"layer": "unit", "satisfied": True, "module": "billing"} -@pytest.mark.parametrize("tag", ["@FC-010"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-010"]) def test_requirement_satisfied_when_module_matches_case_insensitively(tag): view = _view( "F", "S1", @@ -127,7 +127,7 @@ def test_requirement_satisfied_when_module_matches_case_insensitively(tag): assert req == {"layer": "unit", "satisfied": True, "module": "Billing"} -@pytest.mark.parametrize("tag", ["@FC-010"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-010"]) def test_requirement_unsatisfied_when_module_does_not_match(tag): view = _view( "F", "S1", @@ -141,7 +141,7 @@ def test_requirement_unsatisfied_when_module_does_not_match(tag): assert req["satisfied"] is False -@pytest.mark.parametrize("tag", ["@FC-010"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-010"]) def test_requirement_omits_module_key_when_unscoped(tag): view = _view( "F", "S1", @@ -155,7 +155,7 @@ def test_requirement_omits_module_key_when_unscoped(tag): assert "module" not in req -@pytest.mark.parametrize("tag", ["@FC-010"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-010"]) def test_e2e_requirement_satisfied_when_module_matches(tag): view = _view( "F", "S1", @@ -169,7 +169,7 @@ def test_e2e_requirement_satisfied_when_module_matches(tag): assert req == {"layer": "e2e", "satisfied": True, "module": "parsers"} -@pytest.mark.parametrize("tag", ["@FC-010"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-010"]) def test_e2e_requirement_unsatisfied_when_module_does_not_match(tag): view = _view( "F", "S1", @@ -183,7 +183,7 @@ def test_e2e_requirement_unsatisfied_when_module_does_not_match(tag): assert req["satisfied"] is False -@pytest.mark.parametrize("tag", ["@FC-010"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-010"]) def test_features_group_scenarios_and_preserve_first_seen_order(tag): views = [ _view("Zebra", "S1"), @@ -199,7 +199,7 @@ def test_features_group_scenarios_and_preserve_first_seen_order(tag): assert len(zebra["scenarios"]) == 2 -@pytest.mark.parametrize("tag", ["@FC-010"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-010"]) def test_features_use_provided_feature_file_path(tag): view = _view("F", "S1") @@ -208,7 +208,7 @@ def test_features_use_provided_feature_file_path(tag): assert report["features"][0]["file"] == "features/f.feature" -@pytest.mark.parametrize("tag", ["@FC-010"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-010"]) def test_features_default_file_to_empty_string_when_unknown(tag): view = _view("F", "S1") @@ -217,7 +217,7 @@ def test_features_default_file_to_empty_string_when_unknown(tag): assert report["features"][0]["file"] == "" -@pytest.mark.parametrize("tag", ["@FC-010"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-010"]) def test_health_status_green_when_all_checks_pass(tag): report = _build( [], @@ -227,7 +227,7 @@ def test_health_status_green_when_all_checks_pass(tag): assert report["summary"]["health"] == {"status": "green", "reasons": []} -@pytest.mark.parametrize("tag", ["@FC-010"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-010"]) def test_health_status_amber_when_worst_check_warns(tag): report = _build( [], @@ -241,7 +241,7 @@ def test_health_status_amber_when_worst_check_warns(tag): assert report["summary"]["health"]["reasons"] == ["at parity"] -@pytest.mark.parametrize("tag", ["@FC-010"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-010"]) def test_health_status_red_when_any_check_fails(tag): report = _build( [], @@ -255,7 +255,7 @@ def test_health_status_red_when_any_check_fails(tag): assert set(report["summary"]["health"]["reasons"]) == {"needs attention", "inverted"} -@pytest.mark.parametrize("tag", ["@FC-010"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-010"]) def test_pyramid_summary_converts_duration_to_milliseconds(tag): report = _build( [], @@ -265,20 +265,20 @@ def test_pyramid_summary_converts_duration_to_milliseconds(tag): assert report["summary"]["pyramid"]["unit"] == {"testCount": 3, "duration": 250.0, "passRate": 100} -@pytest.mark.parametrize("tag", ["@FC-010"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-010"]) def test_unlinked_tests_included_with_tags_and_optional_module(tag): unlinked = [ - TestResult(layer="unit", name="orphan", tags=["@FC-999"], module="billing"), + TestResult(layer="unit", name="orphan", tags=["@scenario:FC-999"], module="billing"), ] report = _build([], unlinked_results=unlinked) assert report["unlinkedTests"] == [ - {"layer": "unit", "testId": "orphan", "name": "orphan", "tags": ["@FC-999"], "module": "billing"} + {"layer": "unit", "testId": "orphan", "name": "orphan", "tags": ["@scenario:FC-999"], "module": "billing"} ] -@pytest.mark.parametrize("tag", ["@FC-010"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-010"]) def test_config_is_echoed_verbatim(tag): config = {"features": ["./features"], "output": "./out.html", "output_json": "./out.json"} @@ -287,7 +287,7 @@ def test_config_is_echoed_verbatim(tag): assert report["config"] == config -@pytest.mark.parametrize("tag", ["@FC-010"]) +@pytest.mark.parametrize("tag", ["@scenario:FC-010"]) def test_schema_version_and_generated_at_present(tag): report = _build([])