perf: replace linear dict scans with O(1) lookups and cache Lark parser (#302)

* perf: replace linear dict scans with O(1) lookups and cache Lark parser

- Refactor _get_urn_ids_for_svcs() to use dict.get() instead of
  iterating .items()
- Refactor _get_nr_of_impls_for_req() to use len(dict.get()) to
  correctly count multiple annotations per requirement
- Refactor _get_mvr_ids_for_req() to use dict.get() instead of
  nested iteration over .items()
- Cache the Lark parser as a class attribute in GenericELTransformer
  to avoid recreating it on every filter call
- Add unit tests for the refactored statistics_generator methods (TDD)

Signed-off-by: jimisola <jimisola@jimisola.com>

* docs: add regression smoke testing instructions to CLAUDE.md

Signed-off-by: jimisola <jimisola@jimisola.com>

* chore: remove copyright notice from new test file

Signed-off-by: jimisola <jimisola@jimisola.com>

* refactor: move _parser class attribute to top of GenericELTransformer

Signed-off-by: jimisola <jimisola@jimisola.com>

---------

Signed-off-by: jimisola <jimisola@jimisola.com>
Co-authored-by: Jimisola Laursen <jimisola.laursen@resurs.se>

Author: jimisola

CLAUDE.md

@@ -102,6 +102,37 @@ hatch run python src/reqstool/command.py export local -p ../reqstool-demo/docs/r
 hatch run python src/reqstool/command.py report --format markdown local -p ../reqstool-demo/docs/reqstool -o /tmp/report.md
 ```
 
+## Regression Smoke Testing
+
+**Before creating a PR**, capture CLI output from `main` (or the parent branch) and compare it against the feature branch to verify no unintended changes. Strip ANSI codes so diffs are clean.
+
+Run against **both** the in-repo test fixtures and `reqstool-demo`:
+
+```bash
+# Capture baseline output on main/parent branch
+git checkout main
+hatch run python src/reqstool/command.py status local -p tests/resources/test_data/data/local/test_standard/baseline/ms-001 2>&1 | sed 's/\x1b\[[0-9;]*m//g' > /tmp/baseline-status-std.txt
+hatch run python src/reqstool/command.py report --format asciidoc local -p tests/resources/test_data/data/local/test_standard/baseline/ms-001 > /tmp/baseline-report-std.txt 2>&1
+hatch run python src/reqstool/command.py status local -p tests/resources/test_data/data/local/test_basic/baseline/ms-101 2>&1 | sed 's/\x1b\[[0-9;]*m//g' > /tmp/baseline-status-basic.txt
+hatch run python src/reqstool/command.py report --format asciidoc local -p tests/resources/test_data/data/local/test_basic/baseline/ms-101 > /tmp/baseline-report-basic.txt 2>&1
+hatch run python src/reqstool/command.py status local -p ../reqstool-demo/docs/reqstool 2>&1 | sed 's/\x1b\[[0-9;]*m//g' > /tmp/baseline-status-demo.txt
+hatch run python src/reqstool/command.py report --format asciidoc local -p ../reqstool-demo/docs/reqstool > /tmp/baseline-report-demo.txt 2>&1
+
+# Switch to feature branch and capture output
+git checkout <feature-branch>
+# (same commands, writing to /tmp/feature-*.txt)
+
+# Diff — must be identical unless the change intentionally alters output
+diff /tmp/baseline-status-std.txt /tmp/feature-status-std.txt
+diff /tmp/baseline-report-std.txt /tmp/feature-report-std.txt
+diff /tmp/baseline-status-basic.txt /tmp/feature-status-basic.txt
+diff /tmp/baseline-report-basic.txt /tmp/feature-report-basic.txt
+diff /tmp/baseline-status-demo.txt /tmp/feature-status-demo.txt
+diff /tmp/baseline-report-demo.txt /tmp/feature-report-demo.txt
+```
+
+If a diff is expected (e.g. the PR intentionally changes output), note it in the PR description.
+
 ## Key Conventions
 
 - **URN format**: `some:urn:string` — the separator is `:`. `UrnId` is the canonical composite key used throughout indexes.

src/reqstool/commands/status/statistics_generator.py

@@ -60,9 +60,7 @@ def _get_combined_index_data(
     def _calculate(self, cid: CombinedIndexedDataset) -> StatisticsContainer:
         for urn_id in cid.requirements.keys():
             # Get all svc UrnIds related to current requirement
-            svcs_urn_ids: List[UrnId] = self._get_urn_ids_for_svcs(
-                urn_id=urn_id, svcs_from_req=cid.svcs_from_req.items()
-            )
+            svcs_urn_ids: List[UrnId] = self._get_urn_ids_for_svcs(urn_id=urn_id, svcs_from_req=cid.svcs_from_req)
 
             # Get svcs for current requirement
             svcs: List[SVCData] = [cid.svcs[urn_id] for urn_id in svcs_urn_ids]
@@ -116,12 +114,7 @@ def _calculate(self, cid: CombinedIndexedDataset) -> StatisticsContainer:
         return self.stats_container
 
     def _get_urn_ids_for_svcs(self, urn_id: UrnId, svcs_from_req: Dict[UrnId, List[UrnId]]) -> List[UrnId]:
-        svcs_urn_ids: List[UrnId] = []
-        for req_urn_id, svc_list in svcs_from_req:
-            if urn_id == req_urn_id:
-                for svc in svc_list:
-                    svcs_urn_ids.append(svc)
-        return svcs_urn_ids
+        return list(svcs_from_req.get(urn_id, []))
 
     def _check_implementation(self, urn_id: UrnId, nr_of_implementations: int) -> bool:
         implementation = self.cid.requirements[urn_id].implementation
@@ -192,13 +185,8 @@ def _req_verification_equals(self, svcs: List[SVCData], verification: Sequence[V
                 return True
         return False
 
-    # Get the nr of impls for current requirement
     def _get_nr_of_impls_for_req(self, urn_id: UrnId) -> int:
-        nr_of_implementations = 0
-        for annotation_id in self.cid.annotations_impls.keys():
-            if annotation_id == urn_id:
-                nr_of_implementations += 1
-        return nr_of_implementations
+        return len(self.cid.annotations_impls.get(urn_id, []))
 
     def _get_annotated_automated_test_results_for_req(
         self,
@@ -231,13 +219,7 @@ def _get_automated_test_results_for_req(
         return test_results
 
     def _get_mvr_ids_for_req(self, svcs_urn_ids: List[UrnId]) -> List[UrnId]:
-        mvr_ids: List[UrnId] = []
-        for svc_urn_id in svcs_urn_ids:
-            for id, value in self.cid.mvrs_from_svc.items():
-                if id == svc_urn_id:
-                    for urn in value:
-                        mvr_ids.append(urn)
-        return mvr_ids
+        return [urn for svc_urn_id in svcs_urn_ids for urn in self.cid.mvrs_from_svc.get(svc_urn_id, [])]
 
     def _get_mvrs_for_req(self, mvrs: Dict[UrnId, MVRData], mvr_ids: List[UrnId]) -> List[MVRData] | None:
         return [mvrs[mvr_id] for mvr_id in mvr_ids] if mvr_ids else None

src/reqstool/expression_languages/generic_el.py

@@ -16,6 +16,7 @@ class GenericELTransformer(
 ):
     _data: T
     _urn: str
+    _parser: Lark = None
 
     _GRAMMAR = """
     start: expr
@@ -99,8 +100,7 @@ def STRING(self, token) -> str:
 
     @staticmethod
     def parse_el(expression_language: str) -> ParseTree:
-        parser = Lark(GenericELTransformer._GRAMMAR, parser="lalr")
+        if GenericELTransformer._parser is None:
+            GenericELTransformer._parser = Lark(GenericELTransformer._GRAMMAR, parser="lalr")
 
-        tree = parser.parse(expression_language)
-
-        return tree
+        return GenericELTransformer._parser.parse(expression_language)

tests/unit/reqstool/commands/status/test_statistics_generator_methods.py

@@ -0,0 +1,99 @@
+from unittest.mock import MagicMock
+
+from reqstool.commands.status.statistics_generator import StatisticsGenerator
+from reqstool.common.dataclasses.urn_id import UrnId
+
+
+def _make_generator():
+    """Create a StatisticsGenerator bypassing __init__."""
+    gen = object.__new__(StatisticsGenerator)
+    gen.cid = MagicMock()
+    return gen
+
+
+# ---------------------------------------------------------------------------
+# _get_urn_ids_for_svcs
+# ---------------------------------------------------------------------------
+
+REQ_1 = UrnId(urn="urn", id="REQ_001")
+REQ_2 = UrnId(urn="urn", id="REQ_002")
+SVC_1 = UrnId(urn="urn", id="SVC_001")
+SVC_2 = UrnId(urn="urn", id="SVC_002")
+SVC_3 = UrnId(urn="urn", id="SVC_003")
+MVR_1 = UrnId(urn="urn", id="MVR_001")
+MVR_2 = UrnId(urn="urn", id="MVR_002")
+
+
+def test_get_urn_ids_for_svcs_returns_matching():
+    gen = _make_generator()
+    svcs_from_req = {REQ_1: [SVC_1, SVC_2], REQ_2: [SVC_3]}
+    result = gen._get_urn_ids_for_svcs(urn_id=REQ_1, svcs_from_req=svcs_from_req)
+    assert result == [SVC_1, SVC_2]
+
+
+def test_get_urn_ids_for_svcs_returns_empty_for_missing_key():
+    gen = _make_generator()
+    svcs_from_req = {REQ_2: [SVC_3]}
+    result = gen._get_urn_ids_for_svcs(urn_id=REQ_1, svcs_from_req=svcs_from_req)
+    assert result == []
+
+
+def test_get_urn_ids_for_svcs_empty_dict():
+    gen = _make_generator()
+    result = gen._get_urn_ids_for_svcs(urn_id=REQ_1, svcs_from_req={})
+    assert result == []
+
+
+# ---------------------------------------------------------------------------
+# _get_nr_of_impls_for_req
+# ---------------------------------------------------------------------------
+
+
+def test_get_nr_of_impls_for_req_single():
+    gen = _make_generator()
+    gen.cid.annotations_impls = {REQ_1: ["annotation_1"]}
+    assert gen._get_nr_of_impls_for_req(urn_id=REQ_1) == 1
+
+
+def test_get_nr_of_impls_for_req_multiple():
+    gen = _make_generator()
+    gen.cid.annotations_impls = {REQ_1: ["annotation_1", "annotation_2"]}
+    assert gen._get_nr_of_impls_for_req(urn_id=REQ_1) == 2
+
+
+def test_get_nr_of_impls_for_req_missing():
+    gen = _make_generator()
+    gen.cid.annotations_impls = {REQ_2: ["some_annotation"]}
+    assert gen._get_nr_of_impls_for_req(urn_id=REQ_1) == 0
+
+
+def test_get_nr_of_impls_for_req_empty():
+    gen = _make_generator()
+    gen.cid.annotations_impls = {}
+    assert gen._get_nr_of_impls_for_req(urn_id=REQ_1) == 0
+
+
+# ---------------------------------------------------------------------------
+# _get_mvr_ids_for_req
+# ---------------------------------------------------------------------------
+
+
+def test_get_mvr_ids_for_req_returns_matching():
+    gen = _make_generator()
+    gen.cid.mvrs_from_svc = {SVC_1: [MVR_1], SVC_2: [MVR_2]}
+    result = gen._get_mvr_ids_for_req(svcs_urn_ids=[SVC_1, SVC_2])
+    assert result == [MVR_1, MVR_2]
+
+
+def test_get_mvr_ids_for_req_missing_svc():
+    gen = _make_generator()
+    gen.cid.mvrs_from_svc = {SVC_2: [MVR_2]}
+    result = gen._get_mvr_ids_for_req(svcs_urn_ids=[SVC_1])
+    assert result == []
+
+
+def test_get_mvr_ids_for_req_empty():
+    gen = _make_generator()
+    gen.cid.mvrs_from_svc = {}
+    result = gen._get_mvr_ids_for_req(svcs_urn_ids=[])
+    assert result == []