bench(iabench): implement IA-2 retrieval and IA-5 hallucination tasks

.github/workflows/ci.yml

@@ -20,7 +20,7 @@ jobs:
         with:
           python-version: "3.12"
       - name: Install ruff
-        run: pip install ruff==0.4.7
+        run: pip install ruff==0.15.16
       - name: Ruff check
         run: ruff check src/ tests/ benchmarks/ examples/
       - name: Ruff format check

.pre-commit-config.yaml

@@ -15,7 +15,7 @@ repos:
         args: ["--branch", "main"]
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.4.7
+    rev: v0.15.16
     hooks:
       - id: ruff
         args: ["--fix"]

benchmarks/data/ia2_queries.json

@@ -0,0 +1,109 @@
+[
+  {
+    "id": "q001",
+    "query": "What vibration RMS threshold triggers motor bearing replacement on motor_01?",
+    "gold_labels": {
+      "SOP-MAINT-001": 2,
+      "EN-001": 1
+    },
+    "notes": "SOP-MAINT-001 states the 4.5 mm/s threshold directly; EN-001 corroborates with expert context."
+  },
+  {
+    "id": "q002",
+    "query": "What PPE is required when handling bearings during motor maintenance?",
+    "gold_labels": {
+      "SOP-MAINT-001": 2,
+      "EN-001": 1
+    },
+    "notes": "SOP specifies nitrile gloves; EN-001 adds the reason (prevent skin-oil contamination)."
+  },
+  {
+    "id": "q003",
+    "query": "What is the part number for the drive end bearing on motor_01?",
+    "gold_labels": {
+      "SOP-MAINT-001": 2,
+      "EN-001": 1
+    },
+    "notes": "Part BRG-6310-2RS appears in the SOP parts table; EN-001 reaffirms with additional context."
+  },
+  {
+    "id": "q004",
+    "query": "What temperature should bearings be heated to before installation, and what tool is used?",
+    "gold_labels": {
+      "SOP-MAINT-001": 2,
+      "EN-001": 2
+    },
+    "notes": "SOP and EN-001 both specify 80-100 degrees C and induction heater TOOL-HEAT-01."
+  },
+  {
+    "id": "q005",
+    "query": "What hydraulic system pressure should be restored after filter replacement on hydraulic_01?",
+    "gold_labels": {
+      "SOP-MAINT-002": 2,
+      "EN-002": 1
+    },
+    "notes": "SOP-MAINT-002 acceptance: 200 +/- 5 bar; EN-002 elaborates on troubleshooting pressure issues."
+  },
+  {
+    "id": "q006",
+    "query": "When should the hydraulic filter element be replaced according to the PM schedule?",
+    "gold_labels": {
+      "SOP-MAINT-002": 2,
+      "EN-002": 1
+    },
+    "notes": "SOP-MAINT-002 states 2000-hour PM schedule; EN-002 notes typical filter life 1600-1800 hours."
+  },
+  {
+    "id": "q007",
+    "query": "What is the torque specification for the hydraulic filter bowl?",
+    "gold_labels": {
+      "SOP-MAINT-002": 2,
+      "EN-002": 2
+    },
+    "notes": "Both SOP and EN-002 specify 35 Nm with torque wrench."
+  },
+  {
+    "id": "q008",
+    "query": "What qualifications and certifications are required to perform motor bearing replacement?",
+    "gold_labels": {
+      "SOP-MAINT-001": 2
+    },
+    "notes": "SOP-MAINT-001 section 2 specifies Electrical Journeyman (NFPA 70E) and Mechanical Technician Level 2."
+  },
+  {
+    "id": "q009",
+    "query": "What causes hydraulic filter clogging and what are the consequences of ignoring it?",
+    "gold_labels": {
+      "SOP-MAINT-002": 1,
+      "EN-002": 2
+    },
+    "notes": "EN-002 gives expert detail on contamination causes; SOP-MAINT-002 references FM-HYD-003 failure mode."
+  },
+  {
+    "id": "q010",
+    "query": "How do I verify motor alignment after reinstallation, and what is the acceptance tolerance?",
+    "gold_labels": {
+      "SOP-MAINT-001": 2,
+      "EN-001": 2
+    },
+    "notes": "Both specify < 0.05 mm TIR; EN-001 gives practical tips on using dial indicator."
+  },
+  {
+    "id": "q011",
+    "query": "What LOTO and safety steps are required before starting hydraulic filter maintenance?",
+    "gold_labels": {
+      "SOP-MAINT-002": 2,
+      "SOP-MAINT-001": 1
+    },
+    "notes": "SOP-MAINT-002 covers hydraulic depressurization; SOP-MAINT-001 covers LOTO as a general reference."
+  },
+  {
+    "id": "q012",
+    "query": "What are the acceptance criteria for vibration and temperature after motor bearing replacement?",
+    "gold_labels": {
+      "SOP-MAINT-001": 2,
+      "EN-001": 1
+    },
+    "notes": "SOP specifies vibration < 2.5 mm/s, bearing temp < 70 C above ambient; EN-001 describes the run-in procedure."
+  }
+]

benchmarks/data/ia5_grounded_queries.json

@@ -0,0 +1,146 @@
+[
+  {
+    "id": "g001",
+    "question": "What vibration RMS threshold triggers bearing replacement on motor_01?",
+    "source_doc": "SOP-MAINT-001",
+    "expected_key_facts": ["4.5 mm/s", "4.5"],
+    "forbidden_facts": ["5 mm/s", "3 mm/s", "3.5 mm/s", "6 mm/s"],
+    "notes": "Exact value from SOP-MAINT-001 section 1: 'vibration RMS exceeds 4.5 mm/s'"
+  },
+  {
+    "id": "g002",
+    "question": "What is the part number for the drive end bearing on motor_01?",
+    "source_doc": "SOP-MAINT-001",
+    "expected_key_facts": ["BRG-6310-2RS", "6310-2RS"],
+    "forbidden_facts": ["BRG-6208-2RS", "6208-2RS"],
+    "notes": "Part table in SOP-MAINT-001: drive end is BRG-6310-2RS"
+  },
+  {
+    "id": "g003",
+    "question": "What is the torque specification for reinstalling the hydraulic filter bowl on hydraulic_01?",
+    "source_doc": "SOP-MAINT-002",
+    "expected_key_facts": ["35 Nm", "35"],
+    "forbidden_facts": ["25 Nm", "45 Nm", "30 Nm", "50 Nm"],
+    "notes": "SOP-MAINT-002 step 9: torque to 35 Nm"
+  },
+  {
+    "id": "g004",
+    "question": "What maximum temperature must the hydraulic oil be below before starting filter maintenance?",
+    "source_doc": "SOP-MAINT-002",
+    "expected_key_facts": ["50", "50°C", "50 C"],
+    "forbidden_facts": ["60°C", "40°C", "70°C", "80°C"],
+    "notes": "SOP-MAINT-002 safety precaution 3: allow oil to cool below 50 degrees C"
+  },
+  {
+    "id": "g005",
+    "question": "What is the normal operating pressure of the hydraulic system on hydraulic_01?",
+    "source_doc": "SOP-MAINT-002",
+    "expected_key_facts": ["200", "200 bar", "200 ± 5 bar"],
+    "forbidden_facts": ["150 bar", "250 bar", "180 bar", "220 bar"],
+    "notes": "SOP-MAINT-002 acceptance criteria and EN-002: 200 +/- 5 bar"
+  },
+  {
+    "id": "g006",
+    "question": "What vibration level must be achieved during the uncoupled run-in after motor bearing replacement?",
+    "source_doc": "SOP-MAINT-001",
+    "expected_key_facts": ["2.5 mm/s"],
+    "forbidden_facts": ["4.5 mm/s", "5 mm/s", "3 mm/s"],
+    "notes": "SOP-MAINT-001 step 9 and section 6: vibration < 2.5 mm/s"
+  },
+  {
+    "id": "g007",
+    "question": "What bearing temperature triggers replacement on motor_01?",
+    "source_doc": "SOP-MAINT-001",
+    "expected_key_facts": ["90°C", "90 C", "90"],
+    "forbidden_facts": ["70°C", "80°C", "100°C", "60°C"],
+    "notes": "SOP-MAINT-001 section 1: bearing temperature exceeds 90 degrees C"
+  },
+  {
+    "id": "g008",
+    "question": "What temperature range should bearings be heated to before installation?",
+    "source_doc": "SOP-MAINT-001",
+    "expected_key_facts": ["80", "100", "80-100", "80 to 100"],
+    "forbidden_facts": ["150°C", "200°C", "50°C"],
+    "notes": "SOP-MAINT-001 step 6 and EN-001: heat to 80-100 degrees C"
+  },
+  {
+    "id": "g009",
+    "question": "How long must the accumulator bleed valve be held open during hydraulic depressurization?",
+    "source_doc": "SOP-MAINT-002",
+    "expected_key_facts": ["30 second", "30-second", "30"],
+    "forbidden_facts": ["60 second", "15 second", "45 second", "10 second"],
+    "notes": "SOP-MAINT-002 step 3: manual bleed valve 30-second hold"
+  },
+  {
+    "id": "g010",
+    "question": "What is the maximum filter differential pressure accepted after hydraulic filter replacement?",
+    "source_doc": "SOP-MAINT-002",
+    "expected_key_facts": ["0.5 bar"],
+    "forbidden_facts": ["2.5 bar", "1.0 bar", "1 bar", "2 bar"],
+    "notes": "SOP-MAINT-002 step 10 and section 5: filter dP < 0.5 bar"
+  },
+  {
+    "id": "g011",
+    "question": "What is the minimum drain pan capacity required for hydraulic filter maintenance?",
+    "source_doc": "SOP-MAINT-002",
+    "expected_key_facts": ["5L", "5 L", "5 litre", "5 liter", "minimum 5"],
+    "forbidden_facts": ["2L", "10L", "1L"],
+    "notes": "SOP-MAINT-002 step 5: drain pan minimum 5L"
+  },
+  {
+    "id": "g012",
+    "question": "What qualifications are required to perform motor bearing replacement?",
+    "source_doc": "SOP-MAINT-001",
+    "expected_key_facts": ["Electrical Journeyman", "NFPA 70E", "Mechanical Technician"],
+    "forbidden_facts": [],
+    "notes": "SOP-MAINT-001 section 2: Electrical Journeyman (NFPA 70E) and Mechanical Technician Level 2"
+  },
+  {
+    "id": "g013",
+    "question": "What is the FMEA reference number for hydraulic filter clogging?",
+    "source_doc": "SOP-MAINT-002",
+    "expected_key_facts": ["FM-HYD-003"],
+    "forbidden_facts": ["FM-MOT-001", "FM-HYD-001"],
+    "notes": "SOP-MAINT-002 section 6: FM-HYD-003 (Filter Clog / Bypass)"
+  },
+  {
+    "id": "g014",
+    "question": "Which SOP covers bearing replacement for motor_01 and motor_02?",
+    "source_doc": "SOP-MAINT-001",
+    "expected_key_facts": ["SOP-MAINT-001"],
+    "forbidden_facts": ["SOP-MAINT-002"],
+    "notes": "SOP-MAINT-001 header: Applies To motor_01, motor_02"
+  },
+  {
+    "id": "g015",
+    "question": "What is the maximum bearing temperature rise above ambient allowed after motor bearing replacement?",
+    "source_doc": "SOP-MAINT-001",
+    "expected_key_facts": ["70°C", "70 C", "70"],
+    "forbidden_facts": ["90°C", "50°C", "60°C", "40°C"],
+    "notes": "SOP-MAINT-001 section 6: bearing temperature < 70 degrees C above ambient"
+  },
+  {
+    "id": "g016",
+    "question": "How many technicians are required to work on motors greater than 5.5 kW?",
+    "source_doc": "SOP-MAINT-001",
+    "expected_key_facts": ["two", "second technician", "2"],
+    "forbidden_facts": ["one", "three", "alone"],
+    "notes": "SOP-MAINT-001 safety precaution 5 and EN-001: second technician required for motors > 5.5 kW"
+  },
+  {
+    "id": "g017",
+    "question": "What grease is specified for use on motor bearings?",
+    "source_doc": "SOP-MAINT-001",
+    "expected_key_facts": ["Mobil Polyrex EM", "Polyrex EM"],
+    "forbidden_facts": [],
+    "notes": "SOP-MAINT-001 parts table: LUB-POLYREX (Mobil Polyrex EM)"
+  },
+  {
+    "id": "g018",
+    "question": "What is the part number for the non-drive end bearing on motor_01?",
+    "source_doc": "SOP-MAINT-001",
+    "expected_key_facts": ["BRG-6208-2RS", "6208-2RS"],
+    "forbidden_facts": ["BRG-6310-2RS", "6310-2RS"],
+    "notes": "SOP-MAINT-001 parts table: non-drive end is BRG-6208-2RS"
+  }
+]