Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions src/exploit_iq_commons/data_models/checker_status.py
Original file line number Diff line number Diff line change
Expand Up @@ -236,6 +236,14 @@ def format_for_prompt(self) -> str:
lines.append(f"AFFECTED_VERSION_RANGE: {self.affected_version_range}")
if self.fixed_version:
lines.append(f"FIXED_VERSION: {self.fixed_version}")
# Emit version-in-range so the L1 agent can apply the VERSION GUARD
# and VERSION-BASED FALLBACK rules defined in the Case B thought instructions.
# Without this, the agent is told to check TARGET_IN_VULNERABLE_RANGE but never sees it.
if self.target_version_in_vulnerable_range is not None:

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@TamarW0 What happens if self.target_version_in_vulnerable_range is None?
it doesn't include the TARGET_IN_VULNERABLE_RANGE in the formatted prompt.

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@TamarW0 - agree with zvi recommend placing UNKNOWN in that case

label = "YES" if self.target_version_in_vulnerable_range else "NO"
else:
label = "UNKNOWN"
lines.append(f"TARGET_IN_VULNERABLE_RANGE: {label}")
# Identify-phase NVR-vs-fix-NVR signal: guidance only, never a substitute for verification.
if self.identify_phase_fix_signal is True:
lines.append(
Expand Down
2 changes: 1 addition & 1 deletion src/vuln_analysis/functions/cve_verify_vuln_package.py
Original file line number Diff line number Diff line change
Expand Up @@ -348,7 +348,7 @@ class CveProcessingConfig:
4. RPM NEVRA/module strings may embed upstream version separately from distro tags.
5. If description clearly shows the installed version is at or past the fix, answer not vulnerable.

Respond with whether the installed version is vulnerable and explain your reasoning."""
Respond with whether the installed version is vulnerable. Keep the reason to one sentence."""


class CVEVerifyVulnPackageConfig(FunctionBaseConfig, name="cve_verify_vuln_package"):
Expand Down
25 changes: 23 additions & 2 deletions src/vuln_analysis/utils/rpm_checker_prompts.py
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,10 @@
"- State confidence level based on evidence quality."
)

# Case B: vulnerable_patterns is empty, so the agent searches for fix patterns first.
# The VERSION GUARD clause prevents false "PATCHED via rebase" conclusions when a generic
# fix pattern (e.g. a bounds check) matches pre-existing code in a target that is still
# within the vulnerable version range (CVE-2024-48957/libarchive was the triggering case).
L1_AGENT_SYS_PROMPT_UPSTREAM_PATCH_CASE_B = (
"You are a security analyst verifying whether a package remains VULNERABLE to a CVE.\n"
"The TARGET package does NOT contain a CVE-specific patch file.\n"
Expand All @@ -192,7 +196,10 @@
" - The fix must be called and its result must be used.\n"
"3. CONCLUSION:\n"
" - If fix is applied at call site → Package is PATCHED via rebase.\n"
" - If fix is absent or unverified → investigate remaining affected files and use version fallback rules.\n\n"
" - If fix is absent or unverified → investigate remaining affected files and use version fallback rules.\n"
" - VERSION GUARD: If TARGET_IN_VULNERABLE_RANGE is YES, a grep match alone is NOT sufficient\n"
" to conclude PATCHED — the fix must be at the exact code location described in the CVE,\n"
" not a similar pre-existing check. When uncertain, conclude VULNERABLE (version-based).\n\n"
"CRITICAL RULES:\n"
"- Do NOT start with vulnerable-pattern search in this mode.\n"
"- Do NOT assume vulnerability or safety without tool evidence.\n"
Expand Down Expand Up @@ -512,12 +519,26 @@
- ALL AFFECTED_FILES have been searched
- Evidence is sufficient for confident verdict

**VERSION-BASED FALLBACK (when code search is inconclusive):**
**VERSION-BASED FALLBACK:**
If code searches found no fix pattern or no fix call-site verification:
- Check TARGET_IN_VULNERABLE_RANGE in VULNERABILITY_INTEL
- If TARGET_IN_VULNERABLE_RANGE: YES and no fix was verified in code:
→ Conclude VULNERABLE based on version evidence
→ Reason: "Target version is within affected range, and no fix was verified in code."
- If TARGET_IN_VULNERABLE_RANGE is UNKNOWN:
→ Treat the TARGET_IN_VULNERABLE_RANGE as unknown;
→ do not assume YES or NO.

**VERSION GUARD (when fix pattern WAS found):**
If TARGET_IN_VULNERABLE_RANGE is YES AND a fix-pattern grep match was found:
- The match may be a pre-existing similar check, not the actual CVE fix.
- Verify the match is at the EXACT function/location described in the CVE.
- If the match is ambiguous (e.g. a common pattern like a bounds check that could
pre-date the fix), conclude VULNERABLE (version-based).
Comment on lines 200 to +537

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@TamarW0 If TARGET_IN_VULNERABLE_RANGE field is absent in the data in both guards( because of the code that i commented on it previously) , then the LLM is being asked to lookup a field that doesn't exists might hallucinate it wrongly or fallback to some undesired behavior...
Better to explicitly put in the prompts that if field TARGET_IN_VULNERABLE_RANGE is absent from data context , then treat as unknown , or to conclude vulnerable unless fix was verified at the exact CVE location
@RedTanny WDYT?

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@zvigrinberg Agree — omitting the field when it’s None is the wrong default.

The agent is already instructed to check TARGET_IN_VULNERABLE_RANGE, so if the value isn’t in the prompt it may invent YES/NO or skip the guard incorrectly.

I’d prefer we always emit it:

YES / NO when known
TARGET_IN_VULNERABLE_RANGE: UNKNOWN when None
And update both VERSION GUARD and VERSION-BASED FALLBACK to say explicitly: if the field is UNKNOWN/absent, treat as unknown — do not assume YES/NO; conclude VULNERABLE unless the fix is verified at the exact CVE location.

That keeps the VERSION GUARD intent (avoid false PATCHED from a generic grep match) without creating a blind spot when range data is missing.


If TARGET_IN_VULNERABLE_RANGE is UNKNOWN:
- Treat the TARGET_IN_VULNERABLE_RANGE as unknown; do not assume YES or NO.
- Conclude VULNERABLE unless the fix is verified at the exact CVE location.
</INVESTIGATION_PHASES>

<RULES>
Expand Down
158 changes: 158 additions & 0 deletions tests/test_vulnerability_intel_format.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,158 @@
"""Tests for VulnerabilityIntel.format_for_prompt and RPM checker prompt routing."""

from exploit_iq_commons.data_models.checker_status import VulnerabilityIntel
from vuln_analysis.utils.rpm_checker_prompts import (
L1_AGENT_SYS_PROMPT_UPSTREAM_PATCH,
L1_AGENT_SYS_PROMPT_UPSTREAM_PATCH_CASE_B,
L1_AGENT_THOUGHT_UPSTREAM_INSTRUCTIONS,
L1_AGENT_THOUGHT_UPSTREAM_CASE_B_INSTRUCTIONS,
select_upstream_prompt_and_instructions,
)


class TestFormatForPromptVersionRange:

def test_vulnerable_range_true_emits_yes(self):
intel = VulnerabilityIntel(
affected_files=["util.c"],
target_version_in_vulnerable_range=True,
)
output = intel.format_for_prompt()
assert "TARGET_IN_VULNERABLE_RANGE: YES" in output

def test_vulnerable_range_false_emits_no(self):
intel = VulnerabilityIntel(
affected_files=["util.c"],
target_version_in_vulnerable_range=False,
)
output = intel.format_for_prompt()
assert "TARGET_IN_VULNERABLE_RANGE: NO" in output

def test_vulnerable_range_none_omitted(self):
intel = VulnerabilityIntel(
affected_files=["util.c"],
target_version_in_vulnerable_range=None,
)
output = intel.format_for_prompt()
assert "TARGET_IN_VULNERABLE_RANGE: UNKNOWN" in output

def test_all_defaults_returns_empty(self):
intel = VulnerabilityIntel()
output = intel.format_for_prompt()
assert output == "TARGET_IN_VULNERABLE_RANGE: UNKNOWN"

def test_downstream_patch_applied(self):
intel = VulnerabilityIntel(
is_downstream_patch_available=True,
is_patch_applied_in_build=True,
patch_file_name="CVE-2024-48957.patch",
)
output = intel.format_for_prompt()
assert "DOWNSTREAM_PATCH_STATUS: APPLIED" in output
assert "PATCH_FILE: CVE-2024-48957.patch" in output

def test_downstream_patch_available_not_applied(self):
intel = VulnerabilityIntel(
is_downstream_patch_available=True,
is_patch_applied_in_build=False,
)
output = intel.format_for_prompt()
assert "DOWNSTREAM_PATCH_STATUS: AVAILABLE" in output

def test_vulnerable_patterns_formatted_as_indented_list(self):
intel = VulnerabilityIntel(
vulnerable_patterns=["memset(&state, 0, sizeof(state))", "int8_t delta = (int8_t)*src++"],
)
output = intel.format_for_prompt()
assert "VULNERABLE_PATTERNS:" in output
assert " - memset(&state, 0, sizeof(state))" in output
assert " - int8_t delta = (int8_t)*src++" in output

def test_fix_patterns_formatted_as_indented_list(self):
intel = VulnerabilityIntel(
fix_patterns=["if (src >= dst)", "return 0;"],
)
output = intel.format_for_prompt()
assert "FIX_PATTERNS:" in output
assert " - if (src >= dst)" in output
assert " - return 0;" in output

def test_affected_bitness_both_omitted(self):
"""Default bitness 'both' should not appear in output."""
intel = VulnerabilityIntel(affected_files=["a.c"], affected_bitness="both")
output = intel.format_for_prompt()
assert "AFFECTED_BITNESS" not in output

def test_affected_bitness_32bit_emitted(self):
intel = VulnerabilityIntel(affected_files=["a.c"], affected_bitness="32-bit")
output = intel.format_for_prompt()
assert "AFFECTED_BITNESS: 32-bit" in output

def test_cve_2024_48957_scenario(self):
"""Reproduces the CVE-2024-48957/libarchive integration test scenario.
The agent must see TARGET_IN_VULNERABLE_RANGE: YES so the VERSION GUARD
can fire when a generic fix pattern matches pre-existing code."""
intel = VulnerabilityIntel(
affected_files=["archive_read_support_format_rar.c"],
fix_patterns=["if (src >= dst)"],
target_version_in_vulnerable_range=True,
target_package_version="3.7.2",
affected_version_range="< 3.7.7",
fixed_version="3.7.7",
)
output = intel.format_for_prompt()
assert "AFFECTED_FILES: archive_read_support_format_rar.c" in output
assert "if (src >= dst)" in output
assert "TARGET_PACKAGE_VERSION: 3.7.2" in output
assert "AFFECTED_VERSION_RANGE: < 3.7.7" in output
assert "FIXED_VERSION: 3.7.7" in output
assert "TARGET_IN_VULNERABLE_RANGE: YES" in output

def test_version_range_after_fixed_version(self):
"""TARGET_IN_VULNERABLE_RANGE should appear after FIXED_VERSION in the output."""
intel = VulnerabilityIntel(
affected_files=["a.c"],
fixed_version="1.0",
target_version_in_vulnerable_range=True,
)
output = intel.format_for_prompt()
fixed_pos = output.index("FIXED_VERSION")
range_pos = output.index("TARGET_IN_VULNERABLE_RANGE")
assert range_pos > fixed_pos


class TestSelectUpstreamPromptAndInstructions:

def test_empty_vulnerable_patterns_returns_case_b(self):
sys_prompt, thought = select_upstream_prompt_and_instructions([])
assert sys_prompt is L1_AGENT_SYS_PROMPT_UPSTREAM_PATCH_CASE_B
assert thought is L1_AGENT_THOUGHT_UPSTREAM_CASE_B_INSTRUCTIONS

def test_nonempty_vulnerable_patterns_returns_case_a(self):
sys_prompt, thought = select_upstream_prompt_and_instructions(["vuln pattern"])
assert sys_prompt is L1_AGENT_SYS_PROMPT_UPSTREAM_PATCH
assert thought is L1_AGENT_THOUGHT_UPSTREAM_INSTRUCTIONS


class TestCaseBPromptContent:

def test_sys_prompt_contains_version_guard(self):
assert "VERSION GUARD" in L1_AGENT_SYS_PROMPT_UPSTREAM_PATCH_CASE_B

def test_sys_prompt_version_guard_mentions_target_in_vulnerable_range(self):
assert "TARGET_IN_VULNERABLE_RANGE" in L1_AGENT_SYS_PROMPT_UPSTREAM_PATCH_CASE_B

def test_thought_instructions_contain_version_guard(self):
assert "VERSION GUARD" in L1_AGENT_THOUGHT_UPSTREAM_CASE_B_INSTRUCTIONS

def test_thought_instructions_version_guard_warns_about_preexisting_checks(self):
assert "pre-existing" in L1_AGENT_THOUGHT_UPSTREAM_CASE_B_INSTRUCTIONS

def test_thought_instructions_retain_version_based_fallback(self):
assert "VERSION-BASED FALLBACK" in L1_AGENT_THOUGHT_UPSTREAM_CASE_B_INSTRUCTIONS

def test_thought_instructions_retain_example_finish_patched(self):
assert "PATCHED via rebase" in L1_AGENT_THOUGHT_UPSTREAM_CASE_B_INSTRUCTIONS

def test_thought_instructions_retain_example_finish_version_based(self):
assert "VULNERABLE (version-based)" in L1_AGENT_THOUGHT_UPSTREAM_CASE_B_INSTRUCTIONS