diff --git a/README.md b/README.md index 81e4746e..77729b7f 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ The challenge: Produce similar results as Glasswing - using models everyone has **Autonomous vulnerability scanner and source-code hunter.** Built on `genai-pyo3`, a native Rust-backed LLM runtime speaking every major provider (Anthropic, OpenAI, OpenRouter, Ollama, LM Studio, Together, -Groq, DeepSeek, MiniMax, Gemini, any OpenAI-compatible endpoint). +Groq, DeepSeek, Kimi, MiniMax, Gemini, any OpenAI-compatible endpoint). Clearwing is a dual-mode offensive-security tool: @@ -88,7 +88,7 @@ Or skip the wizard and configure directly: export ANTHROPIC_API_KEY=sk-ant-... # Or any OpenAI-compatible endpoint — OpenRouter, Ollama, LM Studio, -# vLLM, Together, Groq, DeepSeek, OpenAI: +# vLLM, Together, Groq, DeepSeek, Kimi, OpenAI: export CLEARWING_BASE_URL=https://openrouter.ai/api/v1 export CLEARWING_API_KEY=sk-or-... export CLEARWING_MODEL=anthropic/claude-opus-4 @@ -305,7 +305,7 @@ Deep dives live in [`docs/`](docs/): |---|---| | [`docs/index.md`](docs/index.md) | Landing page + table of contents | | [`docs/quickstart.md`](docs/quickstart.md) | Full install + first run walkthrough | -| [`docs/providers.md`](docs/providers.md) | OpenRouter / Ollama / LM Studio / vLLM / Together / Groq recipes, per-task routing, env-var precedence | +| [`docs/providers.md`](docs/providers.md) | OpenRouter / Ollama / LM Studio / vLLM / Kimi / Together / Groq recipes, per-task routing, env-var precedence | | [`docs/architecture.md`](docs/architecture.md) | Both pipelines, substrate, capability gating, tool layout | | [`docs/cli.md`](docs/cli.md) | Every subcommand flag, grouped by workflow | | [`docs/api.md`](docs/api.md) | API reference (mkdocstrings autogen) | diff --git a/clearwing/agent/tools/hunt/__init__.py b/clearwing/agent/tools/hunt/__init__.py index 5488e55d..7ea8e673 100644 --- a/clearwing/agent/tools/hunt/__init__.py +++ b/clearwing/agent/tools/hunt/__init__.py @@ -34,6 +34,8 @@ _parse_sanitizer_report, build_analysis_tools, ) +from .candidates import build_candidate_tools, candidate_matches_domain +from .deep_agent import build_deep_agent_tools from .discovery import ( _container_path, _grep_python_fallback, @@ -41,10 +43,10 @@ _parse_rg_output, build_discovery_tools, ) -from .deep_agent import build_deep_agent_tools from .pool_query import build_pool_query_tools from .reporting import build_reporting_tools from .sandbox import HunterContext, _parse_variant_arg +from .windows import build_window_tools def build_hunter_tools(ctx: HunterContext) -> list: @@ -84,12 +86,15 @@ def build_propagation_auditor_tools(ctx: HunterContext) -> list: # Public API "HunterContext", "build_deep_agent_tools", + "build_candidate_tools", + "candidate_matches_domain", "build_hunter_tools", "build_propagation_auditor_tools", # Per-domain builders (for callers that want a narrower tool set) "build_discovery_tools", "build_analysis_tools", "build_reporting_tools", + "build_window_tools", "build_pool_query_tools", # Re-exported helpers for test reach-ins "_container_path", diff --git a/clearwing/agent/tools/hunt/candidates.py b/clearwing/agent/tools/hunt/candidates.py new file mode 100644 index 00000000..bfcf288b --- /dev/null +++ b/clearwing/agent/tools/hunt/candidates.py @@ -0,0 +1,544 @@ +"""Small-model candidate ledger for explicit hypothesis state.""" + +from __future__ import annotations + +import re +from pathlib import Path +from typing import Literal + +from pydantic import Field + +from clearwing.findings.types import TraceStep +from clearwing.llm import NativeToolSpec, ToolInputModel + +from .sandbox import HunterContext + + +def _mentions_identifier(text: str, identifier: str) -> bool: + return bool(re.search(rf"(? bool: + """Return whether candidate prose stays tied to an extracted value domain. + + Extraction can legitimately leave a producer as ``unknown`` and can find + no distinguished literal. Those placeholders are not evidence and must + not become magic words that a hunter is forced to repeat. + """ + + normalized = text.casefold() + placeholders = {"", "none", "unknown", "unresolved"} + stored_state = str(domain.get("stored_state", "")).strip().casefold() + if stored_state in placeholders or not _mentions_identifier(normalized, stored_state): + return False + + producer_names = { + str(value).strip().casefold() + for value in ( + domain.get("producer_state", ""), + *domain.get("producer_tokens", []), + ) + if str(value).strip().casefold() not in placeholders + } + if producer_names and not any( + _mentions_identifier(normalized, name) for name in producer_names + ): + return False + + distinguished_values = { + str(value).strip().casefold() + for value in domain.get("distinguished_tokens", []) + if str(value).strip() + } + if distinguished_values and not any( + value in normalized + for value in distinguished_values | {"reserved", "sentinel", "distinguished"} + ): + return False + return True + + +def _mentions_location(text: str, location: str) -> bool: + path, _separator, line = location.rpartition(":") + short_location = f"{Path(path).name}:{line}" + return ( + location.casefold() in text + or short_location.casefold() in text + or bool(re.search(rf"\bline\s+{re.escape(line)}\b", text)) + ) + + +def _domain_closure_error( + plan: dict, + *, + guard: str, + assessment: str, + evidence: str, +) -> str | None: + if assessment not in {"overlap_blocked", "disjoint"}: + return None + if guard.casefold() == "none observed": + return ( + f"ERROR: {assessment} requires a source-backed dominating guard on the producer " + "value. Record overlap_possible or unresolved instead." + ) + producer_tokens = [ + str(token) + for token in plan.get("producer_tokens", [plan.get("producer_state", "")]) + if token + ] + if not any(_mentions_identifier(guard, token) for token in producer_tokens): + choices = ", ".join(producer_tokens) or "an extracted producer" + return ( + f"ERROR: {assessment} guard must constrain the extracted producer chain " + f"({choices}); unrelated allocation or index bounds do not separate value domains. " + "Record overlap_possible or unresolved instead." + ) + blocking_locations = [ + str(location) for location in plan.get("blocking_guard_locations", []) if location + ] + closure_text = f"{guard} {evidence}".casefold() + location_matched = any( + _mentions_location(closure_text, location) for location in blocking_locations + ) + if not blocking_locations or not location_matched: + choices = ", ".join(blocking_locations) or "none extracted" + return ( + f"ERROR: {assessment} requires an extracted source guard that blocks the producer " + f"before transfer ({choices}). Resets, resource estimates, and practical-impossibility " + "claims do not prove disjoint value domains." + ) + distinguished_tokens = [str(token) for token in plan.get("distinguished_tokens", []) if token] + if distinguished_tokens and not any( + token.casefold() in evidence.casefold() for token in distinguished_tokens + ): + choices = ", ".join(distinguished_tokens) + return ( + f"ERROR: {assessment} evidence must show how the producer guard excludes a " + f"distinguished stored value ({choices})." + ) + return None + + +def _domain_next_check(ctx: HunterContext, domain_id: str, domain: dict) -> str: + producer = str(domain.get("producer_state", "producer")) + if producer.casefold() in {"", "none", "unknown", "unresolved"}: + producer = next( + ( + str(token) + for token in domain.get("producer_tokens", []) + if str(token).casefold() not in {"", "none", "unknown", "unresolved"} + ), + "the producer", + ) + values = "/".join(str(value) for value in domain.get("distinguished_tokens", []) if value) + values = values or "the distinguished value" + guards = ", ".join( + str(location) for location in domain.get("blocking_guard_locations", []) if location + ) + consequence = ctx.domain_consequence_plans.get(domain_id, {}) + impacts = "/".join(str(token) for token in consequence.get("impact_tokens", [])[:2] if token) + locations = ", ".join( + str(location) for location in consequence.get("impact_locations", [])[:2] if location + ) + boundary_facts = consequence.get("boundary_facts", []) + guard_check = ( + f"verify whether the terminating guard at {guards} excludes it" + if guards + else "verify that no terminating producer guard excludes it" + ) + if boundary_facts: + fact = boundary_facts[0] + impact_check = ( + f"then test whether the changed predicate can be true when {fact['token']} is at its " + f"lower bound, allowing [{fact['expression']}] to reach {impacts or 'the memory effect'} " + f"at {locations} without a dominating positive-bound guard" + ) + elif impacts: + impact_check = f"then follow {impacts} at {locations} to the first unsafe effect" + else: + impact_check = "then follow the changed consumer branch to the first unsafe effect" + return f"Determine whether {producer} can equal {values}; {guard_check}; {impact_check}." + + +class RecordCandidateInput(ToolInputModel): + candidate_id: str = Field(description="Short stable identifier, for example C1") + status: Literal["pending", "investigating", "rejected", "validated"] + file: str = Field(description="Repo-relative file containing the current evidence") + line: int = Field(default=0, description="Best current 1-indexed line, or 0 if unknown") + hypothesis: str = Field(description="Concrete vulnerability mechanism") + attacker_control: str = Field(default="", description="Attacker-controlled input or event") + invariant: str = Field(default="", description="Security invariant that may be violated") + effect: str = Field(default="", description="Reachable security-sensitive effect") + counterargument: str = Field( + default="", + description="Strongest guard, bound, or fact that might disprove the candidate", + ) + next_check: str = Field(default="", description="One narrow read/search/test to resolve next") + evidence: str = Field(default="", description="Concise evidence learned so far") + + +class RecordValueDomainInput(ToolInputModel): + domain_id: str = Field(description="Opaque domain ID from read_state_interactions, for example D1") + guard: str = Field(description="Dominating producer-value guard, or exactly 'none observed'") + assessment: Literal["overlap_possible", "overlap_blocked", "disjoint", "unresolved"] + evidence: str = Field(description="At least two packet locations supporting the assessment") + next_check: str = Field(description="One narrow check that could confirm or refute it") + + +class RecordDomainConsequenceInput(ToolInputModel): + domain_id: str = Field(description="Opaque domain ID from read_domain_consequences") + branch_effect: str = Field(description="How branch behavior changes if domains overlap") + state_effect: str = Field(description="Downstream state accepted, rejected, or misclassified") + security_effect: str = Field(description="Potential memory, lifetime, privilege, or availability effect") + assessment: Literal["security_effect_possible", "benign", "unresolved"] + evidence: str = Field(description="At least two consequence-packet locations") + next_check: str = Field(description="One narrow source or runtime check") + + +class RecordDomainProofInput(ToolInputModel): + domain_id: str = Field(description="Opaque domain ID, for example D1") + candidate_id: str = Field(description="Tracked candidate ID, for example C1") + attacker_reaches_producer: bool + producer_reaches_distinguished: bool + changed_branch_reaches_effect: bool + boundary_effect_unguarded: bool + evidence: str = Field(description="Concise source locations supporting every true answer") + counterevidence: str = Field(default="", description="Strongest unresolved counterevidence") + + +def _record_domain_proof( + ctx: HunterContext, + *, + domain_id: str, + candidate_id: str, + attacker_reaches_producer: bool, + producer_reaches_distinguished: bool, + changed_branch_reaches_effect: bool, + boundary_effect_unguarded: bool, + evidence: str, + counterevidence: str, +) -> str: + normalized_id = domain_id.strip().upper() + domain = ctx.value_domains.get(normalized_id) + candidate = ctx.candidates.get(candidate_id) + consequence_plan = ctx.domain_consequence_plans.get(normalized_id) + if domain is None or consequence_plan is None: + return f"ERROR: expand and record domain {normalized_id} before proving it." + if candidate is None or ctx.domain_candidate_ids.get(normalized_id) != candidate_id: + return f"ERROR: {candidate_id} is not the tracked candidate for {normalized_id}." + if domain.get("blocking_guard_locations") and domain.get("assessment") in { + "overlap_possible", + "unresolved", + }: + locations = ", ".join(domain["blocking_guard_locations"]) + return ( + "ERROR: reassess the extracted terminating producer guard with " + f"record_value_domain first ({locations})." + ) + obligations = { + "attacker_reaches_producer": attacker_reaches_producer, + "producer_reaches_distinguished": producer_reaches_distinguished, + "changed_branch_reaches_effect": changed_branch_reaches_effect, + "boundary_effect_unguarded": boundary_effect_unguarded, + } + obligation_labels = { + "attacker_reaches_producer": "attacker reaches producer", + "producer_reaches_distinguished": "producer reaches distinguished value", + "changed_branch_reaches_effect": "changed branch reaches effect", + "boundary_effect_unguarded": "boundary effect lacks a dominating guard", + } + failed = [name for name, satisfied in obligations.items() if not satisfied] + if ctx.enable_domain_proof_refinement: + failed = failed[:1] + ctx.domain_refinement_pending_proof.discard(normalized_id) + if failed: + if ctx.enable_domain_proof_refinement: + ctx.domain_proof_obligations[normalized_id] = failed + failed_labels = [obligation_labels[name] for name in failed] + candidate["status"] = "investigating" + candidate["counterargument"] = counterevidence or ", ".join(failed_labels) + candidate["evidence"] = evidence + candidate["next_check"] = "Resolve: " + "; ".join(failed_labels) + "." + ctx.candidate_revision += 1 + if ctx.enable_domain_proof_refinement: + return ( + f"Domain proof for {candidate_id} remains unresolved: " + f"{'; '.join(failed_labels)}. Call read_domain_proof_refinement for this " + "obligation, then call record_domain_proof again." + ) + return ( + f"Domain proof for {candidate_id} remains unresolved: " + f"{'; '.join(failed_labels)}. Resolve only these obligations." + ) + if not consequence_plan.get("boundary_facts"): + return "ERROR: no source-derived boundary effect was extracted for this domain." + candidate["status"] = "validated" + ctx.domain_proof_obligations.pop(normalized_id, None) + candidate["evidence"] = evidence + candidate["counterargument"] = counterevidence + candidate["next_check"] = ( + "Submit this validated candidate with candidate_id, static_corroboration or stronger " + "evidence, and the seeded trace." + ) + ctx.domain_consequences[normalized_id] = { + **ctx.domain_consequences.get(normalized_id, {}), + "domain_id": normalized_id, + "assessment": "security_effect_possible", + "evidence": evidence, + "next_check": candidate["next_check"], + } + if not ctx.trace_steps: + trace_facts = [ + *domain.get("trace_facts", []), + *consequence_plan.get("trace_facts", []), + ] + ctx.trace_steps.extend( + TraceStep( + file=str(fact["file"]), + line=int(fact["line"]), + code_snippet=str(fact["code_snippet"]), + note=str(fact["role"]), + ) + for fact in trace_facts + ) + ctx.candidate_revision += 1 + return ( + f"Domain proof validated {candidate_id}; seeded {len(ctx.trace_steps)} exact trace steps. " + "Call record_finding now with this candidate_id and " + "evidence_level=static_corroboration or stronger." + ) + + +def _record_domain_consequence( + ctx: HunterContext, + *, + domain_id: str, + branch_effect: str, + state_effect: str, + security_effect: str, + assessment: str, + evidence: str, + next_check: str, +) -> str: + normalized_id = domain_id.strip().upper() + plan = ctx.domain_consequence_plans.get(normalized_id) + if plan is None: + choices = ", ".join(ctx.domain_consequence_plans) or "none; expand a domain first" + return f"ERROR: unknown consequence domain {domain_id!r}. Available: {choices}." + domain = ctx.value_domains.get(normalized_id, {}) + if assessment == "benign" and domain.get("assessment") not in { + "overlap_blocked", + "disjoint", + }: + return ( + "ERROR: an overlapping producer/distinguished domain cannot be marked benign. " + "First prove an extracted producer-blocking guard with record_value_domain, or " + "keep this consequence unresolved while checking the branch-to-effect path." + ) + impact_tokens = [str(token) for token in plan.get("impact_tokens", []) if token] + if assessment == "security_effect_possible" and impact_tokens and not any( + _mentions_identifier(next_check, token) for token in impact_tokens + ): + choices = ", ".join(impact_tokens) + return ( + "ERROR: security_effect_possible next_check must follow an extracted changed " + f"branch toward its memory effect ({choices}); do not switch to an unrelated " + "allocation or index theory." + ) + ctx.domain_consequences[normalized_id] = { + "domain_id": normalized_id, + "branch_effect": branch_effect, + "state_effect": state_effect, + "security_effect": security_effect, + "assessment": assessment, + "evidence": evidence, + "next_check": next_check, + } + return ( + f"Domain consequence saved for {normalized_id}: {assessment}. " + f"Next check: {next_check}" + ) + + +def build_candidate_tools(ctx: HunterContext) -> list[NativeToolSpec]: + """Build one upsert tool that echoes the complete active candidate queue.""" + + def record_candidate( + candidate_id: str, + status: str, + file: str, + line: int = 0, + hypothesis: str = "", + attacker_control: str = "", + invariant: str = "", + effect: str = "", + counterargument: str = "", + next_check: str = "", + evidence: str = "", + **_: object, + ) -> str: + ctx.candidates[candidate_id] = { + "candidate_id": candidate_id, + "status": status, + "file": file, + "line": line, + "hypothesis": hypothesis, + "attacker_control": attacker_control, + "invariant": invariant, + "effect": effect, + "counterargument": counterargument, + "next_check": next_check, + "evidence": evidence, + } + candidate_text = " ".join((hypothesis, invariant, evidence)).casefold() + for domain_id, domain in ctx.value_domains.items(): + consequence = ctx.domain_consequences.get(domain_id, {}) + if consequence.get("assessment") == "benign": + continue + if candidate_matches_domain(candidate_text, domain): + ctx.domain_candidate_ids[domain_id] = candidate_id + if domain.get("assessment") in {"overlap_possible", "unresolved"}: + ctx.candidates[candidate_id]["next_check"] = _domain_next_check( + ctx, + domain_id, + domain, + ) + ctx.candidate_revision += 1 + active = [ + candidate + for candidate in ctx.candidates.values() + if candidate["status"] in {"pending", "investigating", "validated"} + ] + lines = [f"Candidate {candidate_id} saved. Active queue ({len(active)}):"] + for candidate in active: + location = candidate["file"] + if candidate["line"]: + location += f":{candidate['line']}" + lines.append( + f"- {candidate['candidate_id']} [{candidate['status']}] {location}: " + f"{candidate['hypothesis']} | next: {candidate['next_check'] or 'unspecified'}" + ) + return "\n".join(lines) + + def record_value_domain( + domain_id: str, + guard: str, + assessment: str, + evidence: str, + next_check: str, + **_: object, + ) -> str: + normalized_id = domain_id.strip().upper() + plan = ctx.value_domain_plans.get(normalized_id) + if plan is None: + choices = ", ".join(ctx.value_domain_plans) or "none; read a state packet first" + return f"ERROR: unknown value domain {domain_id!r}. Available: {choices}." + normalized = str(plan["stored_state"]) + normalized_guard = guard.strip() + closure_error = _domain_closure_error( + plan, + guard=normalized_guard, + assessment=assessment, + evidence=evidence, + ) + if closure_error is not None: + return closure_error + ctx.value_domains[normalized_id] = { + **plan, + "domain_id": normalized_id, + "guard": normalized_guard, + "assessment": assessment, + "evidence": evidence, + "next_check": next_check, + } + return ( + f"Value-domain comparison saved for {normalized_id} ({normalized}): {assessment}. " + f"Next check: {next_check}" + ) + + def record_domain_consequence( + domain_id: str, + branch_effect: str, + state_effect: str, + security_effect: str, + assessment: str, + evidence: str, + next_check: str, + **_: object, + ) -> str: + return _record_domain_consequence( + ctx, + domain_id=domain_id, + branch_effect=branch_effect, + state_effect=state_effect, + security_effect=security_effect, + assessment=assessment, + evidence=evidence, + next_check=next_check, + ) + + def record_domain_proof( + domain_id: str, + candidate_id: str, + attacker_reaches_producer: bool, + producer_reaches_distinguished: bool, + changed_branch_reaches_effect: bool, + boundary_effect_unguarded: bool, + evidence: str, + counterevidence: str = "", + **_: object, + ) -> str: + return _record_domain_proof( + ctx, + domain_id=domain_id, + candidate_id=candidate_id, + attacker_reaches_producer=attacker_reaches_producer, + producer_reaches_distinguished=producer_reaches_distinguished, + changed_branch_reaches_effect=changed_branch_reaches_effect, + boundary_effect_unguarded=boundary_effect_unguarded, + evidence=evidence, + counterevidence=counterevidence, + ) + + return [ + NativeToolSpec( + name="record_candidate", + description=( + "Create or update one vulnerability hypothesis in the explicit candidate ledger. " + "Use the same candidate_id to update status, counterevidence, and the next check." + ), + schema=RecordCandidateInput.model_json_schema(), + handler=record_candidate, + ), + NativeToolSpec( + name="record_value_domain", + description=( + "Compare one packet state's storage, distinguished states, producer domain, " + "uses, and guards before selecting a vulnerability mechanism." + ), + schema=RecordValueDomainInput.model_json_schema(), + handler=record_value_domain, + ), + NativeToolSpec( + name="record_domain_consequence", + description=( + "Record whether an overlapping value domain changes a consumer branch and " + "propagates to a security-sensitive effect." + ), + schema=RecordDomainConsequenceInput.model_json_schema(), + handler=record_domain_consequence, + ), + NativeToolSpec( + name="record_domain_proof", + description=( + "Resolve the tracked domain candidate with four explicit proof obligations. " + "A complete proof validates the candidate and seeds exact trace steps." + ), + schema=RecordDomainProofInput.model_json_schema(), + handler=record_domain_proof, + ), + ] + + +__all__ = ["build_candidate_tools", "candidate_matches_domain"] diff --git a/clearwing/agent/tools/hunt/reporting.py b/clearwing/agent/tools/hunt/reporting.py index 37b6162f..33686aed 100644 --- a/clearwing/agent/tools/hunt/reporting.py +++ b/clearwing/agent/tools/hunt/reporting.py @@ -59,6 +59,10 @@ class CompatibilityTraceInput(ToolInputModel): class RecordFindingInput(ToolInputModel): + candidate_id: str = Field( + default="", + description="Validated candidate ledger ID when the active scaffold requires one", + ) file: str line_number: int finding_type: str @@ -168,6 +172,7 @@ def record_finding( crypto_attack_class: str = "", key_material_exposed: str = "", trace: dict | None = None, + candidate_id: str = "", **_: object, ) -> str: """Record a finding into the hunter's state. @@ -203,6 +208,31 @@ def record_finding( trace: Optional compatibility trace or summary. Streamed trace steps take precedence when present. """ + if ctx.require_active_candidate_before_finding: + candidate = ctx.candidates.get(candidate_id) + if candidate is None or candidate.get("status") not in { + "pending", + "investigating", + "validated", + }: + return ( + "ERROR: this scaffold requires candidate_id for an active candidate. " + "Do not submit a rejected or absent hypothesis; keep its strongest " + "counterargument and one unresolved next check in the ledger." + ) + if ctx.require_validated_candidate_before_finding: + candidate = ctx.candidates.get(candidate_id) + if candidate is None or candidate.get("status") != "validated": + return ( + "ERROR: this scaffold requires candidate_id for a candidate already marked " + "validated with record_candidate. Resolve its counterargument and exact " + "entry-to-effect trace before submitting." + ) + if evidence_level == "suspicion": + return ( + "ERROR: a validated candidate requires static_corroboration or stronger " + "evidence_level; unresolved suspicion remains in the candidate ledger." + ) explicit_steps = trace.get("steps", []) if trace else [] try: authoritative_steps = ( @@ -256,6 +286,8 @@ def record_finding( }, ) finding_metadata = {"stable_finding_id": stable_finding_id} + if candidate_id: + finding_metadata["candidate_id"] = candidate_id if ctx.work_item_id: finding_metadata["work_item_id"] = ctx.work_item_id diff --git a/clearwing/agent/tools/hunt/sandbox.py b/clearwing/agent/tools/hunt/sandbox.py index 176cb2bf..92e93f34 100644 --- a/clearwing/agent/tools/hunt/sandbox.py +++ b/clearwing/agent/tools/hunt/sandbox.py @@ -30,6 +30,23 @@ class HunterContext: repo_path: str # absolute host path sandbox: SandboxContainer | None = None # primary sandbox; set by hunt loop findings: list[Finding] = field(default_factory=list) + candidates: dict[str, dict] = field(default_factory=dict) + candidate_revision: int = 0 + source_windows_ranked: bool = False + source_window_plan: dict[str, dict] = field(default_factory=dict) + source_windows_read: set[str] = field(default_factory=set) + state_packets_read: set[str] = field(default_factory=set) + value_domain_plans: dict[str, dict] = field(default_factory=dict) + value_domains: dict[str, dict] = field(default_factory=dict) + domain_consequence_plans: dict[str, dict] = field(default_factory=dict) + domain_consequences: dict[str, dict] = field(default_factory=dict) + domain_candidate_ids: dict[str, str] = field(default_factory=dict) + domain_proof_obligations: dict[str, list[str]] = field(default_factory=dict) + domain_refinements_read: set[tuple[str, str]] = field(default_factory=set) + domain_refinement_pending_proof: set[str] = field(default_factory=set) + enable_domain_proof_refinement: bool = False + require_validated_candidate_before_finding: bool = False + require_active_candidate_before_finding: bool = False trace_steps: list = field(default_factory=list) # accumulator for TraceStep dicts files_read: set = field(default_factory=set) # files accessed via read_source_file agent_mode: str = "constrained" # "constrained" | "deep"; deep reads via shell so files_read is not authoritative @@ -48,6 +65,7 @@ class HunterContext: trajectory_dir: object | None = None # Path override for transcript output work_item_id: str | None = None # Stable run-local join key for evaluation instrumentation: object | None = None # SourceHuntInstrumentation, kept generic + context_profile: str = "legacy-context-v1" exploit_result: object | None = None # ExploiterResult slot for exploit agent elaboration_result: object | None = None # ElaborationResult slot for elaboration agent diff --git a/clearwing/agent/tools/hunt/windows.py b/clearwing/agent/tools/hunt/windows.py new file mode 100644 index 00000000..a6ebd5de --- /dev/null +++ b/clearwing/agent/tools/hunt/windows.py @@ -0,0 +1,1184 @@ +"""Generic security-relevant source-window ranking for small-model scaffolds.""" + +from __future__ import annotations + +import re +from collections import Counter +from pathlib import Path +from typing import Literal + +from pydantic import Field + +from clearwing.llm import NativeToolSpec, ToolInputModel +from clearwing.sourcehunt.static_signals import ( + is_production_source_path, + line_security_signals, +) + +from .discovery import _normalize_path +from .sandbox import HunterContext + + +class RankSourceWindowsInput(ToolInputModel): + path: str = Field(description="Repo-relative source file") + max_windows: int = Field(default=12, ge=1, le=30) + window_lines: int = Field(default=80, ge=20, le=200) + + +class ReadRankedWindowInput(ToolInputModel): + window_id: str = Field(description="Opaque ID returned by rank_source_windows, for example W1") + + +class ReadStateInteractionsInput(ToolInputModel): + window_id: str = Field(description="Ranked window whose dominant state should be expanded") + + +class ReadDomainConsequencesInput(ToolInputModel): + domain_id: str = Field(description="Recorded overlapping domain, for example D1") + + +class ReadDomainProofRefinementInput(ToolInputModel): + domain_id: str = Field(description="Tracked unresolved domain, for example D1") + obligation: Literal[ + "attacker_reaches_producer", + "producer_reaches_distinguished", + "changed_branch_reaches_effect", + "boundary_effect_unguarded", + ] = Field(description="One obligation returned unresolved by record_domain_proof") + + +_MEMBER_RE = re.compile(r"(?:->|\.)\s*([A-Za-z_]\w*)") +_SOURCE_SUFFIXES = frozenset({".c", ".cc", ".cpp", ".cxx", ".h", ".hh", ".hpp", ".rs"}) +_STATE_STOPWORDS = frozenset( + { + "const", + "else", + "false", + "return", + "sizeof", + "struct", + "true", + } +) +_INTERACTION_SCORES = { + "declare": 10, + "initialize": 12, + "write": 11, + "compare": 9, + "read": 2, +} +_CONSUMER_BRANCH_RE = re.compile(r"(?:==|!=|\?|\bif\s*\(|\bswitch\s*\()") +_CONSEQUENCE_EFFECT_RE = re.compile( + r"\b(?:memcpy|memmove|memset|strcpy|strncpy|sprintf|snprintf|" + r"XCHG|[A-Z][A-Z0-9_]*(?:COPY|MOVE|XCHG|WRITE|STORE)[A-Z0-9_]*)\s*\(" +) +_IDENTIFIER_RE = re.compile(r"\b[A-Za-z_]\w*\b") +_FUNCTION_DEFINITION_RE = re.compile( + r"(?m)^[ \t]*(?:(?:[A-Za-z_]\w*|\*+)[ \t]+)+" + r"(?P[A-Za-z_]\w*)[ \t]*\([^;{}]*\)[ \t\r\n]*\{" +) +_CALL_RE = re.compile(r"\b([A-Za-z_]\w*)\s*\(") +_LOOP_RE = re.compile(r"\b(?:for|while)\s*\(") + + +def _dominant_anchor_state(lines: list[str], anchor_line: int) -> str | None: + """Choose a mutable state name from an anchor without project knowledge.""" + + line = lines[anchor_line - 1] + scores: Counter[str] = Counter(_MEMBER_RE.findall(line)) + first_argument = re.search( + r"\b(?:memcpy|memmove|memset|strcpy|strncpy|sprintf|snprintf)\s*\(([^,]+)", + line, + ) + if first_argument: + destination_names = _MEMBER_RE.findall(first_argument.group(1)) + if destination_names: + scores[destination_names[0]] += 40 + for name in list(scores): + if name in _STATE_STOPWORDS or len(name) < 3: + del scores[name] + if not scores: + return None + return min(scores, key=lambda name: (-scores[name], name)) + + +def _interaction_kind(line: str, name: str) -> str: + escaped = re.escape(name) + if re.search(rf"\b(?:memset|memcpy|memmove)\s*\([^,]*\b{escaped}\b", line): + return "initialize" + if re.search(rf"\b{escaped}\b(?:\s*\[[^]]*\])?\s*=(?!=)", line): + return "write" + if re.search( + rf"\b(?:u?int(?:8|16|32|64)_t|char|short|int|long|size_t)\b[^=;]*\b{escaped}\b", + line, + ): + return "declare" + without_member_arrows = line.replace("->", ".") + if re.search(rf"\b{escaped}\b[^;]*(?:==|!=|<=|>=|<|>)", without_member_arrows): + return "compare" + return "read" + + +def _source_files(repo: Path, target: Path) -> list[Path]: + """Bound expansion to the target's generic source subsystem.""" + + root = target.parent + return [ + path + for path in sorted(root.rglob("*")) + if path.is_file() + and path.suffix.lower() in _SOURCE_SUFFIXES + and is_production_source_path(path.relative_to(repo).as_posix()) + and path.stat().st_size <= 2_000_000 + ] + + +def _read_source_lines(path: Path) -> list[str]: + try: + return path.read_text(encoding="utf-8", errors="replace").splitlines() + except OSError: + return [] + + +def _following_context(repo: Path, item: dict, *, lines: int = 3) -> str: + source = _read_source_lines(repo / item["path"]) + start = int(item["line"]) + return " ".join( + source[index - 1].strip() + for index in range(start, min(len(source), start + lines - 1) + 1) + ) + + +def _derived_effect_chain(source: list[str], branch_line: int, stored_state: str) -> dict: + branch = source[branch_line - 1] + assignment = re.search( + rf"\b([A-Za-z_]\w*)\s*=(?!=)[^;]*\b{re.escape(stored_state)}\b", + branch, + ) + if assignment is None: + return {} + derived = assignment.group(1) + scan_end = min(len(source), branch_line + 80) + dependent_guards = [ + line_number + for line_number in range(branch_line + 1, scan_end + 1) + if re.search(rf"\bif\s*\([^)]*\b{re.escape(derived)}\b", source[line_number - 1]) + ] + if not dependent_guards: + return {} + effects: list[int] = [] + for guard_line in dependent_guards: + effects.extend( + line_number + for line_number in range(guard_line + 1, min(scan_end, guard_line + 24) + 1) + if _CONSEQUENCE_EFFECT_RE.search(source[line_number - 1]) + ) + effects = list(dict.fromkeys(effects)) + if not effects: + return {} + effect_names = { + match.group(0).split("(", 1)[0].strip() + for line_number in effects + if (match := _CONSEQUENCE_EFFECT_RE.search(source[line_number - 1])) + } + effect_identifiers = { + identifier + for line_number in effects + for identifier in _IDENTIFIER_RE.findall(source[line_number - 1]) + if identifier not in effect_names + } + setup: list[int] = [] + for line_number in range(branch_line + 1, min(effects) + 1): + line = source[line_number - 1] + assigned = re.search(r"\b([A-Za-z_]\w*)\s*=(?!=)", line) + if assigned and assigned.group(1) in effect_identifiers: + setup.append(line_number) + tokens = [derived, *sorted(effect_names)] + tokens.extend( + sorted( + { + re.search(r"\b([A-Za-z_]\w*)\s*=(?!=)", source[line_number - 1]).group(1) + for line_number in setup + } + ) + ) + boundary_facts: list[dict] = [] + for line_number in [branch_line, *setup, *effects]: + line = source[line_number - 1] + for match in re.finditer( + r"\[([^]]*?\b(?:[A-Za-z_]\w*(?:->|\.)\s*)?([A-Za-z_]\w*)\s*-\s*[1-9]\d*[^]]*)\]", + line, + ): + boundary_facts.append( + { + "line": line_number, + "token": match.group(2), + "expression": match.group(1).strip(), + } + ) + return { + "derived": derived, + "branch_line": branch_line, + "guard_lines": dependent_guards[:3], + "setup_lines": setup[:4], + "effect_lines": effects[:5], + "impact_tokens": list(dict.fromkeys(tokens)), + "boundary_facts": boundary_facts, + } + + +def _domain_consequence_packet( + repo: Path, + plan: dict, + *, + max_lines: int = 8, +) -> tuple[str, dict]: + stored_state = str(plan.get("stored_state", "")) + producer_state = str(plan.get("producer_state", "")) + distinguished = [str(value) for value in plan.get("distinguished_tokens", []) if value] + roots = _source_files(repo, repo / plan["target_path"]) + matches: list[dict] = [] + chains: list[dict] = [] + for path in roots: + rel = path.relative_to(repo).as_posix() + source = _read_source_lines(path) + for line_number, line in enumerate(source, start=1): + if not stored_state or not re.search(rf"\b{re.escape(stored_state)}\b", line): + continue + if not _CONSUMER_BRANCH_RE.search(line): + continue + if re.search(r"\b(?:memcpy|memmove|memset)\s*\(", line) and not re.search( + r"\b(?:if|switch)\s*\(", line + ): + continue + downstream = [] + for candidate_line, candidate in enumerate( + source[line_number : line_number + 80], + start=line_number + 1, + ): + signals = line_security_signals(candidate) + if signals: + downstream.append( + ( + sum(weight for _signal, weight in signals), + candidate_line, + candidate.strip(), + ) + ) + downstream = sorted(downstream, key=lambda value: (-value[0], value[1]))[:5] + context = " | ".join( + [f"branch {line_number}: {line.strip()}"] + + [f"effect {number}: {text}" for _score, number, text in downstream] + ) + signal_score = sum(score for score, _number, _text in downstream) + effect_score = 12 * sum( + bool(_CONSEQUENCE_EFFECT_RE.search(text)) + for _score, _number, text in downstream + ) + matches.append( + { + "path": rel, + "line": line_number, + "text": context, + "score": signal_score + + effect_score + + 12 * int(bool(producer_state and producer_state in line)) + + 8 * sum(value.casefold() in context.casefold() for value in distinguished), + } + ) + chain = _derived_effect_chain(source, line_number, stored_state) + if chain: + chains.append({"path": rel, **chain}) + selected: list[dict] = [] + for item in sorted( + matches, + key=lambda value: (-value["score"], value["path"], value["line"]), + ): + if any( + item["path"] == prior["path"] and abs(item["line"] - prior["line"]) < 8 + for prior in selected + ): + continue + selected.append(item) + if len(selected) >= max_lines: + break + body = "\n".join( + f"C{index} | {item['path']}:{item['line']} | {item['text'][:480]}" + for index, item in enumerate(selected, start=1) + ) + chain_lines: list[str] = [] + for index, chain in enumerate(chains[:3], start=1): + path = chain["path"] + source = _read_source_lines(repo / path) + parts = [ + f"branch {path}:{chain['branch_line']} {source[chain['branch_line'] - 1].strip()}" + ] + parts.extend( + f"setup {path}:{line} {source[line - 1].strip()}" for line in chain["setup_lines"] + ) + parts.extend( + f"gate {path}:{line} {source[line - 1].strip()}" for line in chain["guard_lines"] + ) + parts.extend( + f"effect {path}:{line} {source[line - 1].strip()}" + for line in chain["effect_lines"] + ) + parts.extend( + f"boundary {path}:{fact['line']} [{fact['expression']}] can underflow when " + f"{fact['token']} is at its lower bound; check a dominating positive-bound guard" + for fact in chain["boundary_facts"] + ) + chain_lines.append(f"P{index} | " + " | ".join(parts)) + best_chain = chains[0] if chains else {} + consequence_plan = { + "domain_id": plan.get("domain_id", ""), + "impact_tokens": best_chain.get("impact_tokens", []), + "impact_locations": [ + f"{best_chain['path']}:{line}" + for key in ("guard_lines", "effect_lines") + for line in best_chain.get(key, []) + ] + if best_chain + else [], + "boundary_facts": best_chain.get("boundary_facts", []), + "trace_facts": ( + [ + { + "role": "condition", + "file": best_chain["path"], + "line": best_chain["branch_line"], + "code_snippet": _read_source_lines(repo / best_chain["path"])[ + best_chain["branch_line"] - 1 + ].strip(), + }, + *[ + { + "role": "effect sink", + "file": best_chain["path"], + "line": line, + "code_snippet": _read_source_lines(repo / best_chain["path"])[ + line - 1 + ].strip(), + } + for line in best_chain.get("effect_lines", [])[:1] + ], + ] + if best_chain + else [] + ), + } + rendered = ( + "Domain consequence packet (orientation, not evidence). Compare the normal and " + "overlapping branch outcomes, then follow accepted state into a memory, lifetime, " + "privilege, or availability effect.\n" + f"Derived branch-to-effect chains:\n{chr(10).join(chain_lines) or 'None extracted.'}\n" + f"Ranked consumer contexts:\n{body or 'No consumer branches found.'}" + ) + return rendered, consequence_plan + + +def _function_index(repo: Path, paths: list[Path]) -> dict[str, list[dict]]: + """Index ordinary C-family function definitions without compiler metadata.""" + + by_path: dict[str, list[dict]] = {} + for path in paths: + rel = path.relative_to(repo).as_posix() + try: + text = path.read_text(encoding="utf-8", errors="replace") + except OSError: + continue + definitions = [ + { + "name": match.group("name"), + "path": rel, + "line": text.count("\n", 0, match.start("name")) + 1, + } + for match in _FUNCTION_DEFINITION_RE.finditer(text) + ] + line_count = len(text.splitlines()) + for index, definition in enumerate(definitions): + definition["end_line"] = ( + definitions[index + 1]["line"] - 1 + if index + 1 < len(definitions) + else line_count + ) + if definitions: + by_path[rel] = definitions + return by_path + + +def _containing_function(index: dict[str, list[dict]], path: str, line: int) -> dict | None: + return next( + ( + definition + for definition in index.get(path, []) + if int(definition["line"]) <= line <= int(definition["end_line"]) + ), + None, + ) + + +def _attacker_reachability_facts(repo: Path, domain: dict, *, max_facts: int = 6) -> list[dict]: + """Walk a small reverse call chain from the extracted producer function.""" + + source_fact = next( + (fact for fact in domain.get("trace_facts", []) if fact.get("role") == "source"), + None, + ) + if source_fact is None: + return [] + target = repo / str(domain["target_path"]) + paths = _source_files(repo, target) + index = _function_index(repo, paths) + producer_function = _containing_function( + index, + str(source_fact["file"]), + int(source_fact["line"]), + ) + if producer_function is None: + return [] + + known_functions = { + str(definition["name"]) + for definitions in index.values() + for definition in definitions + } + reverse_calls: dict[str, list[dict]] = {} + for path, definitions in index.items(): + lines = _read_source_lines(repo / path) + for definition in definitions: + start = int(definition["line"]) + end = min(int(definition["end_line"]), len(lines)) + for line_number in range(start, end + 1): + text = lines[line_number - 1] + for callee in _CALL_RE.findall(text): + if callee not in known_functions: + continue + if callee == definition["name"] and line_number == start: + continue + reverse_calls.setdefault(callee, []).append( + { + "caller": definition["name"], + "file": path, + "line": line_number, + "code_snippet": text.strip(), + "function_start": start, + } + ) + + facts: list[dict] = [] + queued = [str(producer_function["name"])] + seen = set(queued) + while queued and len(facts) < max_facts: + callee = queued.pop(0) + for call in reverse_calls.get(callee, []): + facts.append( + { + "role": "dispatch", + "file": call["file"], + "line": call["line"], + "code_snippet": call["code_snippet"], + } + ) + lines = _read_source_lines(repo / str(call["file"])) + loop_line = next( + ( + line_number + for line_number in range( + int(call["line"]) - 1, + max(int(call["function_start"]), int(call["line"]) - 80) - 1, + -1, + ) + if _LOOP_RE.search(lines[line_number - 1]) + ), + None, + ) + if loop_line is not None and len(facts) < max_facts: + facts.append( + { + "role": "repetition", + "file": call["file"], + "line": loop_line, + "code_snippet": lines[loop_line - 1].strip(), + } + ) + caller = str(call["caller"]) + if caller not in seen: + seen.add(caller) + queued.append(caller) + if len(facts) >= max_facts: + break + facts.reverse() + return facts + + +def _producer_domain_facts(repo: Path, domain: dict, *, max_facts: int = 6) -> list[dict]: + target = repo / str(domain["target_path"]) + paths = _source_files(repo, target) + selected: list[dict] = [] + seen: set[tuple[str, int]] = set() + for token in domain.get("producer_tokens", []): + occurrences = _state_occurrences(repo, paths, str(token), target=target) + for kind in ("write", "compare", "declare", "initialize"): + item = next((candidate for candidate in occurrences if candidate["kind"] == kind), None) + if item is None or (item["path"], item["line"]) in seen: + continue + context = _following_context(repo, item, lines=8 if kind == "compare" else 2) + role = kind + if kind == "compare": + role = ( + "terminating comparison" + if re.search(r"\b(?:return|break|continue|goto)\b", context) + else "non-terminating comparison" + ) + selected.append( + { + "role": role, + "file": item["path"], + "line": item["line"], + "code_snippet": context, + } + ) + seen.add((item["path"], item["line"])) + if len(selected) >= max_facts: + return selected + return selected + + +def _effect_chain_facts(repo: Path, consequence: dict, *, max_facts: int = 6) -> list[dict]: + facts = list(consequence.get("trace_facts", [])) + seen = {(str(fact.get("file")), int(fact.get("line", 0))) for fact in facts} + for location in consequence.get("impact_locations", []): + path, separator, line_text = str(location).rpartition(":") + if not separator or not line_text.isdigit() or (path, int(line_text)) in seen: + continue + lines = _read_source_lines(repo / path) + line = int(line_text) + if 1 <= line <= len(lines): + facts.append( + { + "role": "effect path", + "file": path, + "line": line, + "code_snippet": lines[line - 1].strip(), + } + ) + seen.add((path, line)) + if len(facts) >= max_facts: + break + return facts[:max_facts] + + +def _boundary_guard_facts(repo: Path, consequence: dict, *, max_lines: int = 22) -> list[dict]: + boundary = next(iter(consequence.get("boundary_facts", [])), None) + condition = next( + (fact for fact in consequence.get("trace_facts", []) if fact.get("role") == "condition"), + None, + ) + if boundary is None or condition is None: + return [] + path = str(condition["file"]) + lines = _read_source_lines(repo / path) + boundary_line = int(boundary["line"]) + condition_line = int(condition["line"]) + start = max(1, min(condition_line, boundary_line) - 2) + end = min(len(lines), max(condition_line, boundary_line) + 3, start + max_lines - 1) + return [ + { + "role": "boundary path", + "file": path, + "line": line_number, + "code_snippet": lines[line_number - 1].strip(), + } + for line_number in range(start, end + 1) + ] + + +def _domain_proof_refinement_packet( + repo: Path, + domain: dict, + consequence: dict, + obligation: str, +) -> str: + extractors = { + "attacker_reaches_producer": lambda: _attacker_reachability_facts(repo, domain), + "producer_reaches_distinguished": lambda: _producer_domain_facts(repo, domain), + "changed_branch_reaches_effect": lambda: _effect_chain_facts(repo, consequence), + "boundary_effect_unguarded": lambda: _boundary_guard_facts(repo, consequence), + } + facts = extractors[obligation]() + body = "\n".join( + f"R{index} {fact['role']} | {fact['file']}:{fact['line']} | " + f"{fact['code_snippet'][:420]}" + for index, fact in enumerate(facts, start=1) + ) + return ( + f"Proof refinement for {obligation} (source-derived orientation only).\n" + f"{body or 'No additional facts extracted; keep this obligation false.'}\n" + "Re-evaluate only this obligation, then call record_domain_proof again." + ) + + +def _state_occurrences( + repo: Path, + paths: list[Path], + name: str, + *, + target: Path, + anchor_line: int | None = None, +) -> list[dict]: + pattern = re.compile(rf"\b{re.escape(name)}\b") + occurrences: list[dict] = [] + for path in paths: + rel = path.relative_to(repo).as_posix() + lines = _read_source_lines(path) + for line_number, line in enumerate(lines, start=1): + if not pattern.search(line): + continue + kind = _interaction_kind(line, name) + score = _INTERACTION_SCORES[kind] + sum( + weight for _signal, weight in line_security_signals(line) + ) + if path == target: + score += 5 + if path == target and line_number == anchor_line: + score += 20 + occurrences.append( + { + "path": rel, + "line": line_number, + "kind": kind, + "text": line.strip(), + "score": score, + } + ) + return sorted( + occurrences, + key=lambda item: (-item["score"], item["path"], item["line"]), + ) + + +def _select_primary_interactions(occurrences: list[dict], *, max_lines: int) -> list[dict]: + selected: list[dict] = [] + seen: set[tuple[str, int]] = set() + + def add(item: dict | None) -> None: + if item is None: + return + location = (item["path"], item["line"]) + if location not in seen and len(selected) < max_lines: + selected.append(item) + seen.add(location) + + for kind in ("declare", "initialize", "write", "compare"): + add(next((item for item in occurrences if item["kind"] == kind), None)) + for item in occurrences: + add(item) + if len(selected) >= 10: + break + return selected + + +def _related_state_names(primary: str, interactions: list[dict]) -> list[str]: + scores: Counter[str] = Counter() + for item in interactions: + if item["kind"] not in {"initialize", "write", "compare"}: + continue + for name in _MEMBER_RE.findall(item["text"]): + if name != primary and name not in _STATE_STOPWORDS and len(name) >= 3: + scores[name] += 1 + return sorted(scores, key=lambda name: (-scores[name], name))[:1] + + +def _best_related_producer( + repo: Path, + paths: list[Path], + name: str, + *, + target: Path, +) -> dict | None: + occurrences = _state_occurrences(repo, paths, name, target=target) + for kind in ("write", "declare", "initialize"): + item = next((item for item in occurrences if item["kind"] == kind), None) + if item is not None: + return item + return None + + +def _related_producer_chain( + repo: Path, + paths: list[Path], + names: list[str], + *, + target: Path, + excluded_names: set[str], +) -> list[dict]: + producers: list[dict] = [] + queued = list(names) + seen_names = set(names) + while queued and len(producers) < 8: + name = queued.pop(0) + item = _best_related_producer(repo, paths, name, target=target) + if item is None: + continue + occurrences = _state_occurrences(repo, paths, name, target=target) + declaration = next( + (candidate for candidate in occurrences if candidate["kind"] == "declare"), + None, + ) + if declaration is not None and declaration != item: + producers.append({**declaration, "kind": f"{name}:declare"}) + producers.append({**item, "kind": f"{name}:{item['kind']}"}) + guard = next( + ( + candidate + for candidate in occurrences + if candidate["kind"] == "compare" + ), + None, + ) + if guard is not None: + producers.append({**guard, "kind": f"{name}:compare"}) + for upstream in _MEMBER_RE.findall(item["text"]): + if ( + upstream != name + and upstream not in excluded_names + and upstream not in seen_names + and upstream not in _STATE_STOPWORDS + and len(upstream) >= 3 + ): + queued.append(upstream) + seen_names.add(upstream) + return producers + + +def _state_interaction_packet( + repo: Path, + planned: dict, + *, + max_lines: int = 18, +) -> tuple[str, dict]: + target = repo / planned["path"] + target_lines = target.read_text(encoding="utf-8", errors="replace").splitlines() + primary = _dominant_anchor_state(target_lines, int(planned["anchor_line"])) + if primary is None: + return ( + "No stable state identifier could be extracted from this anchor; read another window.", + {}, + ) + + paths = _source_files(repo, target) + occurrences = _state_occurrences( + repo, + paths, + primary, + target=target, + anchor_line=int(planned["anchor_line"]), + ) + selected = _select_primary_interactions(occurrences, max_lines=max_lines) + related = _related_state_names(primary, selected) + selected_locations = {(item["path"], item["line"]) for item in selected} + related_chain = _related_producer_chain( + repo, + paths, + related, + target=target, + excluded_names={primary}, + ) + for item in related_chain: + if (item["path"], item["line"]) in selected_locations: + continue + selected.append(item) + selected_locations.add((item["path"], item["line"])) + if len(selected) >= max_lines: + break + + body = "\n".join( + f"{item['kind']:>16} | {item['path']}:{item['line']} | {item['text'][:220]}" + for item in selected + ) + related_label = ", ".join( + dict.fromkeys(item["kind"].partition(":")[0] for item in related_chain) + ) or "none" + primary_declaration = next( + (item for item in selected if item["kind"] == "declare"), + None, + ) + distinguished = [ + item + for item in selected + if item["kind"] in {"initialize", "compare"} + and ("-1" in item["text"] or re.search(r"0x[fF]+", item["text"])) + ] + transfer = next( + ( + item + for item in selected + if item["kind"] == "write" and any(name in item["text"] for name in related) + ), + None, + ) + direct_name = related[0] if transfer and related else "unknown" + producer_assignment = next( + ( + item + for item in related_chain + if item["kind"] == f"{direct_name}:write" + ), + None, + ) + producer_names = { + name + for name in _MEMBER_RE.findall(producer_assignment["text"] if producer_assignment else "") + if name != direct_name + } + producer_tokens = [ + name + for name in (direct_name, *sorted(producer_names)) + if name and name != "unknown" + ] + producer_facts = [ + item + for item in related_chain + if item["kind"] in {f"{direct_name}:declare", f"{direct_name}:write"} + or ( + item["kind"].endswith(":declare") + and item["kind"].partition(":")[0] in producer_names + ) + ] + guards = [ + item + for item in related_chain + if item["kind"].endswith(":compare") + and item["kind"].partition(":")[0] in {direct_name, *producer_names} + ] + distinguished_tokens = sorted( + { + match.group(0) + for item in distinguished + for match in re.finditer(r"(?:0x[0-9a-fA-F]+|(? list[dict]: + """Rank non-overlapping windows using only language-generic signal classes.""" + + lines = source.splitlines() + anchors: list[tuple[int, int, list[str], str]] = [] + for index, line in enumerate(lines, start=1): + matched_signals = line_security_signals(line) + if not matched_signals: + continue + categories = [name for name, _weight in matched_signals] + score = sum(weight for _name, weight in matched_signals) + anchors.append((score, index, categories, line.strip()[:240])) + + selected: list[dict] = [] + half_window = max(10, window_lines // 2) + for score, line, categories, snippet in sorted( + anchors, + key=lambda item: (-item[0], item[1]), + ): + if any(abs(line - existing["anchor_line"]) < half_window for existing in selected): + continue + selected.append( + { + "start_line": max(1, line - half_window), + "end_line": min(len(lines), line + half_window), + "anchor_line": line, + "score": score, + "signals": categories, + "anchor": snippet, + } + ) + if len(selected) >= max_windows: + break + return selected + + +def build_window_tools(ctx: HunterContext) -> list[NativeToolSpec]: # noqa: C901 + def rank_windows(path: str, max_windows: int = 12, window_lines: int = 80, **_: object): + if ctx.source_window_plan: + return { + "path": next(iter(ctx.source_window_plan.values()))["path"], + "windows": list(ctx.source_window_plan.values()), + "instruction": ( + "The reading plan already exists. Do not rank again. Continue with an unread " + "window_id or the active candidate's next_check." + ), + } + try: + rel = _normalize_path(ctx.repo_path, path) + except ValueError as exc: + return {"error": str(exc)} + target = Path(ctx.repo_path) / rel + try: + source = target.read_text(encoding="utf-8", errors="replace") + except OSError as exc: + return {"error": f"could not read {rel}: {exc}"} + windows = rank_source_windows( + source, + max_windows=max(1, min(30, max_windows)), + window_lines=max(20, min(200, window_lines)), + ) + planned_windows = [] + for index, window in enumerate(windows, start=1): + planned = {"window_id": f"W{index}", "path": rel, **window} + ctx.source_window_plan[planned["window_id"]] = planned + planned_windows.append(planned) + ctx.source_windows_ranked = True + return { + "path": rel, + "line_count": len(source.splitlines()), + "windows": planned_windows, + "instruction": ( + "Read windows only through read_ranked_window(window_id). Start with W1, then " + "choose different signal mixes. Anchors are orientation, not evidence." + ), + } + + def read_window(window_id: str, **_: object) -> str: + normalized_id = window_id.strip().upper() + planned = ctx.source_window_plan.get(normalized_id) + if planned is None: + choices = ", ".join(ctx.source_window_plan) or "none; rank first" + return f"ERROR: unknown ranked window {window_id!r}. Available: {choices}." + if not ctx.source_windows_read and normalized_id != "W1": + return "ERROR: read W1 first; it has the strongest composite signal." + target = Path(ctx.repo_path) / planned["path"] + try: + lines = target.read_text(encoding="utf-8", errors="replace").splitlines() + except OSError as exc: + return f"ERROR: could not read {planned['path']}: {exc}" + start = int(planned["start_line"]) + end = int(planned["end_line"]) + ctx.source_windows_read.add(normalized_id) + seen_signals = { + signal + for read_id in ctx.source_windows_read + for signal in ctx.source_window_plan[read_id]["signals"] + } + unread = [ + item + for item in ctx.source_window_plan.values() + if item["window_id"] not in ctx.source_windows_read + ] + next_diverse = max( + unread, + key=lambda item: ( + len(set(item["signals"]) - seen_signals), + item["score"], + -int(item["window_id"][1:]), + ), + default=None, + ) + body = "\n".join( + f"{line_number:6d} | {lines[line_number - 1]}" + for line_number in range(start, min(end, len(lines)) + 1) + ) + continuation = ( + f"\nNext diverse window: {next_diverse['window_id']} " + f"signals={','.join(next_diverse['signals'])}." + if next_diverse is not None + else "" + ) + return ( + f"{normalized_id} {planned['path']}:{start}-{end} " + f"signals={','.join(planned['signals'])}\n{body}{continuation}" + ) + + def read_state_interactions(window_id: str, **_: object) -> str: + normalized_id = window_id.strip().upper() + planned = ctx.source_window_plan.get(normalized_id) + if planned is None: + choices = ", ".join(ctx.source_window_plan) or "none; rank first" + return f"ERROR: unknown ranked window {window_id!r}. Available: {choices}." + if normalized_id not in ctx.source_windows_read: + return f"ERROR: read {normalized_id} before expanding its state interactions." + if normalized_id in ctx.state_packets_read: + return ( + f"ERROR: state interactions for {normalized_id} were already read. " + "Use the packet to form or update a candidate." + ) + try: + packet, domain_plan = _state_interaction_packet(Path(ctx.repo_path), planned) + except OSError as exc: + return f"ERROR: could not build state interactions for {normalized_id}: {exc}" + ctx.state_packets_read.add(normalized_id) + if domain_plan: + ctx.value_domain_plans["D1"] = domain_plan + return packet + + def read_domain_consequences(domain_id: str, **_: object) -> str: + normalized_id = domain_id.strip().upper() + domain = ctx.value_domains.get(normalized_id) + if domain is None: + choices = ", ".join(ctx.value_domains) or "none; record a domain first" + return f"ERROR: unknown recorded domain {domain_id!r}. Available: {choices}." + if domain.get("assessment") not in {"overlap_possible", "unresolved"}: + return f"Domain {normalized_id} is {domain.get('assessment')}; no consequence expansion needed." + packet, consequence_plan = _domain_consequence_packet(Path(ctx.repo_path), domain) + ctx.domain_consequence_plans[normalized_id] = consequence_plan + return packet + + def read_domain_proof_refinement( + domain_id: str, + obligation: str, + **_: object, + ) -> str: + normalized_id = domain_id.strip().upper() + domain = ctx.value_domains.get(normalized_id) + consequence = ctx.domain_consequence_plans.get(normalized_id) + if domain is None or consequence is None: + return f"ERROR: expand and record domain {normalized_id} before refining its proof." + unresolved = ctx.domain_proof_obligations.get(normalized_id, []) + if not unresolved: + return ( + f"ERROR: call record_domain_proof for {normalized_id} first; " + "no unresolved obligation is recorded." + ) + if obligation not in unresolved: + choices = ", ".join(unresolved) + return f"ERROR: refine only a recorded unresolved obligation: {choices}." + refinement_key = (normalized_id, obligation) + if refinement_key in ctx.domain_refinements_read: + return ( + f"ERROR: refinement for {normalized_id}/{obligation} was already read. " + "Call record_domain_proof again and keep the obligation false if unresolved." + ) + packet = _domain_proof_refinement_packet( + Path(ctx.repo_path), + domain, + consequence, + obligation, + ) + ctx.domain_refinements_read.add(refinement_key) + ctx.domain_refinement_pending_proof.add(normalized_id) + return packet + + return [ + NativeToolSpec( + name="rank_source_windows", + description=( + "Rank security-relevant windows in one source file using generic static " + "signals. This prioritizes where to read; it does not identify vulnerabilities." + ), + schema=RankSourceWindowsInput.model_json_schema(), + handler=rank_windows, + ), + NativeToolSpec( + name="read_ranked_window", + description=( + "Read one source window from the current ranked plan by opaque window_id. " + "This avoids manual line/offset translation." + ), + schema=ReadRankedWindowInput.model_json_schema(), + handler=read_window, + ), + NativeToolSpec( + name="read_state_interactions", + description=( + "Expand one read anchor into a compact target-blind packet of declarations, " + "initialization, writes, comparisons, and one-hop state producers." + ), + schema=ReadStateInteractionsInput.model_json_schema(), + handler=read_state_interactions, + ), + NativeToolSpec( + name="read_domain_consequences", + description=( + "Expand a recorded overlapping value domain into compact consumer-branch " + "contexts and nearby security-sensitive effects." + ), + schema=ReadDomainConsequencesInput.model_json_schema(), + handler=read_domain_consequences, + ), + NativeToolSpec( + name="read_domain_proof_refinement", + description=( + "After record_domain_proof returns false, read a small source-derived packet " + "for exactly one unresolved proof obligation." + ), + schema=ReadDomainProofRefinementInput.model_json_schema(), + handler=read_domain_proof_refinement, + ), + ] + + +__all__ = ["build_window_tools", "rank_source_windows"] diff --git a/clearwing/analysis/source_analyzer.py b/clearwing/analysis/source_analyzer.py index fe7559a7..2fbd579e 100644 --- a/clearwing/analysis/source_analyzer.py +++ b/clearwing/analysis/source_analyzer.py @@ -498,10 +498,14 @@ def __init__( *, max_file_size: int | None = None, respect_gitignore: bool = False, + excluded_roots: list[str | Path] | None = None, ): self.repo_path = repo_path self._temp_dir: tempfile.TemporaryDirectory | None = None self.respect_gitignore = respect_gitignore + self.excluded_roots = tuple( + Path(path).expanduser().resolve() for path in (excluded_roots or []) + ) if max_file_size is not None: self.MAX_FILE_SIZE = max_file_size @@ -607,12 +611,18 @@ def analyze(self, path: str | None = None) -> AnalysisResult: def _iter_source_files(self, root: str): """Yield source file paths, skipping irrelevant directories.""" gitignore = _GitignoreMatcher.from_repo(root) if self.respect_gitignore else None + + def excluded(path: str) -> bool: + resolved = Path(path).resolve() + return any(resolved == root or resolved.is_relative_to(root) for root in self.excluded_roots) + for dirpath, dirnames, filenames in os.walk(root): # Prune skip directories dirnames[:] = [ d for d in dirnames if d not in self.SKIP_DIRS + and not excluded(os.path.join(dirpath, d)) and not (gitignore and gitignore.matches_dir(os.path.join(dirpath, d))) ] @@ -620,6 +630,8 @@ def _iter_source_files(self, root: str): if any(fname.endswith(skip) for skip in self.SKIP_FILES): continue full_path = os.path.join(dirpath, fname) + if excluded(full_path): + continue if gitignore and gitignore.matches_file(full_path): continue try: diff --git a/clearwing/eval/__init__.py b/clearwing/eval/__init__.py index 320fe055..6a3beb90 100644 --- a/clearwing/eval/__init__.py +++ b/clearwing/eval/__init__.py @@ -30,9 +30,44 @@ aggregate_baseline, build_ablation_plan, execute_sourcehunt_run, + include_fixed_negative_cases, inspect_ablation_session, run_ablation_campaign, ) +from .sourcehunt_gepa import ( + ClearwingReflectionLM, + SourceHuntGEPAAdapter, + SourceHuntOptimizationExample, + optimize_sourcehunt_prompt, + score_sourcehunt_observation, +) +from .sourcehunt_lair import ( + LairAdapterDataset, + LairAdapterManifest, + LairGoldenChain, + LairSplitConfig, + RouterTrainingRow, + adapt_lair_goldens, + answer_bearing_terms, + lint_router_rows, + load_lair_goldens, + write_lair_adapter_dataset, +) +from .sourcehunt_lair_gepa import ( + LairValidatorGEPAAdapter, + LairValidatorOptimizationExample, + optimize_lair_validator_prompt, + require_generic_validator_prompt, +) +from .sourcehunt_lair_replicates import aggregate_replicates, wilson_interval +from .sourcehunt_lair_validator import ( + LairValidatorCaseResult, + LairValidatorReplaySummary, + build_lair_validator_finding, + replay_lair_validator_case, + run_lair_validator_replay, + summarize_lair_validator_replay, +) __all__ = [ "CounterfactualExpectation", @@ -41,6 +76,7 @@ "CounterfactualScore", "CutoverDecision", "CutoverMetrics", + "ClearwingReflectionLM", "AblationArm", "AblationLevel", "AblationPlan", @@ -49,19 +85,46 @@ "GroundTruth", "GroundTruthManifest", "IntermediateGroundTruth", + "LairAdapterDataset", + "LairAdapterManifest", + "LairGoldenChain", + "LairSplitConfig", + "LairValidatorCaseResult", + "LairValidatorGEPAAdapter", + "LairValidatorOptimizationExample", + "LairValidatorReplaySummary", "ProofEvalObservation", "ProofFunnel", "RunObservation", "SourceHuntCase", + "SourceHuntGEPAAdapter", + "SourceHuntOptimizationExample", + "RouterTrainingRow", "StageFunnel", "ThreatGroundTruth", "aggregate_baseline", + "adapt_lair_goldens", + "aggregate_replicates", + "answer_bearing_terms", "build_ablation_plan", + "build_lair_validator_finding", "execute_sourcehunt_run", + "include_fixed_negative_cases", + "lint_router_rows", + "load_lair_goldens", "evaluate_cutover", "evaluate_counterfactual_sessions", "inspect_proof_session", "inspect_ablation_session", "run_ablation_campaign", + "replay_lair_validator_case", + "run_lair_validator_replay", + "optimize_sourcehunt_prompt", + "optimize_lair_validator_prompt", + "require_generic_validator_prompt", + "score_sourcehunt_observation", "score_counterfactuals", + "summarize_lair_validator_replay", + "write_lair_adapter_dataset", + "wilson_interval", ] diff --git a/clearwing/eval/sourcehunt.py b/clearwing/eval/sourcehunt.py index 5d7225ac..2f028f38 100644 --- a/clearwing/eval/sourcehunt.py +++ b/clearwing/eval/sourcehunt.py @@ -143,6 +143,28 @@ def case(self, case_id: str) -> SourceHuntCase: raise KeyError(case_id) +def include_fixed_negative_cases(manifest: GroundTruthManifest) -> GroundTruthManifest: + """Add patched snapshots as clean controls when a fixed commit is pinned.""" + + cases = list(manifest.cases) + for case in manifest.cases: + if not case.fixed_commit: + continue + fixed_truth = case.ground_truth.model_copy(update={"expected_decision": "disproven"}) + cases.append( + case.model_copy( + update={ + "id": f"{case.id}-fixed-negative", + "cves": [], + "vulnerable_commit": case.fixed_commit, + "fixed_commit": None, + "ground_truth": fixed_truth, + } + ) + ) + return GroundTruthManifest(cases=cases) + + class AblationLevel(IntEnum): REPOSITORY = 1 TARGET_FILE = 2 @@ -157,6 +179,22 @@ class AblationArm(_EvalModel): flow: Literal["legacy", "proof"] model_tier: Literal["local", "frontier"] model: str = Field(min_length=1) + prompt_bundle: str = "legacy-v1" + scaffold_profile: str = "native-v1" + context_profile: str = "legacy-context-v1" + + @model_validator(mode="after") + def _validate_profiles(self) -> AblationArm: + from clearwing.sourcehunt.optimization import ( + get_context_profile, + get_prompt_bundle, + get_scaffold_profile, + ) + + get_prompt_bundle(self.prompt_bundle) + get_scaffold_profile(self.scaffold_profile) + get_context_profile(self.context_profile) + return self class AblationRunSpec(_EvalModel): @@ -169,12 +207,24 @@ class AblationRunSpec(_EvalModel): flow: Literal["legacy", "proof"] model_tier: Literal["local", "frontier"] model: str = Field(min_length=1) + prompt_bundle: str = "legacy-v1" + scaffold_profile: str = "native-v1" + context_profile: str = "legacy-context-v1" level: AblationLevel replicate: int = Field(default=1, ge=1) hints: dict[str, Any] = Field(default_factory=dict) @model_validator(mode="after") def _assign_ids(self) -> AblationRunSpec: + from clearwing.sourcehunt.optimization import ( + get_context_profile, + get_prompt_bundle, + get_scaffold_profile, + ) + + get_prompt_bundle(self.prompt_bundle) + get_scaffold_profile(self.scaffold_profile) + get_context_profile(self.context_profile) context_payload = { "case_id": self.case_id, "case_digest": self.case_digest, @@ -199,6 +249,9 @@ def _assign_ids(self) -> AblationRunSpec: "flow": self.flow, "model_tier": self.model_tier, "model": self.model, + "prompt_bundle": self.prompt_bundle, + "scaffold_profile": self.scaffold_profile, + "context_profile": self.context_profile, "replicate": self.replicate, }, ) @@ -249,11 +302,23 @@ def _validate_matrix(self) -> AblationPlan: run_ids = [run.id for run in self.runs] if len(run_ids) != len(set(run_ids)): raise ValueError("Ablation plan contains duplicate run IDs") - tiers_by_cell: dict[tuple[str, str, int, int], set[str]] = defaultdict(set) - tier_counts_by_cell: dict[tuple[str, str, int, int], Counter[str]] = defaultdict(Counter) - contexts_by_cell: dict[tuple[str, str, int, int], set[str]] = defaultdict(set) + tiers_by_cell: dict[tuple[str, str, str, str, str, int, int], set[str]] = defaultdict(set) + tier_counts_by_cell: dict[ + tuple[str, str, str, str, str, int, int], Counter[str] + ] = defaultdict(Counter) + contexts_by_cell: dict[ + tuple[str, str, str, str, str, int, int], set[str] + ] = defaultdict(set) for run in self.runs: - cell = (run.case_id, run.flow, int(run.level), run.replicate) + cell = ( + run.case_id, + run.flow, + run.prompt_bundle, + run.scaffold_profile, + run.context_profile, + int(run.level), + run.replicate, + ) tiers_by_cell[cell].add(run.model_tier) tier_counts_by_cell[cell][run.model_tier] += 1 contexts_by_cell[cell].add(run.context_id) @@ -363,6 +428,9 @@ def build_ablation_plan( flow=arm.flow, model_tier=arm.model_tier, model=arm.model, + prompt_bundle=arm.prompt_bundle, + scaffold_profile=arm.scaffold_profile, + context_profile=arm.context_profile, level=level, replicate=replicate, hints=ablation_hints(case, level), @@ -405,6 +473,9 @@ class RunObservation(_EvalModel): flow: Literal["legacy", "proof"] model_tier: Literal["local", "frontier"] model: str = "" + prompt_bundle: str = "legacy-v1" + scaffold_profile: str = "native-v1" + context_profile: str = "legacy-context-v1" level: AblationLevel replicate: int session_dir: str @@ -420,6 +491,10 @@ class RunObservation(_EvalModel): cost_usd: float = Field(default=0.0, ge=0.0) input_tokens: int = Field(default=0, ge=0) output_tokens: int = Field(default=0, ge=0) + model_calls: int = Field(default=0, ge=0) + compaction_count: int = Field(default=0, ge=0) + peak_context_tokens: int = Field(default=0, ge=0) + peak_input_tokens: int = Field(default=0, ge=0) report_claim_count: int = Field(default=0, ge=0) unsupported_claims: int = Field(default=0, ge=0) report_failures: int = Field(default=0, ge=0) @@ -443,6 +518,9 @@ class BaselineGroup(_EvalModel): flow: str model_tier: str model: str + prompt_bundle: str + scaffold_profile: str + context_profile: str level: int runs: int true_positives: int @@ -485,8 +563,8 @@ def markdown(self) -> str: f"- Matrix complete: `{str(self.complete).lower()}`", f"- Runs: {self.observed_runs}/{self.expected_runs}", "", - "| Flow | Tier | Model | Level | Runs | Precision | Recall | Mean cost | Mean tokens | Unsupported claims | Report failures | First failures |", - "|---|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---|", + "| Flow | Tier | Model | Prompt | Scaffold | Context | Level | Runs | Precision | Recall | Mean cost | Mean tokens | Unsupported claims | Report failures | First failures |", + "|---|---|---|---|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---|", ] for group in self.groups: failures = ( @@ -495,7 +573,9 @@ def markdown(self) -> str: ) lines.append( f"| {group.flow} | {group.model_tier} | {group.model or '-'} | " - f"{group.level} | {group.runs} | {group.precision:.3f} | " + f"{group.prompt_bundle} | {group.scaffold_profile} | " + f"{group.context_profile} | {group.level} | " + f"{group.runs} | {group.precision:.3f} | " f"{group.recall:.3f} | ${group.mean_cost_usd:.4f} | " f"{group.mean_tokens:.0f} | {group.unsupported_claims}/" f"{group.report_claims} ({group.unsupported_claim_rate:.3f}) | " @@ -542,7 +622,7 @@ def aggregate_baseline( for run_id in sorted(expected_ids & observed_ids): _validate_observation_against_spec(by_id[run_id], specs_by_id[run_id]) - grouped: dict[tuple[str, str, str, int], list[RunObservation]] = defaultdict(list) + grouped: dict[tuple[str, str, str, str, str, str, int], list[RunObservation]] = defaultdict(list) for observation in observed: if observation.run_id not in expected_ids: continue @@ -551,11 +631,22 @@ def aggregate_baseline( observation.flow, observation.model_tier, observation.model, + observation.prompt_bundle, + observation.scaffold_profile, + observation.context_profile, int(observation.level), ) ].append(observation) groups: list[BaselineGroup] = [] - for (flow, tier, model, level), values in sorted(grouped.items()): + for ( + flow, + tier, + model, + prompt_bundle, + scaffold_profile, + context_profile, + level, + ), values in sorted(grouped.items()): tp = sum(item.true_positives for item in values) fp = sum(item.false_positives for item in values) fn = sum(item.false_negatives for item in values) @@ -569,6 +660,9 @@ def aggregate_baseline( flow=flow, model_tier=tier, model=model, + prompt_bundle=prompt_bundle, + scaffold_profile=scaffold_profile, + context_profile=context_profile, level=level, runs=len(values), true_positives=tp, @@ -657,6 +751,9 @@ def _validate_observation_against_spec( "flow": spec.flow, "model_tier": spec.model_tier, "model": spec.model, + "prompt_bundle": spec.prompt_bundle, + "scaffold_profile": spec.scaffold_profile, + "context_profile": spec.context_profile, "level": spec.level, "replicate": spec.replicate, } @@ -686,6 +783,19 @@ async def execute_sourcehunt_run( output_dir: str | Path, provider_manager: Any, budget_usd: float, + input_price_per_million: float | None = None, + output_price_per_million: float | None = None, + max_hunt_files: int | None = None, + max_hunter_steps: int | None = None, + ranker_chunk_size: int | None = None, + ranker_max_inflight_chunks: int | None = None, + ranker_chunk_max_retries: int | None = None, + max_parallel: int = 4, + sandbox_cpus: float | None = None, + starting_band: str = "fast", + redundancy_override: int = 1, + depth: str = "deep", + no_rank: bool = False, compile_commands: str | None = None, validation_manifest: str | None = None, scheduler_calibration: str | None = None, @@ -693,6 +803,8 @@ async def execute_sourcehunt_run( proof_max_actions: int = 200, proof_max_model_calls: int = 40, proof_max_dynamic_actions: int = 20, + prompt_candidate: str | None = None, + session_id: str | None = None, ) -> RunObservation: """Run one planned arm against a pre-positioned immutable checkout.""" @@ -736,16 +848,33 @@ async def execute_sourcehunt_run( if path and not Path(path).expanduser().is_file(): raise ValueError(f"{label} does not exist for {case.id}: {path}") output_root = Path(output_dir).expanduser().resolve() + actual_session_id = session_id or spec.id runner = SourceHuntRunner( repo_url=case.repository, local_path=str(checkout_path), - depth="deep", + depth=depth, budget_usd=budget_usd, + input_price_per_million=input_price_per_million, + output_price_per_million=output_price_per_million, + max_hunt_files=max_hunt_files, + max_hunter_steps=max_hunter_steps, + ranker_chunk_size=ranker_chunk_size, + ranker_max_inflight_chunks=ranker_max_inflight_chunks, + ranker_chunk_max_retries=ranker_chunk_max_retries, + max_parallel=max_parallel, + sandbox_cpus=sandbox_cpus, + starting_band=starting_band, + redundancy_override=redundancy_override, + no_rank=no_rank, output_dir=str(output_root), output_formats=["sarif", "markdown", "json"], - parent_session_id=spec.id, + parent_session_id=actual_session_id, provider_manager=provider_manager, model_override=spec.model, + prompt_bundle=spec.prompt_bundle, + scaffold_profile=spec.scaffold_profile, + context_profile=spec.context_profile, + prompt_candidate=prompt_candidate, campaign_hint=spec.campaign_hint(), flow=spec.flow, proof_compile_commands=compile_commands, @@ -768,7 +897,7 @@ async def execute_sourcehunt_run( enable_behavior_monitor=False, ) await runner.arun() - return inspect_ablation_session(spec, case, output_root / spec.id) + return inspect_ablation_session(spec, case, output_root / actual_session_id) async def run_ablation_campaign( @@ -1034,6 +1163,9 @@ def _inspect_proof_session( flow=spec.flow, model_tier=spec.model_tier, model=spec.model, + prompt_bundle=spec.prompt_bundle, + scaffold_profile=spec.scaffold_profile, + context_profile=spec.context_profile, level=spec.level, replicate=spec.replicate, session_dir=str(root), @@ -1054,6 +1186,43 @@ def _inspect_proof_session( ) +def _legacy_finding_matches_truth(finding: dict[str, Any], truth: IntermediateGroundTruth) -> bool: + """Require mechanism-bearing evidence, not merely a target file and CWE.""" + + target_files = {path.removeprefix("./") for path in truth.target_files} + finding_file = str(finding.get("file") or "").removeprefix("./") + if target_files and finding_file not in target_files: + return False + if truth.expected_cwes and str(finding.get("cwe") or "") not in set(truth.expected_cwes): + return False + if str(finding.get("evidence_level") or "suspicion") == "suspicion": + return False + + trace = finding.get("vulnerability_trace") or finding.get("trace") or {} + steps = trace.get("steps", []) if isinstance(trace, dict) else [] + if not isinstance(steps, list) or len(steps) < 2: + return False + trace_files = { + str(step.get("file") or "").removeprefix("./") + for step in steps + if isinstance(step, dict) + } + if target_files and not target_files.intersection(trace_files): + return False + + serialized = json.dumps(finding, sort_keys=True, default=str).casefold() + distinctive_symbols = [ + value + for value in [*truth.target_functions, *truth.expected_fact_symbols] + if len(value) >= 4 + ] + required_symbol_hits = min(2, len(distinctive_symbols)) + symbol_hits = sum(value.casefold() in serialized for value in distinctive_symbols) + if symbol_hits < required_symbol_hits: + return False + return bool(str(finding.get("description") or "").strip()) + + def _inspect_legacy_session( spec: AblationRunSpec, case: SourceHuntCase, @@ -1075,8 +1244,7 @@ def _inspect_legacy_session( matched = [ finding for finding in findings - if (not truth.target_files or str(finding.get("file") or "") in set(truth.target_files)) - and (not truth.expected_cwes or str(finding.get("cwe") or "") in set(truth.expected_cwes)) + if isinstance(finding, dict) and _legacy_finding_matches_truth(finding, truth) ] expected_positive = truth.expected_decision == "confirmed" true_positive = int(expected_positive and bool(matched)) @@ -1104,6 +1272,7 @@ def _inspect_legacy_session( total_tokens = int(manifest.get("total_tokens", 0) or 0) output_tokens = int(manifest.get("output_tokens", 0) or 0) input_tokens = int(manifest.get("input_tokens", max(0, total_tokens - output_tokens)) or 0) + context_metrics = manifest.get("context_metrics", {}) or {} unsupported_claims = sum( str(finding.get("evidence_level") or "suspicion") == "suspicion" for finding in findings ) @@ -1114,6 +1283,9 @@ def _inspect_legacy_session( flow=spec.flow, model_tier=spec.model_tier, model=spec.model, + prompt_bundle=spec.prompt_bundle, + scaffold_profile=spec.scaffold_profile, + context_profile=spec.context_profile, level=spec.level, replicate=spec.replicate, session_dir=str(root), @@ -1126,6 +1298,12 @@ def _inspect_legacy_session( cost_usd=float(manifest.get("total_spent", 0.0) or 0.0), input_tokens=input_tokens, output_tokens=output_tokens, + model_calls=int(context_metrics.get("model_calls", 0) or 0), + compaction_count=int(context_metrics.get("compaction_count", 0) or 0), + peak_context_tokens=int( + context_metrics.get("peak_context_tokens_estimate", 0) or 0 + ), + peak_input_tokens=int(context_metrics.get("peak_input_tokens", 0) or 0), report_claim_count=len(findings), unsupported_claims=unsupported_claims, report_failures=report_failures, diff --git a/clearwing/eval/sourcehunt_gepa.py b/clearwing/eval/sourcehunt_gepa.py new file mode 100644 index 00000000..ba8846d7 --- /dev/null +++ b/clearwing/eval/sourcehunt_gepa.py @@ -0,0 +1,493 @@ +"""GEPA adapter for leakage-safe SourceHunt prompt optimization. + +GEPA remains an optional dependency. The adapter itself can be imported and +unit-tested without GEPA; when GEPA is installed it returns the framework's +native ``EvaluationBatch`` object. +""" + +from __future__ import annotations + +import asyncio +import json +from collections.abc import Awaitable, Callable, Mapping, Sequence +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +from clearwing.llm import AsyncLLMClient, ChatMessage +from clearwing.sourcehunt.instrumentation import stable_run_id +from clearwing.sourcehunt.optimization import ( + GENERIC_INSTRUCTIONS_V1, + redact_benchmark_terms, + require_generic_prompt, +) + +from .sourcehunt import ( + AblationLevel, + AblationRunSpec, + GroundTruthManifest, + RunObservation, + SourceHuntCase, + execute_sourcehunt_run, +) + +PROMPT_COMPONENT = "discovery_prompt" + + +class ClearwingReflectionLM: + """Expose Clearwing's native client as GEPA's synchronous LM callable.""" + + def __init__(self, client: AsyncLLMClient, *, max_tokens: int = 8192) -> None: + self.client = client + self.max_tokens = max_tokens + + def __call__(self, prompt: str | list[dict[str, Any]]) -> str: + if isinstance(prompt, str): + system = "You improve generic source-code security audit instructions." + messages = [ChatMessage("user", prompt)] + else: + system_parts = [ + str(item.get("content") or "") for item in prompt if item.get("role") == "system" + ] + system = "\n".join(system_parts) or ( + "You improve generic source-code security audit instructions." + ) + messages = [ + ChatMessage(str(item.get("role") or "user"), str(item.get("content") or "")) + for item in prompt + if item.get("role") != "system" + ] + + async def invoke() -> str: + response = await self.client.achat( + messages=messages, + system=system, + max_tokens=self.max_tokens, + ) + return response.first_text or "\n".join(response.texts) + + try: + asyncio.get_running_loop() + except RuntimeError: + return asyncio.run(invoke()) + raise RuntimeError("ClearwingReflectionLM must run outside an active event loop") + + +@dataclass(frozen=True) +class SourceHuntOptimizationExample: + """One immutable benchmark checkout and its execution configuration.""" + + spec: AblationRunSpec + case: SourceHuntCase + checkout: str | Path + output_dir: str | Path + provider_manager: Any = field(repr=False, compare=False) + budget_usd: float = 1.0 + input_price_per_million: float | None = None + output_price_per_million: float | None = None + max_hunt_files: int | None = None + max_hunter_steps: int | None = None + ranker_chunk_size: int = 25 + ranker_max_inflight_chunks: int = 1 + ranker_chunk_max_retries: int = 1 + max_parallel: int = 4 + sandbox_cpus: float | None = None + starting_band: str = "fast" + redundancy_override: int = 1 + depth: str = "standard" + no_rank: bool = True + compile_commands: str | None = None + validation_manifest: str | None = None + scheduler_calibration: str | None = None + learning_registry: str | None = None + proof_max_actions: int = 200 + proof_max_model_calls: int = 40 + proof_max_dynamic_actions: int = 20 + + def __post_init__(self) -> None: + if self.spec.case_id != self.case.id or self.spec.case_digest != self.case.digest: + raise ValueError("Optimization example plan and case do not match") + if self.spec.prompt_bundle != "generic-security-v1": + raise ValueError("GEPA examples must use the generic-security-v1 prompt bundle") + if self.spec.flow != "legacy": + raise ValueError("GEPA prompt examples must use the legacy discovery flow") + if self.spec.level != AblationLevel.REPOSITORY or self.spec.hints: + raise ValueError("GEPA examples must use blind repository-level ablations") + if self.budget_usd <= 0: + raise ValueError("Optimization examples require a positive budget") + if (self.input_price_per_million is None) != ( + self.output_price_per_million is None + ): + raise ValueError("Optimization examples require both token prices or neither") + if self.max_hunt_files is not None and self.max_hunt_files < 1: + raise ValueError("Optimization examples require a positive max_hunt_files") + if self.max_hunter_steps is not None and self.max_hunter_steps < 1: + raise ValueError("Optimization examples require positive max_hunter_steps") + if self.ranker_chunk_size < 1 or self.ranker_max_inflight_chunks < 1: + raise ValueError("Optimization example ranker bounds must be positive") + if self.ranker_chunk_max_retries < 0: + raise ValueError("Optimization example ranker retries cannot be negative") + if self.max_parallel < 1 or self.redundancy_override < 1: + raise ValueError("Optimization example concurrency and redundancy must be positive") + if self.sandbox_cpus is not None and self.sandbox_cpus < 0: + raise ValueError("Optimization example sandbox_cpus cannot be negative") + if self.depth not in {"standard", "deep"}: + raise ValueError("Optimization example depth must be standard or deep") + + +@dataclass +class _CompatEvaluationBatch: + outputs: list[dict[str, Any]] + scores: list[float] + trajectories: list[dict[str, Any]] | None = None + objective_scores: list[dict[str, float]] | None = None + num_metric_calls: int | None = None + + +def _evaluation_batch(**kwargs: Any) -> Any: + try: + from gepa.core.adapter import EvaluationBatch + except ImportError: + return _CompatEvaluationBatch(**kwargs) + return EvaluationBatch(**kwargs) + + +def _funnel_fraction(observation: RunObservation) -> float: + values = [value for value in observation.funnel.model_dump().values() if value is not None] + return sum(value is True for value in values) / len(values) if values else 0.0 + + +def score_sourcehunt_observation( + observation: RunObservation, + case: SourceHuntCase, +) -> tuple[float, dict[str, float]]: + """Score correctness first, then evidence quality and diagnostic progress.""" + + expected_positive = case.ground_truth.expected_decision == "confirmed" + claims = max(1, observation.report_claim_count) + evidence_quality = max(0.0, 1.0 - observation.unsupported_claims / claims) + report_health = float(observation.report_failures == 0) + if expected_positive: + correctness = float(observation.true_positives > 0) + precision = ( + observation.true_positives / (observation.true_positives + observation.false_positives) + if observation.true_positives + observation.false_positives + else 0.0 + ) + score = ( + 0.70 * correctness + + 0.15 * _funnel_fraction(observation) + + 0.10 * evidence_quality + + 0.05 * report_health + ) + else: + correctness = float(observation.finding_count == 0) + precision = correctness + score = 0.85 * correctness + 0.10 * evidence_quality + 0.05 * report_health + + score = max(0.0, min(1.0, score - min(0.25, 0.05 * observation.false_positives))) + objectives = { + "correctness": correctness, + "precision": precision, + "evidence_quality": evidence_quality, + "funnel_progress": _funnel_fraction(observation), + "report_health": report_health, + } + return score, objectives + + +def _trajectory_excerpt(session_dir: str | Path, manifest: GroundTruthManifest) -> str: + root = Path(session_dir) + records: list[str] = [] + for path in sorted(root.rglob("transcript.jsonl"))[:4]: + try: + lines = path.read_text(encoding="utf-8", errors="replace").splitlines() + except OSError: + continue + for line in lines[-24:]: + try: + record = json.loads(line) + except json.JSONDecodeError: + continue + if record.get("event") == "start": + continue + compact = { + key: record.get(key) + for key in ( + "event", + "step", + "message", + "reasoning_content", + "tool_call", + "tool_output", + "status", + ) + if record.get(key) not in (None, "", [], {}) + } + records.append(json.dumps(compact, sort_keys=True, default=str)) + excerpt = "\n".join(records)[-8000:] + return redact_benchmark_terms(excerpt, manifest) + + +_STAGE_FEEDBACK = { + "target_in_working_set": "The run did not focus its exploration on the relevant attack surface.", + "relevant_facts_extracted": "The run read code but missed the state or data facts needed for a concrete hypothesis.", + "true_candidate_generated": "The run failed to turn observations into the relevant vulnerability mechanism.", + "correct_proof_plan_selected": "The candidate lacked a complete set of proof obligations.", + "reachability_dataflow_resolved": "The run did not establish attacker-to-effect reachability.", + "guards_counterevidence_handled": "The run did not resolve guards or the strongest counterevidence.", + "validation_plan_constructed": "The run did not construct a realistic validation plan.", + "expected_evidence_acquired": "The run stopped before acquiring strong supporting evidence.", + "threat_model_classified": "The security boundary and attacker capability remained unclear.", + "correct_certificate_compiled": "The final report did not preserve the proven mechanism and evidence.", +} + + +class SourceHuntGEPAAdapter: + """Execute candidate prompts through Clearwing and emit reflective feedback.""" + + def __init__( + self, + manifest: GroundTruthManifest, + *, + executor: Callable[..., Awaitable[RunObservation]] = execute_sourcehunt_run, + require_negative_controls: bool = True, + ) -> None: + decisions = {case.ground_truth.expected_decision for case in manifest.cases} + if require_negative_controls and not {"confirmed", "disproven"} <= decisions: + raise ValueError( + "GEPA optimization requires both vulnerable positives and fixed/clean negatives" + ) + self.manifest = manifest + self.executor = executor + + def evaluate( + self, + batch: list[SourceHuntOptimizationExample], + candidate: dict[str, str], + capture_traces: bool = False, + ) -> Any: + prompt = candidate.get(PROMPT_COMPONENT, "") + if not prompt.strip(): + raise ValueError(f"Candidate is missing non-empty {PROMPT_COMPONENT!r}") + require_generic_prompt(prompt, manifest=self.manifest) + try: + asyncio.get_running_loop() + except RuntimeError: + return asyncio.run(self._evaluate_async(batch, prompt, capture_traces)) + raise RuntimeError("SourceHuntGEPAAdapter.evaluate must run outside an active event loop") + + async def _evaluate_async( + self, + batch: list[SourceHuntOptimizationExample], + prompt: str, + capture_traces: bool, + ) -> Any: + outputs: list[dict[str, Any]] = [] + scores: list[float] = [] + objectives: list[dict[str, float]] = [] + trajectories: list[dict[str, Any]] = [] + candidate_id = stable_run_id("promptcandidate", prompt) + + for example in batch: + session_id = stable_run_id( + "geparun", + { + "candidate_id": candidate_id, + "run_id": example.spec.id, + "scaffold_profile": example.spec.scaffold_profile, + "context_profile": example.spec.context_profile, + "execution_bounds": { + "max_hunt_files": example.max_hunt_files, + "max_hunter_steps": example.max_hunter_steps, + "ranker_chunk_size": example.ranker_chunk_size, + "ranker_max_inflight_chunks": example.ranker_max_inflight_chunks, + "ranker_chunk_max_retries": example.ranker_chunk_max_retries, + "max_parallel": example.max_parallel, + "sandbox_cpus": example.sandbox_cpus, + "starting_band": example.starting_band, + "redundancy_override": example.redundancy_override, + "depth": example.depth, + "no_rank": example.no_rank, + }, + }, + ) + try: + observation = await self.executor( + example.spec, + example.case, + checkout=example.checkout, + output_dir=example.output_dir, + provider_manager=example.provider_manager, + budget_usd=example.budget_usd, + input_price_per_million=example.input_price_per_million, + output_price_per_million=example.output_price_per_million, + max_hunt_files=example.max_hunt_files, + max_hunter_steps=example.max_hunter_steps, + ranker_chunk_size=example.ranker_chunk_size, + ranker_max_inflight_chunks=example.ranker_max_inflight_chunks, + ranker_chunk_max_retries=example.ranker_chunk_max_retries, + max_parallel=example.max_parallel, + sandbox_cpus=example.sandbox_cpus, + starting_band=example.starting_band, + redundancy_override=example.redundancy_override, + depth=example.depth, + no_rank=example.no_rank, + compile_commands=example.compile_commands, + validation_manifest=example.validation_manifest, + scheduler_calibration=example.scheduler_calibration, + learning_registry=example.learning_registry, + proof_max_actions=example.proof_max_actions, + proof_max_model_calls=example.proof_max_model_calls, + proof_max_dynamic_actions=example.proof_max_dynamic_actions, + prompt_candidate=prompt, + session_id=session_id, + ) + score, objective = score_sourcehunt_observation(observation, example.case) + output = observation.model_dump(mode="json") + trajectory = { + "status": "completed", + "score": score, + "objective_scores": objective, + "first_failure": observation.first_failure, + "metrics": { + "true_positives": observation.true_positives, + "false_positives": observation.false_positives, + "false_negatives": observation.false_negatives, + "unsupported_claims": observation.unsupported_claims, + "report_failures": observation.report_failures, + "model_calls": observation.model_calls, + "compaction_count": observation.compaction_count, + "peak_context_tokens": observation.peak_context_tokens, + "peak_input_tokens": observation.peak_input_tokens, + "input_tokens": observation.input_tokens, + }, + "trace_excerpt": _trajectory_excerpt(observation.session_dir, self.manifest), + "language": example.case.language, + "flow": example.spec.flow, + "scaffold_profile": example.spec.scaffold_profile, + "context_profile": example.spec.context_profile, + } + except Exception as exc: + score = 0.0 + objective = { + "correctness": 0.0, + "precision": 0.0, + "evidence_quality": 0.0, + "funnel_progress": 0.0, + "report_health": 0.0, + } + error = redact_benchmark_terms(str(exc), self.manifest) + output = {"status": "failed", "error": error, "session_id": session_id} + trajectory = { + "status": "failed", + "score": score, + "objective_scores": objective, + "first_failure": "execution_error", + "error": error, + "trace_excerpt": "", + "language": example.case.language, + "flow": example.spec.flow, + "scaffold_profile": example.spec.scaffold_profile, + "context_profile": example.spec.context_profile, + } + outputs.append(output) + scores.append(score) + objectives.append(objective) + trajectories.append(trajectory) + + return _evaluation_batch( + outputs=outputs, + scores=scores, + trajectories=trajectories if capture_traces else None, + objective_scores=objectives, + num_metric_calls=len(batch), + ) + + def make_reflective_dataset( + self, + candidate: dict[str, str], + eval_batch: Any, + components_to_update: list[str], + ) -> Mapping[str, Sequence[Mapping[str, Any]]]: + del candidate + trajectories = eval_batch.trajectories or [] + records: list[dict[str, Any]] = [] + for trajectory in trajectories: + failure = str(trajectory.get("first_failure") or "") + feedback = _STAGE_FEEDBACK.get(failure, "The run completed without a staged failure.") + if trajectory.get("status") == "failed": + feedback = "The candidate caused an execution failure: " + str( + trajectory.get("error") or "unknown error" + ) + feedback += ( + " Improve only generic audit behavior. Do not add repository names, file paths, " + "symbols, commits, CVEs, known mechanisms, or trigger-specific hints." + ) + records.append( + { + "Inputs": { + "language": trajectory.get("language", "unknown"), + "flow": trajectory.get("flow", "unknown"), + "scaffold_profile": trajectory.get("scaffold_profile", "unknown"), + "context_profile": trajectory.get("context_profile", "unknown"), + }, + "Generated Outputs": { + "metrics": trajectory.get("metrics", {}), + "trace_excerpt": trajectory.get("trace_excerpt", ""), + }, + "Feedback": feedback, + "score": trajectory.get("score", 0.0), + "objective_scores": trajectory.get("objective_scores", {}), + } + ) + return { + component: records + for component in components_to_update + if component == PROMPT_COMPONENT + } + + +def optimize_sourcehunt_prompt( + *, + manifest: GroundTruthManifest, + trainset: list[SourceHuntOptimizationExample], + valset: list[SourceHuntOptimizationExample], + reflection_lm: Any, + max_metric_calls: int, + run_dir: str | Path, + seed_prompt: str = GENERIC_INSTRUCTIONS_V1, + seed: int = 0, +) -> Any: + """Run core GEPA without rewriting SourceHunt as a DSPy agent.""" + + try: + import gepa + except ImportError as exc: + raise RuntimeError( + "GEPA is not installed; install Clearwing's 'optimization' extra" + ) from exc + require_generic_prompt(seed_prompt, manifest=manifest) + adapter = SourceHuntGEPAAdapter(manifest) + return gepa.optimize( + seed_candidate={PROMPT_COMPONENT: seed_prompt}, + trainset=trainset, + valset=valset, + adapter=adapter, + reflection_lm=reflection_lm, + max_metric_calls=max_metric_calls, + run_dir=str(run_dir), + seed=seed, + write_agent_state=True, + cache_evaluation=True, + ) + + +__all__ = [ + "ClearwingReflectionLM", + "PROMPT_COMPONENT", + "SourceHuntGEPAAdapter", + "SourceHuntOptimizationExample", + "optimize_sourcehunt_prompt", + "score_sourcehunt_observation", +] diff --git a/clearwing/eval/sourcehunt_lair.py b/clearwing/eval/sourcehunt_lair.py new file mode 100644 index 00000000..a7c06aa0 --- /dev/null +++ b/clearwing/eval/sourcehunt_lair.py @@ -0,0 +1,598 @@ +"""Leakage-safe offline adapter for LAIR source-verified CVE goldens. + +The LAIR goldens are answer-bearing evaluation artifacts. This module keeps +their prose and source coordinates outside the hunter boundary and emits only +delexicalized proof-routing supervision. +""" + +from __future__ import annotations + +import hashlib +import json +import re +from collections import Counter, defaultdict +from collections.abc import Iterable, Mapping, Sequence +from enum import Enum +from pathlib import Path, PurePosixPath +from typing import Any, Literal + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +LAIR_SCHEMA_VERSION = "cwpro.cve-golden-chain.v2" +ROUTER_ROW_SCHEMA_VERSION = "cw.sourcehunt.lair-router-row.v1" +ADAPTER_MANIFEST_SCHEMA_VERSION = "cw.sourcehunt.lair-adapter-manifest.v1" + + +class _StrictModel(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + +class LairRevision(str, Enum): + VULNERABLE = "vulnerable" + FIX = "fix" + + +class LairTraceKind(str, Enum): + ATTACK_SOURCE = "attack_source" + ENTRY_POINT = "entry_point" + PROPAGATION = "propagation" + STATE_TRANSITION = "state_transition" + GUARD_FAILURE = "guard_failure" + VULNERABLE_OPERATION = "vulnerable_operation" + SECURITY_EFFECT = "security_effect" + + +class LairCitation(_StrictModel): + revision: LairRevision + path: str = Field(min_length=1) + line_start: int = Field(ge=1) + line_end: int = Field(ge=1) + excerpt: str = Field(min_length=1) + supports: str = Field(min_length=10) + + @model_validator(mode="after") + def _validate_location(self) -> LairCitation: + if self.line_end < self.line_start or self.line_end - self.line_start >= 80: + raise ValueError("LAIR citations must contain a valid range of at most 80 lines") + path = PurePosixPath(self.path) + if ( + path.is_absolute() + or self.path != path.as_posix() + or self.path in {".", ".."} + or ".." in path.parts + or "\\" in self.path + ): + raise ValueError("LAIR citation paths must be canonical repository-relative paths") + return self + + +class LairCandidate(_StrictModel): + title: str = Field(min_length=1) + location: str = Field(min_length=1) + hypothesis: str = Field(min_length=20) + why_prioritize: str = Field(min_length=20) + evidence: list[LairCitation] = Field(min_length=1) + + +class LairDiscovery(_StrictModel): + id: Literal["discovery"] + candidate: LairCandidate + + +class LairTraceStep(_StrictModel): + id: str = Field(pattern=r"^[a-z][a-z0-9_-]*$") + kind: LairTraceKind + claim: str = Field(min_length=20) + evidence: list[LairCitation] = Field(min_length=1) + + +class LairInvestigation(_StrictModel): + id: Literal["investigation"] + preconditions: list[str] = Field(min_length=1) + causal_trace: list[LairTraceStep] = Field(min_length=3) + security_impact: str = Field(min_length=20) + + @model_validator(mode="after") + def _validate_trace(self) -> LairInvestigation: + ids = [step.id for step in self.causal_trace] + if len(ids) != len(set(ids)): + raise ValueError("LAIR causal trace step ids must be unique") + kinds = [step.kind for step in self.causal_trace] + if kinds[0] not in {LairTraceKind.ATTACK_SOURCE, LairTraceKind.ENTRY_POINT}: + raise ValueError("LAIR causal trace must begin at an attack source or entry point") + try: + operation_index = kinds.index(LairTraceKind.VULNERABLE_OPERATION) + except ValueError as exc: + raise ValueError("LAIR causal trace requires a vulnerable operation") from exc + try: + effect_index = kinds.index(LairTraceKind.SECURITY_EFFECT, operation_index + 1) + except ValueError as exc: + raise ValueError( + "LAIR causal trace requires a security effect after the vulnerable operation" + ) from exc + if effect_index != len(kinds) - 1: + raise ValueError("LAIR causal trace must terminate at the security effect") + return self + + +class LairChallengeCheck(_StrictModel): + assumption: str = Field(min_length=20) + conclusion: str = Field(min_length=20) + evidence: list[LairCitation] = Field(min_length=1) + + +class LairRegressionTest(_StrictModel): + status: Literal["present_in_fix", "proposed"] + description: str = Field(min_length=10) + expected_result: str = Field(min_length=10) + + +class LairFixValidation(_StrictModel): + strategy: str = Field(min_length=20) + behavior_before: str = Field(min_length=20) + behavior_after: str = Field(min_length=20) + changed_files: list[str] = Field(min_length=1) + evidence: list[LairCitation] = Field(min_length=1) + regression_tests: list[LairRegressionTest] = Field(min_length=1) + + +class LairChallenge(_StrictModel): + id: Literal["challenge"] + verdict: Literal["confirmed"] + checks: list[LairChallengeCheck] = Field(min_length=1) + fix_validation: LairFixValidation + + +class LairAgentChain(_StrictModel): + discovery: LairDiscovery + investigation: LairInvestigation + challenge: LairChallenge + + @model_validator(mode="after") + def _validate_evidence_revisions(self) -> LairAgentChain: + vulnerable_groups = [ + self.discovery.candidate.evidence, + *(step.evidence for step in self.investigation.causal_trace), + *(check.evidence for check in self.challenge.checks), + ] + if any( + citation.revision != LairRevision.VULNERABLE + for group in vulnerable_groups + for citation in group + ): + raise ValueError("LAIR discovery, investigation, and challenge must cite vulnerable") + if any( + citation.revision != LairRevision.FIX + for citation in self.challenge.fix_validation.evidence + ): + raise ValueError("LAIR fix validation must cite the fixed revision") + return self + + +class LairGoldenChain(_StrictModel): + schema_version: Literal["cwpro.cve-golden-chain.v2"] + cve: str = Field(pattern=r"^CVE-[0-9]{4}-[0-9]+$") + repo: str = Field(min_length=1) + vulnerable_commit: str = Field(pattern=r"^[0-9a-fA-F]{7,64}$") + fix_commit: str = Field(pattern=r"^[0-9a-fA-F]{7,64}$") + title: str = Field(min_length=1) + vulnerability_class: str = Field(min_length=1) + summary: str = Field(min_length=20) + chain: LairAgentChain + + +class RouterObligation(str, Enum): + ATTACKER_REACHES_ENTRY = "attacker_reaches_entry" + INPUT_REACHES_OPERATION = "input_reaches_operation" + RELEVANT_GUARDS_RESOLVED = "relevant_guards_resolved" + OPERATION_REACHES_SECURITY_EFFECT = "operation_reaches_security_effect" + CANDIDATE_SURVIVES_CHALLENGE = "candidate_survives_challenge" + + +class RouterContextCategory(str, Enum): + INPUT_BOUNDARY_AND_CALLERS = "input_boundary_and_callers" + DATAFLOW_AND_TRANSFERS = "dataflow_and_transfers" + STATE_WRITERS_AND_REPRESENTATION = "state_writers_and_representation" + GUARDS_AND_CONTROL_FLOW = "guards_and_control_flow" + OPERATION_AND_SECURITY_EFFECT = "operation_and_security_effect" + COUNTEREVIDENCE = "counterevidence" + + +class RouterState(_StrictModel): + phase: Literal["investigation", "challenge"] + completed_trace_kinds: list[LairTraceKind] + completed_obligations: list[RouterObligation] + + +class RouterTarget(_StrictModel): + action: Literal["request_context", "challenge_candidate"] + next_trace_kind: LairTraceKind | None + next_obligation: RouterObligation + context_category: RouterContextCategory + + +class RouterTrainingRow(_StrictModel): + schema_version: Literal["cw.sourcehunt.lair-router-row.v1"] = ( + "cw.sourcehunt.lair-router-row.v1" + ) + state: RouterState + target: RouterTarget + + +class LairSplitConfig(_StrictModel): + train: float = Field(default=0.70, ge=0.0, le=1.0) + development: float = Field(default=0.15, ge=0.0, le=1.0) + test: float = Field(default=0.15, ge=0.0, le=1.0) + seed: str = Field(default="lair-sourcehunt-v1", min_length=1) + + @model_validator(mode="after") + def _validate_total(self) -> LairSplitConfig: + if abs(self.train + self.development + self.test - 1.0) > 1e-9: + raise ValueError("LAIR split fractions must sum to 1.0") + return self + + def assign(self, repository: str) -> Literal["train", "development", "test"]: + digest = hashlib.sha256(f"{self.seed}\0{_normalize_repository(repository)}".encode()).digest() + bucket = int.from_bytes(digest[:8], "big") / 2**64 + if bucket < self.train: + return "train" + if bucket < self.train + self.development: + return "development" + return "test" + + +class LairSplitSummary(_StrictModel): + golden_count: int = Field(ge=0) + repository_count: int = Field(ge=0) + row_count: int = Field(ge=0) + file: str + sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + + +class LairAdapterManifest(_StrictModel): + schema_version: Literal["cw.sourcehunt.lair-adapter-manifest.v1"] = ( + "cw.sourcehunt.lair-adapter-manifest.v1" + ) + source_schema_version: Literal["cwpro.cve-golden-chain.v2"] = ( + "cwpro.cve-golden-chain.v2" + ) + corpus_digest: str = Field(pattern=r"^[0-9a-f]{64}$") + split_seed: str + split_fractions: dict[str, float] + reserved_repository_names: list[str] + golden_count: int = Field(ge=0) + excluded_golden_count: int = Field(ge=0) + router_row_count: int = Field(ge=0) + splits: dict[str, LairSplitSummary] + + +class LairLeakage(_StrictModel): + row_index: int = Field(ge=0) + value: str + + +class LairAdapterDataset(_StrictModel): + rows: dict[str, list[RouterTrainingRow]] + manifest: LairAdapterManifest + + +_TRACE_ROUTE: dict[LairTraceKind, tuple[RouterObligation, RouterContextCategory]] = { + LairTraceKind.ATTACK_SOURCE: ( + RouterObligation.ATTACKER_REACHES_ENTRY, + RouterContextCategory.INPUT_BOUNDARY_AND_CALLERS, + ), + LairTraceKind.ENTRY_POINT: ( + RouterObligation.ATTACKER_REACHES_ENTRY, + RouterContextCategory.INPUT_BOUNDARY_AND_CALLERS, + ), + LairTraceKind.PROPAGATION: ( + RouterObligation.INPUT_REACHES_OPERATION, + RouterContextCategory.DATAFLOW_AND_TRANSFERS, + ), + LairTraceKind.STATE_TRANSITION: ( + RouterObligation.INPUT_REACHES_OPERATION, + RouterContextCategory.STATE_WRITERS_AND_REPRESENTATION, + ), + LairTraceKind.GUARD_FAILURE: ( + RouterObligation.RELEVANT_GUARDS_RESOLVED, + RouterContextCategory.GUARDS_AND_CONTROL_FLOW, + ), + LairTraceKind.VULNERABLE_OPERATION: ( + RouterObligation.INPUT_REACHES_OPERATION, + RouterContextCategory.OPERATION_AND_SECURITY_EFFECT, + ), + LairTraceKind.SECURITY_EFFECT: ( + RouterObligation.OPERATION_REACHES_SECURITY_EFFECT, + RouterContextCategory.OPERATION_AND_SECURITY_EFFECT, + ), +} + +_ROUTER_VOCABULARY = { + ROUTER_ROW_SCHEMA_VERSION, + "investigation", + "challenge", + "request_context", + "challenge_candidate", + *(kind.value for kind in LairTraceKind), + *(obligation.value for obligation in RouterObligation), + *(category.value for category in RouterContextCategory), +} + + +def _normalize_repository(repository: str) -> str: + value = repository.strip().lower().removesuffix(".git").rstrip("/") + return value + + +def _repository_name(repository: str) -> str: + return _normalize_repository(repository).rsplit("/", 1)[-1] + + +def _is_reserved(repository: str, reserved_names: set[str]) -> bool: + normalized = _normalize_repository(repository) + return normalized in reserved_names or _repository_name(normalized) in reserved_names + + +def _completed_obligations( + kinds: list[LairTraceKind], + before_index: int, +) -> list[RouterObligation]: + last_index: dict[RouterObligation, int] = {} + for index, kind in enumerate(kinds): + last_index[_TRACE_ROUTE[kind][0]] = index + return [ + obligation + for obligation in RouterObligation + if obligation != RouterObligation.CANDIDATE_SURVIVES_CHALLENGE + and last_index.get(obligation, len(kinds)) < before_index + ] + + +def _router_rows( + golden: LairGoldenChain, +) -> list[RouterTrainingRow]: + kinds = [step.kind for step in golden.chain.investigation.causal_trace] + rows: list[RouterTrainingRow] = [] + for index, kind in enumerate(kinds): + obligation, context_category = _TRACE_ROUTE[kind] + rows.append( + RouterTrainingRow( + state=RouterState( + phase="investigation", + completed_trace_kinds=kinds[:index], + completed_obligations=_completed_obligations(kinds, index), + ), + target=RouterTarget( + action="request_context", + next_trace_kind=kind, + next_obligation=obligation, + context_category=context_category, + ), + ) + ) + rows.append( + RouterTrainingRow( + state=RouterState( + phase="challenge", + completed_trace_kinds=kinds, + completed_obligations=_completed_obligations(kinds, len(kinds)), + ), + target=RouterTarget( + action="challenge_candidate", + next_trace_kind=None, + next_obligation=RouterObligation.CANDIDATE_SURVIVES_CHALLENGE, + context_category=RouterContextCategory.COUNTEREVIDENCE, + ), + ) + ) + return rows + + +def answer_bearing_terms(golden: LairGoldenChain) -> set[str]: + terms = { + golden.cve, + golden.repo, + _repository_name(golden.repo), + golden.vulnerable_commit, + golden.fix_commit, + golden.title, + golden.vulnerability_class, + golden.summary, + golden.chain.discovery.candidate.location, + } + citations = [ + *golden.chain.discovery.candidate.evidence, + *( + citation + for step in golden.chain.investigation.causal_trace + for citation in step.evidence + ), + *(citation for check in golden.chain.challenge.checks for citation in check.evidence), + *golden.chain.challenge.fix_validation.evidence, + ] + for citation in citations: + path = PurePosixPath(citation.path) + terms.update({citation.path, path.name, path.stem, citation.excerpt}) + terms.update( + token + for token in re.findall(r"\b[A-Za-z_][A-Za-z0-9_]{3,}\b", citation.excerpt) + if "_" in token + ) + location = golden.chain.discovery.candidate.location + terms.update(part for part in re.split(r"[:/\\]", location) if len(part) >= 4) + return {term.strip().casefold() for term in terms if term.strip()} + + +def _string_values(value: Any) -> list[str]: + if isinstance(value, Mapping): + return [text for child in value.values() for text in _string_values(child)] + if isinstance(value, list): + return [text for child in value for text in _string_values(child)] + return [value.casefold()] if isinstance(value, str) else [] + + +def lint_router_rows( + rows: Iterable[RouterTrainingRow], + goldens: Sequence[LairGoldenChain], +) -> list[LairLeakage]: + """Return any direct answer-bearing string copied into router row values.""" + + forbidden = set().union(*(answer_bearing_terms(golden) for golden in goldens)) + leaks: list[LairLeakage] = [] + for row_index, row in enumerate(rows): + for value in _string_values(row.model_dump(mode="json")): + # Every valid router row is composed entirely from this fixed, + # case-independent ontology. A source identifier such as + # ``context`` may legitimately be a substring of an ontology value + # such as ``request_context``; that collision is not copied case + # information. Continue to lint any non-ontology value so model + # construction bypasses or future free-text fields fail closed. + if value in _ROUTER_VOCABULARY: + continue + for term in forbidden: + if term in value: + leaks.append(LairLeakage(row_index=row_index, value=term)) + return sorted(leaks, key=lambda leak: (leak.row_index, leak.value)) + + +def adapt_lair_goldens( + goldens: Sequence[LairGoldenChain], + *, + split_config: LairSplitConfig | None = None, + reserved_repository_names: Iterable[str] = ("ffmpeg",), +) -> LairAdapterDataset: + """Build repository-grouped, delexicalized routing rows from LAIR goldens.""" + + config = split_config or LairSplitConfig() + identifiers = [golden.cve for golden in goldens] + duplicates = sorted(name for name, count in Counter(identifiers).items() if count > 1) + if duplicates: + raise ValueError("Duplicate LAIR goldens: " + ", ".join(duplicates)) + + reserved = {_normalize_repository(value) for value in reserved_repository_names} + included = sorted( + (golden for golden in goldens if not _is_reserved(golden.repo, reserved)), + key=lambda golden: (golden.repo.casefold(), golden.cve), + ) + rows: dict[str, list[RouterTrainingRow]] = { + "train": [], + "development": [], + "test": [], + } + golden_counts: Counter[str] = Counter() + repositories: dict[str, set[str]] = defaultdict(set) + for golden in included: + split = config.assign(golden.repo) + golden_counts[split] += 1 + repositories[split].add(_normalize_repository(golden.repo)) + rows[split].extend(_router_rows(golden)) + + all_rows = [row for split_rows in rows.values() for row in split_rows] + leaks = lint_router_rows(all_rows, included) + if leaks: + preview = ", ".join(f"row {leak.row_index}={leak.value!r}" for leak in leaks[:5]) + raise ValueError(f"LAIR router rows contain answer-bearing strings: {preview}") + + source_payloads = [ + json.dumps(golden.model_dump(mode="json"), sort_keys=True, separators=(",", ":")) + for golden in sorted(included, key=lambda item: item.cve) + ] + corpus_digest = hashlib.sha256("\n".join(source_payloads).encode()).hexdigest() + split_summaries = { + split: LairSplitSummary( + golden_count=golden_counts[split], + repository_count=len(repositories[split]), + row_count=len(split_rows), + file=f"router/{split}.jsonl", + sha256=_rows_digest(split_rows), + ) + for split, split_rows in rows.items() + } + manifest = LairAdapterManifest( + corpus_digest=corpus_digest, + split_seed=config.seed, + split_fractions={ + "train": config.train, + "development": config.development, + "test": config.test, + }, + reserved_repository_names=sorted(reserved), + golden_count=len(included), + excluded_golden_count=len(goldens) - len(included), + router_row_count=len(all_rows), + splits=split_summaries, + ) + return LairAdapterDataset(rows=rows, manifest=manifest) + + +def _row_json(row: RouterTrainingRow) -> str: + return json.dumps(row.model_dump(mode="json"), sort_keys=True, separators=(",", ":")) + + +def _rows_digest(rows: Sequence[RouterTrainingRow]) -> str: + payload = "".join(f"{_row_json(row)}\n" for row in rows) + return hashlib.sha256(payload.encode()).hexdigest() + + +def load_lair_goldens(path: str | Path) -> list[LairGoldenChain]: + """Load collected ``GoldenChain`` JSON files from a LAIR output directory.""" + + source = Path(path).expanduser() + if source.is_file(): + files = [source] + else: + golden_root = source / "goldens" if (source / "goldens").is_dir() else source + files = sorted(golden_root.glob("CVE-*.json")) + if not files: + raise ValueError(f"No LAIR CVE golden JSON files found under {source}") + return [ + LairGoldenChain.model_validate(json.loads(file.read_text(encoding="utf-8"))) + for file in files + ] + + +def write_lair_adapter_dataset( + dataset: LairAdapterDataset, + output_dir: str | Path, + *, + overwrite: bool = False, +) -> Path: + """Write audited router splits and a non-answer-bearing manifest.""" + + output = Path(output_dir).expanduser() + targets = [output / summary.file for summary in dataset.manifest.splits.values()] + targets.append(output / "manifest.json") + existing = [path for path in targets if path.exists()] + if existing and not overwrite: + raise FileExistsError(f"Refusing to overwrite LAIR adapter output: {existing[0]}") + (output / "router").mkdir(parents=True, exist_ok=True) + for split, rows in dataset.rows.items(): + target = output / dataset.manifest.splits[split].file + target.write_text("".join(f"{_row_json(row)}\n" for row in rows), encoding="utf-8") + manifest_path = output / "manifest.json" + manifest_path.write_text( + json.dumps(dataset.manifest.model_dump(mode="json"), indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + return manifest_path + + +__all__ = [ + "ADAPTER_MANIFEST_SCHEMA_VERSION", + "LAIR_SCHEMA_VERSION", + "ROUTER_ROW_SCHEMA_VERSION", + "LairAdapterDataset", + "LairAdapterManifest", + "LairGoldenChain", + "LairLeakage", + "LairSplitConfig", + "LairTraceKind", + "RouterContextCategory", + "RouterObligation", + "RouterTrainingRow", + "adapt_lair_goldens", + "answer_bearing_terms", + "lint_router_rows", + "load_lair_goldens", + "write_lair_adapter_dataset", +] diff --git a/clearwing/eval/sourcehunt_lair_gepa.py b/clearwing/eval/sourcehunt_lair_gepa.py new file mode 100644 index 00000000..36e9bbac --- /dev/null +++ b/clearwing/eval/sourcehunt_lair_gepa.py @@ -0,0 +1,368 @@ +"""Leakage-safe GEPA adapter for the LAIR validator development fold.""" + +from __future__ import annotations + +import asyncio +import hashlib +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from clearwing.llm import AsyncLLMClient +from clearwing.sourcehunt.validator import Validator + +from .sourcehunt_gepa import ClearwingReflectionLM, _evaluation_batch +from .sourcehunt_lair import LairGoldenChain, answer_bearing_terms +from .sourcehunt_lair_validator import replay_lair_validator_case + +VALIDATOR_PROMPT_COMPONENT = "validator_system_prompt" +MAX_VALIDATOR_PROMPT_CHARS = 2_000 +_FORBIDDEN_PROTOCOL_TERMS = ( + "paired", + "snapshot", + "benchmark", + "development fold", + "vulnerable_correct", + "fixed_correct", + "pair_correct", + "model_health", + "source_supported_claim", + "source_contradicted_claim", +) +VALIDATOR_REFLECTION_TEMPLATE = """Improve a compact, generic source-code vulnerability validator instruction. + +Current instruction: +``` + +``` + +Abstract evaluation outcomes: +``` + +``` + +Preserve source-first discrimination: accept a flaw present in current source and +reject an allegation when current source breaks its causal chain. Keep +reachability, impact, and deployment judgments independent. Return only a revised +instruction inside ``` blocks. It must be under 2000 characters and must never add +case identities, repositories, CVEs, commits, paths, symbols, excerpts, fixes, known +mechanisms, triggers, or other benchmark-specific content.""" + + +@dataclass(frozen=True) +class LairValidatorOptimizationExample: + """One opaque reference to a private LAIR source pair.""" + + case_id: str + + +class LairValidatorGEPAAdapter: + """Score source-pair discrimination without reflecting benchmark answers.""" + + propose_new_texts = None + + def __init__( + self, + client: AsyncLLMClient, + goldens: Sequence[LairGoldenChain], + campaign_root: str | Path, + *, + model: str, + max_output_tokens: int = 16_384, + temperature: float = 0.0, + max_parallel: int = 2, + context_radius: int = 18, + max_context_chars: int = 20_000, + max_metric_calls: int | None = None, + ) -> None: + if max_parallel < 1: + raise ValueError("GEPA validator max_parallel must be positive") + self.client = client + self.goldens = tuple(goldens) + root = Path(campaign_root) + self._private_cases = { + _opaque_id(golden): (golden, root / "workspaces" / golden.cve / "repo") + for golden in self.goldens + } + if len(self._private_cases) != len(self.goldens): + raise ValueError("LAIR optimization cases must have unique opaque ids") + self.examples = tuple( + LairValidatorOptimizationExample(case_id=case_id) + for case_id in sorted(self._private_cases) + ) + self.model = model + self.max_output_tokens = max_output_tokens + self.temperature = temperature + self.max_parallel = max_parallel + self.context_radius = context_radius + self.max_context_chars = max_context_chars + if max_metric_calls is not None and max_metric_calls < 1: + raise ValueError("GEPA validator metric-call budget must be positive") + self.max_metric_calls = max_metric_calls + self.metric_calls = 0 + + def evaluate( + self, + batch: list[LairValidatorOptimizationExample], + candidate: dict[str, str], + capture_traces: bool = False, + ) -> Any: + prompt = candidate.get(VALIDATOR_PROMPT_COMPONENT, "") + require_generic_validator_prompt(prompt, self.goldens) + if ( + self.max_metric_calls is not None + and self.metric_calls + len(batch) > self.max_metric_calls + ): + raise RuntimeError("GEPA validator metric-call budget exhausted") + self.metric_calls += len(batch) + try: + asyncio.get_running_loop() + except RuntimeError: + return asyncio.run(self._evaluate_async(batch, prompt, capture_traces)) + raise RuntimeError("LairValidatorGEPAAdapter.evaluate cannot run in an event loop") + + async def _evaluate_async( + self, + batch: list[LairValidatorOptimizationExample], + prompt: str, + capture_traces: bool, + ) -> Any: + validator = Validator( + self.client, + gate_threshold=None, + enable_quick_pass=False, + prompt_profile="source-first-compact-v2", + system_prompt=prompt, + max_output_tokens=self.max_output_tokens, + temperature=self.temperature, + ) + semaphore = asyncio.Semaphore(self.max_parallel) + + async def run_one(example: LairValidatorOptimizationExample) -> Any: + async with semaphore: + try: + golden, repo = self._private_cases[example.case_id] + except KeyError as exc: + raise ValueError("unknown opaque LAIR optimization case") from exc + return await replay_lair_validator_case( + golden, + repo, + lambda finding, context: validator.avalidate( + finding, source_context=context + ), + context_radius=self.context_radius, + max_context_chars=self.max_context_chars, + ) + + cases = await asyncio.gather(*(run_one(example) for example in batch)) + outputs: list[dict[str, Any]] = [] + scores: list[float] = [] + objectives: list[dict[str, float]] = [] + trajectories: list[dict[str, Any]] = [] + for case in cases: + error = case.vulnerable.model_error or case.fixed.model_error + vulnerable_score = float(case.vulnerable_correct) + fixed_score = float(case.fixed_correct) + pair_score = float(case.pair_correct) + score = ( + 0.0 + if error + else 0.35 * vulnerable_score + + 0.35 * fixed_score + + 0.30 * pair_score + ) + objective = { + "vulnerable_correct": vulnerable_score, + "fixed_correct": fixed_score, + "pair_correct": pair_score, + "model_health": float(not error), + } + outcome = { + "vulnerable_correct": case.vulnerable_correct, + "fixed_correct": case.fixed_correct, + "pair_correct": case.pair_correct, + "model_error": error, + } + outputs.append(outcome) + scores.append(score) + objectives.append(objective) + trajectories.append( + { + "lesson": _abstract_feedback( + vulnerable_correct=case.vulnerable_correct, + fixed_correct=case.fixed_correct, + model_error=error, + ) + } + ) + return _evaluation_batch( + outputs=outputs, + scores=scores, + trajectories=trajectories if capture_traces else None, + objective_scores=objectives, + num_metric_calls=len(batch), + ) + + def make_reflective_dataset( + self, + candidate: dict[str, str], + eval_batch: Any, + components_to_update: list[str], + ) -> Mapping[str, Sequence[Mapping[str, Any]]]: + del candidate + records = [] + for trajectory in eval_batch.trajectories or []: + records.append( + { + "Inputs": {"task": "source-backed vulnerability validation"}, + "Generated Outputs": "A bounded structured verdict was returned.", + "Feedback": trajectory["lesson"], + } + ) + return { + component: records + for component in components_to_update + if component == VALIDATOR_PROMPT_COMPONENT + } + + +class LairValidatorMetricBudgetStopper: + """Stop before an iteration whose maximum rollout cost exceeds the budget.""" + + def __init__( + self, + adapter: LairValidatorGEPAAdapter, + *, + max_metric_calls: int, + max_iteration_calls: int, + ) -> None: + self.adapter = adapter + self.max_metric_calls = max_metric_calls + self.max_iteration_calls = max_iteration_calls + + def __call__(self, _state: Any) -> bool: + return ( + self.adapter.metric_calls + self.max_iteration_calls + > self.max_metric_calls + ) + + +def require_generic_validator_prompt( + prompt: str, + goldens: Sequence[LairGoldenChain], +) -> None: + """Reject empty, oversized, or answer-bearing validator candidates.""" + + stripped = prompt.strip() + if not stripped: + raise ValueError("validator prompt must be non-empty") + if len(stripped) > MAX_VALIDATOR_PROMPT_CHARS: + raise ValueError( + f"validator prompt exceeds {MAX_VALIDATOR_PROMPT_CHARS} characters" + ) + folded = stripped.casefold() + protocol_leaks = {term for term in _FORBIDDEN_PROTOCOL_TERMS if term in folded} + if protocol_leaks: + raise ValueError("validator prompt leaks evaluation-protocol language") + forbidden = set().union(*(answer_bearing_terms(golden) for golden in goldens)) + leaks = {value for value in forbidden if value in folded} + if leaks: + raise ValueError("validator prompt leaks LAIR benchmark answers") + + +def optimize_lair_validator_prompt( + *, + adapter: LairValidatorGEPAAdapter, + trainset: list[LairValidatorOptimizationExample], + valset: list[LairValidatorOptimizationExample], + reflection_lm: ClearwingReflectionLM, + seed_prompt: str, + max_metric_calls: int, + run_dir: str | Path, + seed: int = 0, +) -> Any: + """Run bounded core GEPA over the opened LAIR development cases.""" + + try: + import gepa + except ImportError as exc: + raise RuntimeError("GEPA is not installed") from exc + require_generic_validator_prompt(seed_prompt, adapter.goldens) + minibatch_size = min(4, len(trainset)) + budget_stopper = LairValidatorMetricBudgetStopper( + adapter, + max_metric_calls=max_metric_calls, + max_iteration_calls=2 * minibatch_size + len(valset), + ) + return gepa.optimize( + seed_candidate={VALIDATOR_PROMPT_COMPONENT: seed_prompt}, + trainset=trainset, + valset=valset, + adapter=adapter, + reflection_lm=reflection_lm, + max_metric_calls=max_metric_calls, + reflection_minibatch_size=minibatch_size, + reflection_prompt_template=VALIDATOR_REFLECTION_TEMPLATE, + run_dir=str(run_dir), + seed=seed, + cache_evaluation=True, + display_progress_bar=True, + frontier_type="objective", + acceptance_criterion="strict_improvement", + raise_on_exception=False, + stop_callbacks=budget_stopper, + ) + + +def _opaque_id(golden: LairGoldenChain) -> str: + material = f"{golden.repo}\0{golden.cve}".encode() + return f"case-{hashlib.sha256(material).hexdigest()[:16]}" + + +def _abstract_feedback( + *, + vulnerable_correct: bool, + fixed_correct: bool, + model_error: bool, +) -> str: + if model_error: + return ( + "The candidate failed to return valid bounded structured output. " + "Make the instruction shorter and make immediate schema compliance explicit. " + "Use only generic validation rules; add no case-specific details." + ) + if vulnerable_correct and fixed_correct: + result = ( + "The candidate correctly accepted source-supported behavior and rejected " + "source-contradicted behavior." + ) + elif not vulnerable_correct and fixed_correct: + result = ( + "The candidate rejected a source-present flaw. Keep source-level reality " + "separate from uncertainty about reachability or deployment prevalence." + ) + elif vulnerable_correct and not fixed_correct: + result = ( + "The candidate accepted a report after current source broke its causal chain. " + "Treat current guards and invariants as authoritative counterevidence." + ) + else: + result = ( + "The candidate failed both source-supported and source-contradicted " + "decisions. Ground REAL in the complete causal chain before judging the " + "remaining axes." + ) + return result + " Improve only generic rules and keep the prompt concise." + + +__all__ = [ + "LairValidatorGEPAAdapter", + "LairValidatorMetricBudgetStopper", + "LairValidatorOptimizationExample", + "MAX_VALIDATOR_PROMPT_CHARS", + "VALIDATOR_PROMPT_COMPONENT", + "VALIDATOR_REFLECTION_TEMPLATE", + "optimize_lair_validator_prompt", + "require_generic_validator_prompt", +] diff --git a/clearwing/eval/sourcehunt_lair_replicates.py b/clearwing/eval/sourcehunt_lair_replicates.py new file mode 100644 index 00000000..15f73b34 --- /dev/null +++ b/clearwing/eval/sourcehunt_lair_replicates.py @@ -0,0 +1,305 @@ +"""Reproducible aggregation for replicated LAIR validator replays.""" + +from __future__ import annotations + +import hashlib +import math +from collections import defaultdict +from collections.abc import Mapping, Sequence +from pathlib import Path +from typing import Any, cast + +from .sourcehunt_lair_validator import ( + REPLAY_CONTEXT_PROFILE, + LairValidatorCaseResult, + LairValidatorReplaySummary, + summarize_lair_validator_replay, +) + +REPLICATION_SCHEMA_VERSION = "cw.sourcehunt.lair-validator-replicates.v1" +_WILSON_95_Z = 1.959963984540054 + + +def file_sha256(path: str | Path) -> str: + """Return the SHA-256 digest of a result exactly as stored.""" + + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + + +def load_replay(path: str | Path) -> LairValidatorReplaySummary: + """Load one replay and reject inconsistent stored metrics or case flags.""" + + result = cast( + LairValidatorReplaySummary, + LairValidatorReplaySummary.model_validate_json(Path(path).read_text()), + ) + _validate_case_flags(result.cases) + calculated = summarize_lair_validator_replay( + result.cases, + model=result.model, + prompt_profile=result.prompt_profile, + max_output_tokens=result.max_output_tokens, + temperature=result.temperature, + ) + for field in ( + "case_count", + "vulnerable_recall", + "fixed_rejection_rate", + "pair_accuracy", + "vulnerable_false_negatives", + "fixed_false_positives", + "model_errors", + "axis_pass_counts", + ): + if getattr(result, field) != getattr(calculated, field): + raise ValueError(f"replay has inconsistent {field}: {path}") + return result + + +def validate_replicate_set( + runs_by_profile: Mapping[str, Sequence[LairValidatorReplaySummary]], + *, + model: str, + replicates: int, + max_output_tokens: int, + temperature: float, +) -> None: + """Require a complete, coordinate-identical replication matrix.""" + + if replicates < 1: + raise ValueError("replicates must be positive") + if not runs_by_profile: + raise ValueError("at least one prompt profile is required") + + expected_coordinates: dict[str, tuple[str, str]] | None = None + expected_cases: set[str] | None = None + for profile, runs in runs_by_profile.items(): + if len(runs) != replicates: + raise ValueError( + f"profile {profile!r} has {len(runs)} runs; expected {replicates}" + ) + for run in runs: + if run.model != model: + raise ValueError(f"replicate model drift: {run.model!r} != {model!r}") + if run.prompt_profile != profile: + raise ValueError( + f"replicate prompt drift: {run.prompt_profile!r} != {profile!r}" + ) + if run.context_profile != REPLAY_CONTEXT_PROFILE: + raise ValueError(f"replicate context drift: {run.context_profile!r}") + if run.max_output_tokens != max_output_tokens: + raise ValueError("replicate output-token cap drift") + if run.temperature != temperature: + raise ValueError("replicate temperature drift") + + cases = {_case_key(case): case for case in run.cases} + if len(cases) != len(run.cases): + raise ValueError("replicate contains duplicate cases") + case_keys = set(cases) + coordinates = { + key: (case.finding_digest, case.source_window_digest) + for key, case in cases.items() + } + if expected_cases is None: + expected_cases = case_keys + expected_coordinates = coordinates + elif case_keys != expected_cases: + raise ValueError("replicate case membership drift") + elif coordinates != expected_coordinates: + raise ValueError("replicate finding or source-coordinate drift") + + +def aggregate_replicates( + run_paths_by_profile: Mapping[str, Sequence[str | Path]], + *, + model: str, + max_output_tokens: int, + temperature: float, + context_radius: int, + max_context_chars: int, + max_parallel: int, + provenance: Mapping[str, str] | None = None, +) -> dict[str, Any]: + """Aggregate replicated binomial metrics and opaque per-case stability.""" + + normalized_paths = { + profile: [Path(path).resolve() for path in paths] + for profile, paths in run_paths_by_profile.items() + } + loaded = { + profile: [load_replay(path) for path in paths] + for profile, paths in normalized_paths.items() + } + replicates = len(next(iter(loaded.values()), ())) + validate_replicate_set( + loaded, + model=model, + replicates=replicates, + max_output_tokens=max_output_tokens, + temperature=temperature, + ) + + inputs: list[dict[str, Any]] = [] + for profile, paths in normalized_paths.items(): + for index, path in enumerate(paths, start=1): + inputs.append( + { + "prompt_profile": profile, + "replicate": index, + "path": str(path), + "sha256": file_sha256(path), + "provenance": (provenance or {}).get(str(path), "generated"), + } + ) + + arms = {profile: _aggregate_arm(runs) for profile, runs in loaded.items()} + return { + "schema_version": REPLICATION_SCHEMA_VERSION, + "model": model, + "configuration": { + "replicates": replicates, + "context_profile": REPLAY_CONTEXT_PROFILE, + "context_radius": context_radius, + "max_context_chars": max_context_chars, + "max_output_tokens": max_output_tokens, + "temperature": temperature, + "max_parallel": max_parallel, + }, + "input_results": inputs, + "arms": arms, + } + + +def wilson_interval(successes: int, trials: int) -> dict[str, float]: + """Return a two-sided 95% Wilson score interval for a binomial rate.""" + + if trials < 1: + raise ValueError("Wilson interval requires at least one trial") + if not 0 <= successes <= trials: + raise ValueError("successes must fall between zero and trials") + proportion = successes / trials + z_squared = _WILSON_95_Z**2 + denominator = 1 + z_squared / trials + center = (proportion + z_squared / (2 * trials)) / denominator + margin = ( + _WILSON_95_Z + * math.sqrt( + proportion * (1 - proportion) / trials + + z_squared / (4 * trials**2) + ) + / denominator + ) + return { + "confidence": 0.95, + "lower": max(0.0, center - margin), + "upper": min(1.0, center + margin), + } + + +def _aggregate_arm(runs: Sequence[LairValidatorReplaySummary]) -> dict[str, Any]: + cases_by_key: dict[str, list[LairValidatorCaseResult]] = defaultdict(list) + replicate_metrics: list[dict[str, Any]] = [] + for index, run in enumerate(runs, start=1): + replicate_metrics.append( + { + "replicate": index, + "vulnerable_recall": run.vulnerable_recall, + "fixed_rejection_rate": run.fixed_rejection_rate, + "pair_accuracy": run.pair_accuracy, + "model_errors": run.model_errors, + } + ) + for case in run.cases: + cases_by_key[_case_key(case)].append(case) + + flattened = [case for run in runs for case in run.cases] + decision_count = 2 * len(flattened) + per_case = [] + for key in sorted(cases_by_key): + cases = cases_by_key[key] + vulnerable = [case.vulnerable_correct for case in cases] + fixed = [case.fixed_correct for case in cases] + pairs = [case.pair_correct for case in cases] + errors = [case.vulnerable.model_error or case.fixed.model_error for case in cases] + per_case.append( + { + "case_id": _opaque_case_id(cases[0]), + "vulnerable_correct_rate": sum(vulnerable) / len(cases), + "fixed_correct_rate": sum(fixed) / len(cases), + "pair_correct_rate": sum(pairs) / len(cases), + "error_run_rate": sum(errors) / len(cases), + "vulnerable_pattern": _bit_pattern(vulnerable), + "fixed_pattern": _bit_pattern(fixed), + "pair_pattern": _bit_pattern(pairs), + "unanimous": len(set(zip(vulnerable, fixed, strict=True))) == 1, + } + ) + + return { + "replicate_metrics": replicate_metrics, + "aggregate": { + "vulnerable_recall": _binomial_metric( + sum(case.vulnerable_correct for case in flattened), len(flattened) + ), + "fixed_rejection_rate": _binomial_metric( + sum(case.fixed_correct for case in flattened), len(flattened) + ), + "pair_accuracy": _binomial_metric( + sum(case.pair_correct for case in flattened), len(flattened) + ), + "model_error_rate": _binomial_metric( + sum( + case.vulnerable.model_error + case.fixed.model_error + for case in flattened + ), + decision_count, + ), + }, + "unanimous_case_count": sum(case["unanimous"] for case in per_case), + "case_count": len(per_case), + "per_case_stability": per_case, + } + + +def _binomial_metric(successes: int, trials: int) -> dict[str, Any]: + return { + "successes": successes, + "trials": trials, + "rate": successes / trials, + "wilson_95": wilson_interval(successes, trials), + } + + +def _validate_case_flags(cases: Sequence[LairValidatorCaseResult]) -> None: + for case in cases: + vulnerable_correct = case.vulnerable.advance and not case.vulnerable.model_error + fixed_correct = not case.fixed.advance and not case.fixed.model_error + if case.vulnerable_correct != vulnerable_correct: + raise ValueError("replay has inconsistent vulnerable correctness flag") + if case.fixed_correct != fixed_correct: + raise ValueError("replay has inconsistent fixed correctness flag") + if case.pair_correct != (vulnerable_correct and fixed_correct): + raise ValueError("replay has inconsistent pair correctness flag") + + +def _case_key(case: LairValidatorCaseResult) -> str: + return f"{case.repository}\0{case.cve}" + + +def _opaque_case_id(case: LairValidatorCaseResult) -> str: + material = f"{case.finding_digest}\0{case.source_window_digest}".encode() + return f"case-{hashlib.sha256(material).hexdigest()[:16]}" + + +def _bit_pattern(values: Sequence[bool]) -> str: + return "".join("1" if value else "0" for value in values) + + +__all__ = [ + "REPLICATION_SCHEMA_VERSION", + "aggregate_replicates", + "file_sha256", + "load_replay", + "validate_replicate_set", + "wilson_interval", +] diff --git a/clearwing/eval/sourcehunt_lair_validator.py b/clearwing/eval/sourcehunt_lair_validator.py new file mode 100644 index 00000000..0bf45b2e --- /dev/null +++ b/clearwing/eval/sourcehunt_lair_validator.py @@ -0,0 +1,421 @@ +"""Offline vulnerable/fixed validator replay over LAIR golden chains. + +This module deliberately feeds the same source-backed alleged finding to both +revisions. Only source text changes, so the model receives no positive/fixed +label and cannot pass by recognizing an evaluation arm. +""" + +from __future__ import annotations + +import asyncio +import hashlib +import json +import re +import subprocess +from collections import defaultdict +from collections.abc import Awaitable, Callable, Sequence +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any + +from pydantic import BaseModel, ConfigDict, Field + +from clearwing.findings.types import Finding +from clearwing.sourcehunt.state import ValidatorVerdict +from clearwing.sourcehunt.validator import Validator + +from .sourcehunt_lair import LairGoldenChain, LairRevision + +REPLAY_SCHEMA_VERSION = "cw.sourcehunt.lair-validator-replay.v2" +REPLAY_CONTEXT_PROFILE = "balanced-anchor-v1" +_REPLAY_ARMS = ("vulnerable", "fixed") +_SOURCE_LINE_SLOT_CHARS = 96 + + +class _ReplayModel(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + +class ReplayVerdict(_ReplayModel): + advance: bool + severity_validated: str | None + evidence_level: str + axes: dict[str, dict[str, Any]] + pro_argument: str + counter_argument: str + tie_breaker: str + model_error: bool + + +class LairValidatorCaseResult(_ReplayModel): + cve: str + repository: str + finding_digest: str = Field(pattern=r"^[0-9a-f]{64}$") + source_window_digest: str = Field(pattern=r"^[0-9a-f]{64}$") + vulnerable: ReplayVerdict + fixed: ReplayVerdict + vulnerable_correct: bool + fixed_correct: bool + pair_correct: bool + + +class LairValidatorReplaySummary(_ReplayModel): + schema_version: str = REPLAY_SCHEMA_VERSION + model: str + prompt_profile: str + context_profile: str = REPLAY_CONTEXT_PROFILE + max_output_tokens: int | None = Field(default=None, ge=1) + temperature: float | None = Field(default=None, ge=0.0, le=2.0) + case_count: int = Field(ge=0) + vulnerable_recall: float = Field(ge=0.0, le=1.0) + fixed_rejection_rate: float = Field(ge=0.0, le=1.0) + pair_accuracy: float = Field(ge=0.0, le=1.0) + vulnerable_false_negatives: int = Field(ge=0) + fixed_false_positives: int = Field(ge=0) + model_errors: int = Field(ge=0) + axis_pass_counts: dict[str, dict[str, int]] + cases: list[LairValidatorCaseResult] + + +@dataclass(frozen=True) +class SourceWindow: + path: str + start: int + end: int + anchors: tuple[int, ...] = () + + +ValidatorCall = Callable[[Finding, str], Awaitable[ValidatorVerdict]] + + +def build_lair_validator_finding(golden: LairGoldenChain) -> Finding: + """Convert a golden into one immutable alleged finding for both snapshots.""" + + trace = golden.chain.investigation.causal_trace + operation = next(step for step in trace if step.kind.value == "vulnerable_operation") + operation_citation = operation.evidence[0] + trace_steps = [ + { + "file": citation.path, + "line": citation.line_start, + "function": "", + "code_snippet": citation.excerpt, + "note": f"{step.kind.value}: {step.claim}", + } + for step in trace + for citation in step.evidence + ] + cwe = re.search(r"\bCWE-[0-9]+\b", golden.vulnerability_class, re.IGNORECASE) + return Finding( + id=f"lair-replay-{hashlib.sha256(golden.cve.encode()).hexdigest()[:16]}", + file=operation_citation.path, + line_number=operation_citation.line_start, + end_line=operation_citation.line_end, + finding_type="source_security", + cwe=cwe.group(0).upper() if cwe else "", + severity="medium", + confidence="high", + description=golden.summary, + code_snippet=operation_citation.excerpt, + discovered_by="sourcehunt_offline_replay", + evidence_level="static_corroboration", + vulnerability_trace={ + "summary": golden.summary, + "steps": trace_steps, + }, + ) + + +def source_windows_for_golden( + golden: LairGoldenChain, + *, + radius: int = 18, +) -> list[SourceWindow]: + """Select revision-independent windows using vulnerable trace coordinates only.""" + + if radius < 0: + raise ValueError("source context radius cannot be negative") + grouped: dict[str, list[tuple[int, int, tuple[int, ...]]]] = defaultdict(list) + for step in golden.chain.investigation.causal_trace: + for citation in step.evidence: + if citation.revision != LairRevision.VULNERABLE: + raise ValueError("investigation trace contains non-vulnerable evidence") + grouped[citation.path].append( + ( + max(1, citation.line_start - radius), + citation.line_end + radius, + ((citation.line_start + citation.line_end) // 2,), + ) + ) + + windows: list[SourceWindow] = [] + for path, ranges in sorted(grouped.items()): + for start, end, anchors in _merge_anchored_windows(ranges): + windows.append( + SourceWindow(path=path, start=start, end=end, anchors=anchors) + ) + return windows + + +def render_revision_context( + repo: str | Path, + revision: str, + windows: Sequence[SourceWindow], + *, + max_chars: int = 20_000, +) -> str: + """Render identical path/line selections from one Git revision.""" + + if max_chars < 1: + raise ValueError("source context max_chars must be positive") + repository = Path(repo) + source_cache: dict[str, list[str] | None] = {} + grouped: dict[str, list[SourceWindow]] = defaultdict(list) + for window in windows: + grouped[window.path].append(window) + path_count = len(grouped) + if not path_count: + return "" + + chunks: list[str] = [] + separator_chars = max(0, len(windows) - 1) * 2 + content_budget = max(1, max_chars - separator_chars) + path_budget, path_remainder = divmod(content_budget, path_count) + for path_index, (path, path_windows) in enumerate(sorted(grouped.items())): + budget = path_budget + (1 if path_index < path_remainder else 0) + window_budget, window_remainder = divmod(budget, len(path_windows)) + source_cache[path] = _git_source(repository, revision, path) + for window_index, window in enumerate(path_windows): + quota = window_budget + (1 if window_index < window_remainder else 0) + chunks.append( + _render_source_window( + window, + source_cache[path], + max_chars=quota, + ) + ) + return "\n\n".join(chunks)[:max_chars] + + +async def replay_lair_validator_case( + golden: LairGoldenChain, + repo: str | Path, + validator_call: ValidatorCall, + *, + context_radius: int = 18, + max_context_chars: int = 20_000, +) -> LairValidatorCaseResult: + """Run one alleged finding against vulnerable and fixed source snapshots.""" + + finding = build_lair_validator_finding(golden) + windows = source_windows_for_golden(golden, radius=context_radius) + vulnerable_context = render_revision_context( + repo, + golden.vulnerable_commit, + windows, + max_chars=max_context_chars, + ) + fixed_context = render_revision_context( + repo, + golden.fix_commit, + windows, + max_chars=max_context_chars, + ) + finding_digest = _digest(asdict(finding)) + window_digest = _digest([asdict(window) for window in windows]) + + vulnerable = await validator_call(finding, vulnerable_context) + fixed = await validator_call(finding, fixed_context) + vulnerable_payload = _verdict_payload(vulnerable) + fixed_payload = _verdict_payload(fixed) + vulnerable_correct = vulnerable.advance and not vulnerable_payload.model_error + fixed_correct = not fixed.advance and not fixed_payload.model_error + return LairValidatorCaseResult( + cve=golden.cve, + repository=golden.repo, + finding_digest=finding_digest, + source_window_digest=window_digest, + vulnerable=vulnerable_payload, + fixed=fixed_payload, + vulnerable_correct=vulnerable_correct, + fixed_correct=fixed_correct, + pair_correct=vulnerable_correct and fixed_correct, + ) + + +async def run_lair_validator_replay( + goldens: Sequence[LairGoldenChain], + campaign_root: str | Path, + validator: Validator, + *, + model: str, + prompt_profile: str = "legacy-v1", + max_output_tokens: int | None = None, + temperature: float | None = None, + max_parallel: int = 2, + context_radius: int = 18, + max_context_chars: int = 20_000, +) -> LairValidatorReplaySummary: + """Replay all goldens with bounded concurrency and aggregate pair metrics.""" + + if max_parallel < 1: + raise ValueError("validator replay max_parallel must be positive") + root = Path(campaign_root) + semaphore = asyncio.Semaphore(max_parallel) + + async def validator_call(finding: Finding, source_context: str) -> ValidatorVerdict: + return await validator.avalidate(finding, source_context=source_context) + + async def run_one(golden: LairGoldenChain) -> LairValidatorCaseResult: + async with semaphore: + repo = root / "workspaces" / golden.cve / "repo" + if not repo.is_dir(): + raise ValueError(f"LAIR replay repository is missing: {repo}") + return await replay_lair_validator_case( + golden, + repo, + validator_call, + context_radius=context_radius, + max_context_chars=max_context_chars, + ) + + cases = await asyncio.gather(*(run_one(golden) for golden in goldens)) + return summarize_lair_validator_replay( + cases, + model=model, + prompt_profile=prompt_profile, + max_output_tokens=max_output_tokens, + temperature=temperature, + ) + + +def summarize_lair_validator_replay( + cases: Sequence[LairValidatorCaseResult], + *, + model: str, + prompt_profile: str = "legacy-v1", + max_output_tokens: int | None = None, + temperature: float | None = None, +) -> LairValidatorReplaySummary: + count = len(cases) + axis_counts: dict[str, dict[str, int]] = {} + for arm in _REPLAY_ARMS: + counts: dict[str, int] = defaultdict(int) + for case in cases: + verdict = getattr(case, arm) + for name, result in verdict.axes.items(): + if result.get("passed") is True: + counts[name] += 1 + axis_counts[arm] = dict(counts) + errors = sum(case.vulnerable.model_error + case.fixed.model_error for case in cases) + return LairValidatorReplaySummary( + model=model, + prompt_profile=prompt_profile, + max_output_tokens=max_output_tokens, + temperature=temperature, + case_count=count, + vulnerable_recall=_fraction(sum(case.vulnerable_correct for case in cases), count), + fixed_rejection_rate=_fraction(sum(case.fixed_correct for case in cases), count), + pair_accuracy=_fraction(sum(case.pair_correct for case in cases), count), + vulnerable_false_negatives=sum(not case.vulnerable_correct for case in cases), + fixed_false_positives=sum(not case.fixed_correct for case in cases), + model_errors=errors, + axis_pass_counts=axis_counts, + cases=list(cases), + ) + + +def _digest(value: Any) -> str: + return hashlib.sha256(json.dumps(value, sort_keys=True, default=str).encode()).hexdigest() + + +def _merge_anchored_windows( + windows: Sequence[tuple[int, int, tuple[int, ...]]], +) -> list[tuple[int, int, tuple[int, ...]]]: + merged: list[tuple[int, int, tuple[int, ...]]] = [] + for start, end, anchors in sorted(windows): + if not merged or start > merged[-1][1] + 4: + merged.append((start, end, tuple(sorted(set(anchors))))) + else: + prior_start, prior_end, prior_anchors = merged[-1] + merged[-1] = ( + prior_start, + max(prior_end, end), + tuple(sorted(set(prior_anchors + anchors))), + ) + return merged + + +def _render_source_window( + window: SourceWindow, + lines: list[str] | None, + *, + max_chars: int, +) -> str: + if max_chars < 1: + return "" + if lines is None: + return f"--- {window.path}: unavailable in current snapshot ---"[:max_chars] + + start = min(window.start, max(1, len(lines))) + end = min(window.end, len(lines)) + anchors = tuple(line for line in window.anchors if start <= line <= end) + if not anchors: + anchors = ((start + end) // 2,) + header = f"--- {window.path}:{start}-{end} ---\n" + candidates = sorted( + range(start, end + 1), + key=lambda line: (min(abs(line - anchor) for anchor in anchors), line), + ) + line_budget = max(1, (max_chars - len(header)) // _SOURCE_LINE_SLOT_CHARS) + selected = sorted(candidates[:line_budget]) + source_chars = _SOURCE_LINE_SLOT_CHARS - 9 + body = "\n".join( + f"{line:6d}: {lines[line - 1][:source_chars]}" for line in selected + ) + return f"{header}{body}"[:max_chars] + + +def _git_source(repo: Path, revision: str, path: str) -> list[str] | None: + result = subprocess.run( + ["git", "-c", "core.hooksPath=/dev/null", "show", f"{revision}:{path}"], + cwd=repo, + text=True, + capture_output=True, + timeout=120, + check=False, + ) + return result.stdout.splitlines() if result.returncode == 0 else None + + +def _verdict_payload(verdict: ValidatorVerdict) -> ReplayVerdict: + return ReplayVerdict( + advance=verdict.advance, + severity_validated=verdict.severity_validated, + evidence_level=verdict.evidence_level, + axes={name: result.model_dump(mode="json") for name, result in verdict.axes.items()}, + pro_argument=verdict.pro_argument, + counter_argument=verdict.counter_argument, + tie_breaker=verdict.tie_breaker, + model_error=not any(True for _ in verdict.axes.items()), + ) + + +def _fraction(numerator: int, denominator: int) -> float: + return numerator / denominator if denominator else 0.0 + + +__all__ = [ + "REPLAY_CONTEXT_PROFILE", + "REPLAY_SCHEMA_VERSION", + "LairValidatorCaseResult", + "LairValidatorReplaySummary", + "ReplayVerdict", + "SourceWindow", + "build_lair_validator_finding", + "render_revision_context", + "replay_lair_validator_case", + "run_lair_validator_replay", + "source_windows_for_golden", + "summarize_lair_validator_replay", +] diff --git a/clearwing/llm/__init__.py b/clearwing/llm/__init__.py index 6ff75786..7863499c 100644 --- a/clearwing/llm/__init__.py +++ b/clearwing/llm/__init__.py @@ -6,6 +6,11 @@ SpendLedger, spend_metadata, ) +from .errors import ( + ProviderExhaustedError, + ProviderExhaustionState, + is_provider_exhausted_error, +) from .messages import ( AIMessage, BaseMessage, @@ -35,11 +40,14 @@ "ChatResponse", "ToolCall", "NativeToolSpec", + "ProviderExhaustedError", + "ProviderExhaustionState", "ToolInputModel", "SpendLedger", "Usage", "extract_json_array", "extract_json_object", "extract_text_content", + "is_provider_exhausted_error", "spend_metadata", ] diff --git a/clearwing/llm/budget.py b/clearwing/llm/budget.py index 2d48b6e3..487fa48f 100644 --- a/clearwing/llm/budget.py +++ b/clearwing/llm/budget.py @@ -36,6 +36,15 @@ class BudgetConfigurationError(ValueError): """Raised when a requested hard cap cannot be enforced safely.""" +@dataclass(slots=True) +class _RestoredRun: + reservations: dict[str, dict[str, Any]] + settlements: dict[str, dict[str, Any]] + status: str | None = None + active_budget_usd: float | None = None + exhausted_budget_usd: float | None = None + + @dataclass(frozen=True, slots=True) class ModelPricing: """USD rates per one million tokens.""" @@ -130,6 +139,7 @@ def __init__( default_max_output_tokens: int = DEFAULT_MAX_OUTPUT_TOKENS, manifest_filename: str = "manifest.json", endpoint: LLMEndpoint | None = None, + resume: bool = False, ) -> None: if not math.isfinite(limit_usd) or limit_usd < 0: raise BudgetConfigurationError("LLM budget must be a finite value >= 0") @@ -188,16 +198,131 @@ def __init__( self.ledger_path = session_dir / "spend-ledger.jsonl" self.manifest_path = session_dir / manifest_filename with self._lock: + if resume: + self._restore_settled_history_locked() self._persist_event_locked( { - "event": "run_started", + "event": "run_resumed" if resume else "run_started", "session_id": self.session_id, "budget_usd": self.limit_usd, + "carried_forward_usd": self._spent_usd, "timestamp": self._timestamp(), } ) self._persist_snapshot_locked() + def _restore_settled_history_locked(self) -> None: + """Restore settled calls and conservatively close orphaned reservations.""" + + if not self.ledger_path.is_file(): + return + restored = self._read_prior_run_locked() + for record in restored.settlements.values(): + self._restore_settlement_locked(record) + for call_id, reservation in restored.reservations.items(): + if call_id not in restored.settlements: + self._recover_orphaned_reservation_locked(call_id, reservation) + + cap_was_not_raised = ( + restored.exhausted_budget_usd is not None + and self.limit_usd <= restored.exhausted_budget_usd + self._EPSILON + ) + exhausted_at_current_cap = self._spent_usd >= self.limit_usd - self._EPSILON + if ( + self.enforcing + and restored.status != "completed" + and (exhausted_at_current_cap or cap_was_not_raised) + ): + self._exhausted = True + self._status = "budget_exhausted" + + def _read_prior_run_locked(self) -> _RestoredRun: + restored = _RestoredRun(reservations={}, settlements={}) + for line in self.ledger_path.read_text(encoding="utf-8").splitlines(): + try: + record = json.loads(line) + except json.JSONDecodeError: + continue + if record.get("event") in {"run_started", "run_resumed"}: + try: + restored.active_budget_usd = float(record.get("budget_usd")) + except (TypeError, ValueError): + restored.active_budget_usd = None + restored.status = None + elif record.get("event") == "budget_exhausted": + restored.exhausted_budget_usd = restored.active_budget_usd + if record.get("event") == "run_finished": + restored.status = str(record.get("status") or "") + if restored.status == "budget_exhausted": + restored.exhausted_budget_usd = restored.active_budget_usd + call_id = str(record.get("call_id") or "") + if not call_id: + continue + if record.get("event") == "call_reserved": + restored.reservations.setdefault(call_id, record) + elif record.get("event") == "call_settled": + restored.settlements.setdefault(call_id, record) + return restored + + def _restore_settlement_locked(self, record: dict[str, Any]) -> None: + try: + cost = float(record["cost_usd"]) + input_tokens = int(record["input_tokens"]) + output_tokens = int(record["output_tokens"]) + cached_tokens = int(record["cached_input_tokens"]) + except (KeyError, TypeError, ValueError): + return + if ( + not math.isfinite(cost) + or cost < 0 + or input_tokens < 0 + or output_tokens < 0 + or cached_tokens < 0 + ): + return + self._records.append(record) + self._spent_usd += cost + self._input_tokens += input_tokens + self._output_tokens += output_tokens + self._cached_input_tokens += cached_tokens + + def _recover_orphaned_reservation_locked( + self, + call_id: str, + reservation: dict[str, Any], + ) -> None: + try: + reserved_usd = float(reservation["reserved_usd"]) + except (KeyError, TypeError, ValueError): + reserved_usd = 0.0 + if not math.isfinite(reserved_usd) or reserved_usd < 0: + reserved_usd = 0.0 + metadata = reservation.get("metadata") + reserved_with_cap = reservation.get("budget_enforcing") + if not isinstance(reserved_with_cap, bool): + reserved_with_cap = reserved_usd > 0 + recovered_cost = reserved_usd if reserved_with_cap else 0.0 + recovered = { + "event": "call_settled", + "call_id": call_id, + "timestamp": self._timestamp(), + "stage": reservation.get("stage"), + "model": reservation.get("model"), + "provider": reservation.get("provider"), + "status": "recovered_ambiguous_failure", + "reserved_usd": reserved_usd, + "cost_usd": recovered_cost, + "cost_source": "reservation" if reserved_with_cap else "none", + "input_tokens": 0, + "cached_input_tokens": 0, + "output_tokens": 0, + "metadata": metadata if isinstance(metadata, dict) else {}, + "error": "Process exited before spend settlement; charged on resume", + } + self._records.append(recovered) + self._spent_usd += recovered_cost + self._persist_event_locked(recovered) + @property def enforcing(self) -> bool: """Whether this run has a finite non-zero dollar cap.""" @@ -349,6 +474,8 @@ def reserve_call( "stage": stage, "model": model, "provider": provider, + "budget_usd": self.limit_usd, + "budget_enforcing": self.enforcing, "reserved_usd": reserved_usd, "input_token_upper_bound": input_token_upper_bound, "max_output_tokens": effective_max_tokens, diff --git a/clearwing/llm/errors.py b/clearwing/llm/errors.py new file mode 100644 index 00000000..bde5d396 --- /dev/null +++ b/clearwing/llm/errors.py @@ -0,0 +1,103 @@ +"""Provider failure classification shared by LLM clients and resumable jobs.""" + +from __future__ import annotations + +import threading +from dataclasses import dataclass, field + + +class ProviderExhaustedError(BaseException): + """A provider rejected work because its account quota is exhausted. + + This is deliberately distinct from authentication and authorization + failures. Callers may safely stop scheduling new work and resume later + after credentials or quota have changed. + Ordinary fallback handlers intentionally do not catch this run-wide stop. + Sourcehunt orchestration catches it only at boundaries where outstanding + tasks can be cancelled and a resumable result can be returned. + """ + + +def _exception_text(exc: BaseException) -> str: + parts: list[str] = [] + seen: set[int] = set() + current: BaseException | None = exc + while current is not None and id(current) not in seen: + seen.add(id(current)) + parts.append(str(current)) + current = current.__cause__ or current.__context__ + return " ".join(parts).lower() + + +def is_provider_exhausted_error(exc: BaseException) -> bool: + """Return whether *exc* is a terminal provider quota rejection. + + Kimi Code reports billing-cycle exhaustion as HTTP 403 with the structured + error type ``access_terminated_error``. Requiring both the structured type + and quota-specific language keeps ordinary 401/403 auth failures out of + this category. + """ + + if isinstance(exc, ProviderExhaustedError): + return True + text = _exception_text(exc) + status_codes: set[int] = set() + seen: set[int] = set() + current: BaseException | None = exc + while current is not None and id(current) not in seen: + seen.add(id(current)) + try: + status_codes.add(int(current.status_code)) # type: ignore[attr-defined] + except (AttributeError, TypeError, ValueError): + pass + response = getattr(current, "response", None) + try: + status_codes.add(int(response.status_code)) # type: ignore[union-attr] + except (AttributeError, TypeError, ValueError): + pass + current = current.__cause__ or current.__context__ + has_http_403 = any( + marker in text + for marker in ( + "http 403", + "error code: 403", + "status code 403", + "status=403", + "status: 403", + ) + ) or 403 in status_codes + has_kimi_type = "access_terminated_error" in text + has_quota_message = any( + marker in text + for marker in ( + "usage limit", + "billing cycle", + "quota exhausted", + "quota exceeded", + ) + ) + return has_http_403 and has_kimi_type and has_quota_message + + +@dataclass +class ProviderExhaustionState: + """Run-shared stop signal used by all bound LLM client views.""" + + _event: threading.Event = field(default_factory=threading.Event) + _message: str = "Provider quota exhausted" + _lock: threading.Lock = field(default_factory=threading.Lock) + + @property + def exhausted(self) -> bool: + return self._event.is_set() + + def mark(self, exc: BaseException) -> ProviderExhaustedError: + with self._lock: + if not self._event.is_set(): + self._message = str(exc) or self._message + self._event.set() + return ProviderExhaustedError(self._message) + + def raise_if_exhausted(self) -> None: + if self._event.is_set(): + raise ProviderExhaustedError(self._message) diff --git a/clearwing/llm/native.py b/clearwing/llm/native.py index 49bf55ab..95ae2018 100644 --- a/clearwing/llm/native.py +++ b/clearwing/llm/native.py @@ -35,6 +35,11 @@ SpendLedger, current_spend_metadata, ) +from .errors import ( + ProviderExhaustedError, + ProviderExhaustionState, + is_provider_exhausted_error, +) logger = logging.getLogger(__name__) @@ -152,14 +157,23 @@ def call_logging_enabled() -> bool: # name. Intentionally empty today; populate as such models ship. _REASONING_EFFORT_OVERRIDE_ALLOW: frozenset[str] = frozenset() +# Per-model defaults for APIs whose accepted reasoning-effort values do not +# include Clearwing's generic "medium" default. Kimi Code K3 accepts +# low/high/max. +_REASONING_EFFORT_MODEL_DEFAULTS: dict[str, str | None] = { + "dsv4-flash-nvfp4": None, + "k3": "high", + "k3-256k": "high", + "kimi-for-coding": None, + "kimi-for-coding-highspeed": None, +} + # Models that must NOT be sent ChatOptions(capture_reasoning_content=True): # genai-pyo3 / the backend errors when reasoning capture is requested for them. # Everything else supports it, so we capture reasoning by default and only skip # for names matching this list. Case-insensitive substring match on the model # name. Add new offenders here as they surface. -_REASONING_CAPTURE_UNSUPPORTED_PATTERNS: tuple[str, ...] = ( - "gpt-5.3-codex-spark", -) +_REASONING_CAPTURE_UNSUPPORTED_PATTERNS: tuple[str, ...] = ("gpt-5.3-codex-spark",) def _model_supports_reasoning_capture(model_name: str) -> bool: @@ -195,7 +209,9 @@ def _is_root_model_type(schema_model: type[BaseModel]) -> bool: def _validate_schema_response(schema_model: type[BaseModel], text: str) -> BaseModel: if not text or not text.strip(): - raise ValueError("LLM returned empty response; expected JSON matching " + schema_model.__name__) + raise ValueError( + "LLM returned empty response; expected JSON matching " + schema_model.__name__ + ) try: return schema_model.model_validate_json(text) except Exception: @@ -237,12 +253,15 @@ class AsyncLLMClient: def _auto_resolve_reasoning_effort(model_name: str) -> str | None: """Return the effective reasoning_effort for *model_name*. - Returns ``None`` (i.e. omit the parameter) when the model name matches - a pattern in :data:`_REASONING_EFFORT_UNSUPPORTED_PATTERNS` and is not + Uses a model-specific supported value when one is registered. Returns + ``None`` (i.e. omit the parameter) when the model name matches a + pattern in :data:`_REASONING_EFFORT_UNSUPPORTED_PATTERNS` and is not in :data:`_REASONING_EFFORT_OVERRIDE_ALLOW`. Returns ``"medium"`` otherwise — the previous default for all callers. """ lower = model_name.lower() + if lower in _REASONING_EFFORT_MODEL_DEFAULTS: + return _REASONING_EFFORT_MODEL_DEFAULTS[lower] if lower in _REASONING_EFFORT_OVERRIDE_ALLOW: return "medium" for pattern in _REASONING_EFFORT_UNSUPPORTED_PATTERNS: @@ -309,9 +328,7 @@ def __init__( account_id = extract_account_id(self.api_key) if not account_id: - raise RuntimeError( - "OpenAI OAuth access token is missing the ChatGPT account id." - ) + raise RuntimeError("OpenAI OAuth access token is missing the ChatGPT account id.") # Store the `.../codex/` base; `_build_client` derives the full # `.../codex/responses` URL and passes it (plus the OAuth headers) @@ -382,8 +399,15 @@ def __init__( self._semaphore = asyncio.Semaphore(max(1, max_concurrency)) self._spend_ledger: SpendLedger | None = None self._spend_stage = "llm" + self._provider_exhaustion_state: ProviderExhaustionState | None = None - def with_spend_ledger(self, ledger: SpendLedger, *, stage: str) -> AsyncLLMClient: + def with_spend_ledger( + self, + ledger: SpendLedger, + *, + stage: str, + provider_exhaustion_state: ProviderExhaustionState | None = None, + ) -> AsyncLLMClient: """Return a run-bound view that shares this client's transport limits. ProviderManager caches native clients process-wide. A shallow view @@ -399,6 +423,7 @@ def with_spend_ledger(self, ledger: SpendLedger, *, stage: str) -> AsyncLLMClien bound = copy.copy(self) bound._spend_ledger = ledger bound._spend_stage = stage + bound._provider_exhaustion_state = provider_exhaustion_state return bound @property @@ -415,6 +440,8 @@ def _reserve_spend_call( tools: list[NativeToolSpec] | None, max_tokens: int | None, ) -> BudgetReservation | None: + if self._provider_exhaustion_state is not None: + self._provider_exhaustion_state.raise_if_exhausted() if self._spend_ledger is None: return None return self._spend_ledger.reserve_call( @@ -476,9 +503,7 @@ def _request_input_token_upper_bound( for tool in tools or [] ], } - serialized_bytes = len( - json.dumps(payload, ensure_ascii=False, default=str).encode("utf-8") - ) + serialized_bytes = len(json.dumps(payload, ensure_ascii=False, default=str).encode("utf-8")) framing_overhead = 256 + 32 * len(messages) + 64 * len(tools or []) return serialized_bytes + framing_overhead @@ -495,9 +520,7 @@ def _settle_spend_call( reservation, input_tokens=usage.prompt_tokens, output_tokens=usage.completion_tokens, - cached_input_tokens=( - details.cached_tokens if details is not None else None - ), + cached_input_tokens=(details.cached_tokens if details is not None else None), ) def _fail_spend_call( @@ -526,6 +549,8 @@ def _is_definitely_unbilled_error(self, exc: BaseException) -> bool: if isinstance(exc, Exception) and self._is_rate_limit_error(exc): return True + if is_provider_exhausted_error(exc): + return True if self._is_unsupported_reasoning_effort_error(exc): return True if isinstance(exc, Exception) and self._is_definitely_unbilled_transport_error(exc): @@ -617,6 +642,8 @@ async def achat( ) async with self._semaphore: + if self._provider_exhaustion_state is not None: + self._provider_exhaustion_state.raise_if_exhausted() client = self._build_client(Client) dispatched = True try: @@ -759,6 +786,8 @@ async def achat_stream( reasoning_effort=self.reasoning_effort, ) async with self._semaphore: + if self._provider_exhaustion_state is not None: + self._provider_exhaustion_state.raise_if_exhausted() client = self._build_client(Client) async def _consume(opts: ChatOptions) -> ChatResponse | None: @@ -801,10 +830,7 @@ async def _consume(opts: ChatOptions) -> ChatResponse | None: options = self._rebuild_options_without_max_tokens(options) response = await _consume(options) elif self._should_try_openai_http_fallback(exc) and ( - not ( - self._spend_ledger is not None - and self._spend_ledger.enforcing - ) + not (self._spend_ledger is not None and self._spend_ledger.enforcing) or self._is_definitely_unbilled_transport_error(exc) ): logger.debug( @@ -823,9 +849,7 @@ async def _consume(opts: ChatOptions) -> ChatResponse | None: else: raise if response is None: - raise RuntimeError( - "LLM stream ended without a terminal usage event" - ) + raise RuntimeError("LLM stream ended without a terminal usage event") except BaseException as exc: elapsed_ms = int((time.monotonic() - started) * 1000) self._fail_spend_call(reservation, exc, dispatched=dispatched) @@ -837,9 +861,8 @@ async def _consume(opts: ChatOptions) -> ChatResponse | None: self._format_exc_chain(exc), ) _record_call(self.model_name, elapsed_ms, None, None, 0, ok=False) - if ( - "without a terminal usage event" in str(exc) - and not (self._spend_ledger is not None and self._spend_ledger.enforcing) + if "without a terminal usage event" in str(exc) and not ( + self._spend_ledger is not None and self._spend_ledger.enforcing ): return await self.achat( messages=messages, @@ -1259,11 +1282,15 @@ async def _collect_openai_sse_response( } elif event_type == "response.function_call_arguments.delta": idx = int(chunk.get("output_index") or 0) - acc = resp_tool_parts.setdefault(idx, {"call_id": chunk.get("call_id") or "", "fn_name": "", "arguments": ""}) + acc = resp_tool_parts.setdefault( + idx, {"call_id": chunk.get("call_id") or "", "fn_name": "", "arguments": ""} + ) acc["arguments"] += chunk.get("delta") or "" elif event_type == "response.function_call_arguments.done": idx = int(chunk.get("output_index") or 0) - acc = resp_tool_parts.setdefault(idx, {"call_id": chunk.get("call_id") or "", "fn_name": "", "arguments": ""}) + acc = resp_tool_parts.setdefault( + idx, {"call_id": chunk.get("call_id") or "", "fn_name": "", "arguments": ""} + ) acc["arguments"] = chunk.get("arguments") or acc["arguments"] continue @@ -1340,11 +1367,13 @@ def _chat_response_from_responses_payload(self, payload: dict[str, Any]) -> Chat elif part_type in ("reasoning", "thinking"): reasoning_parts.append(part.get("text") or part.get("thinking") or "") elif item_type == "function_call": - tool_calls.append({ - "call_id": item.get("call_id") or item.get("id") or "", - "fn_name": item.get("name") or "", - "fn_arguments": self._parse_openai_tool_arguments(item.get("arguments")), - }) + tool_calls.append( + { + "call_id": item.get("call_id") or item.get("id") or "", + "fn_name": item.get("name") or "", + "fn_arguments": self._parse_openai_tool_arguments(item.get("arguments")), + } + ) elif item_type == "reasoning": for summary in item.get("summary") or []: reasoning_parts.append(summary.get("text") or "") @@ -1373,7 +1402,9 @@ def _extract_openai_message_text(self, content: Any) -> str: return "".join(parts) return str(content) - def _openai_tool_calls_from_message(self, tool_calls: list[dict[str, Any]]) -> list[dict[str, Any]]: + def _openai_tool_calls_from_message( + self, tool_calls: list[dict[str, Any]] + ) -> list[dict[str, Any]]: parsed: list[dict[str, Any]] = [] for call in tool_calls: fn = call.get("function") or {} @@ -1493,9 +1524,15 @@ async def _with_retries(self, op) -> ChatResponse: try: return await op() except Exception as exc: + if is_provider_exhausted_error(exc): + if self._provider_exhaustion_state is not None: + raise self._provider_exhaustion_state.mark(exc) from exc + raise ProviderExhaustedError(str(exc)) from exc is_rate_limit = self._is_rate_limit_error(exc) is_transport = self._is_transient_transport_error(exc) - if (not is_rate_limit and not is_transport) or attempt >= self.rate_limit_max_retries: + if ( + not is_rate_limit and not is_transport + ) or attempt >= self.rate_limit_max_retries: raise delay = self._retry_delay_seconds(exc, attempt) diff --git a/clearwing/providers/catalog.py b/clearwing/providers/catalog.py index 70c980d2..0052aed5 100644 --- a/clearwing/providers/catalog.py +++ b/clearwing/providers/catalog.py @@ -234,6 +234,22 @@ class ProviderPreset: alt_models=("deepseek-coder",), provider_adapter="openai", ), + ProviderPreset( + key="kimi-code", + display_name="Kimi Code (membership)", + description="Kimi coding models using a Kimi Code membership key from " + "kimi.com/code. Separate from Open Platform keys and billing.", + docs_url="https://www.kimi.com/code/console", + default_base_url="https://api.kimi.com/coding/v1", + default_model="k3-256k", + api_key_env_var="KIMI_CODE_API_KEY", + alt_models=( + "k3", + "kimi-for-coding", + "kimi-for-coding-highspeed", + ), + provider_adapter="openai", + ), ProviderPreset( key="minimax", display_name="MiniMax", @@ -277,6 +293,7 @@ def preset_by_key(key: str) -> ProviderPreset | None: "anthropic_oauth": "anthropic-oauth", "claude-code": "anthropic-oauth", "claude_code": "anthropic-oauth", + "kimi_code": "kimi-code", } key_lower = aliases.get(key_lower, key_lower) for preset in PROVIDER_PRESETS: diff --git a/clearwing/providers/env.py b/clearwing/providers/env.py index 3cba9ba9..a05ad693 100644 --- a/clearwing/providers/env.py +++ b/clearwing/providers/env.py @@ -117,7 +117,7 @@ def is_openai_compat(self) -> bool: """True if this endpoint talks the OpenAI-compatible dialect. That covers: OpenRouter, Ollama (via /v1), LM Studio, vLLM, - Together, Fireworks, Groq, Anyscale, SiliconFlow, DeepSeek, + Together, Fireworks, Groq, Anyscale, SiliconFlow, DeepSeek, Kimi, and OpenAI direct. ChatGPT/Codex OAuth is not OpenAI-compatible. """ return self.provider == "openai_compat" @@ -519,6 +519,8 @@ def _default_openai_compat_model(base_url: str) -> str: return "gpt-4o" if "api.deepseek.com" in host: return "deepseek-chat" + if "api.kimi.com" in host and "/coding" in host: + return "k3-256k" # Catch-all return "default" diff --git a/clearwing/sourcehunt/__init__.py b/clearwing/sourcehunt/__init__.py index 3f8fca27..27e698bd 100644 --- a/clearwing/sourcehunt/__init__.py +++ b/clearwing/sourcehunt/__init__.py @@ -28,6 +28,16 @@ SourceHuntConfig, TargetConfig, ) +from .optimization import ( + CONTEXT_PROFILES, + PROMPT_BUNDLES, + SCAFFOLD_PROFILES, + ContextProfile, + PromptBundle, + ScaffoldProfile, + lint_prompt_candidate, + require_generic_prompt, +) from .runner import SourceHuntProgress, SourceHuntProgressCallback, SourceHuntRunner from .state import ( EVIDENCE_LEVELS, @@ -42,6 +52,8 @@ __all__ = [ "BudgetConfig", + "CONTEXT_PROFILES", + "ContextProfile", "EvidenceLevel", "FeatureFlags", "FileTag", @@ -49,7 +61,11 @@ "Finding", "HuntTuning", "OutputConfig", + "PROMPT_BUNDLES", "ProofConfig", + "PromptBundle", + "SCAFFOLD_PROFILES", + "ScaffoldProfile", "SourceHuntConfig", "SourceHuntProgress", "SourceHuntProgressCallback", @@ -60,4 +76,6 @@ "evidence_at_or_above", "evidence_compare", "filter_by_evidence", + "lint_prompt_candidate", + "require_generic_prompt", ] diff --git a/clearwing/sourcehunt/campaign.py b/clearwing/sourcehunt/campaign.py index 832d81b0..facda895 100644 --- a/clearwing/sourcehunt/campaign.py +++ b/clearwing/sourcehunt/campaign.py @@ -374,6 +374,9 @@ async def _run_project( campaign_hint=target.campaign_hint or self.config.campaign_hint, parent_session_id=ps.session_id, prompt_mode=self.config.prompt_mode, + prompt_bundle=self.config.prompt_bundle, + scaffold_profile=self.config.scaffold_profile, + context_profile=self.config.context_profile, output_dir=str(self._checkpoint_dir), enable_findings_pool=True, enable_subsystem_hunt=bool(target.focus), diff --git a/clearwing/sourcehunt/campaign_config.py b/clearwing/sourcehunt/campaign_config.py index 99763cef..be39bc8a 100644 --- a/clearwing/sourcehunt/campaign_config.py +++ b/clearwing/sourcehunt/campaign_config.py @@ -41,6 +41,9 @@ class CampaignConfig: max_concurrent_containers: int = 200 depth: str = "deep" prompt_mode: str = "unconstrained" + prompt_bundle: str = "legacy-v1" + scaffold_profile: str = "native-v1" + context_profile: str = "legacy-context-v1" campaign_hint: str | None = None targets: list[CampaignTargetConfig] = field(default_factory=list) oss_fuzz_corpus: OSSFuzzCorpusConfig | None = None @@ -102,6 +105,9 @@ def load_campaign_config(path: str | Path) -> CampaignConfig: max_concurrent_containers=int(raw.get("max_concurrent_containers", 200)), depth=raw.get("depth", "deep"), prompt_mode=raw.get("prompt_mode", "unconstrained"), + prompt_bundle=raw.get("prompt_bundle", "legacy-v1"), + scaffold_profile=raw.get("scaffold_profile", "native-v1"), + context_profile=raw.get("context_profile", "legacy-context-v1"), campaign_hint=raw.get("campaign_hint"), targets=targets, oss_fuzz_corpus=oss_fuzz, @@ -137,6 +143,11 @@ def validate_campaign_config(config: CampaignConfig) -> None: f"Invalid campaign depth '{config.depth}', " f"must be one of: {', '.join(sorted(_VALID_DEPTHS))}", ) + from .optimization import get_context_profile, get_prompt_bundle, get_scaffold_profile + + get_prompt_bundle(config.prompt_bundle) + get_scaffold_profile(config.scaffold_profile) + get_context_profile(config.context_profile) for t in config.targets: if not t.repo: raise ValueError("Target repo URL is required") diff --git a/clearwing/sourcehunt/config.py b/clearwing/sourcehunt/config.py index afe57b7d..b87a68f7 100644 --- a/clearwing/sourcehunt/config.py +++ b/clearwing/sourcehunt/config.py @@ -8,7 +8,7 @@ from __future__ import annotations -from dataclasses import dataclass, field +from dataclasses import asdict, dataclass, field, fields from typing import Any @@ -78,6 +78,9 @@ class FeatureFlags: exploit_mode: bool = False agent_mode: str = "auto" # "auto" | "constrained" | "deep" prompt_mode: str = "unconstrained" # "unconstrained" | "specialist" + prompt_bundle: str = "legacy-v1" + scaffold_profile: str = "native-v1" + context_profile: str = "legacy-context-v1" @dataclass(frozen=True) @@ -92,8 +95,11 @@ class HuntTuning: seed_corpus_sources: list[str] | None = None subsystem_paths: list[str] | None = None campaign_hint: str | None = None + mechanism_store_path: str | None = None + historical_db_path: str | None = None gvisor_runtime: str | None = None sandbox_cpus: float | None = None # None = auto, 0 = unlimited + respect_gitignore: bool = False @dataclass(frozen=True) @@ -137,3 +143,65 @@ class SourceHuntConfig: features: FeatureFlags = field(default_factory=FeatureFlags) tuning: HuntTuning = field(default_factory=HuntTuning) proof: ProofConfig = field(default_factory=ProofConfig) + + @classmethod + def from_options(cls, options: dict[str, Any]) -> SourceHuntConfig: + """Group the runner's effective legacy options into typed config.""" + + def build( + model: type[Any], + prefix: str = "", + source: dict[str, Any] = options, + ) -> Any: + values = { + item.name: source[f"{prefix}{item.name}"] + for item in fields(model) + if f"{prefix}{item.name}" in source + } + return model(**values) + + proof_options = dict(options) + for name in ("flow", "retain_incomplete_certificates", "emit_rejection_certificates", "falsify"): + proof_options[f"proof_{name}"] = options[name] + return cls( + target=build(TargetConfig), + budget=build(BudgetConfig), + output=build(OutputConfig), + features=build(FeatureFlags), + tuning=build(HuntTuning), + proof=build(ProofConfig, "proof_", proof_options), + ) + + def to_dict(self) -> dict[str, Any]: + """Return the JSON-compatible representation persisted with a session.""" + + return asdict(self) + + @classmethod + def from_dict(cls, payload: dict[str, Any]) -> SourceHuntConfig: + """Restore a config previously produced by :meth:`to_dict`.""" + + budget = dict(payload.get("budget") or {}) + tier_budget = budget.get("tier_budget") + if isinstance(tier_budget, dict): + from .pool import TierBudget + + budget["tier_budget"] = TierBudget(**tier_budget) + return cls( + target=TargetConfig(**payload["target"]), + budget=BudgetConfig(**budget), + output=OutputConfig(**(payload.get("output") or {})), + features=FeatureFlags(**(payload.get("features") or {})), + tuning=HuntTuning(**(payload.get("tuning") or {})), + proof=ProofConfig(**(payload.get("proof") or {})), + ) + + +@dataclass(frozen=True) +class SourceHuntResumeOptions: + """Runtime-only options permitted while resuming a saved hunt plan.""" + + session_id: str + output_dir: str + model_override: str | None = None + live: bool = False diff --git a/clearwing/sourcehunt/context.py b/clearwing/sourcehunt/context.py new file mode 100644 index 00000000..e41d48e3 --- /dev/null +++ b/clearwing/sourcehunt/context.py @@ -0,0 +1,284 @@ +"""Bounded, deterministic context assembly for SourceHunt hunters.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from typing import Any + +from clearwing.llm import ChatMessage, NativeToolSpec + +from .optimization import ContextProfile + +_CHECKPOINT_PREFIX = "[SourceHunt durable checkpoint]" + + +def _content(message: ChatMessage) -> str: + return str(getattr(message, "content", None) or "") + + +def _clip(value: Any, limit: int) -> str: + text = str(value or "").strip() + if len(text) <= limit: + return text + return text[: max(0, limit - 1)].rstrip() + "…" + + +def _value(item: Any, name: str, default: Any = "") -> Any: + if isinstance(item, dict): + return item.get(name, default) + return getattr(item, name, default) + + +def estimate_messages_tokens(messages: list[ChatMessage]) -> int: + """Cheap, provider-independent estimate suitable for deterministic policy.""" + + total_chars = 0 + for message in messages: + total_chars += len(str(getattr(message, "role", ""))) + len(_content(message)) + total_chars += len(str(getattr(message, "tool_response_call_id", "") or "")) + for tool_call in getattr(message, "tool_calls", None) or []: + total_chars += len(str(getattr(tool_call, "fn_name", "") or "")) + total_chars += len(str(getattr(tool_call, "fn_arguments_json", "") or "")) + return max(1, total_chars // 4) + + +def estimate_request_tokens( + messages: list[ChatMessage], + *, + system: str, + tools: list[NativeToolSpec], +) -> int: + static_chars = len(system) + for tool in tools: + static_chars += len(tool.name) + len(tool.description) + static_chars += len(json.dumps(tool.schema, sort_keys=True, separators=(",", ":"))) + return estimate_messages_tokens(messages) + static_chars // 4 + + +def compact_tool_specs(tools: list[NativeToolSpec]) -> list[NativeToolSpec]: + """Remove provider-facing schema prose while preserving callable contracts.""" + + def compact_schema(value: Any) -> Any: + if isinstance(value, dict): + result = {} + for key, child in value.items(): + if key == "title": + continue + if key == "description": + result[key] = _clip(child, 120) + else: + result[key] = compact_schema(child) + return result + if isinstance(value, list): + return [compact_schema(item) for item in value] + return value + + return [ + NativeToolSpec( + name=tool.name, + description=_clip(tool.description, 160), + schema=compact_schema(tool.schema), + handler=tool.handler, + ) + for tool in tools + ] + + +def durable_checkpoint(ctx: Any, *, max_chars: int) -> str: + """Render tool-maintained investigation state without an extra model call.""" + + candidates = [] + for candidate_id in sorted(ctx.candidates): + candidate = ctx.candidates[candidate_id] + candidates.append( + { + "id": _clip(candidate.get("candidate_id", candidate_id), 24), + "status": _clip(candidate.get("status"), 20), + "location": _clip( + f"{candidate.get('file', '')}:{candidate.get('line', 0) or '?'}", 180 + ), + "hypothesis": _clip(candidate.get("hypothesis"), 360), + "attacker_control": _clip(candidate.get("attacker_control"), 220), + "invariant": _clip(candidate.get("invariant"), 220), + "effect": _clip(candidate.get("effect"), 220), + "counterargument": _clip(candidate.get("counterargument"), 300), + "evidence": _clip(candidate.get("evidence"), 360), + "next_check": _clip(candidate.get("next_check"), 260), + } + ) + + trace = [] + for index, step in enumerate(ctx.trace_steps, start=1): + trace.append( + { + "step": index, + "location": _clip( + f"{_value(step, 'file')}:{_value(step, 'line', 0) or '?'}", 180 + ), + "function": _clip(_value(step, "function"), 120), + "code": _clip(_value(step, "code_snippet"), 320), + "note": _clip(_value(step, "note"), 280), + } + ) + + payload = { + "target": str(ctx.file_path or "unknown"), + "source_windows_ranked": bool(ctx.source_windows_ranked), + "ranked_windows_read": sorted(ctx.source_windows_read), + "state_packets_read": sorted(ctx.state_packets_read), + "value_domains": [ctx.value_domains[name] for name in sorted(ctx.value_domains)], + "domain_consequences": [ + ctx.domain_consequences[name] for name in sorted(ctx.domain_consequences) + ], + "candidates": candidates, + "trace": trace, + "findings_recorded": len(ctx.findings), + } + active_next_checks = [ + candidate["next_check"] + for candidate in candidates + if candidate["status"] in {"pending", "investigating", "validated"} + and candidate["next_check"] + ] + payload["continue"] = ( + f"Do not rerank. Resolve this next: {active_next_checks[0]}" + if active_next_checks + else "Do not rerank. Continue with one unread ranked window or form a candidate." + ) + rendered = _CHECKPOINT_PREFIX + "\n" + json.dumps( + payload, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ) + if len(rendered) <= max_chars: + return rendered + + # Active state and the newest rejected/trace evidence are most actionable. + active = [item for item in candidates if item["status"] != "rejected"] + rejected = [item for item in candidates if item["status"] == "rejected"] + payload["candidates"] = active + rejected[-4:] + payload["trace"] = trace[:2] + trace[-6:] if len(trace) > 8 else trace + rendered = _CHECKPOINT_PREFIX + "\n" + json.dumps( + payload, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ) + return _clip(rendered, max_chars) + + +def _protocol_groups(messages: list[ChatMessage]) -> list[list[ChatMessage]]: + """Group assistant calls with all matching tool results.""" + + groups: list[list[ChatMessage]] = [] + index = 0 + while index < len(messages): + message = messages[index] + if getattr(message, "role", "") != "assistant": + groups.append([message]) + index += 1 + continue + group = [message] + expected = len(getattr(message, "tool_calls", None) or []) + index += 1 + while expected > 0 and index < len(messages): + next_message = messages[index] + if getattr(next_message, "role", "") != "tool": + break + group.append(next_message) + expected -= 1 + index += 1 + groups.append(group) + return groups + + +@dataclass +class ContextCompaction: + messages: list[ChatMessage] + before_tokens: int + after_tokens: int + dropped_messages: int + + +class SourceHuntContextManager: + """Apply a versioned context policy to a hunter transcript.""" + + def __init__(self, profile: ContextProfile, ctx: Any) -> None: + self.profile = profile + self.ctx = ctx + + def should_compact( + self, + messages: list[ChatMessage], + *, + system: str, + tools: list[NativeToolSpec], + ) -> bool: + return ( + self.profile.strategy != "legacy" + and estimate_request_tokens(messages, system=system, tools=tools) + >= self.profile.compact_at_tokens + ) + + def compact( + self, + messages: list[ChatMessage], + *, + system: str, + tools: list[NativeToolSpec], + ) -> ContextCompaction: + before = estimate_request_tokens(messages, system=system, tools=tools) + checkpoint = ChatMessage( + "system", + durable_checkpoint(self.ctx, max_chars=self.profile.checkpoint_chars), + ) + initial = next( + ( + message + for message in messages + if getattr(message, "role", "") == "user" + and not _content(message).startswith(_CHECKPOINT_PREFIX) + ), + None, + ) + eligible = [ + message + for message in messages + if message is not initial and not _content(message).startswith(_CHECKPOINT_PREFIX) + ] + groups = _protocol_groups(eligible) + base = ([initial] if initial is not None else []) + [checkpoint] + static_tokens = estimate_request_tokens(base, system=system, tools=tools) + remaining = max(0, self.profile.compact_to_tokens - static_tokens) + recent: list[list[ChatMessage]] = [] + used = 0 + for group in reversed(groups): + if len(recent) >= self.profile.recent_protocol_groups: + break + group_tokens = estimate_messages_tokens(group) + if recent and used + group_tokens > remaining: + break + if not recent or group_tokens <= remaining: + recent.append(group) + used += group_tokens + recent.reverse() + compacted = [*base, *(message for group in recent for message in group)] + after = estimate_request_tokens(compacted, system=system, tools=tools) + return ContextCompaction( + messages=compacted, + before_tokens=before, + after_tokens=after, + dropped_messages=max(0, len(messages) - len(compacted)), + ) + + +__all__ = [ + "ContextCompaction", + "SourceHuntContextManager", + "compact_tool_specs", + "durable_checkpoint", + "estimate_messages_tokens", + "estimate_request_tokens", +] diff --git a/clearwing/sourcehunt/findings_pool.py b/clearwing/sourcehunt/findings_pool.py index 9f367a96..94cc728b 100644 --- a/clearwing/sourcehunt/findings_pool.py +++ b/clearwing/sourcehunt/findings_pool.py @@ -227,6 +227,60 @@ def query( def all_findings(self) -> list[Finding]: return list(self._findings.values()) + def restore( + self, + findings: list[Finding], + clusters: list[dict[str, Any]] | None = None, + ) -> None: + """Restore findings and their clustering state without writing JSONL.""" + + for item in clusters or []: + cluster_id = str(item.get("cluster_id") or "") + if not cluster_id: + continue + self._clusters[cluster_id] = FindingCluster( + cluster_id=cluster_id, + root_cause_summary=str(item.get("root_cause_summary") or ""), + primitive_type=str(item.get("primitive_type") or "unknown"), + cwe=str(item.get("cwe") or ""), + ) + for finding in findings: + finding_id = finding.get("id", "") + if not finding_id: + continue + self._findings[finding_id] = finding + cluster_id = finding.get("cluster_id", "") + if not cluster_id: + continue + cluster = self._clusters.get(cluster_id) + if cluster is None: + cluster = FindingCluster( + cluster_id=cluster_id, + root_cause_summary=finding.get("description", "")[:200], + primitive_type=finding.get("primitive_type", "unknown"), + cwe=finding.get("cwe", ""), + ) + self._clusters[cluster_id] = cluster + if finding_id not in cluster.finding_ids: + cluster.finding_ids.append(finding_id) + file_path = finding.get("file", "") + if file_path: + cluster.file_paths.add(file_path) + + def cluster_state(self, cluster_ids: set[str]) -> list[dict[str, Any]]: + """Serialize only the clusters referenced by one completed work item.""" + + return [ + { + "cluster_id": cluster.cluster_id, + "root_cause_summary": cluster.root_cause_summary, + "primitive_type": cluster.primitive_type, + "cwe": cluster.cwe, + } + for cluster_id in sorted(cluster_ids) + if (cluster := self._clusters.get(cluster_id)) is not None + ] + def clusters(self) -> list[FindingCluster]: return list(self._clusters.values()) diff --git a/clearwing/sourcehunt/hunter.py b/clearwing/sourcehunt/hunter.py index 02ab7e9f..eb62bc7f 100644 --- a/clearwing/sourcehunt/hunter.py +++ b/clearwing/sourcehunt/hunter.py @@ -20,9 +20,12 @@ from clearwing.agent.tools.hunt import ( HunterContext, + build_candidate_tools, build_deep_agent_tools, build_hunter_tools, build_propagation_auditor_tools, + build_window_tools, + candidate_matches_domain, ) from clearwing.core.events import EventBus, EventType from clearwing.data.memory import ContextSummarizer @@ -31,7 +34,20 @@ from clearwing.observability.telemetry import CostTracker from clearwing.sandbox.container import SandboxContainer +from .context import ( + SourceHuntContextManager, + compact_tool_specs, + estimate_request_tokens, +) from .instrumentation import stable_run_id +from .optimization import ( + GENERIC_INSTRUCTIONS_COMPACT_V1, + GENERIC_PROMPT_HEADER, + GENERIC_PROMPT_HEADER_COMPACT, + get_context_profile, + get_prompt_bundle, + get_scaffold_profile, +) from .state import FileTarget, Finding, SubsystemTarget logger = logging.getLogger(__name__) @@ -251,6 +267,7 @@ def for_hunter( "specialist": ctx.specialist, "prompt": prompt, "tools": [tool.name for tool in tools], + "context_profile": getattr(ctx, "context_profile", "legacy-context-v1"), "seeded_crash": ctx.seeded_crash, }, ) @@ -916,6 +933,11 @@ def _choose_specialist(file_target: FileTarget) -> str: """ +COMPACT_TRACE_INSTRUCTIONS = """Trace: record exact source-backed entry, relevant conditions/flow, and sink as you +read them. Submit only after the trace is coherent and counterevidence is resolved. +""" + + _SPECIALIST_PROMPTS = { "general": GENERAL_HUNTER_PROMPT, "memory_safety": MEMORY_SAFETY_HUNTER_PROMPT, @@ -1171,6 +1193,84 @@ def _build_unconstrained_prompt( return prompt +def _build_generic_prompt( + file_target: FileTarget, + project_name: str, + seeded_crash: dict | None, + semgrep_hints: list[dict] | None, + *, + template: str, + prompt_candidate: str | None = None, + campaign_hint: str | None = None, + exploit_mode: bool = False, + entry_point: Any = None, + findings_pool: Any = None, + agent_mode: str = "constrained", + compact_static: bool = False, +) -> str: + """Render a versioned generic prompt without solution-derived context.""" + + seed_parts: list[str] = [] + if seeded_crash: + report = seeded_crash.get("report", "") + seed_parts.append( + "\nA pre-run harness produced this crash. Treat it as evidence to explain, " + "not as proof of a particular root cause:\n" + f"{report[:2000]}\n" + ) + if semgrep_hints: + hint_lines = [ + f" - line {hint.get('line', '?')}: {hint.get('description', '')}" + for hint in semgrep_hints[:5] + ] + seed_parts.append( + "\nMachine-generated static hints (untrusted starting points):\n" + + "\n".join(hint_lines) + + "\n" + ) + + render_values = { + "project_name": project_name, + "file_path": file_target.get("path", "unknown"), + "language": file_target.get("language", "unknown"), + "tags": ", ".join(file_target.get("tags", [])) or "none", + "seed_context_block": "".join(seed_parts), + } + if compact_static: + prompt = GENERIC_PROMPT_HEADER_COMPACT.format(**render_values) + ( + prompt_candidate + if prompt_candidate is not None + else GENERIC_INSTRUCTIONS_COMPACT_V1 + ) + elif prompt_candidate is not None: + prompt = GENERIC_PROMPT_HEADER.format(**render_values) + prompt_candidate + else: + prompt = template.format(**render_values) + if campaign_hint: + prompt += "\n" + CAMPAIGN_HINT_TEMPLATE.format(objective=campaign_hint) + if exploit_mode: + prompt += "\n" + EXPLOIT_EXTENSION + "\n" + MITIGATION_REASONING + if entry_point is not None: + prompt += "\n" + ENTRY_POINT_FOCUS.format( + entry_point=entry_point.function_name, + file_path=file_target.get("path", "unknown"), + start_line=entry_point.start_line, + end_line=entry_point.end_line, + entry_type=entry_point.entry_type, + ) + if findings_pool is not None: + count = len(findings_pool.all_findings()) + if count > 0: + prompt += "\n" + POOL_ACCESS_BLOCK.format(count=count) + if compact_static: + prompt += "\n" + COMPACT_TRACE_INSTRUCTIONS + else: + prompt += "\n" + ( + DEEP_TRACE_INSTRUCTIONS if agent_mode == "deep" else TRACE_BUILDING_INSTRUCTIONS + ) + return prompt + + SEED_TRANSCRIPT_BLOCK = """ A previous investigation of this file found the following: {transcript} @@ -1365,8 +1465,14 @@ class HunterRunResult: findings: list[Finding] cost_usd: float tokens_used: int - stop_reason: str # "completed" | "budget_exhausted" | "max_steps" | "degenerate_loop" + stop_reason: str # completed | budget_exhausted | max_steps | degenerate_loop | no_source_action transcript_summary: str = "" + input_tokens: int = 0 + output_tokens: int = 0 + model_calls: int = 0 + compaction_count: int = 0 + peak_context_tokens: int = 0 + peak_input_tokens: int = 0 @dataclass @@ -1378,9 +1484,24 @@ class NativeHunter: max_steps: int = 20 agent_mode: str = "constrained" # "constrained" | "deep" budget_usd: float = 0.0 # 0 = unlimited (bounded by max_steps) + input_price_per_million: float | None = None + output_price_per_million: float | None = None initial_user_message: str = "" # spec 006: override default first message max_repeated_skips: int = 15 # hard cap on total skipped degenerate-loop calls before giving up + candidate_gate_after_source_actions: int = 0 + require_source_windows: bool = False + ranked_windows_before_candidate: int = 0 + state_packets_before_candidate: int = 0 + value_domains_before_candidate: int = 0 + enable_domain_proof_refinement: bool = False summarizer: ContextSummarizer | None = field(default=None) + context_profile: str = "legacy-context-v1" + context_manager: SourceHuntContextManager | None = field(default=None) + tool_result_chars: int = 0 + temperature: float | None = None + max_output_tokens: int | None = None + closing_steps: int = 0 + initial_source_action_retries: int = 0 def _should_stop(self, step: int, cost_usd: float) -> str | None: """Return a stop reason string, or None to continue.""" @@ -1390,6 +1511,30 @@ def _should_stop(self, step: int, cost_usd: float) -> str | None: return "max_steps" return None + def _request_tools(self) -> list[NativeToolSpec]: + """Expose the refinement schema only after a proof records a gap.""" + + if self.enable_domain_proof_refinement and self.ctx.domain_proof_obligations: + return self.tools + return [ + tool for tool in self.tools if tool.name != "read_domain_proof_refinement" + ] + + def _request_system(self, step: int) -> str: + """Add a tiny, generic closure signal only near the hard step cap.""" + + remaining = self.max_steps - step + 1 + if self.closing_steps < 1 or remaining > self.closing_steps: + return self.prompt + return ( + self.prompt + + f"\n\nBudget closure: {remaining} model call(s) remain, including this one. " + "Focus only on the strongest active candidate. Use at most one narrow " + "check to resolve its decisive counterargument. If coherent, record the " + "exact trace and submit now; if disproved, reject it. If none survives, " + "finish without a finding. Do not start broad exploration." + ) + async def arun(self) -> HunterRunResult: user_msg = ( self.initial_user_message @@ -1409,6 +1554,14 @@ async def arun(self) -> HunterRunResult: total_repeated_skips = 0 tools_by_name = {tool.name: tool for tool in self.tools} last_assistant_text = "" + candidate_revision_seen = self.ctx.candidate_revision + source_actions_since_candidate_update = 0 + model_calls = 0 + compaction_count = 0 + peak_context_tokens = 0 + peak_input_tokens = 0 + source_actions_completed = 0 + source_action_retries = 0 step = 0 while True: @@ -1440,6 +1593,12 @@ async def arun(self) -> HunterRunResult: tokens_used=total_input_tokens + total_output_tokens, stop_reason=stop_reason, transcript_summary=last_assistant_text[-500:], + input_tokens=total_input_tokens, + output_tokens=total_output_tokens, + model_calls=model_calls, + compaction_count=compaction_count, + peak_context_tokens=peak_context_tokens, + peak_input_tokens=peak_input_tokens, ) model_call_id = stable_run_id( @@ -1450,16 +1609,49 @@ async def arun(self) -> HunterRunResult: "step": step, }, ) + request_tools = self._request_tools() + request_system = self._request_system(step) with spend_metadata(model_call_id=model_call_id): - if self.summarizer and self.summarizer.should_summarize(messages): + if self.context_manager and self.context_manager.should_compact( + messages, + system=request_system, + tools=request_tools, + ): + compaction = self.context_manager.compact( + messages, + system=request_system, + tools=request_tools, + ) + messages = compaction.messages + compaction_count += 1 + trajectory.log( + "context_compaction", + { + "step": step, + "context_profile": self.context_profile, + "before_tokens_estimate": compaction.before_tokens, + "after_tokens_estimate": compaction.after_tokens, + "dropped_messages": compaction.dropped_messages, + "compaction_count": compaction_count, + }, + ) + elif self.summarizer and self.summarizer.should_summarize(messages): pre = len(messages) messages = await self.summarizer.summarize(messages, self.llm) logger.info("Hunter context summarized: %d → %d messages", pre, len(messages)) + estimated_context_tokens = estimate_request_tokens( + messages, + system=request_system, + tools=request_tools, + ) + peak_context_tokens = max(peak_context_tokens, estimated_context_tokens) response = await self.llm.achat( messages=messages, - system=self.prompt, - tools=self.tools, + system=request_system, + tools=request_tools, + temperature=self.temperature, + max_tokens=self.max_output_tokens, ) # Preserve the provider's reasoning_content alongside the # visible text. `response.first_text` only returns the @@ -1486,21 +1678,36 @@ async def arun(self) -> HunterRunResult: "total_tokens": response.usage.total_tokens or 0, }, "model": response.provider_model_name, + "context_profile": self.context_profile, + "estimated_context_tokens": estimated_context_tokens, + "compaction_count": compaction_count, }, ) + model_calls += 1 total_input_tokens += response.usage.prompt_tokens or 0 total_output_tokens += response.usage.completion_tokens or 0 + peak_input_tokens = max(peak_input_tokens, response.usage.prompt_tokens or 0) # Older genai-pyo3 responses and lightweight test doubles may not # expose prompt_tokens_details at all. Treat that the same as a # response where nothing was cache-served. details = getattr(response.usage, "prompt_tokens_details", None) cached_tokens = (getattr(details, "cached_tokens", None) or 0) if details else 0 - total_cost_usd += _estimate_cost_usd( - response.usage.prompt_tokens or 0, - response.usage.completion_tokens or 0, - self.llm.model_name, - cached_tokens, - ) + if ( + self.input_price_per_million is not None + and self.output_price_per_million is not None + ): + total_cost_usd += ( + (response.usage.prompt_tokens or 0) * self.input_price_per_million + + (response.usage.completion_tokens or 0) + * self.output_price_per_million + ) / 1_000_000 + else: + total_cost_usd += _estimate_cost_usd( + response.usage.prompt_tokens or 0, + response.usage.completion_tokens or 0, + self.llm.model_name, + cached_tokens, + ) last_assistant_text = response.first_text or "" if last_assistant_text: @@ -1552,13 +1759,353 @@ async def arun(self) -> HunterRunResult: # two tools' arguments literal — only tools without an # inherent legitimate-pagination shape benefit from # normalizing away incrementing digits. - if tool_call.fn_name in ("read_file", "read_source_file"): + if tool_call.fn_name in ( + "read_file", + "read_ranked_window", + "read_source_file", + ): key = (tool_call.fn_name, tool_call.fn_arguments_json[:300]) else: normalized_args = re.sub(r"\d+", "#", tool_call.fn_arguments_json) key = (tool_call.fn_name, normalized_args[:300]) repeated_tool_calls[key] = repeated_tool_calls.get(key, 0) + 1 skipped = repeated_tool_calls[key] > 3 + if self.ctx.candidate_revision != candidate_revision_seen: + candidate_revision_seen = self.ctx.candidate_revision + source_actions_since_candidate_update = 0 + source_action = tool_call.fn_name in { + "read_domain_consequences", + "read_ranked_window", + "read_state_interactions", + "read_source_file", + "grep_source", + "read_file", + "execute", + } + ranked_coverage_target = min( + self.ranked_windows_before_candidate, + len(self.ctx.source_window_plan), + ) + ranked_coverage_incomplete = bool( + ranked_coverage_target > 0 + and len(self.ctx.source_windows_read) < ranked_coverage_target + ) + state_packet_coverage_incomplete = bool( + self.state_packets_before_candidate > 0 + and len(self.ctx.state_packets_read) + < self.state_packets_before_candidate + ) + value_domain_coverage_incomplete = bool( + self.value_domains_before_candidate > 0 + and len(self.ctx.value_domains) < self.value_domains_before_candidate + ) + unresolved_domain_ids = { + domain_id + for domain_id, domain in self.ctx.value_domains.items() + if domain.get("assessment") in {"overlap_possible", "unresolved"} + } + consequence_coverage_incomplete = bool( + unresolved_domain_ids + - set(self.ctx.domain_consequences) + ) + candidate_gate_blocked = bool( + source_action + and self.candidate_gate_after_source_actions > 0 + and source_actions_since_candidate_update + >= self.candidate_gate_after_source_actions + and not ranked_coverage_incomplete + and tool_call.fn_name != "read_domain_consequences" + ) + proof_checkpoint = next( + ( + (domain_id, candidate_id) + for domain_id, candidate_id in self.ctx.domain_candidate_ids.items() + if "record_domain_proof" in tools_by_name + and self.ctx.value_domains.get(domain_id, {}).get("assessment") + in {"overlap_possible", "unresolved"} + and not self.ctx.value_domains.get(domain_id, {}).get( + "blocking_guard_locations" + ) + and self.ctx.domain_consequences.get(domain_id, {}).get("assessment") + != "benign" + and self.ctx.domain_consequence_plans.get(domain_id, {}).get( + "boundary_facts" + ) + and self.ctx.candidates.get(candidate_id, {}).get("status") + in {"pending", "investigating"} + ), + None, + ) + domain_proof_gate_blocked = bool( + proof_checkpoint + and ( + bool(self.ctx.domain_proof_obligations.get(proof_checkpoint[0])) + or ( + self.candidate_gate_after_source_actions > 0 + and source_actions_since_candidate_update + >= self.candidate_gate_after_source_actions + ) + ) + and (source_action or tool_call.fn_name == "record_candidate") + ) + domain_refinement_pending = bool( + self.enable_domain_proof_refinement + and proof_checkpoint + and proof_checkpoint[0] in self.ctx.domain_refinement_pending_proof + ) + window_gate_blocked = bool( + source_action + and self.require_source_windows + and not self.ctx.source_windows_ranked + ) + ranked_coverage_blocked = bool( + source_action + and self.require_source_windows + and self.ctx.source_window_plan + and ranked_coverage_incomplete + and tool_call.fn_name != "read_ranked_window" + ) + candidate_window_gate_blocked = bool( + tool_call.fn_name == "record_candidate" + and ranked_coverage_incomplete + ) + state_packet_gate_blocked = bool( + source_action + and not ranked_coverage_incomplete + and state_packet_coverage_incomplete + and tool_call.fn_name != "read_state_interactions" + ) + candidate_state_packet_gate_blocked = bool( + tool_call.fn_name == "record_candidate" + and not ranked_coverage_incomplete + and state_packet_coverage_incomplete + ) + value_domain_gate_blocked = bool( + source_action + and not ranked_coverage_incomplete + and not state_packet_coverage_incomplete + and value_domain_coverage_incomplete + ) + candidate_value_domain_gate_blocked = bool( + tool_call.fn_name == "record_candidate" + and not ranked_coverage_incomplete + and not state_packet_coverage_incomplete + and value_domain_coverage_incomplete + ) + consequence_gate_blocked = bool( + source_action + and not value_domain_coverage_incomplete + and consequence_coverage_incomplete + and tool_call.fn_name != "read_domain_consequences" + ) + candidate_consequence_gate_blocked = bool( + tool_call.fn_name == "record_candidate" + and consequence_coverage_incomplete + ) + candidate_text = " ".join( + str(tool_arguments.get(field, "")) + for field in ("hypothesis", "invariant", "evidence") + ).casefold() + domain_rejection_blocked = bool( + tool_call.fn_name == "record_candidate" + and str(tool_arguments.get("status", "")) == "rejected" + and any( + consequence.get("assessment") != "benign" + and candidate_matches_domain(candidate_text, domain) + for domain_id, consequence in self.ctx.domain_consequences.items() + if (domain := self.ctx.value_domains.get(domain_id)) is not None + ) + ) + domain_candidate_mismatch = False + domain_guard_reassessment_blocked = False + if tool_call.fn_name == "record_candidate": + candidate_id = str(tool_arguments.get("candidate_id", "")) + domain_guard_reassessment_blocked = any( + tracked_id == candidate_id + and self.ctx.value_domains.get(domain_id, {}).get("assessment") + in {"overlap_possible", "unresolved"} + and bool( + self.ctx.value_domains.get(domain_id, {}).get( + "blocking_guard_locations" + ) + ) + for domain_id, tracked_id in self.ctx.domain_candidate_ids.items() + ) + tracked_domain = next( + ( + self.ctx.value_domains[domain_id] + for domain_id, tracked_id in self.ctx.domain_candidate_ids.items() + if tracked_id == candidate_id + and self.ctx.value_domains[domain_id].get("assessment") + in {"overlap_possible", "unresolved"} + and self.ctx.domain_consequences.get(domain_id, {}).get( + "assessment" + ) + != "benign" + ), + None, + ) + unresolved_domains = [ + domain + for domain in self.ctx.value_domains.values() + if domain.get("assessment") in {"overlap_possible", "unresolved"} + ] + domain = tracked_domain or ( + unresolved_domains[0] + if not self.ctx.candidates and unresolved_domains + else None + ) + if domain is not None: + domain_candidate_mismatch = not candidate_matches_domain( + candidate_text, + domain, + ) + gate_name: str | None = None + gate_message: str | None = None + if candidate_window_gate_blocked: + gate_name = "candidate_window_gate" + gate_message = ( + "inspect at least " + f"{ranked_coverage_target} " + "ranked windows before " + "forming a candidate; compare different signal mixes first." + ) + elif candidate_state_packet_gate_blocked: + gate_name = "candidate_state_packet_gate" + gate_message = ( + "expand the first ranked anchor before forming a candidate. " + "Call read_state_interactions on a ranked window you already read." + ) + elif candidate_value_domain_gate_blocked: + gate_name = "candidate_value_domain_gate" + gate_message = ( + "record one value-domain comparison before forming a candidate. " + "Call record_value_domain using only the interaction packet." + ) + elif candidate_consequence_gate_blocked: + gate_name = "candidate_consequence_gate" + gate_message = ( + "expand and assess the overlapping domain before forming a candidate. " + "Call read_domain_consequences, then record_domain_consequence." + ) + elif domain_rejection_blocked: + gate_name = "domain_rejection_gate" + gate_message = ( + "do not reject this domain candidate until its recorded consequence is " + "benign or the candidate's next_check falsifies the security effect." + ) + elif domain_guard_reassessment_blocked: + gate_name = "domain_guard_reassessment_gate" + gate_message = ( + "the packet extracted a terminating producer guard. Reassess the " + "value domain with record_value_domain before updating this candidate; " + "cite that guard's source location and the distinguished value." + ) + elif ( + domain_refinement_pending + and proof_checkpoint is not None + and tool_call.fn_name != "record_domain_proof" + ): + domain_id, candidate_id = proof_checkpoint + gate_name = "domain_refinement_proof_gate" + gate_message = ( + "proof refinement consumed. Call record_domain_proof now with " + f"domain_id={domain_id} and candidate_id={candidate_id}; update the " + "refined obligation and preserve the other answers." + ) + elif ( + self.enable_domain_proof_refinement + and + proof_checkpoint is not None + and self.ctx.domain_proof_obligations.get(proof_checkpoint[0]) + and tool_call.fn_name + not in {"read_domain_proof_refinement", "record_domain_proof"} + ): + domain_id, _candidate_id = proof_checkpoint + unresolved = self.ctx.domain_proof_obligations[domain_id] + gate_name = "domain_proof_refinement_gate" + gate_message = ( + "domain proof has one prerequisite unresolved. Call " + "read_domain_proof_refinement with " + f"domain_id={domain_id} and obligation={unresolved[0]}." + ) + elif domain_proof_gate_blocked and proof_checkpoint is not None: + domain_id, candidate_id = proof_checkpoint + unresolved = [ + obligation + for obligation in self.ctx.domain_proof_obligations.get(domain_id, []) + if (domain_id, obligation) not in self.ctx.domain_refinements_read + ] + if ( + self.enable_domain_proof_refinement + and unresolved + and "read_domain_proof_refinement" in tools_by_name + and tool_call.fn_name != "read_domain_proof_refinement" + ): + gate_name = "domain_proof_refinement_gate" + choices = ", ".join(unresolved) + gate_message = ( + "domain proof has unresolved obligations. Call " + "read_domain_proof_refinement with " + f"domain_id={domain_id} and one of: {choices}." + ) + elif tool_call.fn_name != "read_domain_proof_refinement": + gate_name = "domain_proof_gate" + gate_message = ( + "domain proof checkpoint required before more source actions or " + "candidate rewrites. Call record_domain_proof now with " + f"domain_id={domain_id} and candidate_id={candidate_id}. Answer " + "its four booleans from source read so far; use false for anything " + "unresolved. It will narrow the next check or validate and seed " + "the exact trace." + ) + elif domain_candidate_mismatch: + gate_name = "candidate_domain_mismatch" + gate_message = ( + "the candidate must stay tied to the concrete state identifiers " + "extracted by the domain packet and, when present, its distinguished " + "value. The scaffold supplies its next check automatically." + ) + elif window_gate_blocked: + gate_name = "window_gate" + gate_message = ( + "source-window ranking required before direct source actions. " + "Call rank_source_windows on the assigned file, then choose a " + "small, signal-diverse set of windows to inspect." + ) + elif state_packet_gate_blocked: + gate_name = "state_packet_gate" + gate_message = ( + "expand the first ranked anchor before free-form source actions. " + "Call read_state_interactions on a ranked window you already read." + ) + elif value_domain_gate_blocked: + gate_name = "value_domain_gate" + gate_message = ( + "record one value-domain comparison before free-form source actions. " + "Call record_value_domain using only the interaction packet." + ) + elif consequence_gate_blocked: + gate_name = "domain_consequence_gate" + gate_message = ( + "assess the overlapping domain before free-form source actions. Call " + "read_domain_consequences, then record_domain_consequence." + ) + elif ranked_coverage_blocked: + gate_name = "ranked_window_gate" + gate_message = ( + "finish ranked-window coverage before free-form source actions. " + "Call read_ranked_window using the next diverse window_id from " + "the previous result." + ) + elif candidate_gate_blocked: + gate_name = "candidate_gate" + gate_message = ( + "candidate ledger checkpoint required before more source actions. " + "Call record_candidate now. Create the best concrete hypothesis if " + "the queue is empty; otherwise update the selected candidate with " + "new evidence, counterevidence, status, and one narrow next_check." + ) if skipped: total_repeated_skips += 1 @@ -1571,7 +2118,7 @@ async def arun(self) -> HunterRunResult: "final summary and no tool calls to finish this hunt." ) } - tool_summary = _tool_output_text( + tool_summary = self._tool_output_text( tool_call.fn_name, tool_arguments, tool_output, @@ -1586,6 +2133,23 @@ async def arun(self) -> HunterRunResult: "repeated_skip": True, }, ) + elif gate_message is not None: + tool_output = {"error": gate_message} + tool_summary = self._tool_output_text( + tool_call.fn_name, + tool_arguments, + tool_output, + ) + gate_payload = { + "step": step, + "tool_call": _serialize_tool_call(tool_call), + "tool_output": tool_output, + "tool_summary": tool_summary, + "repeated_skip": False, + } + if gate_name is not None: + gate_payload[gate_name] = True + trajectory.log("tool_result", gate_payload) else: trajectory.log( "tool_call", @@ -1594,8 +2158,14 @@ async def arun(self) -> HunterRunResult: "tool_call": _serialize_tool_call(tool_call), }, ) + if source_action: + source_actions_since_candidate_update += 1 tool_output = await self._run_tool(tools_by_name, tool_call) - tool_summary = _tool_output_text( + if source_action and not ( + isinstance(tool_output, dict) and tool_output.get("error") + ): + source_actions_completed += 1 + tool_summary = self._tool_output_text( tool_call.fn_name, tool_arguments, tool_output, @@ -1651,9 +2221,56 @@ async def arun(self) -> HunterRunResult: tokens_used=total_input_tokens + total_output_tokens, stop_reason="degenerate_loop", transcript_summary=last_assistant_text[-500:], + input_tokens=total_input_tokens, + output_tokens=total_output_tokens, + model_calls=model_calls, + compaction_count=compaction_count, + peak_context_tokens=peak_context_tokens, + peak_input_tokens=peak_input_tokens, ) continue + if source_actions_completed == 0 and self.initial_source_action_retries > 0: + if last_assistant_text: + messages.append(ChatMessage("assistant", last_assistant_text)) + if source_action_retries < self.initial_source_action_retries: + source_action_retries += 1 + retry_message = ( + "No source tool ran. Use one available source-read or search tool " + "now. Do not write or simulate a tool call in text." + ) + messages.append(ChatMessage("user", retry_message)) + trajectory.log( + "source_action_retry", + {"step": step, "retry": source_action_retries}, + ) + continue + + trajectory.log( + "finish", + { + "step": step, + "status": "no_source_action", + "findings": [], + "total_input_tokens": total_input_tokens, + "total_output_tokens": total_output_tokens, + "total_cost_usd": total_cost_usd, + }, + ) + return HunterRunResult( + findings=[], + cost_usd=total_cost_usd, + tokens_used=total_input_tokens + total_output_tokens, + stop_reason="no_source_action", + transcript_summary=last_assistant_text[-500:], + input_tokens=total_input_tokens, + output_tokens=total_output_tokens, + model_calls=model_calls, + compaction_count=compaction_count, + peak_context_tokens=peak_context_tokens, + peak_input_tokens=peak_input_tokens, + ) + if last_assistant_text: messages.append(ChatMessage("assistant", last_assistant_text)) logger.info( @@ -1679,8 +2296,20 @@ async def arun(self) -> HunterRunResult: tokens_used=total_input_tokens + total_output_tokens, stop_reason="completed", transcript_summary=last_assistant_text[-500:], + input_tokens=total_input_tokens, + output_tokens=total_output_tokens, + model_calls=model_calls, + compaction_count=compaction_count, + peak_context_tokens=peak_context_tokens, + peak_input_tokens=peak_input_tokens, ) + def _tool_output_text(self, tool_name: str, arguments: dict[str, Any], value: Any) -> str: + rendered = _tool_output_text(tool_name, arguments, value) + if self.tool_result_chars > 0: + return _clip_text(rendered, self.tool_result_chars) + return rendered + async def _run_tool( self, tools_by_name: dict[str, NativeToolSpec], @@ -1877,7 +2506,16 @@ def build_hunter_agent( default_sanitizers: tuple = ("asan", "ubsan"), # v0.4: primary sanitizer combo agent_mode: str = "constrained", # "constrained" | "deep" budget_usd: float = 0.0, + input_price_per_million: float | None = None, + output_price_per_million: float | None = None, prompt_mode: str = "unconstrained", # "unconstrained" | "specialist" + prompt_bundle: str = "legacy-v1", + scaffold_profile: str = "native-v1", + context_profile: str = "legacy-context-v1", + prompt_candidate: str | None = None, + max_steps_override: int | None = None, + temperature: float | None = None, + max_output_tokens: int | None = None, campaign_hint: str | None = None, exploit_mode: bool = False, seed_transcript: str | None = None, @@ -1903,8 +2541,15 @@ def build_hunter_agent( variant_seed: v0.3 — variant hunter loop seed. agent_mode: "constrained" (legacy 9-tool) or "deep" (full-shell 4+1 tool). budget_usd: Per-agent budget in USD (0 = unlimited, bounded by max_steps). + input_price_per_million: Optional run-scoped input token price override. + output_price_per_million: Optional run-scoped output token price override. prompt_mode: "unconstrained" (simple discovery prompt) or "specialist" (legacy prescriptive checklists with execution rules). + prompt_bundle: Versioned prompt unit. ``generic-security-v1`` disables + solution-derived heuristics and historical CVE context. + scaffold_profile: Versioned tool/control surface. ``minimal-linear-v1`` + exposes a Pi-like narrow tool set. + prompt_candidate: Optional optimizer-proposed generic instruction text. campaign_hint: Optional campaign objective, e.g. "bugs reachable from unauthenticated remote input". exploit_mode: When True, append exploit-writing and mitigation-reasoning @@ -1917,6 +2562,17 @@ def build_hunter_agent( (native_hunter, hunter_context). The caller owns the context and reads ctx.findings after the run completes. """ + bundle = get_prompt_bundle(prompt_bundle) + scaffold = get_scaffold_profile(scaffold_profile) + context_policy = get_context_profile(context_profile) + if prompt_candidate is not None and not bundle.is_generic: + raise ValueError("prompt_candidate requires a generic prompt bundle") + if max_steps_override is not None and max_steps_override < 1: + raise ValueError("max_steps_override must be positive when provided") + if temperature is not None and not 0.0 <= temperature <= 2.0: + raise ValueError("temperature must be between 0 and 2") + if max_output_tokens is not None and max_output_tokens < 1: + raise ValueError("max_output_tokens must be positive when provided") tier = file_target.get("tier", "B") if specialist is None: if tier == "C": @@ -1924,24 +2580,55 @@ def build_hunter_agent( else: specialist = _choose_specialist(file_target) + if bundle.is_generic: + context_specialist = "generic" + elif prompt_mode == "specialist" or specialist == "propagation": + context_specialist = specialist + else: + context_specialist = "unconstrained" + ctx = HunterContext( repo_path=repo_path, sandbox=sandbox, findings=[], file_path=file_target.get("path"), session_id=session_id, - specialist=( - specialist - if prompt_mode == "specialist" or specialist == "propagation" - else "unconstrained" - ), + specialist=context_specialist, seeded_crash=seeded_crash, sandbox_manager=sandbox_manager, default_sanitizers=tuple(default_sanitizers), findings_pool=findings_pool, + require_validated_candidate_before_finding=( + scaffold.require_validated_candidate_before_finding + ), + require_active_candidate_before_finding=( + scaffold.require_active_candidate_before_finding + ), ) - if specialist == "propagation": + if bundle.is_generic: + combined_hints = list(semgrep_hints or []) + prompt = _build_generic_prompt( + file_target, + project_name, + seeded_crash, + combined_hints, + template=bundle.discovery_template or "", + prompt_candidate=prompt_candidate, + campaign_hint=campaign_hint, + exploit_mode=exploit_mode, + entry_point=entry_point, + findings_pool=findings_pool, + agent_mode=agent_mode, + compact_static=context_policy.compact_static_instructions, + ) + if agent_mode == "deep": + tools = build_deep_agent_tools(ctx) + max_steps = 500 + else: + tools = build_hunter_tools(ctx) + max_steps = 20 + elif specialist == "propagation": tools = build_propagation_auditor_tools(ctx) prompt = _build_propagation_prompt(file_target) max_steps = 20 @@ -1982,7 +2669,7 @@ def build_hunter_agent( else: tools = build_hunter_tools(ctx) combined_hints = list(semgrep_hints or []) - if specialist == "memory_safety": + if specialist == "memory_safety" and bundle.allow_solution_heuristics: combined_hints = _memory_safety_heuristic_hints(repo_path, file_target) + combined_hints prompt = _build_hunter_prompt( file_target, @@ -1993,9 +2680,46 @@ def build_hunter_agent( ) max_steps = 20 + allowed_tools = scaffold.tool_names(agent_mode) + if allowed_tools is not None: + if { + "record_candidate", + "record_value_domain", + "record_domain_consequence", + "record_domain_proof", + } & allowed_tools: + tools.extend(build_candidate_tools(ctx)) + if { + "rank_source_windows", + "read_state_interactions", + "read_domain_consequences", + "read_domain_proof_refinement", + } & allowed_tools: + tools.extend(build_window_tools(ctx)) + tools = [tool for tool in tools if tool.name in allowed_tools] + scaffold_instructions = ( + scaffold.compact_instructions + if context_policy.compact_static_instructions and scaffold.compact_instructions + else scaffold.instructions + ) + if scaffold_instructions: + prompt += "\n\n" + scaffold_instructions + if max_steps_override is not None: + max_steps = max_steps_override + if seed_transcript: prompt += "\n\n" + SEED_TRANSCRIPT_BLOCK.format(transcript=seed_transcript) + if context_policy.compact_tool_specs: + tools = compact_tool_specs(tools) + ctx.context_profile = context_policy.name + ctx.enable_domain_proof_refinement = scaffold.enable_domain_proof_refinement + context_manager = ( + SourceHuntContextManager(context_policy, ctx) + if context_policy.strategy != "legacy" + else None + ) + return NativeHunter( llm=llm, prompt=prompt, @@ -2004,5 +2728,21 @@ def build_hunter_agent( max_steps=max_steps, agent_mode=agent_mode, budget_usd=budget_usd, - summarizer=ContextSummarizer(), + input_price_per_million=input_price_per_million, + output_price_per_million=output_price_per_million, + initial_user_message=("Begin." if context_policy.strategy != "legacy" else ""), + summarizer=ContextSummarizer() if context_policy.strategy == "legacy" else None, + candidate_gate_after_source_actions=scaffold.candidate_gate_after_source_actions, + require_source_windows=scaffold.require_source_windows, + ranked_windows_before_candidate=scaffold.ranked_windows_before_candidate, + state_packets_before_candidate=scaffold.state_packets_before_candidate, + value_domains_before_candidate=scaffold.value_domains_before_candidate, + enable_domain_proof_refinement=scaffold.enable_domain_proof_refinement, + context_profile=context_policy.name, + context_manager=context_manager, + tool_result_chars=context_policy.tool_result_chars, + temperature=temperature, + max_output_tokens=max_output_tokens, + closing_steps=scaffold.closing_steps, + initial_source_action_retries=scaffold.initial_source_action_retries, ), ctx diff --git a/clearwing/sourcehunt/optimization.py b/clearwing/sourcehunt/optimization.py new file mode 100644 index 00000000..93f348c5 --- /dev/null +++ b/clearwing/sourcehunt/optimization.py @@ -0,0 +1,720 @@ +"""Versioned, leakage-safe prompt and scaffold profiles for SourceHunt. + +The optimization campaign treats prompts and agent topology as independent +variables. This module keeps those variables explicit and provides a linter +for rejecting candidates that contain benchmark answers. +""" + +from __future__ import annotations + +import re +from collections.abc import Iterable +from dataclasses import dataclass +from typing import Any + +GENERIC_PROMPT_HEADER = """You are conducting a blind source-code security audit. + +Project: {project_name} +Starting file: {file_path} +Language: {language} +Tags: {tags} +{seed_context_block} +""" + + +GENERIC_PROMPT_HEADER_COMPACT = """Blind source-code security audit. +Target: {file_path} ({language}; {tags}) +{seed_context_block} +""" + + +GENERIC_INSTRUCTIONS_V1 = """Goal: find as many distinct, real, attacker-reachable vulnerabilities as possible. +Do not assume the target contains a bug, do not invent evidence, and do not report +the same root cause more than once. + +Use this investigation loop: +1. ORIENT: read a narrow source window and identify external inputs, state, guards, + allocations, lifetimes, and security-sensitive effects. +2. CANDIDATES: maintain a short queue of concrete hypotheses. For each, track the + attacker-controlled value or event, violated invariant, reachable effect, and + the strongest fact that might disprove it. +3. INVESTIGATE: choose the highest-value unresolved candidate. Follow callers, + callees, definitions, and relevant state transitions with narrow reads/searches. +4. CHALLENGE: actively try to falsify the candidate. Check dominating validation, + type/range limits, ownership and lifetime rules, error paths, configuration, + reachability, and whether the supposed sink is actually security-relevant. +5. SUBMIT: when the entry-to-effect mechanism survives challenge, record its exact + trace and submit it with the strongest evidence available. Then continue hunting + for independent root causes until the budget is exhausted. + +Prefer mechanisms over surface patterns. A suspicious API call is not a finding +without a reachable violating input and a security consequence. Dynamic evidence is +valuable but not mandatory when the source establishes the complete mechanism. +""" + + +GENERIC_INSTRUCTIONS_COMPACT_V1 = """Find distinct, real, attacker-reachable vulnerabilities; never invent evidence. +Read narrowly. Keep 1-3 concrete candidates: attacker control, violated invariant, +security effect, strongest counterargument, and one next check. Follow only the best +candidate through callers/callees and state transitions. Try to disprove it with +guards, bounds, ownership, lifetime, configuration, and reachability. Submit an +exact entry-to-effect trace only when it survives; then continue with independent +root causes. Suspicious APIs alone are not findings. +""" + + +GENERIC_DISCOVERY_V1 = GENERIC_PROMPT_HEADER + GENERIC_INSTRUCTIONS_V1 + + +MINIMAL_LINEAR_INSTRUCTIONS = """Scaffold protocol: +- Keep the interaction linear: READ -> CANDIDATES -> INVESTIGATE -> CHALLENGE -> SUBMIT. +- Use the smallest useful source window or search result; expand only to resolve a + named uncertainty. +- Keep candidate state in your reasoning instead of producing long progress essays. +- Before submitting, state the best counterargument and resolve it with source or + runtime evidence. +- A submission must identify an attacker-controlled entry, violated invariant, + reachable security effect, and exact supporting trace. +""" + + +MINIMAL_LINEAR_COMPACT_INSTRUCTIONS = """Protocol: READ -> CANDIDATES -> INVESTIGATE -> CHALLENGE -> SUBMIT. +Use narrow reads, expand only to answer a named uncertainty, and resolve the best +counterargument before submitting an exact attacker-entry-to-effect trace. +""" + + +CANDIDATE_LEDGER_INSTRUCTIONS = """Scaffold protocol: +- Use narrow READ and search actions. Do not sweep the file sequentially. +- Within the first two source actions, call record_candidate for one to three + concrete hypotheses. If no strong candidate exists yet, record the best weak + hypothesis and the exact search that would strengthen or reject it. +- Before each additional source action, select one candidate and resolve only its + next_check. Update that candidate immediately after learning new evidence. +- Reject candidates aggressively when a guard or invariant disproves them. Replace + rejected candidates with new hypotheses instead of continuing the same search. +- Validate the best surviving candidate, record its exact trace, submit it, and then + continue with the remaining queue for independent root causes. +""" + + +CANDIDATE_LEDGER_COMPACT_INSTRUCTIONS = """Protocol: use narrow reads. Within two source actions, record 1-3 concrete +candidates. Resolve one candidate's next_check at a time; update or reject it +immediately. Persist an exact entry-to-effect trace before submitting, then seek +independent root causes. +""" + + +WINDOW_LEDGER_INSTRUCTIONS = """Scaffold protocol: +- Start by calling rank_source_windows on the assigned file. Treat its output only + as a reading plan, never as vulnerability evidence. +- Read a small, diverse set of ranked windows before committing to a hypothesis; + include representation/state, input-boundary, and effect/lifetime signals when + available instead of selecting only familiar unsafe APIs. +- Use record_candidate to maintain concrete hypotheses and their strongest + counterarguments. Resolve one narrow next_check at a time and reject aggressively. +- Validate the best surviving candidate, record an exact entry-to-effect trace, + submit it, then continue with independent candidates. +""" + + +WINDOW_LEDGER_COMPACT_INSTRUCTIONS = """Protocol: call rank_source_windows first; it is only a reading plan. Inspect a +few signal-diverse narrow windows, then maintain 1-3 candidates with a strongest +counterargument and one next_check. Update or reject after each check. Persist an +exact entry-to-effect trace before submitting, then seek independent root causes. +""" + + +GUIDED_WINDOW_LEDGER_INSTRUCTIONS = """Protocol: +- Call rank_source_windows once, then read W1 with read_ranked_window. +- Before forming a candidate, read two more windows with different signal mixes. +- The anchors are only a reading plan. Compare the three mechanisms; do not commit + to the first familiar pattern. +- Maintain 1-3 candidates. Resolve one next_check, then update or reject it. +- After compaction, obey the durable checkpoint's continuation; never rerank. +- Persist an exact entry-to-effect trace before submitting, then seek independent +root causes. +""" + + +STATE_INTERACTION_LEDGER_INSTRUCTIONS = """Protocol: +- Call rank_source_windows once, read W1, then call read_state_interactions(W1). +- The packet is only orientation. Call record_value_domain once using packet lines + to decide whether producer and distinguished stored domains overlap. Treat them + as overlapping unless a source-backed dominating guard excludes the value. +- If overlap is possible, call read_domain_consequences then + record_domain_consequence before the first candidate. Form that candidate from + the stored-state/producer interaction and its changed consumer branch. +- Resolve one next_check, then update or reject it. Do not rerank or sweep. +- Keep every update tied to the distinguished value. After source checks, call + record_domain_proof. If it validates and seeds the exact trace, submit with + candidate_id and static corroboration or stronger evidence. +""" + + +@dataclass(frozen=True) +class PromptBundle: + """One immutable prompt candidate used as an optimization unit.""" + + name: str + discovery_template: str | None + allow_solution_heuristics: bool + allow_historical_context: bool + + @property + def is_generic(self) -> bool: + return self.discovery_template is not None + + +@dataclass(frozen=True) +class ScaffoldProfile: + """Tool surface and control instructions independent of prompt wording.""" + + name: str + instructions: str = "" + compact_instructions: str = "" + constrained_tools: frozenset[str] | None = None + deep_tools: frozenset[str] | None = None + candidate_gate_after_source_actions: int = 0 + require_source_windows: bool = False + ranked_windows_before_candidate: int = 0 + state_packets_before_candidate: int = 0 + value_domains_before_candidate: int = 0 + require_validated_candidate_before_finding: bool = False + require_active_candidate_before_finding: bool = False + enable_domain_proof_refinement: bool = False + closing_steps: int = 0 + initial_source_action_retries: int = 0 + + def tool_names(self, agent_mode: str) -> frozenset[str] | None: + return self.deep_tools if agent_mode == "deep" else self.constrained_tools + + +@dataclass(frozen=True) +class ContextProfile: + """Versioned request-context assembly policy for one hunter.""" + + name: str + strategy: str = "legacy" + compact_at_tokens: int = 150_000 + compact_to_tokens: int = 120_000 + recent_protocol_groups: int = 3 + checkpoint_chars: int = 6_000 + tool_result_chars: int = 0 + compact_static_instructions: bool = False + compact_tool_specs: bool = False + + +PROMPT_BUNDLES: dict[str, PromptBundle] = { + "legacy-v1": PromptBundle( + name="legacy-v1", + discovery_template=None, + allow_solution_heuristics=True, + allow_historical_context=True, + ), + "generic-security-v1": PromptBundle( + name="generic-security-v1", + discovery_template=GENERIC_DISCOVERY_V1, + allow_solution_heuristics=False, + allow_historical_context=False, + ), +} + + +SCAFFOLD_PROFILES: dict[str, ScaffoldProfile] = { + "native-v1": ScaffoldProfile(name="native-v1"), + "minimal-linear-v1": ScaffoldProfile( + name="minimal-linear-v1", + instructions=MINIMAL_LINEAR_INSTRUCTIONS, + compact_instructions=MINIMAL_LINEAR_COMPACT_INSTRUCTIONS, + constrained_tools=frozenset( + { + "read_source_file", + "grep_source", + "record_trace_step", + "record_finding", + } + ), + deep_tools=frozenset( + { + "execute", + "read_file", + "record_trace_step", + "record_finding", + } + ), + ), + "candidate-ledger-v1": ScaffoldProfile( + name="candidate-ledger-v1", + instructions=CANDIDATE_LEDGER_INSTRUCTIONS, + compact_instructions=CANDIDATE_LEDGER_COMPACT_INSTRUCTIONS, + constrained_tools=frozenset( + { + "read_source_file", + "grep_source", + "record_candidate", + "record_trace_step", + "record_finding", + } + ), + deep_tools=frozenset( + { + "execute", + "read_file", + "record_candidate", + "record_trace_step", + "record_finding", + } + ), + candidate_gate_after_source_actions=2, + ), + "candidate-ledger-closure-v1": ScaffoldProfile( + name="candidate-ledger-closure-v1", + instructions=CANDIDATE_LEDGER_INSTRUCTIONS, + compact_instructions=CANDIDATE_LEDGER_COMPACT_INSTRUCTIONS, + constrained_tools=frozenset( + { + "read_source_file", + "grep_source", + "record_candidate", + "record_trace_step", + "record_finding", + } + ), + deep_tools=frozenset( + { + "execute", + "read_file", + "record_candidate", + "record_trace_step", + "record_finding", + } + ), + candidate_gate_after_source_actions=2, + closing_steps=3, + ), + "candidate-ledger-source-retry-v1": ScaffoldProfile( + name="candidate-ledger-source-retry-v1", + instructions=CANDIDATE_LEDGER_INSTRUCTIONS, + compact_instructions=CANDIDATE_LEDGER_COMPACT_INSTRUCTIONS, + constrained_tools=frozenset( + { + "read_source_file", + "grep_source", + "record_candidate", + "record_trace_step", + "record_finding", + } + ), + deep_tools=frozenset( + { + "execute", + "read_file", + "record_candidate", + "record_trace_step", + "record_finding", + } + ), + candidate_gate_after_source_actions=2, + initial_source_action_retries=1, + ), + "candidate-ledger-source-retry-active-v1": ScaffoldProfile( + name="candidate-ledger-source-retry-active-v1", + instructions=CANDIDATE_LEDGER_INSTRUCTIONS, + compact_instructions=CANDIDATE_LEDGER_COMPACT_INSTRUCTIONS, + constrained_tools=frozenset( + { + "read_source_file", + "grep_source", + "record_candidate", + "record_trace_step", + "record_finding", + } + ), + deep_tools=frozenset( + { + "execute", + "read_file", + "record_candidate", + "record_trace_step", + "record_finding", + } + ), + candidate_gate_after_source_actions=2, + initial_source_action_retries=1, + require_active_candidate_before_finding=True, + ), + "window-ledger-v1": ScaffoldProfile( + name="window-ledger-v1", + instructions=WINDOW_LEDGER_INSTRUCTIONS, + compact_instructions=WINDOW_LEDGER_COMPACT_INSTRUCTIONS, + constrained_tools=frozenset( + { + "rank_source_windows", + "read_source_file", + "grep_source", + "record_candidate", + "record_trace_step", + "record_finding", + } + ), + deep_tools=frozenset( + { + "rank_source_windows", + "execute", + "read_file", + "record_candidate", + "record_trace_step", + "record_finding", + } + ), + candidate_gate_after_source_actions=2, + require_source_windows=True, + ), + "guided-window-ledger-v1": ScaffoldProfile( + name="guided-window-ledger-v1", + instructions=GUIDED_WINDOW_LEDGER_INSTRUCTIONS, + compact_instructions=GUIDED_WINDOW_LEDGER_INSTRUCTIONS, + constrained_tools=frozenset( + { + "rank_source_windows", + "read_ranked_window", + "read_source_file", + "grep_source", + "record_candidate", + "record_trace_step", + "record_finding", + } + ), + deep_tools=frozenset( + { + "rank_source_windows", + "read_ranked_window", + "execute", + "read_file", + "record_candidate", + "record_trace_step", + "record_finding", + } + ), + candidate_gate_after_source_actions=3, + require_source_windows=True, + ranked_windows_before_candidate=3, + ), + "state-interaction-ledger-v1": ScaffoldProfile( + name="state-interaction-ledger-v1", + instructions=STATE_INTERACTION_LEDGER_INSTRUCTIONS, + compact_instructions=STATE_INTERACTION_LEDGER_INSTRUCTIONS, + constrained_tools=frozenset( + { + "rank_source_windows", + "read_ranked_window", + "read_state_interactions", + "read_domain_consequences", + "record_value_domain", + "record_domain_consequence", + "record_domain_proof", + "read_source_file", + "grep_source", + "record_candidate", + "record_trace_step", + "record_finding", + } + ), + deep_tools=frozenset( + { + "rank_source_windows", + "read_ranked_window", + "read_state_interactions", + "read_domain_consequences", + "record_value_domain", + "record_domain_consequence", + "record_domain_proof", + "execute", + "read_file", + "record_candidate", + "record_trace_step", + "record_finding", + } + ), + candidate_gate_after_source_actions=2, + require_source_windows=True, + ranked_windows_before_candidate=1, + state_packets_before_candidate=1, + value_domains_before_candidate=1, + require_validated_candidate_before_finding=True, + ), + "proof-refinement-ledger-v1": ScaffoldProfile( + name="proof-refinement-ledger-v1", + instructions=STATE_INTERACTION_LEDGER_INSTRUCTIONS, + compact_instructions=STATE_INTERACTION_LEDGER_INSTRUCTIONS, + constrained_tools=frozenset( + { + "rank_source_windows", + "read_ranked_window", + "read_state_interactions", + "read_domain_consequences", + "read_domain_proof_refinement", + "record_value_domain", + "record_domain_consequence", + "record_domain_proof", + "read_source_file", + "grep_source", + "record_candidate", + "record_trace_step", + "record_finding", + } + ), + deep_tools=frozenset( + { + "rank_source_windows", + "read_ranked_window", + "read_state_interactions", + "read_domain_consequences", + "read_domain_proof_refinement", + "record_value_domain", + "record_domain_consequence", + "record_domain_proof", + "execute", + "read_file", + "record_candidate", + "record_trace_step", + "record_finding", + } + ), + candidate_gate_after_source_actions=2, + require_source_windows=True, + ranked_windows_before_candidate=1, + state_packets_before_candidate=1, + value_domains_before_candidate=1, + require_validated_candidate_before_finding=True, + enable_domain_proof_refinement=True, + ), +} + + +CONTEXT_PROFILES: dict[str, ContextProfile] = { + "legacy-context-v1": ContextProfile(name="legacy-context-v1"), + "compact-small-model-v1": ContextProfile( + name="compact-small-model-v1", + strategy="deterministic-checkpoint", + compact_at_tokens=12_000, + compact_to_tokens=8_000, + recent_protocol_groups=3, + checkpoint_chars=6_000, + tool_result_chars=3_500, + compact_static_instructions=True, + compact_tool_specs=True, + ), +} + + +def get_prompt_bundle(name: str) -> PromptBundle: + try: + return PROMPT_BUNDLES[name] + except KeyError as exc: + choices = ", ".join(sorted(PROMPT_BUNDLES)) + raise ValueError(f"Unknown prompt bundle {name!r}; choose one of: {choices}") from exc + + +def get_scaffold_profile(name: str) -> ScaffoldProfile: + try: + return SCAFFOLD_PROFILES[name] + except KeyError as exc: + choices = ", ".join(sorted(SCAFFOLD_PROFILES)) + raise ValueError(f"Unknown scaffold profile {name!r}; choose one of: {choices}") from exc + + +def get_context_profile(name: str) -> ContextProfile: + try: + return CONTEXT_PROFILES[name] + except KeyError as exc: + choices = ", ".join(sorted(CONTEXT_PROFILES)) + raise ValueError(f"Unknown context profile {name!r}; choose one of: {choices}") from exc + + +@dataclass(frozen=True) +class PromptLeakage: + category: str + value: str + + +_CVE_RE = re.compile(r"\bCVE-\d{4}-\d{4,}\b", re.IGNORECASE) +_COMMIT_RE = re.compile(r"(? Any: + return case.get(name, default) if isinstance(case, dict) else getattr(case, name, default) + + +def _distinctive_symbol(value: str) -> bool: + return len(value) >= 4 or any(marker in value for marker in ("_", ".", "::", "/")) + + +def manifest_forbidden_terms(manifest: Any) -> dict[str, set[str]]: + """Extract answer-bearing terms without importing the evaluator models.""" + + cases = _case_value(manifest, "cases", []) or [] + terms: dict[str, set[str]] = { + "case_id": set(), + "repository": set(), + "cve": set(), + "commit": set(), + "target_file": set(), + "target_symbol": set(), + "solution_phrase": set(), + "mechanism": set(), + "expected_cwe": set(), + } + for case in cases: + case_id = str(_case_value(case, "id", "")).strip() + if case_id: + terms["case_id"].add(case_id) + repository = str(_case_value(case, "repository", "")).strip() + if repository: + terms["repository"].add(repository) + repository_name = repository.rstrip("/").rsplit("/", 1)[-1].removesuffix(".git") + if len(repository_name) >= 4: + terms["repository"].add(repository_name) + terms["cve"].update(str(value) for value in (_case_value(case, "cves", []) or [])) + for field in ("vulnerable_commit", "fixed_commit"): + value = str(_case_value(case, field, "") or "").strip() + if value: + terms["commit"].add(value) + + truth = _case_value(case, "ground_truth", {}) or {} + terms["target_file"].update( + str(value) for value in (_case_value(truth, "target_files", []) or []) + ) + for field in ("target_functions", "expected_fact_symbols"): + for raw in _case_value(truth, field, []) or []: + value = str(raw).strip() + if value and _distinctive_symbol(value): + terms["target_symbol"].add(value) + for step in _case_value(truth, "trace", []) or []: + value = str(_case_value(step, "symbol", "")).strip() + if value and _distinctive_symbol(value): + terms["target_symbol"].add(value) + for field in ( + "entry_points", + "sources", + "sinks", + "transformations", + "invariants", + "guards", + "trigger_constraints", + "reproduction_behavior", + "expected_proof_plans", + "expected_predicates", + "expected_evidence_kinds", + ): + terms["solution_phrase"].update( + str(value) for value in (_case_value(truth, field, []) or []) + ) + threat = _case_value(truth, "threat_model", {}) or {} + for field in ( + "attacker_principal", + "attacker_capabilities", + "trust_boundary", + "protected_asset", + "capability_gained", + "security_property_violated", + "deployment_assumptions", + ): + raw = _case_value(threat, field, "") + values = raw if isinstance(raw, list) else [raw] + terms["solution_phrase"].update(str(value) for value in values) + terms["mechanism"].update( + str(value) for value in (_case_value(truth, "expected_mechanisms", []) or []) + ) + terms["expected_cwe"].update( + str(value) for value in (_case_value(truth, "expected_cwes", []) or []) + ) + return terms + + +def lint_prompt_candidate( + candidate: str, + *, + manifest: Any | None = None, + forbidden_terms: dict[str, Iterable[str]] | None = None, +) -> list[PromptLeakage]: + """Return every benchmark-answer leak in a proposed generic prompt.""" + + leaks: set[PromptLeakage] = set() + for match in _CVE_RE.finditer(candidate): + leaks.add(PromptLeakage("cve", match.group(0))) + for match in _COMMIT_RE.finditer(candidate): + leaks.add(PromptLeakage("commit", match.group(0))) + + terms: dict[str, set[str]] = {} + if manifest is not None: + terms.update(manifest_forbidden_terms(manifest)) + if forbidden_terms: + for category, values in forbidden_terms.items(): + terms.setdefault(category, set()).update(str(value) for value in values) + + normalized = candidate.casefold() + for category, values in terms.items(): + for raw in values: + value = raw.strip() + if value and value.casefold() in normalized: + leaks.add(PromptLeakage(category, value)) + return sorted(leaks, key=lambda item: (item.category, item.value.casefold())) + + +def require_generic_prompt( + candidate: str, + *, + manifest: Any | None = None, + forbidden_terms: dict[str, Iterable[str]] | None = None, +) -> None: + leaks = lint_prompt_candidate( + candidate, + manifest=manifest, + forbidden_terms=forbidden_terms, + ) + if not leaks: + return + preview = ", ".join(f"{item.category}={item.value!r}" for item in leaks[:8]) + suffix = f" (+{len(leaks) - 8} more)" if len(leaks) > 8 else "" + raise ValueError(f"Prompt candidate leaks benchmark answers: {preview}{suffix}") + + +def redact_benchmark_terms(text: str, manifest: Any) -> str: + """Remove answer-bearing strings before trajectories reach a reflection LM.""" + + redacted = _CVE_RE.sub("[case-specific-cve]", text) + redacted = _COMMIT_RE.sub("[case-specific-commit]", redacted) + terms = manifest_forbidden_terms(manifest) + values = { + value.strip() + for category_values in terms.values() + for value in category_values + if value.strip() + } + for value in sorted(values, key=len, reverse=True): + redacted = re.sub(re.escape(value), "[case-specific]", redacted, flags=re.IGNORECASE) + return redacted + + +__all__ = [ + "CONTEXT_PROFILES", + "GENERIC_DISCOVERY_V1", + "GENERIC_INSTRUCTIONS_V1", + "GENERIC_INSTRUCTIONS_COMPACT_V1", + "GENERIC_PROMPT_HEADER", + "PROMPT_BUNDLES", + "SCAFFOLD_PROFILES", + "ContextProfile", + "PromptBundle", + "PromptLeakage", + "ScaffoldProfile", + "get_prompt_bundle", + "get_scaffold_profile", + "get_context_profile", + "lint_prompt_candidate", + "manifest_forbidden_terms", + "redact_benchmark_terms", + "require_generic_prompt", +] diff --git a/clearwing/sourcehunt/pool.py b/clearwing/sourcehunt/pool.py index 89f44691..7ccacbc2 100644 --- a/clearwing/sourcehunt/pool.py +++ b/clearwing/sourcehunt/pool.py @@ -19,7 +19,7 @@ import asyncio import logging import uuid -from collections import Counter +from collections import Counter, deque from collections.abc import Callable from dataclasses import dataclass, field from pathlib import Path @@ -27,6 +27,7 @@ from clearwing.core.event_payloads import HuntProgressPayload from clearwing.core.events import EventBus +from clearwing.llm import ProviderExhaustedError, ProviderExhaustionState from clearwing.llm.budget import BudgetExceeded, spend_metadata from clearwing.runners.parallel.executor import ( TargetResult, @@ -36,6 +37,7 @@ ) from .instrumentation import stable_run_id +from .resume import CompletedWork, SourceHuntResumeStore, deterministic_work_id from .state import FileTarget, Finding logger = logging.getLogger(__name__) @@ -73,19 +75,16 @@ class WorkItem: entry_point: Any = None # EntryPoint | None — spec 004 seed_context: str | None = None # spec 004 seed corpus - def stable_identifier(self, run_id: str) -> str: - entry_point = self.entry_point - return stable_run_id( - "work", - { - "run_id": run_id, - "file": self.file_target.get("path", ""), - "band": self.band, - "attempt": self.attempt, - "entry_point": ( - getattr(entry_point, "function_name", "") if entry_point is not None else "" - ), - }, + def stable_identifier(self, run_id: str, tier: str) -> str: + return deterministic_work_id( + run_id, + file=str(self.file_target.get("path") or ""), + tier=tier, + band=self.band, + attempt=self.attempt, + entry_point=HunterPool._entry_point_payload(self), + seed_context=self.seed_context, + seed_transcript=self.seed_transcript, ) @@ -186,6 +185,8 @@ class HuntPoolConfig: llm: object | None = None # Required if hunter_factory is None max_parallel: int = 8 budget_usd: float = 0.0 + input_price_per_million: float | None = None + output_price_per_million: float | None = None tier_budget: TierBudget = field(default_factory=TierBudget) cost_limit_per_file_a: float = 0.25 cost_limit_per_file_b: float = 0.15 @@ -200,6 +201,13 @@ class HuntPoolConfig: semgrep_hints_by_file: dict = field(default_factory=dict) agent_mode: str = "constrained" # "constrained" | "deep" prompt_mode: str = "unconstrained" # "unconstrained" | "specialist" + prompt_bundle: str = "legacy-v1" + scaffold_profile: str = "native-v1" + context_profile: str = "legacy-context-v1" + prompt_candidate: str | None = None + max_hunter_steps: int | None = None + hunter_temperature: float | None = None + hunter_max_output_tokens: int | None = None campaign_hint: str | None = None exploit_mode: bool = False starting_band: str = "fast" # "fast" | "standard" | "deep" @@ -212,6 +220,9 @@ class HuntPoolConfig: findings_pool: Any = None # FindingsPool | None — spec 005 trajectory_root: str | Path | None = None instrumentation: Any = None # SourceHuntInstrumentation | None + resume_store: SourceHuntResumeStore | None = None + prior_spend_per_tier: dict[str, float] = field(default_factory=dict) + provider_exhaustion_state: ProviderExhaustionState | None = None def _format_seed_context(entries: list) -> str | None: @@ -274,7 +285,10 @@ def __init__(self, config: HuntPoolConfig): for ft in self.config.files: ft["tier"] = assign_tier(ft) self._results: dict[str, TargetResult] = {} - self._spent_per_tier: dict[str, float] = {"A": 0.0, "B": 0.0, "C": 0.0} + self._spent_per_tier: dict[str, float] = { + tier: max(0.0, float(self.config.prior_spend_per_tier.get(tier, 0.0))) + for tier in ("A", "B", "C") + } self._spent_per_band: dict[str, float] = {"fast": 0.0, "standard": 0.0, "deep": 0.0} self._runs_per_band: dict[str, int] = {"fast": 0, "standard": 0, "deep": 0} self._promotion_counts: dict[str, int] = {"fast→standard": 0, "standard→deep": 0} @@ -329,9 +343,88 @@ def _expand_to_work_items(self, files: list[FileTarget], band: str) -> list[Work ) return items + @staticmethod + def _entry_point_payload(item: WorkItem) -> dict[str, Any] | None: + entry_point = item.entry_point + if entry_point is None: + return None + return { + "file_path": getattr(entry_point, "file_path", ""), + "function_name": getattr(entry_point, "function_name", ""), + "start_line": getattr(entry_point, "start_line", 0), + "end_line": getattr(entry_point, "end_line", 0), + "entry_type": getattr(entry_point, "entry_type", ""), + "description": getattr(entry_point, "description", ""), + } + + def _pending_work_items( + self, + initial: list[WorkItem], + tier: str, + ) -> list[WorkItem]: + store = self.config.resume_store + if store is None: + return initial + completed = store.load_completed_work() + pending: list[WorkItem] = [] + queue = deque(initial) + seen: set[str] = set() + while queue: + item = queue.popleft() + work_id = item.stable_identifier(self.config.session_id_prefix, tier) + if work_id in seen: + continue + seen.add(work_id) + result = completed.get(work_id) + if result is None: + pending.append(item) + continue + self._restore_result(result) + next_band = promotion_decision( + result.findings, + result.stop_reason, + item.band, + self.config.max_band, + ) + if next_band: + self._promotion_counts[f"{item.band}→{next_band}"] += 1 + queue.append( + WorkItem( + file_target=item.file_target, + band=next_band, + attempt=item.attempt, + seed_transcript=result.promotion_transcript, + entry_point=item.entry_point, + seed_context=item.seed_context, + ) + ) + return pending + + def _restore_result(self, result: CompletedWork) -> None: + key = result.work_id + if key in self._results: + return + self._results[key] = TargetResult( + target=result.file, + status="completed", + findings=cast(list[dict], result.findings), + cost_usd=0.0, + tokens_used=0, + tier=result.tier, + band=result.band, + stop_reason=result.stop_reason, + ) + async def arun(self) -> list[Finding]: """Run the full A → B → C pipeline with band promotion. Returns merged findings.""" logger.info("HunterPool dispatching %d tiered file tasks", len(self.config.files)) + if self.config.provider_exhaustion_state is not None: + self.config.provider_exhaustion_state.raise_if_exhausted() + if self.config.resume_store is not None and self.config.findings_pool is not None: + self.config.findings_pool.restore( + self.config.resume_store.completed_findings(), + self.config.resume_store.completed_clusters(), + ) by_tier: dict[str, list[FileTarget]] = {"A": [], "B": [], "C": []} for item in self.config.files: by_tier[item.get("tier", "C")].append(item) @@ -348,25 +441,42 @@ async def arun(self) -> list[Finding]: total_budget = self.config.budget_usd tb = self.config.tier_budget + prior_spend = dict(self._spent_per_tier) if total_budget <= 0: budget_a = budget_b = budget_c = float("inf") else: - budget_a = total_budget * tb.tier_a_fraction - budget_b = total_budget * tb.tier_b_fraction - budget_c = total_budget * tb.tier_c_fraction + allocation_a = total_budget * tb.tier_a_fraction + allocation_b = total_budget * tb.tier_b_fraction + budget_a = max(0.0, allocation_a - prior_spend["A"]) + # Rollover is based on lifetime spend, including completed work + # restored from an earlier invocation. + budget_b = max( + 0.0, + allocation_a + allocation_b - prior_spend["A"] - prior_spend["B"], + ) + budget_c = max( + 0.0, + total_budget - sum(prior_spend.values()), + ) starting_band = self.config.starting_band - work_items_a = self._expand_to_work_items(by_tier["A"], starting_band) + work_items_a = self._pending_work_items( + self._expand_to_work_items(by_tier["A"], starting_band), "A" + ) spent_a = await self._run_tier_phase(work_items_a, "A", budget_a) - budget_b += max(0.0, budget_a - spent_a) - work_items_b = self._expand_to_work_items(by_tier["B"], starting_band) + work_items_b = self._pending_work_items( + self._expand_to_work_items(by_tier["B"], starting_band), "B" + ) + budget_b = max(0.0, budget_b - spent_a) spent_b = await self._run_tier_phase(work_items_b, "B", budget_b) - budget_c += max(0.0, budget_b - spent_b) + budget_c = max(0.0, budget_c - spent_a - spent_b) if by_tier["C"] and tb.tier_c_fraction > 0: - work_items_c = self._expand_to_work_items(by_tier["C"], starting_band) + work_items_c = self._pending_work_items( + self._expand_to_work_items(by_tier["C"], starting_band), "C" + ) await self._run_tier_phase(work_items_c, "C", budget_c) target_results = list(self._results.values()) @@ -379,6 +489,8 @@ async def arun(self) -> list[Finding]: all_findings: list[Finding] = [] if self.config.findings_pool is not None: all_findings = self.config.findings_pool.all_findings() + elif self.config.resume_store is not None: + all_findings = self.config.resume_store.completed_findings() else: for tr in target_results: if tr.status == "completed": @@ -427,8 +539,17 @@ def total_spent(self) -> float: def budget_exhausted(self) -> bool: return any(result.status == "budget_exhausted" for result in self._results.values()) + @property + def provider_exhausted(self) -> bool: + state = self.config.provider_exhaustion_state + return bool(state is not None and state.exhausted) or any( + result.status == "provider_exhausted" for result in self._results.values() + ) + @property def completed_target_count(self) -> int: + if self.config.resume_store is not None: + return self.config.resume_store.completed_target_count() return len( {result.target for result in self._results.values() if result.status == "completed"} ) @@ -450,22 +571,29 @@ async def _run_tier_phase( spent = 0.0 in_flight: dict[asyncio.Task[TargetResult], WorkItem] = {} item_iter = iter(work_items) - promotion_queue: list[WorkItem] = [] + promotion_queue: deque[WorkItem] = deque() def _submit_next() -> bool: nonlocal spent - if self._cancelled or spent >= budget: + if ( + self._cancelled + or spent >= budget + or ( + self.config.provider_exhaustion_state is not None + and self.config.provider_exhaustion_state.exhausted + ) + ): return False wi: WorkItem | None = None try: wi = next(item_iter) except StopIteration: if promotion_queue: - wi = promotion_queue.pop(0) + wi = promotion_queue.popleft() if wi is None: return False band_cost = self.config.band_budget.for_band(wi.band) - work_item_id = wi.stable_identifier(self.config.session_id_prefix) + work_item_id = wi.stable_identifier(self.config.session_id_prefix, tier) task = asyncio.create_task( self._run_file_task( wi.file_target, @@ -535,6 +663,17 @@ def _submit_next() -> bool: band=wi.band, stop_reason="budget_exhausted", ) + except ProviderExhaustedError as exc: + logger.warning("tier %s hunter for %s exhausted provider quota", tier, key) + self._cancelled = True + result = TargetResult( + target=key, + status="provider_exhausted", + error=str(exc), + tier=tier, + band=wi.band, + stop_reason="provider_exhausted", + ) except Exception as exc: logger.warning("tier %s hunter for %s failed: %s", tier, key, exc) result = TargetResult( @@ -545,6 +684,70 @@ def _submit_next() -> bool: band=wi.band, ) ep_suffix = f":{wi.entry_point.function_name}" if wi.entry_point else "" + next_band = None + promoted_item = None + if result.status == "completed": + next_band = promotion_decision( + cast(list[Finding], result.findings), + result.stop_reason, + wi.band, + self.config.max_band, + ) + if next_band: + promoted_item = WorkItem( + file_target=wi.file_target, + band=next_band, + attempt=wi.attempt, + seed_transcript=_extract_transcript(result), + entry_point=wi.entry_point, + seed_context=wi.seed_context, + ) + if result.status == "completed": + try: + if self.config.findings_pool is not None: + for finding in cast(list[Finding], result.findings): + await self.config.findings_pool.add(finding) + if self.config.resume_store is not None: + self.config.resume_store.save_work_result( + work_id=wi.stable_identifier(self.config.session_id_prefix, tier), + file=str(wi.file_target.get("path") or ""), + tier=tier, + band=wi.band, + attempt=wi.attempt, + entry_point=self._entry_point_payload(wi), + seed_context=wi.seed_context, + seed_transcript=wi.seed_transcript, + findings=cast(list[Finding], result.findings), + clusters=( + self.config.findings_pool.cluster_state( + { + finding.cluster_id + for finding in cast(list[Finding], result.findings) + if finding.cluster_id + } + ) + if self.config.findings_pool is not None + else [] + ), + cost_usd=result.cost_usd, + tokens_used=result.tokens_used, + stop_reason=result.stop_reason, + promotion_transcript=( + promoted_item.seed_transcript + if promoted_item is not None + else None + ), + ) + except ProviderExhaustedError as exc: + self._cancelled = True + result = TargetResult( + target=key, + status="provider_exhausted", + error=str(exc), + tier=tier, + band=wi.band, + stop_reason="provider_exhausted", + ) async with self._state_lock: self._results[f"{key}{ep_suffix}:{wi.band}:{wi.attempt}"] = result self._spent_per_tier[tier] += result.cost_usd @@ -593,20 +796,8 @@ def _submit_next() -> bool: f.get("description", "") or "", trace_chain, ) - if self.config.findings_pool is not None: - try: - await self.config.findings_pool.add(f) - except Exception: - logger.debug("findings_pool.add failed", exc_info=True) - if result.status == "completed": - next_band = promotion_decision( - cast(list[Finding], result.findings), - result.stop_reason, - wi.band, - self.config.max_band, - ) - if next_band: + if next_band and promoted_item is not None: promo_key = f"{wi.band}→{next_band}" async with self._state_lock: self._promotion_counts[promo_key] = ( @@ -618,17 +809,17 @@ def _submit_next() -> bool: wi.band, next_band, ) - promotion_queue.append( - WorkItem( - file_target=wi.file_target, - band=next_band, - attempt=wi.attempt, - seed_transcript=_extract_transcript(result), - ) - ) + promotion_queue.append(promoted_item) + + if result.status == "provider_exhausted": + for active_task in in_flight: + if not active_task.done(): + active_task.cancel() _submit_next() + if self.provider_exhausted: + raise ProviderExhaustedError("Provider quota exhausted during hunt") return spent # --- Internals: hunter-specific logic the runner delegates back to ---- @@ -686,12 +877,15 @@ async def _run_file_task( seed_context=seed_context, work_item_id=work_item_id, ) - except Exception as exc: + except BaseException as exc: + status = ( + "provider_exhausted" if isinstance(exc, ProviderExhaustedError) else "failed" + ) if instrumentation is not None: instrumentation.record( "work_item", stage="hunt", - status="failed", + status=status, files=[file_target.get("path", "")], symbols=[entry_symbol] if entry_symbol else [], work_item_id=work_item_id, @@ -834,9 +1028,18 @@ def _build_hunter_for_file( sandbox_manager=self.config.sandbox_manager, agent_mode=self.config.agent_mode, prompt_mode=self.config.prompt_mode, + prompt_bundle=self.config.prompt_bundle, + scaffold_profile=self.config.scaffold_profile, + context_profile=self.config.context_profile, + prompt_candidate=self.config.prompt_candidate, + max_steps_override=self.config.max_hunter_steps, + temperature=self.config.hunter_temperature, + max_output_tokens=self.config.hunter_max_output_tokens, campaign_hint=self.config.campaign_hint, exploit_mode=self.config.exploit_mode, budget_usd=budget_usd, + input_price_per_million=self.config.input_price_per_million, + output_price_per_million=self.config.output_price_per_million, seed_transcript=seed_transcript, entry_point=entry_point, seed_context=seed_context, diff --git a/clearwing/sourcehunt/preprocessor.py b/clearwing/sourcehunt/preprocessor.py index fea0e6e1..585dfebc 100644 --- a/clearwing/sourcehunt/preprocessor.py +++ b/clearwing/sourcehunt/preprocessor.py @@ -164,6 +164,7 @@ def _count_imports_by( repo_path: str, file_path: str, language: str, + source_files: list[str], gitignore: _GitignoreMatcher | None = None, ) -> int: """Cheap heuristic for `imports_by`: grep the repo for references to this @@ -189,28 +190,20 @@ def _count_imports_by( return 0 count = 0 - for dirpath, dirnames, filenames in os.walk(repo_path): - dirnames[:] = [ - d - for d in dirnames - if d not in SourceAnalyzer.SKIP_DIRS - and not (gitignore and gitignore.matches_dir(os.path.join(dirpath, d))) - ] - for fname in filenames: - other = os.path.join(dirpath, fname) - if other == file_path: - continue - if gitignore and gitignore.matches_file(other): - continue - try: - if os.path.getsize(other) > SourceAnalyzer.MAX_FILE_SIZE: - continue - with open(other, encoding="utf-8", errors="ignore") as f: - head = f.read(64 * 1024) # only scan the first 64 KB - if pattern.search(head): - count += 1 - except OSError: + for other in source_files: + if other == file_path: + continue + if gitignore and gitignore.matches_file(other): + continue + try: + if os.path.getsize(other) > SourceAnalyzer.MAX_FILE_SIZE: continue + with open(other, encoding="utf-8", errors="ignore") as f: + head = f.read(64 * 1024) # only scan the first 64 KB + if pattern.search(head): + count += 1 + except OSError: + continue return count @@ -244,6 +237,7 @@ def __init__( run_taint: bool = False, # v0.4: tree-sitter taint analysis max_imports_by_files: int = 1000, # cap the imports_by walk respect_gitignore: bool = False, + excluded_roots: list[str | Path] | None = None, ): self.repo_url = repo_url self.branch = branch @@ -256,6 +250,7 @@ def __init__( self.run_taint = run_taint self.max_imports_by_files = max_imports_by_files self.respect_gitignore = respect_gitignore + self.excluded_roots = list(excluded_roots or []) self._analyzer: SourceAnalyzer | None = None self._cloner: SourceAnalyzer | None = None @@ -268,6 +263,7 @@ def run(self) -> PreprocessResult: self._analyzer = SourceAnalyzer( repo_path=repo_path, respect_gitignore=self.respect_gitignore, + excluded_roots=self.excluded_roots, ) gitignore = _GitignoreMatcher.from_repo(repo_path) if self.respect_gitignore else None analysis_result = self._analyzer.analyze() @@ -337,7 +333,13 @@ def run(self) -> PreprocessResult: # v0.1 imports_by — capped to keep large repos snappy imports_by = 0 if len(file_targets) < imports_by_budget: - imports_by = _count_imports_by(repo_path, abs_path, language, gitignore) + imports_by = _count_imports_by( + repo_path, + abs_path, + language, + source_files, + gitignore, + ) target: FileTarget = { "path": rel_path, @@ -373,7 +375,7 @@ def run(self) -> PreprocessResult: try: builder = CallGraphBuilder() if builder.available: - callgraph = builder.build(repo_path) + callgraph = builder.build(repo_path, files=source_files) self._populate_callgraph_signals(file_targets, callgraph) else: logger.info("tree-sitter grammars not available; callgraph skipped") @@ -390,7 +392,10 @@ def run(self) -> PreprocessResult: try: sidecar = SemgrepSidecar(respect_gitignore=self.respect_gitignore) if sidecar.available: - semgrep_findings_objs = sidecar.run_scan(repo_path) + semgrep_findings_objs = sidecar.run_scan( + repo_path, + files=source_files, + ) semgrep_findings = [_semgrep_finding_to_dict(f) for f in semgrep_findings_objs] self._apply_semgrep_hints(file_targets, semgrep_findings) else: @@ -407,7 +412,7 @@ def run(self) -> PreprocessResult: try: analyzer = TaintAnalyzer() if analyzer.available: - taint_result = analyzer.analyze_repo(repo_path) + taint_result = analyzer.analyze_repo(repo_path, files=source_files) taint_paths = taint_result.paths self._apply_taint_signals(file_targets, taint_paths) else: diff --git a/clearwing/sourcehunt/ranker.py b/clearwing/sourcehunt/ranker.py index 98160507..2b19942a 100644 --- a/clearwing/sourcehunt/ranker.py +++ b/clearwing/sourcehunt/ranker.py @@ -17,6 +17,7 @@ import json import logging from dataclasses import dataclass +from pathlib import Path from typing import Annotated, Any, Literal from pydantic import BaseModel, ConfigDict, Field, ValidationError @@ -25,6 +26,7 @@ from clearwing.llm.native import extract_json_array, extract_json_object from .state import FileTarget +from .static_signals import is_production_source_path, score_source_security_signals logger = logging.getLogger(__name__) @@ -133,17 +135,26 @@ class Ranker: def __init__( self, - llm: AsyncLLMClient, + llm: AsyncLLMClient | None, config: RankerConfig | None = None, ): self.llm = llm self.config = config or RankerConfig() + self.completed_successfully = False + + def rank_heuristically(self, files: list[FileTarget]) -> list[FileTarget]: + """Apply only deterministic static ranking, without an LLM call.""" + + self._apply_heuristic_baseline(files) + return files def rank(self, files: list[FileTarget]) -> list[FileTarget]: return asyncio.run(self.arank(files)) async def arank(self, files: list[FileTarget]) -> list[FileTarget]: + self.completed_successfully = False if not files: + self.completed_successfully = True return files # Seed the whole corpus with cheap heuristic scores first so large @@ -153,21 +164,28 @@ async def arank(self, files: list[FileTarget]) -> list[FileTarget]: llm_candidates = self._select_llm_candidates(files) if llm_candidates: + if self.llm is None: + raise ValueError("LLM candidates require a configured ranker client") chunks = self._chunk(llm_candidates, self.config.chunk_size) scores_by_chunk = await self._rank_chunks_bounded(chunks) + chunks_complete = all( + set(scores) == {str(target.get("path") or "") for target in chunk} + for chunk, scores in zip(chunks, scores_by_chunk, strict=True) + ) + if not chunks_complete: + logger.warning( + "At least one rank chunk was incomplete; discarding partial LLM ranks " + "and using the heuristic plan for every file" + ) + scores_by_chunk = [{} for _ in chunks] for chunk, scores in zip(chunks, scores_by_chunk, strict=False): self._apply_scores(chunk, scores) # Apply floors and compute priority for every file for ft in files: - self._apply_floors(ft) - ft["priority"] = self._compute_priority(ft) - # v0.4 fuzz-harness rank boost: back-propagate the harness - # generator's selection criteria into the priority score so - # fuzzable parsers outrank non-fuzzable code at the same - # surface+influence+reachability level. - self._apply_fuzzable_boost(ft) + self._finalize_scores(ft) + self.completed_successfully = not llm_candidates or chunks_complete return files async def _rank_chunks_bounded( @@ -201,7 +219,7 @@ async def run_one( completed, total_chunks, ) - except BudgetExceeded: + except BaseException: for task in tasks: if not task.done(): task.cancel() @@ -209,6 +227,13 @@ async def run_one( raise return scores_by_chunk + def _finalize_scores(self, file_target: FileTarget) -> None: + """Apply score floors, priority, and the one-time fuzzable boost.""" + + self._apply_floors(file_target) + file_target["priority"] = self._compute_priority(file_target) + self._apply_fuzzable_boost(file_target) + def _apply_heuristic_baseline(self, files: list[FileTarget]) -> None: """Populate cheap baseline scores before any LLM reranking.""" for ft in files: @@ -219,6 +244,35 @@ def _apply_heuristic_baseline(self, files: list[FileTarget]) -> None: self._apply_floors(ft) ft["priority"] = self._compute_priority(ft) self._apply_fuzzable_boost(ft) + self._apply_source_signal_score(ft) + + @staticmethod + def _apply_source_signal_score(ft: FileTarget) -> None: + """Attach an auditable, target-blind source score for deterministic ranking.""" + + path = str(ft.get("path") or "") + absolute_path = str(ft.get("absolute_path") or "") + supported_language = ft.get("language") in {"c", "cpp", "rust"} + if not absolute_path or not supported_language or not is_production_source_path(path): + ft["security_signal_score"] = 0.0 + ft["deterministic_rank_score"] = float(ft.get("priority", 0.0)) + ft["security_signal_counts"] = {} + return + + try: + source = Path(absolute_path).read_text(encoding="utf-8", errors="replace") + except OSError: + ft["security_signal_score"] = 0.0 + ft["deterministic_rank_score"] = float(ft.get("priority", 0.0)) + ft["security_signal_counts"] = {} + return + + evidence = score_source_security_signals(source) + ft["security_signal_score"] = evidence.score + # Preserve tier/band semantics while ranking a bounded no-LLM run by + # source evidence instead of filename-derived ties. + ft["deterministic_rank_score"] = evidence.score + float(ft.get("priority", 0.0)) + ft["security_signal_counts"] = evidence.counts def _select_llm_candidates(self, files: list[FileTarget]) -> list[FileTarget]: if len(files) <= self.config.large_repo_file_threshold: @@ -337,7 +391,7 @@ async def _rank_chunk( return {} last_exc = exc if attempt < max_attempts - 1: - delay = max(0.0, self.config.chunk_retry_backoff_seconds) * (2 ** attempt) + delay = max(0.0, self.config.chunk_retry_backoff_seconds) * (2**attempt) logger.warning( "Ranker chunk %d/%d attempt %d failed; retrying in %.1fs", idx, @@ -361,9 +415,8 @@ async def _rank_chunk( def _is_retryable_structured_output_error(exc: Exception) -> bool: if isinstance(exc, (json.JSONDecodeError, ValidationError)): return True - return ( - isinstance(exc, ValueError) - and str(exc).startswith("LLM returned empty response; expected JSON matching") + return isinstance(exc, ValueError) and str(exc).startswith( + "LLM returned empty response; expected JSON matching" ) def _build_user_message(self, chunk: list[FileTarget]) -> str: diff --git a/clearwing/sourcehunt/resume.py b/clearwing/sourcehunt/resume.py new file mode 100644 index 00000000..353c526a --- /dev/null +++ b/clearwing/sourcehunt/resume.py @@ -0,0 +1,658 @@ +"""Immutable completion records for resumable standalone sourcehunts.""" + +from __future__ import annotations + +import fcntl +import hashlib +import json +import math +import os +import re +import tempfile +from collections.abc import Iterable +from dataclasses import asdict, dataclass, is_dataclass +from pathlib import Path +from typing import Any + +from clearwing.findings.types import Finding + +SESSION_SCHEMA_VERSION = 1 +RANK_PLAN_SCHEMA_VERSION = 1 +WORK_RESULT_SCHEMA_VERSION = 1 +SESSION_FILENAME = "session.json" +RANK_PLAN_FILENAME = "rank-plan.json" +WORK_RESULTS_DIRNAME = "work-results" +SESSION_LOCK_FILENAME = ".sourcehunt.lock" +_SESSION_ID = re.compile(r"^sh-[A-Za-z0-9_-]+$") +_WORK_ID = re.compile(r"^work-[a-f0-9]{16}$") +_RANK_INTEGER_FIELDS = ( + "surface", + "influence", + "reachability", + "loc", + "static_hint", + "semgrep_hint", + "taint_hits", + "imports_by", + "transitive_callers", +) +_RANK_RATIONALE_FIELDS = ( + "surface_rationale", + "influence_rationale", + "reachability_rationale", +) + + +class SourceHuntResumeError(ValueError): + """A resumable session is missing, malformed, incompatible, or busy.""" + + +@dataclass(frozen=True, slots=True) +class CompletedWork: + """One successfully committed hunt work result.""" + + work_id: str + file: str + tier: str + band: str + attempt: int + entry_point: dict[str, Any] | None + seed_context: str | None + seed_transcript: str | None + findings: list[Finding] + clusters: list[dict[str, Any]] + cost_usd: float + tokens_used: int + stop_reason: str + promotion_transcript: str | None + + +class SourceHuntSessionLock: + """Advisory process lock preventing concurrent writers to one session.""" + + def __init__(self, session_dir: str | Path): + self.path = Path(session_dir) / SESSION_LOCK_FILENAME + self._stream: Any = None + + def acquire(self) -> None: + self.path.parent.mkdir(parents=True, exist_ok=True) + stream = open(self.path, "a+", encoding="utf-8") + try: + fcntl.flock(stream.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError as exc: + stream.close() + raise SourceHuntResumeError( + f"Sourcehunt session {self.path.parent.name!r} is already running" + ) from exc + stream.seek(0) + stream.truncate() + stream.write(f"pid={os.getpid()}\n") + stream.flush() + self._stream = stream + + def release(self) -> None: + if self._stream is None: + return + try: + fcntl.flock(self._stream.fileno(), fcntl.LOCK_UN) + finally: + self._stream.close() + self._stream = None + + +def resolve_session_dir(output_dir: str | Path, session_id: str) -> Path: + """Resolve a safe bare session ID below the configured output root.""" + + if not _SESSION_ID.fullmatch(session_id): + raise SourceHuntResumeError( + f"Invalid sourcehunt session ID {session_id!r}; expected a value like sh-535ed81b" + ) + return Path(output_dir).expanduser().resolve() / session_id + + +def _json_value(value: Any) -> Any: + if is_dataclass(value) and not isinstance(value, type): + return {key: _json_value(item) for key, item in asdict(value).items()} + if isinstance(value, dict): + return {str(key): _json_value(item) for key, item in value.items()} + if isinstance(value, (list, tuple, set)): + return [_json_value(item) for item in value] + if isinstance(value, Path): + return str(value) + return value + + +def _read_json(path: Path) -> dict[str, Any] | None: + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError, UnicodeDecodeError): + return None + return payload if isinstance(payload, dict) else None + + +def _atomic_json(path: Path, payload: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + descriptor, temporary = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) + try: + with os.fdopen(descriptor, "w", encoding="utf-8") as stream: + json.dump(payload, stream, indent=2, sort_keys=True) + stream.write("\n") + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, path) + parent_descriptor = os.open(path.parent, os.O_RDONLY) + try: + os.fsync(parent_descriptor) + finally: + os.close(parent_descriptor) + finally: + if os.path.exists(temporary): + os.unlink(temporary) + + +def deterministic_work_id( + session_id: str, + *, + file: str, + tier: str, + band: str, + attempt: int, + entry_point: dict[str, Any] | None, + seed_context: str | None, + seed_transcript: str | None, +) -> str: + """Identify one hunt invocation from all behavior-affecting inputs.""" + + payload = { + "session_id": session_id, + "file": file, + "tier": tier, + "band": band, + "attempt": attempt, + "entry_point": entry_point, + "seed_context": seed_context, + "seed_transcript": seed_transcript, + } + encoded = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8") + return f"work-{hashlib.sha256(encoded).hexdigest()[:16]}" + + +def source_input_identity( + repo_path: str | Path, + file_targets: Iterable[dict[str, Any]], +) -> dict[str, Any]: + """Hash exactly the selected source paths and their complete contents.""" + + repository = Path(repo_path).resolve() + selected: list[tuple[str, Path]] = [] + for target in file_targets: + relative = Path(str(target.get("path") or "")).as_posix() + if not relative: + continue + relative_path = Path(relative) + absolute = Path(str(target.get("absolute_path") or repository / relative_path)).resolve() + expected_absolute = (repository / relative_path).resolve() + if ( + relative_path.is_absolute() + or ".." in relative_path.parts + or absolute != expected_absolute + or not absolute.is_relative_to(repository) + ): + raise SourceHuntResumeError( + f"Selected source input {relative!r} is outside the repository" + ) + selected.append((relative, absolute)) + + if len(selected) != len({relative for relative, _ in selected}): + raise SourceHuntResumeError("Selected source inputs contain duplicate paths") + + digest = hashlib.sha256() + paths: list[str] = [] + for relative, absolute in sorted(selected): + try: + content = absolute.read_bytes() + except OSError as exc: + raise SourceHuntResumeError( + f"Unable to fingerprint selected source input {relative!r}: {exc}" + ) from exc + encoded_path = relative.encode("utf-8") + digest.update(len(encoded_path).to_bytes(8, "big")) + digest.update(encoded_path) + digest.update(len(content).to_bytes(8, "big")) + digest.update(content) + paths.append(relative) + return { + "algorithm": "sha256-path-content-v1", + "fingerprint": digest.hexdigest(), + "paths": paths, + } + + +class SourceHuntResumeStore: + """Session metadata plus immutable rank and work completion records.""" + + def __init__(self, session_dir: str | Path, session: dict[str, Any] | None = None): + self.session_dir = Path(session_dir) + self.session_path = self.session_dir / SESSION_FILENAME + self.rank_plan_path = self.session_dir / RANK_PLAN_FILENAME + self.work_results_dir = self.session_dir / WORK_RESULTS_DIRNAME + self._session = session + self._completed_work: dict[str, CompletedWork] | None = None + + @property + def session_id(self) -> str: + return self.session_dir.name + + @classmethod + def load(cls, session_dir: str | Path) -> SourceHuntResumeStore: + store = cls(session_dir) + payload = _read_json(store.session_path) + if payload is None: + raise SourceHuntResumeError( + f"Session {store.session_id!r} has no valid {SESSION_FILENAME}; " + "legacy standalone sourcehunt sessions are not resumable" + ) + store._validate_session(payload) + store._session = payload + return store + + def create_session( + self, + *, + repository: dict[str, Any], + config: dict[str, Any], + source_identity: dict[str, Any], + ) -> None: + if self.session_path.exists(): + raise SourceHuntResumeError(f"Session metadata already exists: {self.session_path}") + payload = { + "schema_version": SESSION_SCHEMA_VERSION, + "session_id": self.session_id, + "repository": _json_value(repository), + "config": _json_value(config), + "source_identity": _json_value(source_identity), + } + self._validate_session(payload) + _atomic_json(self.session_path, payload) + self._session = payload + + def config(self) -> dict[str, Any]: + return dict(self._required_session()["config"]) + + def validate_source_identity(self, actual: dict[str, Any]) -> None: + expected = self._required_session()["source_identity"] + if actual != expected: + raise SourceHuntResumeError( + "Selected source inputs changed since this sourcehunt session began" + ) + + def load_rank_plan(self) -> list[dict[str, Any]] | None: + payload = _read_json(self.rank_plan_path) + if payload is None or payload.get("schema_version") != RANK_PLAN_SCHEMA_VERSION: + return None + targets = payload.get("targets") + if not isinstance(targets, list): + return None + validated = [self._rank_target(target) for target in targets] + if any(target is None for target in validated): + return None + complete_targets = [target for target in validated if target is not None] + expected_paths = self._required_session()["source_identity"]["paths"] + actual_paths = [target["path"] for target in complete_targets] + if len(actual_paths) != len(set(actual_paths)) or sorted(actual_paths) != sorted( + expected_paths + ): + return None + return complete_targets + + def save_rank_plan(self, targets: list[dict[str, Any]]) -> None: + if self.load_rank_plan() is not None: + return + payload = { + "schema_version": RANK_PLAN_SCHEMA_VERSION, + "targets": _json_value(targets), + } + _atomic_json(self.rank_plan_path, payload) + if self.load_rank_plan() is None: + raise SourceHuntResumeError("Unable to validate committed rank plan") + + def load_completed_work(self) -> dict[str, CompletedWork]: + if self._completed_work is not None: + return dict(self._completed_work) + completed: dict[str, CompletedWork] = {} + if self.work_results_dir.is_dir(): + for path in sorted(self.work_results_dir.glob("work-*.json")): + result = self._load_work_result(path) + if result is not None: + completed[result.work_id] = result + self._completed_work = completed + return dict(completed) + + def save_work_result( + self, + *, + work_id: str, + file: str, + tier: str, + band: str, + attempt: int, + entry_point: dict[str, Any] | None, + seed_context: str | None, + seed_transcript: str | None, + findings: Iterable[Finding], + clusters: Iterable[dict[str, Any]] = (), + cost_usd: float, + tokens_used: int, + stop_reason: str, + promotion_transcript: str | None, + ) -> CompletedWork: + if not _WORK_ID.fullmatch(work_id): + raise SourceHuntResumeError(f"Invalid sourcehunt work ID {work_id!r}") + payload = { + "schema_version": WORK_RESULT_SCHEMA_VERSION, + "work_id": work_id, + "work": { + "file": file, + "tier": tier, + "band": band, + "attempt": attempt, + "entry_point": _json_value(entry_point), + "seed_context": seed_context, + "seed_transcript": seed_transcript, + }, + "result": { + "status": "completed", + "findings": _json_value(list(findings)), + "clusters": _json_value(list(clusters)), + "cost_usd": cost_usd, + "tokens_used": tokens_used, + "stop_reason": stop_reason, + "promotion_transcript": promotion_transcript, + }, + } + path = self.work_results_dir / f"{work_id}.json" + if path.exists(): + existing = self._load_work_result(path) + if existing is not None: + return existing + # Invalid/truncated records are not completions. Replace them + # only after this rerun completes successfully. + _atomic_json(path, payload) + result = self._load_work_result(path) + if result is None: # pragma: no cover - validates our own serialization + raise SourceHuntResumeError(f"Unable to validate committed work result {path}") + if self._completed_work is not None: + self._completed_work[work_id] = result + return result + + def completed_findings(self) -> list[Finding]: + findings: dict[str, Finding] = {} + for result in self.load_completed_work().values(): + for finding in result.findings: + findings.setdefault(finding.id, finding) + return list(findings.values()) + + def completed_clusters(self) -> list[dict[str, Any]]: + """Merge the referenced cluster descriptors stored with work results.""" + + clusters: dict[str, dict[str, Any]] = {} + for result in self.load_completed_work().values(): + for item in result.clusters: + cluster_id = item["cluster_id"] + clusters.setdefault( + cluster_id, + { + "cluster_id": cluster_id, + "root_cause_summary": item["root_cause_summary"], + "primitive_type": item["primitive_type"], + "cwe": item["cwe"], + "finding_ids": [], + "file_paths": [], + }, + ) + return list(clusters.values()) + + def completed_target_count(self) -> int: + return len({result.file for result in self.load_completed_work().values()}) + + def _load_work_result(self, path: Path) -> CompletedWork | None: + payload = _read_json(path) + if payload is None or payload.get("schema_version") != WORK_RESULT_SCHEMA_VERSION: + return None + work_id = payload.get("work_id") + work = payload.get("work") + result = payload.get("result") + if ( + not isinstance(work_id, str) + or not _WORK_ID.fullmatch(work_id) + or path.name != f"{work_id}.json" + or not isinstance(work, dict) + or not isinstance(result, dict) + or result.get("status") != "completed" + or not isinstance(result.get("findings"), list) + or not isinstance(result.get("clusters", []), list) + ): + return None + try: + file = work["file"] + tier = work["tier"] + band = work["band"] + attempt = work["attempt"] + if ( + not isinstance(file, str) + or file not in self._required_session()["source_identity"]["paths"] + or not isinstance(tier, str) + or not isinstance(band, str) + or not isinstance(attempt, int) + or isinstance(attempt, bool) + ): + return None + entry_point = ( + dict(work["entry_point"]) + if isinstance(work.get("entry_point"), dict) + else None + ) + if ( + work.get("entry_point") is not None + and ( + entry_point is None + or not self._valid_entry_point(entry_point) + ) + ): + return None + seed_context = work.get("seed_context") + seed_transcript = work.get("seed_transcript") + if not isinstance(seed_context, (str, type(None))) or not isinstance( + seed_transcript, + (str, type(None)), + ): + return None + expected_id = deterministic_work_id( + self.session_id, + file=file, + tier=tier, + band=band, + attempt=attempt, + entry_point=entry_point, + seed_context=seed_context, + seed_transcript=seed_transcript, + ) + if ( + work_id != expected_id + or tier not in {"A", "B", "C"} + or band not in {"fast", "standard", "deep"} + or attempt < 0 + ): + return None + findings = [self._finding(item) for item in result["findings"]] + if any(finding is None for finding in findings): + return None + clusters = [self._cluster(item) for item in result.get("clusters", [])] + if any(cluster is None for cluster in clusters): + return None + raw_cost = result.get("cost_usd", 0.0) + tokens_used = result.get("tokens_used", 0) + stop_reason = result.get("stop_reason", "completed") + promotion_transcript = result.get("promotion_transcript") + if ( + not isinstance(raw_cost, (int, float)) + or isinstance(raw_cost, bool) + or not isinstance(tokens_used, int) + or isinstance(tokens_used, bool) + or not isinstance(stop_reason, str) + or not isinstance(promotion_transcript, (str, type(None))) + ): + return None + cost_usd = float(raw_cost) + if not math.isfinite(cost_usd) or cost_usd < 0 or tokens_used < 0: + return None + return CompletedWork( + work_id=work_id, + file=file, + tier=tier, + band=band, + attempt=attempt, + entry_point=entry_point, + seed_context=seed_context, + seed_transcript=seed_transcript, + findings=[finding for finding in findings if finding is not None], + clusters=[cluster for cluster in clusters if cluster is not None], + cost_usd=cost_usd, + tokens_used=tokens_used, + stop_reason=stop_reason, + promotion_transcript=promotion_transcript, + ) + except (KeyError, TypeError, ValueError): + return None + + @staticmethod + def _finding(value: Any) -> Finding | None: + if ( + not isinstance(value, dict) + or not isinstance(value.get("id"), str) + or not value["id"] + ): + return None + fields = Finding.__dataclass_fields__ + try: + return Finding(**{key: item for key, item in value.items() if key in fields}) + except Exception: + return None + + @staticmethod + def _rank_target(value: Any) -> dict[str, Any] | None: + if not isinstance(value, dict) or "absolute_path" in value: + return None + path = value.get("path") + if ( + not isinstance(path, str) + or not path + or any( + not isinstance(value.get(field), int) + or isinstance(value.get(field), bool) + or value[field] < 0 + for field in _RANK_INTEGER_FIELDS + ) + or not isinstance(value.get("priority"), (int, float)) + or isinstance(value.get("priority"), bool) + or not math.isfinite(float(value["priority"])) + or value.get("tier") not in {"A", "B", "C"} + or not isinstance(value.get("tags"), list) + or not all(isinstance(tag, str) for tag in value["tags"]) + or not isinstance(value.get("language"), str) + or any( + not isinstance(value.get(field), str) + for field in _RANK_RATIONALE_FIELDS + ) + or not isinstance(value.get("defines_constants"), bool) + or not isinstance(value.get("has_fuzz_entry_point"), bool) + or not isinstance(value.get("fuzz_harness_path"), (str, type(None))) + ): + return None + return dict(value) + + @staticmethod + def _valid_entry_point(value: dict[str, Any]) -> bool: + return ( + set(value) + == { + "file_path", + "function_name", + "start_line", + "end_line", + "entry_type", + "description", + } + and all( + isinstance(value[field], str) + for field in ( + "file_path", + "function_name", + "entry_type", + "description", + ) + ) + and all( + isinstance(value[field], int) + and not isinstance(value[field], bool) + and value[field] >= 0 + for field in ("start_line", "end_line") + ) + ) + + @staticmethod + def _cluster(value: Any) -> dict[str, Any] | None: + if not isinstance(value, dict): + return None + if ( + not isinstance(value.get("cluster_id"), str) + or not isinstance(value.get("root_cause_summary"), str) + or not isinstance(value.get("primitive_type"), str) + or not isinstance(value.get("cwe"), str) + ): + return None + return { + "cluster_id": value["cluster_id"], + "root_cause_summary": value["root_cause_summary"], + "primitive_type": value["primitive_type"], + "cwe": value["cwe"], + # Membership is reconstructed from canonical findings, so only + # bounded descriptors are accepted from completion files. + "finding_ids": [], + "file_paths": [], + } + + def _required_session(self) -> dict[str, Any]: + if self._session is None: + raise SourceHuntResumeError("Sourcehunt session metadata has not been loaded") + return self._session + + def _validate_session(self, payload: dict[str, Any]) -> None: + if payload.get("schema_version") != SESSION_SCHEMA_VERSION: + raise SourceHuntResumeError( + f"Unsupported sourcehunt session schema {payload.get('schema_version')!r}" + ) + if payload.get("session_id") != self.session_id or not _SESSION_ID.fullmatch( + str(payload.get("session_id") or "") + ): + raise SourceHuntResumeError("Sourcehunt session ID does not match its directory") + for key in ("repository", "config", "source_identity"): + if not isinstance(payload.get(key), dict): + raise SourceHuntResumeError(f"Sourcehunt session field {key!r} is invalid") + identity = payload["source_identity"] + paths = identity.get("paths") + if ( + identity.get("algorithm") != "sha256-path-content-v1" + or not isinstance(identity.get("fingerprint"), str) + or re.fullmatch(r"[a-f0-9]{64}", identity["fingerprint"]) is None + or not isinstance(paths, list) + or not all( + isinstance(path, str) + and path + and not Path(path).is_absolute() + and ".." not in Path(path).parts + for path in paths + ) + or len(paths) != len(set(paths)) + ): + raise SourceHuntResumeError("Sourcehunt source identity is invalid") diff --git a/clearwing/sourcehunt/runner.py b/clearwing/sourcehunt/runner.py index 92659f7c..e4a197ce 100644 --- a/clearwing/sourcehunt/runner.py +++ b/clearwing/sourcehunt/runner.py @@ -26,6 +26,7 @@ from clearwing.core.event_payloads import SourcehuntStagePayload from clearwing.core.events import EventBus from clearwing.llm.budget import BudgetExceeded, SpendLedger +from clearwing.llm.errors import ProviderExhaustedError, ProviderExhaustionState from clearwing.llm.native import AsyncLLMClient from clearwing.providers import ( ProviderManager, @@ -53,6 +54,13 @@ from .pool import HunterPool, HuntPoolConfig, TierBudget from .preprocessor import Preprocessor, PreprocessResult from .ranker import Ranker, RankerConfig +from .resume import ( + SourceHuntResumeError, + SourceHuntResumeStore, + SourceHuntSessionLock, + resolve_session_dir, + source_input_identity, +) from .state import ( EvidenceLevel, FileTarget, @@ -185,9 +193,67 @@ def _apply_elaboration(finding: Finding, elab_result) -> Finding: } +class _DeferredInstrumentation: + """Create session instrumentation only after the runner holds its lock.""" + + def __init__(self, session_dir: Path, run_id: str): + self._session_dir = session_dir + self._run_id = run_id + self._instrumentation: SourceHuntInstrumentation | None = None + + def _get(self) -> SourceHuntInstrumentation: + if self._instrumentation is None: + self._instrumentation = SourceHuntInstrumentation( + self._session_dir, + self._run_id, + ) + return self._instrumentation + + def __getattr__(self, name: str) -> Any: + return getattr(self._get(), name) + + class SourceHuntRunner: """Public entry point for the sourcehunt pipeline.""" + @classmethod + def resume( + cls, + session_id: str, + *, + output_dir: str | None = None, + provider_manager: ProviderManager | None = None, + model_override: str | None = None, + live: bool = False, + sandbox_factory: Any = None, + ) -> SourceHuntRunner: + """Restore run behavior while accepting fresh runtime dependencies.""" + + if output_dir is None: + from clearwing.core.config import default_results_dir + + output_dir = default_results_dir("sourcehunt") + session_dir = resolve_session_dir(output_dir, session_id) + store = SourceHuntResumeStore.load(session_dir) + config = SourceHuntConfig.from_dict(store.config()) + if config.proof.flow != "legacy": + raise SourceHuntResumeError("Standalone sourcehunt resume supports only flow=legacy") + + payload = config.to_dict() + payload["output"]["output_dir"] = output_dir + runner = cls( + config=SourceHuntConfig.from_dict(payload), + provider_manager=provider_manager, + model_override=model_override, + live=live, + sandbox_factory=sandbox_factory, + parent_session_id=session_id, + ) + runner._completion_store = store + runner._standalone_session = True + runner._resume_session = session_id + return runner + def __init__( self, repo_url: str = "", @@ -233,9 +299,23 @@ def __init__( parent_session_id: str | None = None, agent_mode: str = "auto", # "auto" | "constrained" | "deep" prompt_mode: str = "unconstrained", # "unconstrained" | "specialist" + prompt_bundle: str = "legacy-v1", + scaffold_profile: str = "native-v1", + context_profile: str = "legacy-context-v1", + prompt_candidate: str | None = None, campaign_hint: str | None = None, exploit_mode: bool = False, starting_band: str | None = None, # "fast" | "standard" | "deep" | None (auto) + max_hunt_files: int | None = None, + hunt_file_offset: int = 0, + hunt_file_offsets: list[int] | None = None, + hunt_file_paths: list[str] | None = None, + max_hunter_steps: int | None = None, + hunter_temperature: float | None = None, + hunter_max_output_tokens: int | None = None, + ranker_chunk_size: int | None = None, + ranker_max_inflight_chunks: int | None = None, + ranker_chunk_max_retries: int | None = None, redundancy_override: int | None = None, shard_entry_points: bool | None = None, # None = auto (deep depth) min_shard_rank: int = 4, @@ -361,6 +441,17 @@ def __init__( exploit_mode = exploit_mode or f.exploit_mode agent_mode = agent_mode if agent_mode != "auto" else f.agent_mode prompt_mode = prompt_mode if prompt_mode != "unconstrained" else f.prompt_mode + prompt_bundle = ( + prompt_bundle if prompt_bundle != "legacy-v1" else f.prompt_bundle + ) + scaffold_profile = ( + scaffold_profile if scaffold_profile != "native-v1" else f.scaffold_profile + ) + context_profile = ( + context_profile + if context_profile != "legacy-context-v1" + else f.context_profile + ) # Hunt tuning starting_band = starting_band if starting_band is not None else h.starting_band redundancy_override = ( @@ -376,8 +467,15 @@ def __init__( ) subsystem_paths = subsystem_paths if subsystem_paths is not None else h.subsystem_paths campaign_hint = campaign_hint if campaign_hint is not None else h.campaign_hint + mechanism_store_path = ( + mechanism_store_path if mechanism_store_path is not None else h.mechanism_store_path + ) + historical_db_path = ( + historical_db_path if historical_db_path is not None else h.historical_db_path + ) gvisor_runtime = gvisor_runtime if gvisor_runtime is not None else h.gvisor_runtime sandbox_cpus = sandbox_cpus if sandbox_cpus is not None else h.sandbox_cpus + respect_gitignore = respect_gitignore or h.respect_gitignore p = config.proof flow = flow if flow != "legacy" else p.flow proof_compile_commands = ( @@ -442,6 +540,57 @@ def __init__( raise ValueError("sandbox_cpus must be a finite number greater than or equal to 0") if flow not in {"legacy", "proof"}: raise ValueError("flow must be 'legacy' or 'proof'") + if max_hunt_files is not None and max_hunt_files < 1: + raise ValueError("max_hunt_files must be positive when provided") + if hunt_file_offset < 0: + raise ValueError("hunt_file_offset cannot be negative") + if hunt_file_offsets is not None: + if not hunt_file_offsets: + raise ValueError("hunt_file_offsets cannot be empty when provided") + if any(offset < 0 for offset in hunt_file_offsets): + raise ValueError("hunt_file_offsets cannot contain negative offsets") + if len(set(hunt_file_offsets)) != len(hunt_file_offsets): + raise ValueError("hunt_file_offsets cannot contain duplicates") + if hunt_file_offset or max_hunt_files is not None: + raise ValueError( + "hunt_file_offsets cannot be combined with hunt_file_offset " + "or max_hunt_files" + ) + if hunt_file_paths is not None: + if not hunt_file_paths: + raise ValueError("hunt_file_paths cannot be empty when provided") + if any(not path for path in hunt_file_paths): + raise ValueError("hunt_file_paths cannot contain empty paths") + if len(set(hunt_file_paths)) != len(hunt_file_paths): + raise ValueError("hunt_file_paths cannot contain duplicates") + if ( + hunt_file_offset + or max_hunt_files is not None + or hunt_file_offsets is not None + ): + raise ValueError( + "hunt_file_paths cannot be combined with rank window options" + ) + if max_hunter_steps is not None and max_hunter_steps < 1: + raise ValueError("max_hunter_steps must be positive when provided") + if hunter_temperature is not None and not 0.0 <= hunter_temperature <= 2.0: + raise ValueError("hunter_temperature must be between 0 and 2") + if hunter_max_output_tokens is not None and hunter_max_output_tokens < 1: + raise ValueError("hunter_max_output_tokens must be positive when provided") + for name, value, minimum in ( + ("ranker_chunk_size", ranker_chunk_size, 1), + ("ranker_max_inflight_chunks", ranker_max_inflight_chunks, 1), + ("ranker_chunk_max_retries", ranker_chunk_max_retries, 0), + ): + if value is not None and value < minimum: + raise ValueError(f"{name} must be at least {minimum} when provided") + from .optimization import get_context_profile, get_prompt_bundle, get_scaffold_profile + + get_prompt_bundle(prompt_bundle) + get_scaffold_profile(scaffold_profile) + get_context_profile(context_profile) + if prompt_candidate is not None and prompt_bundle == "legacy-v1": + raise ValueError("prompt_candidate requires a generic prompt bundle") if proof_max_actions < 1: raise ValueError("proof_max_actions must be positive") if proof_max_model_calls < 0 or proof_max_dynamic_actions < 0: @@ -453,8 +602,13 @@ def __init__( if proof_structured_fraction + proof_exploration_fraction > 1.000001: raise ValueError("proof structured and exploration budgets exceed 100%") - # Store the config for introspection (None if constructed the old way) - self._config = config + tier_budget = tier_budget or TierBudget() + if output_dir is None: + from clearwing.core.config import default_results_dir + + output_dir = default_results_dir("sourcehunt") + output_formats = output_formats or ["sarif", "markdown", "json"] + effective_config = SourceHuntConfig.from_options(locals()) self.repo_url = repo_url self.branch = branch @@ -464,13 +618,9 @@ def __init__( self.input_price_per_million = input_price_per_million self.output_price_per_million = output_price_per_million self.max_parallel = max_parallel - self.tier_budget = tier_budget or TierBudget() - if output_dir is None: - from clearwing.core.config import default_results_dir - - output_dir = default_results_dir("sourcehunt") + self.tier_budget = tier_budget self.output_dir = output_dir - self.output_formats = output_formats or ["sarif", "markdown", "json"] + self.output_formats = output_formats self.no_verify = no_verify self.no_exploit = no_exploit self._exploit_budget_override = exploit_budget @@ -514,9 +664,25 @@ def __init__( self._session_id = parent_session_id or f"sh-{uuid.uuid4().hex[:8]}" self._agent_mode_override = agent_mode self._prompt_mode = prompt_mode + self._prompt_bundle = prompt_bundle + self._scaffold_profile = scaffold_profile + self._context_profile = context_profile + self._prompt_candidate = prompt_candidate self._campaign_hint = campaign_hint self._exploit_mode = exploit_mode self._starting_band_override = starting_band + self._max_hunt_files = max_hunt_files + self._hunt_file_offset = hunt_file_offset + self._hunt_file_offsets = ( + sorted(hunt_file_offsets) if hunt_file_offsets is not None else None + ) + self._hunt_file_paths = list(hunt_file_paths) if hunt_file_paths is not None else None + self._max_hunter_steps = max_hunter_steps + self._hunter_temperature = hunter_temperature + self._hunter_max_output_tokens = hunter_max_output_tokens + self._ranker_chunk_size = ranker_chunk_size + self._ranker_max_inflight_chunks = ranker_max_inflight_chunks + self._ranker_chunk_max_retries = ranker_chunk_max_retries self._redundancy_override = redundancy_override self._shard_entry_points_override = shard_entry_points self._min_shard_rank = min_shard_rank @@ -542,7 +708,12 @@ def __init__( self._live = live self._spend_ledger: SpendLedger | None = None self._spend_instrumented = False + self._provider_exhaustion = ProviderExhaustionState() self._metered_clients: dict[tuple[int, str], AsyncLLMClient] = {} + self._completion_store: SourceHuntResumeStore | None = None + self._standalone_session = parent_session_id is None + self._resume_session: str | None = None + self._session_lock: SourceHuntSessionLock | None = None self._flow = flow self._proof_compile_commands = proof_compile_commands self._proof_validation_manifest = proof_validation_manifest @@ -558,13 +729,14 @@ def __init__( self._retain_incomplete_certificates = retain_incomplete_certificates self._emit_rejection_certificates = emit_rejection_certificates self._falsify = falsify - self._instrumentation = SourceHuntInstrumentation( + self._instrumentation = _DeferredInstrumentation( Path(self.output_dir) / self._session_id, self._session_id, ) self._instrumentation_finalized = False self._last_reporting_error: dict[str, str] | None = None self._on_progress = on_progress + self._config = effective_config @staticmethod def _check_runtime_available(runtime: str | None) -> str | None: @@ -643,6 +815,31 @@ def _run_spent_usd(self) -> float: return 0.0 return self._spend_ledger.spent_usd + @staticmethod + def _resolved_commit(repo_path: str) -> str: + """Resolve a Git commit for useful repository metadata.""" + + try: + result = subprocess.run( + ["git", "-C", repo_path, "rev-parse", "HEAD"], + check=True, + capture_output=True, + text=True, + timeout=30, + ) + except (OSError, subprocess.SubprocessError): + return "" + return result.stdout.strip().lower() + + @staticmethod + def _ranked_targets(files: list[FileTarget]) -> list[dict[str, Any]]: + targets = [] + for target in files: + payload = dict(target) + payload.pop("absolute_path", None) + targets.append(payload) + return targets + @property def _shard_entry_points(self) -> bool: if self._shard_entry_points_override is not None: @@ -730,6 +927,7 @@ def _ensure_spend_ledger(self) -> SpendLedger: manifest_filename=( "spend-summary.json" if self._flow == "proof" else "manifest.json" ), + resume=self._resume_session is not None, ) return self._spend_ledger @@ -742,7 +940,11 @@ def _preflight_budget_clients(self) -> None: if self._spend_ledger is None or not self._spend_ledger.enforcing: return roles: list[tuple[str, AsyncLLMClient | None, str]] = [] - if not self._no_rank: + rank_complete = ( + self._completion_store is not None + and self._completion_store.load_rank_plan() is not None + ) + if not self._no_rank and not rank_complete: roles.append(("ranker", self.ranker_llm, "rank")) if self.depth != "quick": roles.append(("hunter", self.hunter_llm, "hunt")) @@ -769,11 +971,9 @@ def _finalize_spend_ledger(self, status: str | None = None) -> dict[str, Any]: def run(self) -> SourceHuntResult: from clearwing.ui.llm_activity import llm_activity_panel - self._ensure_spend_ledger() with llm_activity_panel( live=self._live, budget_usd=self.budget_usd or None, - spend_ledger=self._spend_ledger, ): return asyncio.run(self.arun()) @@ -1009,23 +1209,51 @@ async def arun(self) -> SourceHuntResult: finally: self._finalize_instrumentation("failed") start_time = time.monotonic() - self._ensure_output_dir_layout() - self._ensure_spend_ledger() - pipeline_status = PipelineStatus() - logger.info("Sourcehunt session %s starting on %s", self._session_id, self.repo_url) - self._instrumentation.record( - "run", - stage="run", - status="started", - metadata={"flow": self._flow, "repository": self.repo_url}, - ) + session_dir = self._ensure_output_dir_layout() + try: + if self._standalone_session: + self._acquire_session_lock(session_dir) + self._refresh_resume_store_locked() + self._ensure_spend_ledger() + pipeline_status = PipelineStatus() + if self._standalone_session and self._completion_store is None: + self._completion_store = SourceHuntResumeStore(session_dir) + logger.info("Sourcehunt session %s starting on %s", self._session_id, self.repo_url) + self._instrumentation.record( + "run", + stage="run", + status="started", + metadata={"flow": self._flow, "repository": self.repo_url}, + ) + except BaseException: + self._release_session_lock() + raise try: self._preflight_budget_clients() # 1. Preprocess self._emit_stage("preprocess", "started") preprocess_result = self._preprocess() repo_path = preprocess_result.repo_path + resolved_commit = self._resolved_commit(repo_path) files = preprocess_result.file_targets + identity = source_input_identity( + repo_path, + files, + ) + if self._completion_store is not None: + if self._resume_session is not None: + self._completion_store.validate_source_identity(identity) + elif self._standalone_session: + self._completion_store.create_session( + repository={ + "url": self.repo_url, + "branch": self.branch, + "local_path": self.local_path, + "resolved_commit": resolved_commit or None, + }, + config=self._config.to_dict(), + source_identity=identity, + ) files_ranked = len(files) logger.info("Preprocessor enumerated %d files", files_ranked) stage_files = [str(file_target.get("path") or "") for file_target in files] @@ -1038,9 +1266,25 @@ async def arun(self) -> SourceHuntResult: self._ensure_sandbox_factory(repo_path, files) # 2. Rank — unless depth=quick AND no LLM available, or --no-rank + restored_targets = ( + self._completion_store.load_rank_plan() + if self._completion_store is not None + else None + ) + rank_complete = restored_targets is not None + rank_plan_ready = rank_complete + if restored_targets is not None: + current_by_path = {target["path"]: target for target in files} + files = [ + { + **target, + "absolute_path": current_by_path[target["path"]]["absolute_path"], + } + for target in restored_targets + ] ranker_llm = ( None - if self._no_rank + if self._no_rank or rank_complete else self._get_native_client( "ranker", self.ranker_llm, @@ -1053,29 +1297,33 @@ async def arun(self) -> SourceHuntResult: detail=f"{len(files)} files", files=stage_files, ) - if self._no_rank: - logger.info("Ranker skipped (--no-rank); assigning default priority scores") - for ft in files: - ft["surface"] = ft.get("surface") or 3 - ft["influence"] = ft.get("influence") or 2 - ft["reachability"] = ft.get("reachability") or 3 - ft["priority"] = ( - ft["surface"] * 0.5 + ft["influence"] * 0.2 + ft["reachability"] * 0.3 - ) + if rank_complete: + logger.info("Ranker skipped; restored complete ranked target plan") + elif self._no_rank: + logger.info("Ranker LLM skipped (--no-rank); using deterministic static ranking") + Ranker(None).rank_heuristically(files) pipeline_status.record_degraded( "ranker", - "All files assigned default priority scores (--no-rank)", + "LLM ranking skipped; deterministic static ranking used (--no-rank)", ) self._emit_stage( "rank", "degraded", - detail="Skipped (--no-rank)", + detail="Deterministic static ranking (--no-rank)", files=stage_files, ) + rank_plan_ready = True elif ranker_llm is not None and files: logger.info("Ranker starting on %d files", len(files)) + unranked_files = [dict(target) for target in files] try: ranker_config = RankerConfig() + if self._ranker_chunk_size is not None: + ranker_config.chunk_size = self._ranker_chunk_size + if self._ranker_max_inflight_chunks is not None: + ranker_config.max_inflight_chunks = self._ranker_max_inflight_chunks + if self._ranker_chunk_max_retries is not None: + ranker_config.chunk_max_retries = self._ranker_chunk_max_retries if not self._preprocessing: ranker_config.include_static_hints = False ranker_config.include_imports_by = False @@ -1088,16 +1336,27 @@ async def arun(self) -> SourceHuntResult: ranker_config.chunk_size, ranker_config.max_inflight_chunks, ) - await Ranker(ranker_llm, ranker_config).arank(files) + ranker = Ranker(ranker_llm, ranker_config) + await ranker.arank(files) logger.info("Ranker completed") - pipeline_status.record_succeeded("ranker") + if ranker.completed_successfully: + pipeline_status.record_succeeded("ranker") + else: + pipeline_status.record_degraded( + "ranker", + "Incomplete rank output; the whole plan used heuristic scores", + ) self._emit_stage( "rank", - "completed", + "completed" if ranker.completed_successfully else "degraded", detail=f"Ranked {len(files)} files", files=stage_files, ) + rank_plan_ready = ranker.completed_successfully + except ProviderExhaustedError: + raise except BudgetExceeded: + files = unranked_files logger.info("Ranker stopped because the run budget is exhausted") pipeline_status.record( "ranker", @@ -1111,6 +1370,7 @@ async def arun(self) -> SourceHuntResult: files=stage_files, ) except Exception: + files = unranked_files logger.warning("Ranker failed", exc_info=True) pipeline_status.record_degraded( "ranker", @@ -1142,6 +1402,7 @@ async def arun(self) -> SourceHuntResult: detail="No ranker model available; default priority scores used", files=stage_files, ) + rank_plan_ready = not files # Ensure a partial/failed rank pass still leaves every file # schedulable by the deterministic fallback. @@ -1153,6 +1414,84 @@ async def arun(self) -> SourceHuntResult: ft["surface"] * 0.5 + ft["influence"] * 0.2 + ft["reachability"] * 0.3 ) + if self._completion_store is not None and not rank_complete and rank_plan_ready: + self._completion_store.save_rank_plan(self._ranked_targets(files)) + + if ( + self._hunt_file_offset + or self._max_hunt_files is not None + or self._hunt_file_offsets is not None + or self._hunt_file_paths is not None + ): + rank_field = "deterministic_rank_score" if self._no_rank else "priority" + ranked_files = sorted( + files, + key=lambda item: ( + -float(item.get(rank_field, item.get("priority", 0.0))), + str(item.get("path") or ""), + ), + ) + if self._hunt_file_paths is not None: + ranked_by_path = { + str(item.get("path") or ""): item for item in ranked_files + } + missing_paths = [ + path for path in self._hunt_file_paths if path not in ranked_by_path + ] + if missing_paths: + raise ValueError( + "requested hunt_file_paths are absent from ranked source: " + + ", ".join(missing_paths) + ) + files = [ranked_by_path[path] for path in self._hunt_file_paths] + logger.info( + "Selecting %d exact hunter paths by sealed manifest", + len(files), + ) + selection_detail = ( + f"Selected {len(files)} exact files from a sealed path manifest" + ) + elif self._hunt_file_offsets is not None: + files = [ + ranked_files[offset] + for offset in self._hunt_file_offsets + if offset < len(ranked_files) + ] + logger.info( + "Selecting hunter rank offsets %s by %s (%d files)", + self._hunt_file_offsets, + rank_field, + len(files), + ) + selection_detail = ( + f"Selected {len(files)} files from exact rank offsets " + f"{self._hunt_file_offsets} by {rank_field}" + ) + else: + stop = ( + None + if self._max_hunt_files is None + else self._hunt_file_offset + self._max_hunt_files + ) + files = ranked_files[self._hunt_file_offset : stop] + logger.info( + "Selecting hunter rank window [%d, %s) by %s (%d files)", + self._hunt_file_offset, + stop if stop is not None else "end", + rank_field, + len(files), + ) + selection_detail = ( + f"Selected {len(files)} files from rank offset " + f"{self._hunt_file_offset} by {rank_field}" + ) + selected_files = [str(item.get("path") or "") for item in files] + self._emit_stage( + "rank", + "bounded", + detail=selection_detail, + files=selected_files, + ) # depth=quick exits here with the static_findings as-is if self.depth == "quick": return self._build_quick_result( @@ -1283,8 +1622,7 @@ async def arun(self) -> SourceHuntResult: from .findings_pool import FindingsPool from .historical_findings_db import HistoricalFindingsDB - checkpoint_path = Path(self.output_dir) / self._session_id / "findings_pool.jsonl" - findings_pool = FindingsPool(checkpoint_path=checkpoint_path) + findings_pool = FindingsPool() try: historical_db = HistoricalFindingsDB(path=self._historical_db_path) prior = historical_db.query_prior(repo_url=self.repo_url) @@ -1300,9 +1638,20 @@ async def arun(self) -> SourceHuntResult: self.hunter_llm, budget_stage="hunt", ) - all_findings: list[Finding] = [] - files_hunted = 0 - spent_per_tier: dict[str, float] = {"A": 0.0, "B": 0.0, "C": 0.0} + all_findings = ( + self._completion_store.completed_findings() + if self._completion_store is not None + else [] + ) + files_hunted = ( + self._completion_store.completed_target_count() + if self._completion_store is not None + else 0 + ) + lifetime_spend_by_tier = self._ensure_spend_ledger().spent_by("tier") + spent_per_tier: dict[str, float] = { + tier: lifetime_spend_by_tier.get(tier, 0.0) for tier in ("A", "B", "C") + } band_stats: dict | None = None hunt_symbols = sorted( { @@ -1341,12 +1690,21 @@ async def arun(self) -> SourceHuntResult: llm=hunter_llm, max_parallel=self.max_parallel, budget_usd=self.budget_usd, + input_price_per_million=self.input_price_per_million, + output_price_per_million=self.output_price_per_million, tier_budget=self.tier_budget, session_id_prefix=self._session_id, seeded_crashes_by_file=seeded_by_file, semgrep_hints_by_file=semgrep_hints_by_file, agent_mode=self._effective_agent_mode, prompt_mode=self._prompt_mode, + prompt_bundle=self._prompt_bundle, + scaffold_profile=self._scaffold_profile, + context_profile=self._context_profile, + prompt_candidate=self._prompt_candidate, + max_hunter_steps=self._max_hunter_steps, + hunter_temperature=self._hunter_temperature, + hunter_max_output_tokens=self._hunter_max_output_tokens, campaign_hint=self._campaign_hint, exploit_mode=self._exploit_mode, starting_band=self._starting_band, @@ -1358,6 +1716,9 @@ async def arun(self) -> SourceHuntResult: findings_pool=findings_pool, trajectory_root=(Path(self.output_dir) / self._session_id / "trajectories"), instrumentation=self._instrumentation, + resume_store=self._completion_store, + prior_spend_per_tier=self._ensure_spend_ledger().spent_by("tier"), + provider_exhaustion_state=self._provider_exhaustion, ) ) try: @@ -1391,6 +1752,10 @@ async def arun(self) -> SourceHuntResult: symbols=self._finding_symbols(all_findings), finding_ids=[finding.id for finding in all_findings], ) + except ProviderExhaustedError: + if self._completion_store is not None: + all_findings = self._completion_store.completed_findings() + raise except BudgetExceeded: logger.info("HunterPool stopped because the run budget is exhausted") pipeline_status.record( @@ -1603,6 +1968,7 @@ async def arun(self) -> SourceHuntResult: Path(self.output_dir) / self._session_id / "trajectories" ), instrumentation=self._instrumentation, + provider_exhaustion_state=self._provider_exhaustion, ) ) try: @@ -2205,6 +2571,8 @@ async def arun(self) -> SourceHuntResult: "instrumentation_events": str(self._instrumentation.events_path), } ) + if self._completion_store is not None: + output_paths["session"] = str(self._completion_store.session_path) duration = time.monotonic() - start_time exit_findings = all_findings if self.no_verify else verified @@ -2231,6 +2599,47 @@ async def arun(self) -> SourceHuntResult: status=run_status, budget_usd=self.budget_usd, ) + except ProviderExhaustedError: + pipeline_status = locals().get("pipeline_status", PipelineStatus()) + summary = self._finalize_spend_ledger("provider_exhausted") + self._finalize_instrumentation("provider_exhausted") + findings = ( + self._completion_store.completed_findings() + if self._completion_store is not None + else [] + ) + partial_preprocess = locals().get("preprocess_result") + if isinstance(partial_preprocess, PreprocessResult): + findings = self._merge_static_findings(findings, partial_preprocess) + return SourceHuntResult( + exit_code=3, + repo_url=self.repo_url, + repo_path=locals().get("repo_path", self.local_path or self.repo_url), + findings=findings, + verified_findings=[finding for finding in findings if finding.verified], + exploited_findings=[finding for finding in findings if finding.exploit_success], + files_ranked=locals().get("files_ranked", 0), + files_hunted=( + self._completion_store.completed_target_count() + if self._completion_store is not None + else 0 + ), + duration_seconds=round(time.monotonic() - start_time, 2), + cost_usd=summary["total_spent"], + spent_per_tier=self._ensure_spend_ledger().spent_by("tier"), + tokens_used=summary["total_tokens"], + output_paths=( + {"session": str(self._completion_store.session_path)} + if self._completion_store is not None + else {} + ), + session_id=self._session_id, + pipeline_status=pipeline_status, + status="provider_exhausted", + budget_usd=self.budget_usd, + ) + except (KeyboardInterrupt, asyncio.CancelledError): + raise finally: if self._spend_ledger is not None: self._finalize_spend_ledger("failed") @@ -2243,11 +2652,22 @@ async def arun(self) -> SourceHuntResult: if self._preprocessor is not None: self._preprocessor.cleanup() self._preprocessor = None + self._release_session_lock() @property def session_id(self) -> str: return self._session_id + @property + def flow(self) -> str: + return self._flow + + @property + def session_path(self) -> Path: + """Public path to the immutable standalone session metadata.""" + + return Path(self.output_dir) / self._session_id / "session.json" + # --- Pipeline helpers --------------------------------------------------- def _ensure_output_dir_layout(self) -> Path: @@ -2255,6 +2675,25 @@ def _ensure_output_dir_layout(self) -> Path: session_dir.mkdir(parents=True, exist_ok=True) return session_dir + def _acquire_session_lock(self, session_dir: Path) -> None: + if self._session_lock is None: + lock = SourceHuntSessionLock(session_dir) + lock.acquire() + self._session_lock = lock + + def _release_session_lock(self) -> None: + if self._session_lock is not None: + self._session_lock.release() + self._session_lock = None + + def _refresh_resume_store_locked(self) -> None: + if self._resume_session is None: + return + if self._session_lock is None: + raise SourceHuntResumeError("Sourcehunt resume must refresh its store under lock") + session_dir = Path(self.output_dir) / self._session_id + self._completion_store = SourceHuntResumeStore.load(session_dir) + def _export_disclosure_bundle( self, verified_findings: list[Finding], @@ -2617,6 +3056,7 @@ def _preprocess(self) -> PreprocessResult: run_semgrep=(self.depth != "quick" and self._preprocessing), run_taint=(self.depth != "quick" and self._preprocessing), respect_gitignore=self._respect_gitignore, + excluded_roots=[Path(self.output_dir) / self._session_id], ) return self._preprocessor.run() @@ -2777,6 +3217,8 @@ def _build_quick_result( "instrumentation_events": str(self._instrumentation.events_path), } ) + if self._completion_store is not None: + output_paths["session"] = str(self._completion_store.session_path) duration = time.monotonic() - start_time return SourceHuntResult( exit_code=(3 if run_status == "budget_exhausted" else self._exit_code(all_findings)), @@ -2911,7 +3353,11 @@ def _get_native_client( key = (id(client), stage) bound = self._metered_clients.get(key) if bound is None: - bound = client.with_spend_ledger(self._spend_ledger, stage=stage) + bound = client.with_spend_ledger( + self._spend_ledger, + stage=stage, + provider_exhaustion_state=self._provider_exhaustion, + ) self._metered_clients[key] = bound return bound @@ -2925,6 +3371,48 @@ def _build_native_from_model_string(self, model: str) -> AsyncLLMClient | None: # --- Reporting ---------------------------------------------------------- + def _context_metrics(self) -> dict[str, Any]: + """Aggregate per-hunter context events already persisted in trajectories.""" + + trajectory_root = Path(self.output_dir) / self._session_id / "trajectories" + metrics = { + "context_profile": self._context_profile, + "model_calls": 0, + "compaction_count": 0, + "peak_context_tokens_estimate": 0, + "peak_input_tokens": 0, + "total_input_tokens": 0, + "total_output_tokens": 0, + } + if not trajectory_root.is_dir(): + return metrics + for transcript in trajectory_root.rglob("transcript.jsonl"): + try: + lines = transcript.read_text(encoding="utf-8").splitlines() + except OSError: + continue + for line in lines: + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + if event.get("event") == "context_compaction": + metrics["compaction_count"] += 1 + if event.get("event") != "message" or "usage" not in event: + continue + usage = event.get("usage") or {} + input_tokens = int(usage.get("input_tokens", 0) or 0) + output_tokens = int(usage.get("output_tokens", 0) or 0) + metrics["model_calls"] += 1 + metrics["total_input_tokens"] += input_tokens + metrics["total_output_tokens"] += output_tokens + metrics["peak_input_tokens"] = max(metrics["peak_input_tokens"], input_tokens) + metrics["peak_context_tokens_estimate"] = max( + metrics["peak_context_tokens_estimate"], + int(event.get("estimated_context_tokens", 0) or 0), + ) + return metrics + def _record_reporting_failure( self, exc: Exception, @@ -2969,6 +3457,9 @@ def _write_report( self._record_reporting_failure(exc, findings) return {} try: + report_budget_summary = dict(budget_summary or {}) + report_budget_summary["context_profile"] = self._context_profile + report_budget_summary["context_metrics"] = self._context_metrics() return write_sourcehunt_report( output_dir=self.output_dir, session_id=self._session_id, @@ -2981,7 +3472,7 @@ def _write_report( pool_stats=pool_stats, subsystem_stats=subsystem_stats, pipeline_status=pipeline_status, - budget_summary=budget_summary, + budget_summary=report_budget_summary, ) except Exception as exc: logger.warning("Reporter failed", exc_info=True) diff --git a/clearwing/sourcehunt/semgrep_sidecar.py b/clearwing/sourcehunt/semgrep_sidecar.py index ab997450..c35c9242 100644 --- a/clearwing/sourcehunt/semgrep_sidecar.py +++ b/clearwing/sourcehunt/semgrep_sidecar.py @@ -63,7 +63,11 @@ def __init__( def available(self) -> bool: return shutil.which(self.binary) is not None - def run_scan(self, repo_path: str) -> list[SemgrepFinding]: + def run_scan( + self, + repo_path: str, + files: list[str] | None = None, + ) -> list[SemgrepFinding]: """Invoke `semgrep --json --config `. Returns a list of normalized findings. On any failure, logs and @@ -85,7 +89,10 @@ def run_scan(self, repo_path: str) -> list[SemgrepFinding]: ] if not self.respect_gitignore: cmd.append("--no-git-ignore") # also scan ignored files — v0.1 choice - cmd = cmd + self.extra_args + [repo_path] + targets = files if files is not None else [repo_path] + if not targets: + return [] + cmd = cmd + self.extra_args + targets try: proc = subprocess.run( diff --git a/clearwing/sourcehunt/state.py b/clearwing/sourcehunt/state.py index 9c9ef42e..391641a8 100644 --- a/clearwing/sourcehunt/state.py +++ b/clearwing/sourcehunt/state.py @@ -85,6 +85,9 @@ class FileTarget(TypedDict, total=False): reachability: int # 1-5 — attacker-reachability through callgraph # v0.1: defaults to 3 (unknown); v0.2: real propagation priority: float # surface*0.5 + influence*0.2 + reachability*0.3 + security_signal_score: float # target-blind static source evidence + deterministic_rank_score: float # signal score + legacy priority tie-break + security_signal_counts: dict[str, int] tier: Literal["A", "B", "C"] tags: list[FileTag] # v0.1: heuristic tagger; v0.2: + LLM polish language: str diff --git a/clearwing/sourcehunt/static_signals.py b/clearwing/sourcehunt/static_signals.py new file mode 100644 index 00000000..8be007cf --- /dev/null +++ b/clearwing/sourcehunt/static_signals.py @@ -0,0 +1,161 @@ +"""Target-blind static security signals shared by ranking and hunter tools.""" + +from __future__ import annotations + +import math +import re +from collections import Counter +from dataclasses import dataclass +from pathlib import PurePosixPath + + +@dataclass(frozen=True) +class StaticSignal: + """A language-generic source pattern used only to prioritize review.""" + + name: str + pattern: re.Pattern[str] + weight: int + + +@dataclass(frozen=True) +class StaticSignalScore: + """Auditable evidence behind one file's deterministic ranking score.""" + + score: float + counts: dict[str, int] + top_anchor_scores: tuple[int, ...] + + @property + def diversity(self) -> int: + return len(self.counts) + + +STATIC_SECURITY_SIGNALS: tuple[StaticSignal, ...] = ( + StaticSignal( + "memory_operation", + re.compile(r"\b(?:memcpy|memmove|memset|strcpy|strncpy|sprintf|snprintf)\s*\("), + 7, + ), + StaticSignal( + "allocation_lifetime", + re.compile(r"\b(?:malloc|calloc|realloc|free|new|delete|alloc|release|unref)\b"), + 6, + ), + StaticSignal( + "representation_transition", + re.compile( + r"(?:\(\s*(?:u?int(?:8|16|32)_t|char|short)\s*\)|" + r"=\s*(?:\+\+|--)[^;]+|[^;]+(?:\+\+|--)\s*;)" + ), + 5, + ), + StaticSignal( + "reserved_value_state", + re.compile(r"(?:\bmemset\b[^;]*(?:-1|0x[fF]{2,})|(?:==|!=)\s*(?:-1|0x[fF]{2,}))"), + 5, + ), + StaticSignal( + "size_arithmetic", + re.compile( + r"\b(?:size|len|count|width|height|offset|stride|index|idx)\w*\b" + r"[^;]*(?:\+|-|\*|<<|>>)" + ), + 4, + ), + StaticSignal( + "input_boundary", + re.compile(r"\b(?:parse|decode|read|get_bits|packet|header|request|input)\w*\b", re.I), + 3, + ), + StaticSignal( + "pointer_index", + re.compile(r"(?:->\w+\s*\[[^]]+\]|\b\w+\s*\[[^]]+\]\s*=|\*\s*\([^)]*[+-][^)]*\))"), + 3, + ), + StaticSignal( + "security_guard", + re.compile(r"\bif\s*\([^)]*(?:<=|>=|<|>|==|!=)[^)]*\)"), + 1, + ), +) + + +_NON_PRODUCTION_COMPONENT = re.compile( + r"^(?:tests?|docs?|examples?|benchmarks?|fuzz(?:er|ers|ing)?)$", + re.I, +) +_NON_PRODUCTION_TOKEN = re.compile( + r"(?:^|[_-])(?:test|tests|example|examples|benchmark|benchmarks|fuzz|fuzzer|fuzzers)" + r"(?:[_-]|$)", + re.I, +) + + +def is_production_source_path(path: str) -> bool: + """Exclude generic test, documentation, example, benchmark, and fuzz paths.""" + + parts = PurePosixPath(path.replace("\\", "/")).parts + return not any( + _NON_PRODUCTION_COMPONENT.match(part) + or _NON_PRODUCTION_TOKEN.search(PurePosixPath(part).stem) + for part in parts + ) + + +def line_security_signals(line: str) -> list[tuple[str, int]]: + """Return generic signal categories and weights present on one line.""" + + return [ + (signal.name, signal.weight) + for signal in STATIC_SECURITY_SIGNALS + if signal.pattern.search(line) + ] + + +def score_source_security_signals( + source: str, + *, + saturation_count: int = 5, + max_anchors: int = 8, +) -> StaticSignalScore: + """Score source without target, repository, or known-answer hints. + + Each category saturates after repeated evidence so large files and dense + memory-operation loops cannot dominate by volume alone. A bounded set of + the strongest individual lines preserves local signal interactions. + """ + + counts: Counter[str] = Counter() + anchor_scores: list[int] = [] + for line in source.splitlines(): + matches = line_security_signals(line) + if not matches: + continue + counts.update(name for name, _weight in matches) + anchor_scores.append(sum(weight for _name, weight in matches)) + + saturation_denominator = math.log1p(max(1, saturation_count)) + coverage_score = sum( + signal.weight + * min(1.0, math.log1p(counts[signal.name]) / saturation_denominator) + for signal in STATIC_SECURITY_SIGNALS + ) + top_anchor_scores = tuple(sorted(anchor_scores, reverse=True)[:max_anchors]) + diversity_score = 2.0 * len(counts) + local_evidence_score = 0.5 * sum(top_anchor_scores) + return StaticSignalScore( + score=coverage_score + diversity_score + local_evidence_score, + counts=dict(sorted(counts.items())), + top_anchor_scores=top_anchor_scores, + ) + + +__all__ = [ + "STATIC_SECURITY_SIGNALS", + "StaticSignal", + "StaticSignalScore", + "is_production_source_path", + "line_security_signals", + "score_source_security_signals", +] diff --git a/clearwing/sourcehunt/subsystem.py b/clearwing/sourcehunt/subsystem.py index b1e508c7..b01915c5 100644 --- a/clearwing/sourcehunt/subsystem.py +++ b/clearwing/sourcehunt/subsystem.py @@ -18,6 +18,7 @@ from typing import Any from clearwing.llm.budget import BudgetExceeded, spend_metadata +from clearwing.llm.errors import ProviderExhaustedError, ProviderExhaustionState from clearwing.sourcehunt.state import FileTarget, Finding, SubsystemTarget from .instrumentation import stable_run_id @@ -172,6 +173,7 @@ class SubsystemHuntConfig: project_name: str = "target" trajectory_root: str | Path | None = None instrumentation: Any = None + provider_exhaustion_state: ProviderExhaustionState | None = None class SubsystemHuntRunner: @@ -200,6 +202,8 @@ async def arun(self) -> list[Finding]: async def _guarded_run(subsystem: SubsystemTarget) -> list[Finding]: async with sem: + if self.config.provider_exhaustion_state is not None: + self.config.provider_exhaustion_state.raise_if_exhausted() if self.config.total_budget_usd > 0 and self._spent >= self.config.total_budget_usd: logger.info( "Subsystem %s skipped: total budget exhausted", @@ -242,22 +246,35 @@ async def _guarded_run(subsystem: SubsystemTarget) -> list[Finding]: tasks = [asyncio.create_task(_guarded_run(s)) for s in self.config.subsystems] + try: + await self._collect_tasks(tasks, all_findings) + except ProviderExhaustedError: + for task in tasks: + if not task.done(): + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + raise + + return all_findings + + @staticmethod + async def _collect_tasks( + tasks: list[asyncio.Task[list[Finding]]], + all_findings: list[Finding], + ) -> None: for coro in asyncio.as_completed(tasks): try: - findings = await coro - all_findings.extend(findings) + all_findings.extend(await coro) except BudgetExceeded: logger.info("Subsystem hunt stopped because the run budget is exhausted") for task in tasks: if not task.done(): task.cancel() await asyncio.gather(*tasks, return_exceptions=True) - break + return except Exception: logger.warning("Subsystem hunt task failed", exc_info=True) - return all_findings - async def _run_one_subsystem( self, subsystem: SubsystemTarget, diff --git a/clearwing/sourcehunt/validator.py b/clearwing/sourcehunt/validator.py index 68d30ae6..31fec22f 100644 --- a/clearwing/sourcehunt/validator.py +++ b/clearwing/sourcehunt/validator.py @@ -130,6 +130,50 @@ class in verifier.py stays for backward compatibility. }""" +VALIDATOR_RETRY_PROMPT = """\ +Independently validate the reported vulnerability against the supplied source. +Return the structured verdict immediately. Decide whether it is real, reachable +from attacker input, security-impactful, and realistic in common configurations. +State the strongest case for and against it. Do not omit required fields.""" + + +VALIDATOR_SOURCE_FIRST_PROMPT = """\ +You are an independent source-first security validator. The report and trace are +allegations; the supplied current source is authoritative. Verify the complete +causal chain in that source. For REAL, compare the alleged operation and missing +checks with the current code. If a guard, bound, exact-identity check, accounting +update, early return, or other invariant breaks a required step, REAL is false. +Do not affirm a historical bug merely because the report describes it plausibly. + +Judge TRIGGERABLE, IMPACTFUL, and GENERAL independently. Optional features, +compile flags, or plausible deployment or environment preconditions reduce +prevalence; they do not alone erase a source-level flaw. Remote memory corruption, +confidentiality loss, authentication bypass, and material resource exhaustion are +security impacts. Reject when the chain is absent, unreachable, non-security- +relevant, or requires implausible conditions. Return the structured verdict.""" + + +VALIDATOR_SOURCE_FIRST_COMPACT_PROMPT = """\ +Validate one alleged vulnerability against the supplied current source. The +report is untrusted; current source is authoritative. REAL is false if a guard, +bound, exact-identity check, accounting update, early return, or other invariant +breaks any required causal step. Never affirm a historical flaw against +contradictory current code. + +Judge reachability, security impact, and realistic deployment independently. +Optional builds or plausible environment conditions are assumptions, not +automatic rejection. Set advance=true only when REAL and IMPACTFUL pass and +TRIGGERABLE and GENERAL either pass or have medium/high confidence with explicit +assumptions. Otherwise set it false. Return the schema now with concise reasons.""" + + +VALIDATOR_PROMPT_PROFILES = { + "legacy-v1": VALIDATOR_SYSTEM_PROMPT, + "source-first-high-recall-v1": VALIDATOR_SOURCE_FIRST_PROMPT, + "source-first-compact-v2": VALIDATOR_SOURCE_FIRST_COMPACT_PROMPT, +} + + # --- Enforced structured output schema --------------------------------------- # The prompts already ask for exactly this JSON; passing it as a schema_model to # aask_json turns it into a genai-pyo3 response_json_spec so the gateway emits it @@ -257,51 +301,80 @@ def __init__( *, gate_threshold: EvidenceLevel | None = "static_corroboration", enable_quick_pass: bool = True, + prompt_profile: str = "legacy-v1", + system_prompt: str | None = None, + max_output_tokens: int | None = None, + temperature: float | None = None, ): + if prompt_profile not in VALIDATOR_PROMPT_PROFILES: + choices = ", ".join(sorted(VALIDATOR_PROMPT_PROFILES)) + raise ValueError( + f"Unknown validator prompt profile {prompt_profile!r}; choose from {choices}" + ) + if max_output_tokens is not None and max_output_tokens < 1: + raise ValueError("validator max_output_tokens must be positive") + if temperature is not None and not 0.0 <= temperature <= 2.0: + raise ValueError("validator temperature must be between 0 and 2") self.llm = llm self.gate_threshold = gate_threshold self.enable_quick_pass = enable_quick_pass + self.prompt_profile = prompt_profile + self.system_prompt = system_prompt + self.max_output_tokens = max_output_tokens + self.temperature = temperature def _prompt_for_finding(self, finding: Finding) -> str: + full_prompt = self.system_prompt or VALIDATOR_PROMPT_PROFILES[self.prompt_profile] if not self.enable_quick_pass: - return VALIDATOR_SYSTEM_PROMPT + return full_prompt if self.gate_threshold is None: - return VALIDATOR_SYSTEM_PROMPT + return full_prompt level = cast(EvidenceLevel, finding.get("evidence_level", "suspicion")) try: above = evidence_at_or_above(level, self.gate_threshold) except KeyError: above = False - return VALIDATOR_SYSTEM_PROMPT if above else VALIDATOR_QUICK_PROMPT + return full_prompt if above else VALIDATOR_QUICK_PROMPT async def avalidate( self, finding: Finding, file_content: str = "", + source_context: str = "", ) -> ValidatorVerdict: - user_msg = self._build_user_message(finding, file_content) + user_msg = self._build_user_message(finding, file_content, source_context) system_prompt = self._prompt_for_finding(finding) - # Enforced structured output. response_schema becomes a genai-pyo3 - # response_json_spec (constrained decoding), so the model emits a JSON - # object matching _VerdictSchema — even reasoning models that would - # otherwise return empty text under free-form prompting. We validate to - # the typed wire object and map it to the domain verdict directly (no - # dict round-trip). Requires a model/gateway with constrained decoding. + # Enforced structured output. Some small reasoning models can consume + # their entire default output budget before emitting the final JSON. + # Retry once with a much shorter generic instruction and more output + # headroom; never retry a budget refusal. try: - response = await self.llm.aask_text( - system=system_prompt, - user=user_msg, - response_schema=_VerdictSchema, - response_schema_name="ValidatorVerdict", + schema = await self._request_schema( + system_prompt, + user_msg, + max_tokens=self.max_output_tokens, ) - schema = _VerdictSchema.model_validate_json(response_text(response)) verdict = schema.to_verdict(finding.get("id", "unknown")) except BudgetExceeded: raise - except Exception as e: - logger.warning("Validator LLM call failed", exc_info=True) - verdict = self._error_verdict(finding, f"validator error: {e}") + except Exception as first_error: + logger.info("Validator response invalid; retrying with compact prompt") + try: + schema = await self._request_schema( + VALIDATOR_RETRY_PROMPT, + user_msg, + max_tokens=self.max_output_tokens or 8192, + ) + verdict = schema.to_verdict(finding.get("id", "unknown")) + except BudgetExceeded: + raise + except Exception as retry_error: + logger.warning("Validator LLM call failed after retry", exc_info=True) + verdict = self._error_verdict( + finding, + f"validator error: {first_error}; retry error: {retry_error}", + ) EventBus().emit_validation_result(ValidationResultPayload( finding_id=verdict.finding_id, @@ -313,7 +386,32 @@ async def avalidate( return verdict - def _build_user_message(self, finding: Finding, file_content: str) -> str: + async def _request_schema( + self, + system: str, + user: str, + *, + max_tokens: int | None = None, + ) -> _VerdictSchema: + response = await self.llm.aask_text( + system=system, + user=user, + temperature=self.temperature, + max_tokens=max_tokens, + response_schema=_VerdictSchema, + response_schema_name="ValidatorVerdict", + ) + return cast( + _VerdictSchema, + _VerdictSchema.model_validate_json(response_text(response)), + ) + + def _build_user_message( + self, + finding: Finding, + file_content: str, + source_context: str = "", + ) -> str: finding_view = { "id": finding.get("id"), "file": finding.get("file"), @@ -327,13 +425,25 @@ def _build_user_message(self, finding: Finding, file_content: str) -> str: "poc": finding.get("poc"), "exploit": finding.get("exploit"), "discovered_by": finding.get("discovered_by"), + "vulnerability_trace": finding.get("vulnerability_trace"), } - msg = "Validate the following bug report:\n\n" + msg = ( + "Validate the following bug report. Treat vulnerability_trace as the " + "reporter's alleged source chain: use it to locate the claim, but independently " + "verify every step against the supplied current source before relying on it.\n\n" + ) msg += json.dumps(finding_view, indent=2) if file_content: excerpts = self._build_file_context(finding, file_content) if excerpts: msg += f"\n\nRelevant file excerpts:\n{excerpts}" + if source_context: + msg += ( + "\n\nIndependently collected current source snapshot. Treat this " + "source as authoritative and re-check the report's alleged snippets " + "and mechanism against it:\n" + f"{source_context[:24000]}" + ) return msg def _build_file_context(self, finding: Finding, file_content: str) -> str: @@ -432,8 +542,14 @@ async def arun_patch_oracle( from .verifier import Verifier temp_v = Verifier(self.llm) - return await temp_v.arun_patch_oracle( - finding, file_content, sandbox, rerun_poc, + return cast( + tuple[bool, str, str], + await temp_v.arun_patch_oracle( + finding, + file_content, + sandbox, + rerun_poc, + ), ) diff --git a/clearwing/ui/cli.py b/clearwing/ui/cli.py index df8bb87c..2b1b31cd 100644 --- a/clearwing/ui/cli.py +++ b/clearwing/ui/cli.py @@ -8,6 +8,7 @@ import argparse import logging +import sys from typing import TYPE_CHECKING from rich.console import Console @@ -51,7 +52,9 @@ def engine(self) -> CoreEngine: def run(self, args: list | None = None) -> None: """Run the CLI.""" parser = self._create_parser() - parsed_args = parser.parse_args(args) + raw_args = list(args) if args is not None else sys.argv[1:] + parsed_args = parser.parse_args(raw_args) + parsed_args._raw_args = raw_args # Dispatch to the matching command module. A module's base name # matches the subcommand it registers; command modules may also diff --git a/clearwing/ui/commands/setup.py b/clearwing/ui/commands/setup.py index c0f606e6..f94ccbe4 100644 --- a/clearwing/ui/commands/setup.py +++ b/clearwing/ui/commands/setup.py @@ -49,7 +49,8 @@ def add_parser(subparsers): help=( "Skip the menu and configure this provider directly " "(e.g. openrouter, ollama, lmstudio, anthropic, openai, " - "openai-oauth, together, groq, deepseek, fireworks, custom)" + "openai-oauth, together, groq, deepseek, kimi-code, " + "fireworks, custom)" ), ) parser.add_argument( @@ -458,6 +459,7 @@ def _write_config( existing["provider"] = provider_section path.write_text(yaml.safe_dump(existing, default_flow_style=False, sort_keys=True)) + path.chmod(0o600) # Keep the in-memory Config object in sync for the current process. cli.config.set("provider", value=provider_section) @@ -555,12 +557,20 @@ def _run_test_invoke( client = ProviderManager.for_endpoint(endpoint).get_native_client("default") start = time.monotonic() - resp = asyncio.run( - client.aask_text(system="", user="Reply with exactly the word PONG.") - ) + resp = asyncio.run(client.aask_text(system="", user="Reply with exactly the word PONG.")) elapsed_ms = int((time.monotonic() - start) * 1000) except Exception as exc: console.print(f"\n[red]Test failed: {exc}[/red]") + error_text = str(exc).lower() + if preset.key == "kimi-code" and ( + "401" in error_text or "invalid authentication" in error_text + ): + console.print( + "[yellow]Kimi Code keys are separate from Open Platform keys. " + "Use a membership key created at https://www.kimi.com/code/console " + "with the default https://api.kimi.com/coding/v1 endpoint, and " + "confirm that your membership tier includes the selected model.[/yellow]" + ) console.print( "[yellow]The config was still written. " "Run `clearwing doctor` for a fuller diagnosis.[/yellow]" diff --git a/clearwing/ui/commands/sourcehunt.py b/clearwing/ui/commands/sourcehunt.py index 5362713c..a92ad36f 100644 --- a/clearwing/ui/commands/sourcehunt.py +++ b/clearwing/ui/commands/sourcehunt.py @@ -40,6 +40,12 @@ def add_parser(subparsers): help="Source-code vulnerability hunting (source-hunt pipeline)", ) parser.add_argument("repo", nargs="?", help="Git URL or local path to a repository") + parser.add_argument( + "--resume", + metavar="SESSION_ID", + default=None, + help="Continue a resumable standalone session from its completed work results", + ) parser.add_argument("--machine-fd", type=int, help=argparse.SUPPRESS) parser.add_argument( "--flow", @@ -199,6 +205,36 @@ def add_parser(subparsers): help="Prompt mode: 'unconstrained' uses a simple discovery prompt " "(default), 'specialist' uses legacy prescriptive checklists", ) + parser.add_argument( + "--prompt-bundle", + choices=["legacy-v1", "generic-security-v1"], + default="legacy-v1", + dest="prompt_bundle", + help="Versioned prompt candidate. 'generic-security-v1' enforces the " + "solution-independent optimization baseline", + ) + parser.add_argument( + "--scaffold-profile", + choices=[ + "native-v1", + "minimal-linear-v1", + "candidate-ledger-v1", + "window-ledger-v1", + "guided-window-ledger-v1", + "state-interaction-ledger-v1", + "proof-refinement-ledger-v1", + ], + default="native-v1", + dest="scaffold_profile", + help="Versioned agent tool/control surface (default: native-v1)", + ) + parser.add_argument( + "--context-profile", + choices=["legacy-context-v1", "compact-small-model-v1"], + default="legacy-context-v1", + dest="context_profile", + help="Versioned request-context policy (default: legacy-context-v1)", + ) parser.add_argument( "--campaign-hint", default=None, @@ -666,12 +702,36 @@ def add_parser(subparsers): return parser +def _parse_resume_options(raw_args, *, default_output_dir): + """Use argparse to enforce the intentionally small resume CLI surface.""" + + from ...sourcehunt.config import SourceHuntResumeOptions + + parser = argparse.ArgumentParser(prog="clearwing sourcehunt --resume", add_help=False) + parser.add_argument("command", choices=["sourcehunt"]) + parser.add_argument("--resume", required=True, dest="session_id") + parser.add_argument("--output-dir", default=default_output_dir) + parser.add_argument("--model", dest="model_override") + parser.add_argument("--base-url") + parser.add_argument("--api-key") + parser.add_argument("--live", action="store_true") + parser.add_argument("--log-level") + parser.add_argument("--verbose", "-v", action="store_true") + parsed = parser.parse_args(raw_args) + return SourceHuntResumeOptions( + session_id=parsed.session_id, + output_dir=parsed.output_dir, + model_override=parsed.model_override, + live=parsed.live, + ) + + def handle(cli, args): """Run the sourcehunt pipeline.""" if args.machine_fd is not None: raise SystemExit(_handle_machine(args.machine_fd)) - if not args.repo: - args._command_parser.error("the following arguments are required: repo") + if not args.repo and not args.resume: + args._command_parser.error("repo is required unless --resume SESSION_ID is used") from ...core.config import default_results_dir from ...providers import ProviderManager, resolve_llm_endpoint @@ -681,6 +741,24 @@ def handle(cli, args): if args.output_dir is None: args.output_dir = default_results_dir("sourcehunt") + if args.resume: + incompatible_modes = { + "--retro-hunt": args.retro_hunt, + "--nday": args.nday, + "--reveng": args.reveng, + "--watch": args.watch, + "--webhook": args.webhook, + "--calibrate": args.calibrate, + "--elaborate": args.elaborate or args.elaborate_auto, + } + conflicts = [name for name, enabled in incompatible_modes.items() if enabled] + if conflicts: + args._command_parser.error(f"--resume cannot be combined with {', '.join(conflicts)}") + resume_options = _parse_resume_options( + args._raw_args, + default_output_dir=args.output_dir, + ) + _log_level_name = "DEBUG" if args.verbose else args.log_level logging.basicConfig( level=getattr(logging, _log_level_name), @@ -1130,7 +1208,7 @@ def handle(cli, args): ) sys.exit(1) - runner = SourceHuntRunner( + runner_options = dict( repo_url=args.repo, branch=args.branch, local_path=args.local_path, @@ -1165,6 +1243,9 @@ def handle(cli, args): provider_manager=provider_manager, agent_mode=args.agent_mode, prompt_mode=args.prompt_mode, + prompt_bundle=args.prompt_bundle, + scaffold_profile=args.scaffold_profile, + context_profile=args.context_profile, campaign_hint=args.campaign_hint, exploit_mode=args.exploit_mode, starting_band=args.starting_band, @@ -1199,6 +1280,27 @@ def handle(cli, args): emit_rejection_certificates=args.emit_rejection_certificates, falsify=args.falsify, ) + if args.resume: + try: + runner = SourceHuntRunner.resume( + resume_options.session_id, + output_dir=resume_options.output_dir, + provider_manager=provider_manager, + model_override=resume_options.model_override, + live=resume_options.live, + ) + except ValueError as exc: + args._command_parser.error(str(exc)) + args.repo = runner.repo_url + args.branch = runner.branch + args.local_path = runner.local_path + args.flow = runner.flow + args.depth = runner.depth + args.max_parallel = runner.max_parallel + args.budget = runner.budget_usd + formats = runner.output_formats + else: + runner = SourceHuntRunner(**runner_options) cli.console.print( f"[bold blue]Sourcehunt: {args.repo} flow={args.flow} depth={args.depth} " @@ -1243,7 +1345,13 @@ def _on_tool_start(data): bus.unsubscribe(EventType.TOOL_START, _on_tool_start) # Summary - if result.status == "budget_exhausted": + if result.status == "provider_exhausted": + cli.console.print( + "\n[bold yellow]Sourcehunt stopped: provider quota exhausted[/bold yellow]" + ) + cli.console.print(" Status: provider_exhausted (resumable)") + cli.console.print(f" Resume: clearwing sourcehunt --resume {result.session_id}") + elif result.status == "budget_exhausted": cli.console.print("\n[bold yellow]Sourcehunt stopped at budget[/bold yellow]") cli.console.print(" Status: partial (budget exhausted)") else: diff --git a/docs/architecture.md b/docs/architecture.md index ba3bca6d..964846da 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -132,6 +132,27 @@ input for the next: writes pre-filled MITRE CVE-request and HackerOne templates for every verified finding `>= root_cause_explained`. +### Standalone sourcehunt resume + +Legacy-flow standalone sessions use immutable completion records rather than a +mutable workflow snapshot. `session.json` records the schema, behavior-affecting +configuration, repository metadata, and the hash of exactly the source files +selected by preprocessing. `rank-plan.json` appears only after the complete +ranking pass succeeds. Each successful hunt invocation atomically creates one +`work-results/.json`, including successful zero-finding +work and any context needed to derive a promoted band. `spend-ledger.jsonl` +remains the only authority for lifetime model spend. A single advisory lock +prevents concurrent writers. + +The recovery rule is: if a valid atomic work result exists, reuse it; otherwise +run that work again. Missing, truncated, or interrupted work therefore restarts +from its beginning, while completed findings and cluster descriptors are loaded +into the live findings pool before unfinished hunters run. Missing or invalid +rank plans cause the whole ranking pass to restart; partial rank chunks are not +stored. Verification, exploitation, reporting, and later enrichment may rerun. +This does not restore a coroutine, provider request, sandbox, or mid-agent +transcript. Provider credentials are never written to the session. + ## The shared Finding type `clearwing.findings.Finding` is the single canonical finding diff --git a/docs/cli.md b/docs/cli.md index d43665fd..007adc36 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -53,6 +53,52 @@ clearwing sourcehunt See [LLM providers](providers.md) for the full precedence rules and provider-specific snippets. +### Resuming a standalone session + +Every new legacy-flow standalone run writes `session.json`, a complete atomic +`rank-plan.json`, and one atomic JSON result per completed hunt work item in its +`sh-*` session directory. Continue that exact session after provider quota +exhaustion, interruption, or process failure with: + +```bash +clearwing sourcehunt --resume sh-deadbeef +``` + +The command deliberately has a narrow surface: `--output-dir`, provider/model +options, `--live`, and logging options may be supplied. The repository and all +behavior-affecting options come from `session.json`. + +Resume reruns preprocessing and hashes exactly the selected relative paths and +their complete contents. The selected-input hash is authoritative; a recorded +Git commit is metadata and is not used as a substitute for detecting dirty +source changes. The session output directory is excluded from preprocessing, +so regenerated reports and other session artifacts do not invalidate the +identity check. + +A complete rank plan is restored exactly. Missing, invalid, or interrupted +ranking restarts from the beginning. A valid atomic work result is skipped, +including zero-finding work; missing, invalid, or interrupted work runs from +the beginning. Completed findings and cluster state are restored before new +hunters start, and completed promotions deterministically reconstruct their +next band. Verification, exploitation, reporting, and later enrichment stages +may rerun. Resume does not restore an exact coroutine, provider request, +sandbox, or mid-agent transcript. Only one process may run a session at a time. + +Provider/model credentials are deliberately absent from session metadata and are +resolved fresh from the current CLI flags, environment, and config on every +resume. This permits a replenished key or a replacement endpoint: + +```bash +clearwing sourcehunt --resume sh-deadbeef --api-key "$REPLACEMENT_API_KEY" +``` + +The saved budget remains the total lifetime session cap and cannot be overridden +on resume. Settled calls are restored once from `spend-ledger.jsonl`; an orphaned +reservation is handled according to whether the cap was active when it was +reserved. Resume currently supports standalone `--flow legacy` sessions created +with session schema 1. Older sessions without `session.json` are not resumable +and produce an explicit unsupported-session error. + Depths: - **`quick`** — preprocessor + ranker + static findings. No LLM hunters. Free. Useful as a sanity check or for CI. @@ -431,7 +477,7 @@ clearwing init # alias — same wizard Walks through LLM backend selection, credential entry, optional connection testing, and persistence to `~/.clearwing/config.yaml`. The menu currently lists Anthropic, OpenRouter, Ollama, LM Studio, -OpenAI, Together, Groq, Fireworks, DeepSeek, and a "custom +OpenAI, Together, Groq, Fireworks, DeepSeek, Kimi Code, MiniMax, and a "custom OpenAI-compatible endpoint" catch-all. Safe to re-run — existing config is shown and can be overwritten. diff --git a/docs/index.md b/docs/index.md index a93698e5..2981d49b 100644 --- a/docs/index.md +++ b/docs/index.md @@ -3,7 +3,7 @@ **Autonomous vulnerability scanner and source-code hunter.** Built on `genai-pyo3`, a native Rust-backed LLM runtime speaking every major provider (Anthropic, OpenAI, OpenRouter, Ollama, LM Studio, Together, -Groq, DeepSeek, MiniMax, Gemini, any OpenAI-compatible endpoint). +Groq, DeepSeek, Kimi, MiniMax, Gemini, any OpenAI-compatible endpoint). Clearwing is a dual-mode offensive-security tool: @@ -27,7 +27,7 @@ Clearwing is a dual-mode offensive-security tool: | Page | What you'll learn | |---|---| | [**Quickstart**](quickstart.md) | Install, run a network scan, run a sourcehunt pass, read the results | -| [**LLM providers**](providers.md) | OpenRouter / Ollama / LM Studio / vLLM / Together / Groq / DeepSeek / OpenAI — CLI + env + config.yaml recipes for each | +| [**LLM providers**](providers.md) | OpenRouter / Ollama / LM Studio / vLLM / Kimi / Together / Groq / DeepSeek / OpenAI — CLI + env + config.yaml recipes for each | | [**Sourcehunt evaluation and rollout**](eval_rollout.md) | Run the paired proof/legacy empirical campaign, evaluate cutover gates, and roll out proof flow safely | | [**Architecture**](architecture.md) | How the ReAct loops, sandboxes, capabilities layer, Finding dataclass, and knowledge graph fit together | | [**CLI reference**](cli.md) | Every `clearwing ` flag, with examples | diff --git a/docs/providers.md b/docs/providers.md index 60877b2d..43e14e21 100644 --- a/docs/providers.md +++ b/docs/providers.md @@ -18,8 +18,8 @@ Backends covered below: - **OpenAI (Responses API)** — GPT-5.x / o-series via `/v1/responses` (`openai_resp` adapter). Required for GPT-5.x and o-series. - **OpenAI OAuth (ChatGPT)** — Plus/Pro login via Codex backend (`openai_codex`). -- **OpenRouter, Together, Groq, Fireworks, DeepSeek, LM Studio, vLLM, custom** — - anything that speaks `/v1/chat/completions` (`openai` adapter). +- **OpenRouter, Together, Groq, Fireworks, DeepSeek, Kimi, LM Studio, vLLM, + custom** — anything that speaks `/v1/chat/completions` (`openai` adapter). - **MiniMax** — M2.7 / M2.5 via the Anthropic-compatible endpoint (`anthropic` adapter at `api.minimax.io/anthropic`). - **Ollama** — local models via the native rust-genai Ollama adapter @@ -47,6 +47,7 @@ Direct variants: ```bash clearwing setup --provider openrouter +clearwing setup --provider kimi-code clearwing setup --provider ollama --no-test clearwing init # alias ``` @@ -88,6 +89,41 @@ clearwing config --show-provider which prints the effective model, base URL, API key status, and source (`cli` / `env` / `config` / `default`). +## Recovering a sourcehunt from provider exhaustion + +Standalone legacy-flow sourcehunt sessions stop promptly with the distinct +resumable status `provider_exhausted` when a provider reports terminal account +quota exhaustion. This includes Kimi Code's HTTP 403 response whose structured +type is `access_terminated_error` and whose message states that the usage or +billing-cycle limit was reached. Ordinary HTTP 401 and non-quota HTTP 403 +responses remain authentication/authorization errors. Temporary HTTP 429 and +transport errors keep their existing bounded exponential retries. + +The stopped command prints the session ID and exact recovery command: + +```bash +clearwing sourcehunt --resume sh-deadbeef +``` + +Provider resolution happens again when the resume command starts. Update the +environment or `~/.clearwing/config.yaml`, replenish the original account, or +provide replacement CLI credentials before resuming. API keys are never stored +in `session.json`. + +```bash +export CLEARWING_API_KEY="$REPLACEMENT_API_KEY" +clearwing sourcehunt --resume sh-deadbeef + +# Or use a replacement compatible endpoint for this resume: +clearwing sourcehunt --resume sh-deadbeef \ + --base-url https://example.invalid/v1 \ + --api-key "$REPLACEMENT_API_KEY" \ + --model replacement-model +``` + +See [CLI reference](cli.md#resuming-a-standalone-session) for selected-source +identity, configuration, budget, and legacy-session compatibility rules. + ## Anthropic direct (default) No setup beyond the API key. This is what Clearwing used before @@ -321,6 +357,32 @@ For OpenAI itself, use the [Chat Completions](#openai-chat-completions-api) or [Responses](#openai-responses-api) section above — the model generation decides which. +## Kimi Code (membership) + +Use this option for API keys created in the Kimi Code Console and backed +by a Kimi Code membership: + +```bash +export KIMI_CODE_API_KEY=your-key +clearwing setup --provider kimi-code +``` + +The generated provider section uses Kimi Code's separate endpoint and +model IDs: + +```yaml +provider: + base_url: https://api.kimi.com/coding/v1 + api_key: ${KIMI_CODE_API_KEY} + model: k3-256k + adapter: openai +``` + +`k3-256k` is the recommended default because it provides K3 capability +while consuming less membership quota than the 1M-context `k3` model. +The model picker also offers `k3`, `kimi-for-coding`, and +`kimi-for-coding-highspeed`; availability depends on membership tier. + ## MiniMax MiniMax's M-series reasoning models are served via an diff --git a/docs/quickstart.md b/docs/quickstart.md index e08018ba..f23d92be 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -26,7 +26,7 @@ export ANTHROPIC_API_KEY=sk-ant-... ``` Or use an OpenAI-compatible endpoint (OpenRouter, Ollama, LM Studio, -vLLM, Together, Groq, DeepSeek, OpenAI, ...): +vLLM, Together, Groq, DeepSeek, Kimi, OpenAI, ...): ```bash # Per-command @@ -137,4 +137,4 @@ make gate # full CI gate locally: lint + type + test + build - [**Architecture**](architecture.md) — how the pieces fit together. - [**CLI reference**](cli.md) — every flag, with examples. - [**LLM providers**](providers.md) — OpenRouter, Ollama, LM Studio, - vLLM, Together, Groq, DeepSeek, OpenAI direct. + vLLM, Kimi, Together, Groq, DeepSeek, OpenAI direct. diff --git a/docs/sourcehunt_optimization.md b/docs/sourcehunt_optimization.md new file mode 100644 index 00000000..a72a7731 --- /dev/null +++ b/docs/sourcehunt_optimization.md @@ -0,0 +1,1722 @@ +# SourceHunt optimization campaign + +This campaign optimizes Clearwing for small local models while keeping every +candidate prompt solution-independent. The immediate task model is +`dsv4-flash-nvfp4` at: + +```text +http://tinybox.taile6728b.ts.net:30000/v1 +``` + +The vLLM server must enable native tools, for example with +`--enable-auto-tool-choice --tool-call-parser deepseek_v3`. Both automatic and +required tool selection have been verified against this endpoint. + +## Experiment variables + +Prompt, scaffold, and context policy are independent variables: + +- `legacy-v1`: existing behavior, retained for compatibility. +- `generic-security-v1`: generic discovery loop with solution heuristics and + historical CVE context disabled. +- `native-v1`: the existing Clearwing tool surface. +- `minimal-linear-v1`: Read/Grep/Trace/Submit only. +- `candidate-ledger-v1`: minimal tools plus typed, persistent hypothesis state. +- `candidate-ledger-closure-v1`: the candidate ledger unchanged until the final + three model calls, when a short generic countdown forces submit/reject/finish + closure on the strongest candidate. +- `candidate-ledger-source-retry-v1`: candidate ledger plus one runtime repair + turn when the first response invokes no source tool. A second text-only + response stops as `no_source_action`; the retry is absent from the standing + prompt. +- `candidate-ledger-source-retry-active-v1`: source retry plus a generic + submission invariant: `record_finding` must cite a pending, investigating, or + validated ledger candidate. This blocks findings after every hypothesis was + rejected without requiring the stronger proof scaffold. +- `window-ledger-v1`: candidate ledger plus generic static source-window ranking. +- `guided-window-ledger-v1`: opaque ranked-window reads with mandatory initial + coverage, removing line-plan translation from the model. +- `state-interaction-ledger-v1`: expands the strongest read anchor into a compact, + target-blind packet of declarations, initialization, writes, comparisons, and + one-hop state producers before candidate commitment. +- `proof-refinement-ledger-v1`: keeps the state-interaction packet unchanged and + exposes one bounded, obligation-specific source packet only after a failed + structured domain proof. +- `legacy-context-v1`: existing growing transcript, retained as a control. +- `compact-small-model-v1`: shorter static instructions/tool schemas, 3.5k-char + tool-result clipping, and deterministic compaction near 12k estimated request + tokens. It keeps a small complete assistant/tool tail plus durable active and + rejected candidates, counterevidence, next checks, and trace evidence. Raw + source windows remain disposable because they can be reread. + +Select them with `--prompt-bundle`, `--scaffold-profile`, and +`--context-profile`. Ablation IDs and baseline groups include all three values, +so results from different treatments cannot be mixed accidentally. + +## Leakage boundary + +Optimization candidates are rejected if they contain benchmark CVEs, commits, +target files, distinctive symbols, expected CWEs, known mechanism labels, or +ground-truth phrases. Reflection traces are redacted by the same manifest-derived +policy before reaching the reflection model. + +The generic bundle also disables the legacy FFmpeg-specific memory-safety hints +and CVE seed context. Do not use `--seed-cves`, case-specific campaign hints, or +the specialist prompt when collecting optimization data. + +## LAIR offline supervision + +LAIR's `training/cve_data_extraction` goldens are useful outside the hunter +boundary. They contain source-verified `discovery -> investigation -> challenge` +chains, but they also contain answer-bearing commits, changed files, citations, +symbols, root-cause prose, and fix behavior. Never attach a LAIR task, golden, or +retrieved excerpt to a blind hunter request. + +`clearwing.eval.sourcehunt_lair` validates the upstream `GoldenChain` contract and +converts each causal trace into abstract next-action rows. A row contains only: + +- completed trace kinds and completed generic proof obligations; +- the next generic proof obligation; +- one bounded context category to request next. + +It never copies a CVE, repository, commit, path, symbol, citation, claim, CWE, +fix fact, vulnerability description, case key, or repository key. Before output, a leakage audit compares +every emitted string with case identifiers, paths, source excerpts, and +source-level identifiers. Splits are deterministic by repository, so two CVEs +from the same codebase cannot cross train/development/test. `ffmpeg` is reserved +and excluded by default even if a future corpus contains it. + +After LAIR has generated and audited its collected `goldens/CVE-*.json` files, +build SourceHunt routing supervision with: + +```bash +uv run python evaluations/build_sourcehunt_lair_dataset.py \ + --goldens /data/lair-cve-training-data \ + --output-dir /data/sourcehunt-lair-router-v1 +``` + +The output contains `router/{train,development,test}.jsonl` and `manifest.json`. +The manifest records corpus and split digests without retaining reversible case +metadata. Treat the development fold as permanently opened once it participates +in scaffold or prompt selection. Keep the test fold sealed until the router and +validator policies are frozen. + +The first intended consumer is a small deterministic or learned router choosing +the next on-demand context category. It is not a source-text generator and must +not lengthen the standing hunter prompt. A second consumer can score validator +decisions against LAIR challenge outcomes and exact citations, but this must run +offline; only the blind finding and source-derived trace enter production +validation. + +### Differential validator replay + +`evaluations/run_sourcehunt_lair_validator.py` measures the existing validator +on LAIR goldens without exposing a snapshot label. It builds one alleged finding +from the vulnerable golden, selects bounded source windows from that trace, and +submits the identical finding twice. Only the source text at the pinned revision +changes. A correct pair advances the vulnerable snapshot and rejects the fixed +snapshot. + +```bash +uv run python evaluations/run_sourcehunt_lair_validator.py \ + --campaign-root /data/lair-cve-training-data \ + --base-url http://tinybox.taile6728b.ts.net:30000/v1 \ + --model dsv4-flash-nvfp4 \ + --prompt-profile legacy-v1 \ + --temperature 0 \ + --max-output-tokens 16384 \ + --output results/sourcehunt-optimization/lair-validator-pilot.json +``` + +The replay reports vulnerable recall, fixed rejection rate, pair accuracy, +per-axis pass counts, model errors, false negatives, and fixed false positives. +Use the same completed golden set for every validator treatment. Do not optimize +on the replay test fold or use fix citations to choose the source windows. + +The replay uses `balanced-anchor-v1` context. It derives all paths, coordinates, +and anchors from the vulnerable trace, allocates the character budget fairly +across causal paths, and selects fixed-width line slots nearest those anchors. +Both revisions therefore receive exactly the same line coordinates; only source +text changes. This prevents early alphabetical paths from consuming the context +budget and hiding the authoritative operation or guard in a later path. + +Always set an explicit output cap for small reasoning models. An uncapped pilot +request generated roughly 568,000 tokens without reaching the constrained JSON. +The replay now defaults to 8,192 output tokens, retries invalid output once under +the same cap, records the cap and temperature in its v2 result, and never scores +a model-error rejection as a correct fixed negative. + +### LAIR development pilot (2026-08-12) + +The opened pilot contains 12 completed source-verified C/C++ chains from 12 +repositories. The leakage-safe routing adapter emitted 107 development rows: + +- trace targets: 12 attack sources, 12 entry points, 20 propagations, 13 state + transitions, 14 guard failures, 12 vulnerable operations, 12 security effects, + and 12 challenge actions; +- context targets: 24 input/caller, 20 dataflow, 13 state/representation, 14 + guard/control-flow, 24 operation/effect, and 12 counterevidence requests; +- serialized vocabulary: 22 fixed ontology strings, with no CVE, repository, + commit, path, symbol, excerpt, vulnerability prose, or FFmpeg text. + +The original legacy diagnostic, before balanced selection and explicit sampling, +scored 83.3% vulnerable recall, 75.0% fixed rejection, and 58.3% pair accuracy +with no model errors. Its three fixed false positives all omitted a decisive late +path from the capped context, so it is useful failure evidence but not the final +controlled baseline. + +The controlled temperature-zero comparison used identical `balanced-anchor-v1` +context and a 16,384-token cap: + +| Validator prompt | Vulnerable recall | Fixed rejection | Pair accuracy | Errors | +|---|---:|---:|---:|---:| +| `legacy-v1` | 91.7% | 58.3% | 50.0% | 0 | +| `source-first-compact-v2` | 91.7% | 58.3% | 58.3% | 0 | + +The compact prompt aligned one additional pair, but 1/12 on an opened development +fold is not enough to promote it. Keep `legacy-v1` as the production default and +treat `source-first-compact-v2` as the leading optimization candidate. A longer +`source-first-high-recall-v1` ablation overcorrected: at an 8,192-token cap it +scored 33.3% vulnerable recall, 91.7% fixed rejection, 25.0% pair accuracy, and +two model errors. Do not promote it. + +Five temperature-zero replicates per arm then showed that the one-pass advantage +was not promotion evidence: + +| Validator prompt | Vulnerable recall | Fixed rejection | Pair accuracy | Errors | +|---|---:|---:|---:|---:| +| `legacy-v1` | 86.7% (52/60) | 58.3% (35/60) | 46.7% (28/60) | 0/120 | +| `source-first-compact-v2` | 88.3% (53/60) | 56.7% (34/60) | 51.7% (31/60) | 1/120 | + +The compact arm gained three paired decisions but regressed fixed rejection and +produced one capped structured-output failure. Wilson 95% intervals overlap +broadly. The aggregate, exact input digests, opaque per-case stability, and seed +provenance are recorded in +`results/sourcehunt-optimization/lair-validator-replicates/summary.json` (SHA256 +`efbbbd14a936192e1b546844aef48d0f19a0699b5151c43dc6e7abf0917e5468`). + +The replication runner alternates arm order, checkpoints each complete arm, +resumes without repeating finished work, fails on configuration/case/coordinate +drift, and uses pooled Wilson intervals. Promotion requires all of the following: + +- vulnerable recall at least the replicated legacy control; +- fixed rejection at least the replicated legacy control; +- mean pair accuracy above the control with stable per-case behavior; +- zero model errors at the 16,384-token cap. + +### Leakage-safe validator GEPA result + +Core GEPA ran directly over a deterministic 6/6 split of the same opened 12-case +development fold. Each case scored 35% vulnerable correctness, 35% fixed +correctness, and a 30% same-case bonus; any model error scored zero. GEPA saw only +opaque case handles. Reflection saw only a generic natural-language lesson—never +golden prose, source, identifiers, coordinates, axes, metric keys, or arm labels. +Candidates were limited to 2,000 characters and rejected if they contained any +answer-bearing term or evaluation-protocol wording. + +This boundary caught two optimizer shortcuts before promotion: one candidate +mentioned paired snapshots, and another copied an internal metric-field name. +Those diagnostic runs are preserved under explicitly rejected result directories. +The hardened run used 48 of a hard 60-call budget, evaluated four candidates, and +kept the 725-character `source-first-compact-v2` seed as best. Its stochastic seed +score was 0.892; every accepted mutation scored 0.783 on the six-case validation +half. Result: no GEPA mutation advances to replicated replay, and `legacy-v1` +remains the production default. Do not reopen FFmpeg or consume a sealed test fold +for this validator treatment. + +## Evaluation design + +Never optimize on positives alone. Use `include_fixed_negative_cases()` to add +every pinned patched snapshot as a negative control. The current manifest yields +an FFmpeg vulnerable/fixed pair; add more fixed pairs and clean repositories before +trusting transfer results. + +Legacy scoring requires all of the following: + +- correct target file and CWE; +- evidence stronger than suspicion; +- a multi-step trace linked to the target; +- at least two mechanism-bearing target symbols; +- a non-empty root-cause description. + +File+CWE-only reports are false positives. + +## GEPA + +Install the optional integration: + +```bash +uv sync --extra dev --extra optimization +``` + +`clearwing.eval.sourcehunt_gepa.SourceHuntGEPAAdapter` runs the unchanged +SourceHunt system, scores each observation, and returns redacted actionable side +information. Core GEPA is used directly; SourceHunt is not rewritten as DSPy +ReAct, which keeps the runtime/sandbox constant during prompt experiments. + +Use `optimize_sourcehunt_prompt()` with train/validation +`SourceHuntOptimizationExample` lists. The adapter fails closed unless the +manifest contains both confirmed positives and disproven fixed/clean controls. +For a no-charge local endpoint, set both token-price fields to `0.0`; action, +model-call, and scaffold limits still bound execution independently of cost. +These run-scoped prices propagate through the spend ledger, HunterPool, and +each native hunter. This is required: otherwise the global fallback price for +an unknown model can prematurely stop file dispatch even while the authoritative +local-endpoint ledger correctly records zero dollars. +Set `max_hunt_files` to cap repository-wide fan-out after generic ranking; the +selection uses priority descending with a stable path tie-break. +Use `max_hunter_steps`, `max_parallel`, `starting_band`, and +`redundancy_override` to hold per-file work constant across scaffold arms. +Use standard depth for scaffold optimization: it retains deep sandboxed hunters +but excludes the separate deep-depth harness generator from the comparison. +For small models, use bounded ranker chunks (currently 25 files, one chunk in +flight, one retry) so structured rankings fit comfortably in one response. +DeepSeek V4 Flash still returned truncated/empty ranker JSON at that size, so +the scaffold comparison disables LLM reranking and holds Clearwing's generic +deterministic static ranking constant across every arm. +The deterministic ranker scans production C/C++/Rust source with the same +repository-independent signal categories exposed by `rank_source_windows`. +Category counts saturate, signal diversity is rewarded, and only eight strong +line-local anchors contribute; this prevents large files or repeated memory +operations from monopolizing a bounded campaign. Test, documentation, example, +benchmark, and fuzz paths are excluded generically. The score and per-category +counts are retained on each file target for auditability. +Examples also fail closed unless they use blind repository-level ablations with +no assisted hint packet; target-file and later ablations are diagnostics only. +The current prompt/scaffold seam belongs to the legacy hunter, so GEPA examples +also reject the proof flow instead of silently scoring a prompt it does not use. +The reflection model may be a GEPA/LiteLLM model or `ClearwingReflectionLM` around +an existing native Clearwing client. + +Optimization order: + +1. Compare scaffolds with the fixed seed prompt. +2. Freeze the winning scaffold and optimize discovery instructions. +3. Freeze discovery and optimize investigation/challenge instructions. +4. Re-evaluate all candidates on held-out projects and fixed controls. +5. Run the winning generic configuration blindly across FFmpeg, then independently + validate and deduplicate every finding. + +## Local DeepSeek diagnostics (2026-08-11) + +The endpoint emitted valid native tools through both raw OpenAI-compatible calls +and Clearwing's `AsyncLLMClient`. On the small heap-overflow fixture: + +| Scaffold | Tokens | Result | +|---|---:|---| +| `minimal-linear-v1` | 13,420 | Correct overflow, suspicion evidence | +| `native-v1` | 20,926 | Correct overflow, suspicion evidence | + +On the pinned vulnerable FFmpeg target-file ablation, 12-14-step static runs did +not find the labeled issue. Linear mode swept source windows; the enforced ledger +created explicit hypotheses but selected other mechanisms. This is a diagnostic +failure, not a prompt hint: retain it as negative optimization feedback. + +Full proof-flow evaluation is not available on a host without Docker and Bear. +Do not treat static target-file diagnostics as the final FFmpeg campaign baseline. + +Docker and Bear are now available through the local Colima Docker socket. The +pinned vulnerable and fixed FFmpeg checkouts each have a 2,174-entry compile +database, so repository-level proof-flow baselines are the authoritative results +below. + +### Compact repository baselines + +Both treatments used the same blind 24-file target set, 40 steps per hunter, +parallelism 4, standard depth, fast band, redundancy 1, deterministic static file +ranking, and zero local token prices. + +| Scaffold | Vulnerable | Fixed | Vuln input | Fixed input | Result | +|---|---:|---:|---:|---:|---| +| `window-ledger-v1` | 0.175 | 1.0 | 9,039,148 | 8,359,653 | no findings | +| `guided-window-ledger-v1` | 0.175 | 1.0 | 9,310,919 | 9,440,150 | no findings | + +Both vulnerable arms failed first at `true_candidate_generated`; both fixed arms +remained free of false reports. Guided ranked reads correctly forced W1/W2/W3 and +reduced the vulnerable target file from 389,660 to 358,643 input tokens, but the +model still isolated familiar memory operations, rejected two sound bounds +hypotheses, then repeated the first one. Navigation improved; semantic state +interaction did not. + +The next scaffold therefore keeps the prompt short and moves generic context +assembly into a deterministic tool. For each anchor it extracts the dominant +mutable state and a compact role-labeled packet: representation, reserved-state +initialization, writers, comparisons, the stored producer, and that producer's +upstream state and guards. It derives every line from the current checkout and +uses no case manifest, patch, CVE, known filename, symbol, value, or mechanism. +On the pinned pair, the same generic algorithm includes the relevant state chain +in the vulnerable packet and independently includes the terminating producer +guard only in the fixed packet. No patch or answer-derived term participates in +packet construction. + +### Proof-gated state interaction result + +The state-interaction treatment initially found the right value-domain candidate +but spent its remaining turns rereading source and rewriting the candidate. A +generic checkpoint now activates after two source actions for a tracked, +unresolved domain. It blocks further source reads and candidate rewrites until +the hunter calls `record_domain_proof` with four explicit obligations: + +1. attacker input reaches the producer; +2. the producer can reach the distinguished stored value; +3. the changed consumer branch reaches the extracted effect; and +4. the boundary effect lacks a dominating guard. + +A false answer narrows the candidate's next check to the unresolved obligation. +Four true answers validate the candidate, record +`security_effect_possible`, seed exact source/transfer/condition/effect trace +steps, and permit a `static_corroboration` report. This small amount of enforced +control flow eliminated the model's executive loop without lengthening the +standing prompt. + +The 40-step target-pair diagnostic at +`domain-proof-gated-40-diagnostics/summary.json` separated the snapshots: + +- vulnerable: one validated finding with a four-step exact source trace; +- fixed: zero findings, with the extracted terminating guard correctly proving + the producer and distinguished domains disjoint. + +The blind repository run at +`domain-proof-gated-v1/state-interaction-ledger-v1--compact-small-model-v1.json` +used the same 24-file target set on each snapshot. It submitted exactly one +finding on the vulnerable target and none across the fixed files. Vulnerable and +fixed input totals were 8,546,607 and 8,468,558 tokens respectively. The recorded +scores were 0.125 and 1.0 because independent validation suppressed the positive, +not because discovery missed it. + +The original validator call returned an empty response, so structured parsing +rejected the report. Validation now retries invalid or empty structured output +once with a short generic instruction, 8,192 output tokens, and the hunter's +exact vulnerability trace. A manual retry produced structured output but still +rejected the candidate on an incorrect practical-reachability claim. Treat that +as a validator false negative: the discovery and fixed-side differential remain +the useful scaffold signal, while validator reachability reasoning needs its own +bounded refinement. + +### Context expansion ablation + +Two subsequent target-pair ablations front-loaded additional, correctly +source-derived context about repeated-input reachability, dispatch, allocation +extent, and boundary addressing. Both regressed vulnerable discovery from one +finding to zero: + +- `reachability-proof-gated-40-diagnostics`; +- `reachability-allocation-proof-40-diagnostics`. + +The failure is important: correct context is not automatically useful context +for this model. Keep the winning initial packet compact. The next experiment +now exposes `read_domain_proof_refinement` only after a particular domain-proof +obligation remains false. Its schema is absent from earlier model requests, it +accepts only a recorded unresolved obligation, returns a bounded source-derived +packet for that obligation, and can be called only once before another structured +proof is required. The initial state-interaction packet is unchanged. Measure +this on-demand treatment against the same vulnerable/fixed controls before +beginning GEPA prompt search. + +The first 40-step paired run of this treatment, +`on-demand-refinement-40-diagnostics`, produced zero findings on both snapshots. +The vulnerable hunter used three refinements but processed independent false +obligations in an inefficient order and temporarily treated a non-terminating +comparison as a producer cap. It corrected that interpretation only on its final +turn. The refinement loop now exposes one prerequisite at a time and labels +comparison context as terminating or non-terminating based on nearby control-flow +effects. This revision requires another paired diagnostic; do not promote it to a +repository run on the first result. + +The revised run, `on-demand-refinement-ordered-40-diagnostics`, restored clean +separation. The vulnerable hunter submitted one validated finding after 23 model +calls and 230,190 input tokens; its seeded source/state-sink/condition/effect-sink +trace matched the compact winning run. The fixed hunter submitted zero findings +after 40 calls and 398,625 input tokens, correctly treating the extracted +terminating producer guard as proof that the domains are disjoint. Peak estimated +contexts were 11,389 and 11,621 tokens. This passes the target-pair promotion gate; +the next meaningful test was a blind repository run, not another target-file +replicate. + +That blind 24-file run completed at +`proof-refinement-ledger-v1/proof-refinement-ledger-v1--compact-small-model-v1.json`. +It retained one vulnerable finding, zero fixed findings, and the same `[0.125, +1.0]` scores as the unchanged `state-interaction-ledger-v1` control. Independent +validation again suppressed the vulnerable positive, so correctness did not +improve. The refinement treatment used 8,425,157 vulnerable and 8,123,672 fixed +input tokens with 890 and 864 model calls. Relative to control, that is 121,450 +and 344,886 fewer input tokens (about 1.4% and 4.1%) and 37 and 36 fewer calls. +Keep it as an efficiency candidate, not a promoted correctness winner. Do not +begin GEPA prompt search on this result; use the LAIR development fold to improve +generic routing and validator evaluation first. + +## Blind FFmpeg discovery campaign (2026-08-13) + +DeepSeek V4 Flash has meaningfully examined all 180 attempted deterministic +FFmpeg rank slots. Ranks 1–24 came from the earlier +repository baseline. Ranks 25–96 used the proof-refinement scaffold: 72 files, +24,751,510 tokens, 563 raw candidate updates, and no submitted findings. Ranks +97–108 were then +replayed as a controlled scaffold comparison using the same 12 files, 24 steps, +temperature 0, and a 4,096-token output cap: + +| Scaffold | Tokens | Candidate calls | Step-cap hunters | Findings | +|---|---:|---:|---:|---:| +| `minimal-linear-v1` | 2,345,124 | 0 | 9/12 | 0 | +| `candidate-ledger-v1` | 2,455,745 | 74 | 10/12 | 0 | +| `proof-refinement-ledger-v1` | 2,577,673 | 64 | 12/12 | 0 | + +Use `candidate-ledger-v1` as the discovery baseline. Its 4.7% token overhead +over minimal-linear retains a reviewable hypothesis funnel. Proof refinement +cost 5.0% more than candidate ledger without improving submission yield. These +results do not establish a correctness winner because every arm submitted zero +findings. The exact inputs, campaign hashes, token components, call counts, and +decision are recorded in +`results/sourcehunt-optimization/ffmpeg-blind-campaign/matched-scaffold-ranks-0097-0108.json`. + +Most hunters reached the hard step cap, so end-of-budget behavior is now an +independent scaffold variable. `candidate-ledger-closure-v1` preserves the +candidate-ledger standing prompt and tools, then supplies a generic remaining-call +count only for the final three calls. It tells the hunter to resolve, submit, or +reject its strongest candidate and forbids new broad exploration. Keep this as a +separate treatment so prior `candidate-ledger-v1` artifacts remain reproducible; +promote it only after a matched replay. + +The matched closure replay did not pass that promotion gate. It used 2,306,771 +tokens (6.1% fewer than candidate ledger) and retained 76 candidate calls, but +still submitted zero findings and left 10/12 hunters at the step cap. In the +final three steps the hunters made 28 source actions and 11 candidate updates, +but no trace or finding calls. The small model did not reliably obey the textual +closure request. Retain `candidate-ledger-v1` for unseen coverage; if closure is +revisited, make it a bounded runtime phase rather than adding more standing +prompt text. + +Offline source tracing independently supports two deduplicated root causes: + +- H.264 slice ownership stores an unbounded slice counter in a 16-bit table where + slice 65,535 aliases the unavailable-entry sentinel. The fixed control adds the + missing counter cap, and the production validator advanced the vulnerable report + at high severity. +- Vulkan HEVC RPS construction may write more than the public Khronos array + capacity of eight because the write loop uses FFmpeg's larger `nb_refs` without + a destination-capacity guard. The production validator advanced the report at + medium severity; no executable Vulkan trigger has yet established more than + eight live current references. + +At that checkpoint, neither issue had a crash-confirmed reproducer. Survivor details remain offline in +`evaluations/sourcehunt_ffmpeg_survivors.json`; never include that file or its +derived hypotheses in hunter context. + +### Blind continuation and first crash-confirmed discovery + +Candidate-ledger continuation attempted ranks 109–180. Source-action audit—not +the runner's historical `files_hunted` counter—confirms that all 24 files at +ranks 109–132, 19/24 at ranks 133–156, and 9/24 at ranks 157–180 actually ran a +source-bearing tool. Together with ranks 1–108, that is 160 meaningfully examined +files and 20 false completion slots. The failures were DeepSeek responses that +printed DSML/XML-like calls as ordinary text; the provider returned no native +tool call, and the old loop treated that as a clean finish. The unusually cheap +last waves are therefore parser/tool-call failures, not efficiency gains. + +The missed ranks were 138, 146, 153, 154, 156, 158, 161, 162, 164, 167–169, +172, and 174–180. `candidate-ledger-source-retry-v1` repairs this generically at +runtime. A first sparse-offset replay exposed deterministic-ranker drift: current +offsets no longer resolved to all historical files after ranking code changed. +The final replay therefore used a sealed exact-path manifest and failed closed +if any requested file was absent. The first pinned attempt source-confirmed +15/20 files; bounded replays recovered the remaining 5, including one path that +needed three attempts. The recovery runs used 3,688,534 tokens and 111 candidate +calls in total. Coverage accounting records `files_examined` and +`source_action_files`; selected or dispatched files alone are never counted as +examined. + +The pinned replay's only hunter-submitted report, in `hdsenc.c`, was rejected +offline. `parse_header` bounds every tag, returns `AVERROR_INVALIDDATA` whenever +metadata is absent, and processes header output produced by the nested FLV muxer. +No NULL-metadata manifest path or attacker-controlled memory violation survives. +The hunter had already marked both relevant candidates rejected before filing; +`candidate-ledger-source-retry-active-v1` now prevents that control-flow error at +runtime without adding standing prompt text. + +The completed continuation consumed 9,025,918 tokens and 261 raw candidate calls +through rank 156, with no hunter-submitted findings. Offline source triage rejected +the usual incomplete hypotheses but recovered one complete new root cause from +the candidate funnel: + +- `af_arnndn.c` accepts `denoise_output->nb_neurons` from 0 through 128 while + validating only the separate VAD output shape. During non-silent inference, + `compute_dense` writes that many floats into `g[NB_BANDS]`, a 22-float stack + array. A syntactically valid model declaring 23 outputs deterministically causes + an ASan stack-buffer-overflow in `compute_dense`; the ASan object report names + `g` as the overwritten buffer. The production validator independently advanced + the source report at high severity. + +Build and run the target-blind reproducer with: + +```bash +uv run python evaluations/run_ffmpeg_arnndn_reproducer.py \ + --ffmpeg .reference/ffmpeg/ffmpeg \ + --model-output /tmp/clearwing-arnndn-23.model \ + --output results/sourcehunt-optimization/ffmpeg-dynamic-validation/arnndn-denoise-output-stack-overflow.json +``` + +The structured crash artifact is +`results/sourcehunt-optimization/ffmpeg-dynamic-validation/arnndn-denoise-output-stack-overflow.json`. +The vulnerable and fixed-control `af_arnndn.c` files are byte-identical, so the +issue persists in both checked snapshots. + +The three-case validator replay advanced Vulkan at medium and arnndn at high, but +rejected H.264 after incorrectly treating `top_borders[-1]` as a previous-row +element. The upstream H.264 fix commit documents that `top_borders` has only +`mb_width` elements and the access underflows its allocation by 96 bytes (with +writes at negative offsets). Keep H.264 as source-confirmed and record this replay +as a validator false negative. Current honest totals are three confirmed root +causes, one dynamically crash-confirmed. + +### Exact-path wave 181–204 and three blind dynamic discoveries + +The next sealed manifest selected 24 previously unseen production files by exact +path. Thirteen targets performed successful source-bearing actions on the first +attempt. Exact-path replays recovered seven, then two, then the final two; the +full wave is therefore 24/24 source-confirmed. The three recovery passes used +2,336,138 tokens and 66 raw candidate calls. As in the earlier recovery, +`files_hunted` was not accepted as coverage evidence. + +This wave produced three source-supported issues, all found without CVE, patch, +fixed-checkout, historical-location, or survivor context and all independently +reproduced under ASan: + +- RTP/QDM2 accepts a packet-controlled `block_size` smaller than its mandatory + reconstructed header. A one-byte block yields `negative-size-param (size=-1)` + in `qdm2_parse_packet`. Linking the later upstream minimum-size check ahead of + the sealed static library makes the identical harness return invalid data with + no sanitizer report. The later upstream commit describes the same invariant as + an out-of-array access. +- `ahistogram` signed-logarithmic binning maps a valid positive full-scale sample + to `bin == width` for even widths. A constant `+1.0` source with + `ascale=log:hmode=sign` produces an eight-byte heap-buffer-overflow exactly one + element past the histogram allocation. The issue is present in both sealed + snapshots. +- XPSNR's downsampled high-pass stencil steps by two over odd active boundary + dimensions but reads through `x + 3` and `y + 3`. Two valid 2049x1153 YUV444 + frames produce a two-byte heap-buffer-overflow read beyond the tightly allocated + temporal source plane. The issue is present in both sealed snapshots. + +Structured artifacts live under +`results/sourcehunt-optimization/ffmpeg-dynamic-validation/`, and the generic +reproducer entry points are `run_ffmpeg_qdm2_reproducer.py`, +`run_ffmpeg_ahistogram_reproducer.py`, and `run_ffmpeg_xpsnr_reproducer.py` in +`evaluations/`. The vulnerable-source-only survivor file now contains six cases. +Repeated validator calls show meaningful variance on the H.264 and Vulkan impact +axes even while source reality/triggerability remain supported, so no single +validator vote should be used as the prompt-optimization objective. Keep upstream +fix evidence, dynamic evidence, and independent source-only validation as separate +signals. + +Current honest totals are 204 distinct source-confirmed files, six confirmed root +causes, and four dynamically crash-confirmed issues. H.264 and Vulkan still need +their older dynamic proofs; broad FFmpeg coverage beyond rank 204 remains open. + +### Blind wave 205–228 + +The next sealed exact-path wave completed 24/24 source-confirmed on its first pass, +using 5,028,744 tokens and 147 candidate calls. It made no formal submissions, but +offline terminal-ledger triage recovered one complete denial-of-service root cause: + +- `ff_rdt_parse_header` accepts a zero-length RDT status packet because it checks + only `pkt_len > len`. The loop then advances `buf`, `len`, and `consumed` by zero + and sees the identical status header forever. A 16-byte direct harness times out + deterministically on both sealed snapshots. This is a remotely supplied CPU + denial rather than memory corruption. + +The candidate-ledger remains useful even when the small model does not complete a +formal report: XPSNR and RDT were both recovered from source-grounded terminal +hypotheses. Current totals are 228 distinct source-confirmed files, seven confirmed +root causes, and five dynamically reproduced issues. + +### Blind waves 229–276 and four additional dynamic confirmations + +Two more sealed exact-path waves completed with the unchanged +`candidate-ledger-source-retry-active-v1` scaffold, the +`compact-small-model-v1` context, 24 hunter steps, a 4,096-token output cap, +temperature zero, and no answer-bearing inputs. Ranks 229–252 were 24/24 +source-confirmed, used 4,952,368 tokens, and produced 155 candidate calls. +Ranks 253–276 were also 24/24 source-confirmed, used 4,563,131 tokens, and +produced 158 candidate calls plus one formal finding. + +Offline triage and production-code reproducer work confirmed four new root +causes: + +- RDT's AAC cache path can copy a 9,212-byte unconsumed record tail into an + 8,256-byte `PayloadContext.buffer`. RTSP admits and forwards substantially + larger interleaved records. A direct production-parser harness reports the + heap-buffer-overflow on both sealed snapshots. +- A zero-sized `tfra` atom for an unknown track makes `ismindex` seek to the + same position and repeat `read_tfra` forever. Both snapshot harnesses time + out. This is a real but low-impact issue in a local indexing tool. +- The `entropy` filter allocates `1 << depth` histogram entries and indexes + them with an unmasked stored 16-bit sample. A gray10le sample of 1,024 + accesses the first eight bytes beyond its 1,024-entry allocation. The formal + blind hunter report was reproduced through the production ffmpeg filter graph + under ASan on both snapshots. +- CAF's variable-packet seek callback fails to handle the documented negative + result from `av_index_search_timestamp`. A forward seek beyond the final + packet dereferences `index_entries[-1]`. A 107-byte syntactically parsed CAF + followed by public `av_seek_frame` produces an eight-byte ASan heap-buffer- + overflow read in `read_seek` on both snapshots. + +The CAF case is especially useful scaffold evidence: the small model recorded +and validated the exact negative-index candidate in its terminal ledger but did +not submit it before the step cap. As with XPSNR and RDT, terminal candidate +state therefore remains valuable even when formal finding yield understates +discovery quality. + +The ten-case vulnerable-source-only survivor replay (before adding CAF) advanced +8/10 reports. Entropy passed all four axes at high confidence. H.264 remained a +validator false negative, and the validator correctly identified the `ismindex` +loop as real and triggerable while rejecting it on security impact. This again +supports keeping source proof, dynamic reproduction, and validator votes as +separate signals instead of optimizing toward one validator decision. + +At this checkpoint, ranks 1–276 are source-confirmed. The honest totals are +eleven confirmed root causes, nine dynamically reproduced issues, and two +source-only issues awaiting older dynamic proofs. The next manifest excludes +every prior bounded target found across all instrumentation ledgers and seals +the deterministic ranks 277–300 by exact path. + +### Blind waves 277–324 and three more confirmed root causes + +The unchanged frozen treatment completed two further exact-path waves: + +| Ranks | Source-confirmed | Tokens | Candidate calls | Formal findings | +|---|---:|---:|---:|---:| +| 277–300 | 24/24 | 4,589,393 | 165 | 0 | +| 301–324 | 24/24 | 4,344,701 | 129 | 0 | + +Before selecting ranks 301–324, the deterministic selector was audited by +excluding every bounded target recorded in all campaign instrumentation +ledgers. It reproduced the already sealed 277–300 manifest exactly, then +selected the next 24 paths. This is now the required procedure for future +waves; rank offsets and historical `files_hunted` counters are not coverage +evidence. + +Terminal-ledger triage and independent production-code validation confirmed +three additional root causes: + +- The Dolby Vision RPU parser accepts long signed fixed-point coefficients, but + the generator passes their integer components to a signed Golomb writer whose + documented domain is only 16 bits. The generator also reserves a constant 177 + bytes for an MMR piece despite coefficient-dependent Golomb lengths. A + parser-to-metadata-to-generator harness produces undefined bit-writer shifts + followed by the always-on `flush_put_bits` assertion on both snapshots. Treat + this conservatively as denial of service, not demonstrated heap corruption. +- `showfreqs=data=delay` starts its group-delay loop at frequency bin zero while + reading `fft_data[ch][f-1]`. A normal production filter invocation produces a + four-byte ASan heap-buffer-overflow read eight bytes before the FFT allocation + on both snapshots. This is an unambiguous memory-safety defect but has limited + security impact because the value affects only visualization output. +- HLS SAMPLE-AES accepts the 13-bit ADTS frame length without checking it against + the remaining packet. That length directly determines the in-place AES block + count. A 64-byte packet declaring an 8,191-byte frame produces an ASan heap- + buffer-overflow in the production AES decryption routine on both sealed + snapshots. Although later repository history contains a validating change, + neither history nor later source was exposed to the hunter. + +The corresponding generic recorders are +`run_ffmpeg_dovi_rpu_reproducer.py`, +`run_ffmpeg_showfreqs_reproducer.py`, and +`run_ffmpeg_hls_sample_aes_reproducer.py` under `evaluations/`. Structured +artifacts are retained under +`results/sourcehunt-optimization/ffmpeg-dynamic-validation/`. + +The complete vulnerable-source-only `v11` survivor replay evaluated all 14 +cases. It advanced 10/14: HLS SAMPLE-AES advanced at high severity, while +`showfreqs`, `ismindex`, XPSNR, and Dolby Vision were rejected on impact or +general reachability rather than source reality. These votes are useful triage, +not ground truth. The replay artifact is +`results/sourcehunt-optimization/ffmpeg-survivor-validation/deepseek-v4-flash-0731-vulnerable-v11.json`. + +Current honest totals are 324 distinct source-confirmed files, 14 confirmed root +causes, and 12 dynamically reproduced issues. H.264 and Vulkan HEVC remain the +two source-only cases. Open-ended FFmpeg exhaustion is not close: this is strong +bounded progress, not a claim that the remaining production source has been +comprehensively audited. + +### Blind wave 325–348 and Musepack SV7 confirmation + +The audited selector excluded 324 unique bounded paths from every campaign +instrumentation ledger, then reproduced the sealed 301–324 manifest exactly +when that set was withheld from the exclusion. Only after that replay passed did +it seal ranks 325–348. The unchanged treatment completed all 24 targets with +source-bearing actions, using 4,352,353 tokens and 129 candidate/finding calls. + +This wave produced one successful formal report and dynamic confirmation: + +- Musepack SV7 stores an 11-bit `lastframelen` from file extradata without + constraining it to `MPC_FRAME_SIZE` (1,152). The decoder allocates planar S16 + storage for 1,152 samples, synthesizes that fixed amount, and then publishes + as many as 2,047 samples when the packet marks itself as the last frame. A + public `avcodec_send_packet`/`avcodec_receive_frame` harness receives + `nb_samples=2047`; a normal consumer loop immediately produces a two-byte + ASan heap-buffer-overflow read at the end of each 2,304-byte channel plane on + both snapshots. + +The reproducer and recorder are +`evaluations/ffmpeg_mpc7_lastframelen_reproducer.c` and +`evaluations/run_ffmpeg_mpc7_lastframelen_reproducer.py`. Current honest totals +are 348 distinct source-confirmed files, 15 confirmed root causes, and 13 +dynamically reproduced issues. H.264 and Vulkan HEVC remain source-only. + +The complete vulnerable-source-only `v12` survivor replay evaluated all 15 +cases and advanced 14/15. Musepack advanced at medium severity. `showfreqs` was +the only rejection, failing the general and impactful axes while passing real +and triggerable. XPSNR, `ismindex`, and Dolby Vision advanced in this replay +after being rejected in `v11`, reinforcing that validator votes are variable +triage signals rather than ground truth. The replay artifact is +`results/sourcehunt-optimization/ffmpeg-survivor-validation/deepseek-v4-flash-0731-vulnerable-v12.json`. + +### Blind wave 349–372 + +The audited selector found 348 unique bounded paths across all campaign +instrumentation ledgers. With the 325–348 set withheld, it reproduced that +sealed manifest exactly before selecting the next 24 deterministic unseen +paths. The unchanged frozen treatment completed 24/24 targets with +source-bearing actions, using 4,334,219 tokens and 132 candidate calls. It +produced no formal findings, and focused source triage did not establish a new +root cause. All 24 target files are byte-identical in the independent control +snapshot, so that comparison provides no differential evidence. Current honest +totals are 372 distinct source-confirmed files, 15 confirmed root causes, and +13 dynamically reproduced issues. + +### Blind wave 373–396 + +The selector audit collected 396 unique bounded paths from every SourceHunt +optimization instrumentation ledger. With the 373–396 set withheld, the current +deterministic ranker reproduced the sealed manifest exactly. The same audit also +reproduced the 325–348 and 349–372 manifests when each was withheld, so the +selection state is internally consistent across the last three waves. + +The unchanged frozen treatment completed all 24 targets with successful +source-bearing actions. It used 4,335,583 tokens, made 137 candidate calls, and +submitted no formal findings. Focused review rejected the strongest terminal +leads: image-dimension validation prevents the suspected BMP stride overflow; +AFIR's exponential partition growth keeps its segment count below 1,024; +signature category counts sum exactly to the fixed 380/348 capacities; x264's +SEI callback owns and frees the transferred payloads; and Pixlet's scratch copies +are byte counts covered by its 16-element margin. The FFV1 Vulkan encoder would +benefit from defensive CPU-side validation of GPU-returned slice lengths, but no +attacker-controlled route to an overlarge length or concrete violated allocation +invariant was established, so it is not promoted. + +All 24 wave files are byte-identical in the independent control snapshot. No new +root cause was established. Current honest totals are 396 distinct +source-confirmed files, 15 confirmed root causes, and 13 dynamically reproduced +issues. Relative to the 4,995-file deterministic ranked corpus, bounded coverage +is 7.9%, with 4,599 ranked files remaining. + +### Blind wave 397–420 and LCL/ZLIB disclosure confirmation + +The unchanged frozen treatment completed all 24 exact-path targets with +source-bearing actions. It used 4,453,854 tokens, made 135 candidate calls, and +submitted no formal finding. The all-ledger audit found exactly 420 bounded +paths, and withholding this wave reproduced its sealed manifest exactly (as did +withholding each earlier exact-path wave from ranks 205–396). All 24 files are +byte-identical in the independent control snapshot. + +Offline terminal-ledger triage confirmed one additional root cause: + +- LCL/ZLIB's multithread decoder accepts valid zlib streams whose actual outputs + are shorter than the packet's claimed half-frame size. `zlib_decomp` returns + the short positive count, both call sites discard it, and the decoder then + marks the entire decompression allocation as valid image data. A public + 16x16 RGB24 decoder harness supplies two three-byte streams; FFmpeg returns a + 768-byte frame containing the six supplied bytes plus all 762 bytes of the + otherwise uninitialized allocation. ASan's deterministic malloc-fill marker + confirms the complete disclosure on both sealed snapshots. + +The generic reproducer and recorder are +`evaluations/ffmpeg_lcl_multithread_reproducer.c` and +`evaluations/run_ffmpeg_lcl_multithread_reproducer.py`; structured artifacts +are under `results/sourcehunt-optimization/reproducers/`. The case is retained +as dynamically reproduced even though no sanitizer crash is expected: the +security effect is exposure of initialized allocator contents through a public +decoded-frame contract. + +The other terminal leads were rejected after source proof. PAF motion blocks +have explicit source-page end checks; bounded bytestream access prevents GDV LZ +copies from leaving its frame allocation; HEVC intra-prediction callers provide +the required doubled top/left border arrays; swscale swizzles are internally +constructed from component indices zero through three; and the reviewed VVC, +RTSP, Vulkan FFV1, RALF, median, MJPEG Huffman, H.264 CAVLC, and V4L2 candidates +are covered by the producer bounds described in the campaign triage record. + +Current honest totals are 420 distinct source-confirmed files, 16 confirmed root +causes, and 14 dynamically reproduced issues. H.264 and Vulkan HEVC remain the +two source-only cases. Relative to the 4,995-file deterministic corpus, bounded +coverage is 8.4%, with 4,575 files remaining. The audited next exact-path +manifest seals ranks 421–444 without changing prompts, scaffold, or context. + +### Blind wave 421–444 + +The unchanged treatment completed 24/24 targets with successful source-bearing +actions, using 4,601,980 tokens and 140 candidate calls with no formal findings. +Peak estimated context remained 11,956 tokens; the largest individual input was +16,049 tokens. All 24 source files are byte-identical in the control snapshot. + +Focused terminal-ledger review did not establish another root cause. The AAC +encoder's largest scale-factor band is exactly the 96-entry quantization scratch +capacity, and short-window grouping keeps its `w * 16 + g` metadata within 128 +entries. TrueMotion2 checks Huffman literal counts and token bounds, constrains +motion blocks to the picture, and allocates explicit luma/chroma edge padding. +DV stops at `pos >= 64` before indexing the coefficient block. D3D12 decode's +32-entry barrier stack cannot overflow under any registered codec because each +sets `max_num_ref` to 17 or less, leaving room for its two leading output +barriers. E-AC-3's copy span and extension span are multiples of 12 bins, keeping +the maximum copy-section count below `SPX_MAX_BANDS` (17). TIFF's LZW encoder +checks its remaining output bound and propagates failure. Android MediaCodec's +input/output capacity observations depend on the platform codec violating its +own buffer contract and are not attacker-media routes in this encoder path. + +The all-ledger selector audit now finds exactly 444 unique bounded paths, none +outside the deterministic corpus. Withholding each sealed exact-path wave from +ranks 205–444 reproduces it exactly. Current totals remain 16 confirmed root +causes and 14 dynamically reproduced issues. Coverage is 444/4,995 (8.9%), with +4,551 ranked files remaining. The next sealed manifest contains ranks 445–468. + +### Blind wave 445–468 + +The first pass completed successful source-bearing actions for 19/24 targets. +Two exact-path recovery passes covered the remaining five and then two paths, +so the merged wave is 24/24 source-confirmed. Across all three passes it used +3,563,833 tokens, made 103 candidate calls, and submitted no formal findings. +All 24 target files are byte-identical in the independent control snapshot. + +Focused terminal-ledger review did not establish another root cause. VC-1's +bitplanes are allocated for `mb_stride * FFALIGN(mb_height, 2)`, covering the +decoder's `mb_stride * mb_height` writes. MS Video 1 visits only complete 4x4 +blocks and its bottom-up row arithmetic stays inside those blocks. Rawvideo's +sub-16-bit expansion writes either at most the copied packet size or exactly +`width * height` 16-bit samples into an allocation at least as large as the +computed frame. The audio equalizer validates channel indices and its four +section-history accesses match the four-element arrays. DTS-to-PTS creates a +fresh null tree-node holder on every removal iteration, so the suspected reuse +of a non-null removal node does not occur. HQX dispatches exactly 16 slice jobs +for its 16-element slice state, ZMBV's previous-frame allocation includes its +configured motion-search margins, and ClearVideo's copy helpers reject source +or destination rectangles outside the coded frame. + +The all-ledger selector audit now finds exactly 468 unique bounded paths, none +outside the 4,995-file deterministic corpus. Withholding every sealed +exact-path wave from ranks 205–468 regenerates it exactly. Totals remain 16 +confirmed root causes and 14 dynamically reproduced issues. Coverage is +468/4,995 (9.4%), with 4,527 ranked files remaining. The unchanged next exact- +path manifest seals ranks 469–492. + +### Blind wave 469–492 + +The first pass completed successful source-bearing actions for 11/24 targets. +Several other hunters incorrectly claimed that source tools were unavailable +despite receiving their schemas, then reasoned from model memory and mentioned +historical CVEs. Those trajectories are not coverage or vulnerability evidence. +An exact-path replay recovered all 13 misses, making the merged wave 24/24 +source-confirmed. The two passes used 4,102,069 tokens and made 132 candidate +calls, with no formal findings. All 24 target files are byte-identical in the +independent control snapshot. + +Focused terminal-ledger review did not establish another root cause. HEVC's +unguarded center collocated-MV lookup remains inside the picture: SPS parsing +requires width and height to be multiples of the minimum coding block, boundary +quadtrees must split until their leaves fit, and every prediction partition is +contained by such a leaf. AAC rejects `max_sfb > num_swb`, selects matching +short- or long-window offset tables, and bounds the flattened group arrays to +128 entries. H.261 constrains each motion estimate to -15 through 15, so the +successive-vector difference stays within the 64-entry VLC table. PP7 allocates +its temporary surface for aligned luma geometry, which dominates every chroma +stride and height used by the same buffer. Spectrumsynth allocates a two-window +overlap buffer. Unsharp allocates `width + 2 * steps_x` columns and per-thread +row state matching its inclusive loop ranges and dispatch count. DXVA2 copies +slices into driver buffers only after checking each slice plus start code +against the remaining capacity. + +RTMP-over-HTTP's signed capacity arithmetic deserves hardening but did not +survive the promotion gate. The normal RTMPT write path flushes after ten FLV +packets by default; even with a caller-selected larger flush interval, the +capacity-doubling expression reaches signed overflow while the requested total +is still below `INT_MAX`, causing allocation failure and state reset before a +demonstrated `out_size + size` wrap can reach `memcpy`. The reviewed swscale, +test-source, motion-estimation, IDet, Vulkan H.265, ASF, 4XM, WAV, WavPack, SGA, +MVHA, Fraps, SDP, and qt-faststart leads likewise resolved to producer bounds, +format/plane contracts, bounded allocation failure, or no attacker-media route. + +The all-ledger selector audit finds exactly 492 unique bounded paths, none +outside the 4,995-file deterministic corpus. Withholding ranks 469–492 +regenerates the sealed manifest exactly. Totals remain 16 confirmed root causes +and 14 dynamically reproduced issues. Coverage is 492/4,995 (9.85%), with +4,503 ranked files remaining. The unchanged next exact-path manifest seals +ranks 493–516. + +### Blind wave 493–516 and `af_join` use-after-free confirmation + +The unchanged frozen treatment completed all 24 exact-path targets with +successful source-bearing actions on its first pass. It used 4,318,609 tokens, +made 133 candidate/finding calls, and submitted one formal finding. The +WS-SND1 fast path does logically read four bytes beyond the declared packet +payload because its length check does not subtract the four-byte header. It is +not retained as a security finding: FFmpeg's decoder API requires at least +`AV_INPUT_BUFFER_PADDING_SIZE` initialized zero bytes after packet data, so the +bounded read stays inside the caller-provided padding and cannot fault or expose +adjacent data under the contract. + +Offline terminal-ledger triage recovered and dynamically confirmed a different +root cause in `libavfilter/af_join.c`. Its unique-buffer loop compares `j == i` +after searching only `nb_buffers` entries. With a valid map that sends one input +plane to two early output channels and a different input plane to a later output +channel, the duplicate makes `i` diverge from `nb_buffers`; the later unique +plane is omitted from the output frame's `AVBufferRef` list. `try_push_frame` +then frees both input frames after delivering the output, leaving that later +plane dangling. + +A public libavfilter graph with two mono `abuffer` inputs, +`join=inputs=2:channel_layout=3.0:map=0.0-FL|0.0-FR|1.0-FC`, and an +`abuffersink` demonstrates the complete lifetime violation. The sink receives a +three-channel frame whose first two data pointers are identical and whose third +plane has no owning buffer reference. Reading the third channel normally after +retrieval produces an ASan heap-use-after-free, with the free stack in +`af_join`'s activation path. This reproduces on both sealed snapshots because +the source is identical; it is a real valid-configuration bug, not differential +evidence for the selected commits. The generic harness and recorder are +`evaluations/ffmpeg_af_join_uaf_reproducer.c` and +`evaluations/run_ffmpeg_af_join_uaf_reproducer.py`; structured vulnerable and +control records are under `results/sourcehunt-optimization/reproducers/`. + +The remaining strongest terminal candidates did not pass the promotion gate. +MV30's coefficient and motion-vector consumers use checked `bytestream2` reads, +which stop at the end and return zero. SIPR's mode-specific first-subframe pitch +indexes decode to at most `PITCH_DELAY_MAX`, and later indexes are clipped +around that prior lag. Xstack includes every configured placement rectangle +when deriving output dimensions. RTP iLBC accepts only 38- or 50-byte frames and +derives its per-packet frame cap from payload capacity. VP9's parsed three-bit +reference indexes address eight reference-frame state entries; the four-entry +array is for semantic reference types and is indexed independently. Network +parallelism is clamped to its three-entry arrays, VQC remains within its full +frame-sized vector allocation, and the reviewed LV2, packet dictionary, Dirac, +Theora, NVDEC, Vulkan encode, FLV, Bink, Opus PVQ, RL2, CBS VP9, and SRTP paths +did not establish an attacker-controlled violated allocation or lifetime +invariant. + +All 24 wave files are byte-identical in the independent control snapshot. The +all-ledger selector audit now finds exactly 516 unique bounded paths, none +outside the 4,995-file deterministic corpus. Withholding ranks 493–516 +regenerates its sealed manifest exactly. Current totals are 17 confirmed root +causes and 15 dynamically reproduced issues; H.264 and Vulkan HEVC remain the +two source-only cases. Coverage is 516/4,995 (10.33%), with 4,479 ranked files +remaining. The unchanged exact-path manifest for ranks 517–540 has SHA-256 +`a0ec534215e2811bd88e5e2a903fb9efab4471f981e63fcb7fa60d931dcf9031`. + +### Blind wave 517–540 and DNN output-shape overflow confirmation + +The unchanged frozen treatment completed all 24 exact-path targets with +successful source-bearing actions on its first pass. It used 4,375,440 tokens, +made 127 candidate/finding calls, and submitted one formal finding. Peak +estimated context was 11,972 tokens. All 24 target files are byte-identical in +the independent control snapshot. + +The formal JV report was rejected dynamically. A 64x64 frame with the minimum +accepted 16-byte video stream and recursive block selectors passes the +decoder's initial size guard and returns a decoded frame without a sanitizer +report on either snapshot. This build uses `CONFIG_SAFE_BITSTREAM_READER=1`; +the bit index clamps at `size_in_bits + 8`, and cache reads remain within the +required packet padding. The production diagnostic and structured records are +`evaluations/ffmpeg_jv_bitstream_reproducer.c`, +`evaluations/run_ffmpeg_jv_bitstream_reproducer.py`, and the corresponding +`ffmpeg_jv_bitstream_*` JSON files under +`results/sourcehunt-optimization/reproducers/`. + +Offline terminal-ledger triage instead confirmed a model-file-reachable heap +overflow in `libavfilter/dnn/dnn_io_proc.c`. Both the TensorFlow and OpenVINO +backends pass model-declared output tensor dimensions to +`ff_proc_from_dnn_to_frame` without validating the output channel count against +the RGB frame format. In the NCHW path, `middle_data` is allocated as +`frame_width * frame_height * output_channels`, but RGB conversion always +processes `frame_width * 3` elements per row and later addresses three planes. +A one-channel NCHW output tensor for a four-by-four RGB24 frame therefore +produces an ASan heap-buffer-overflow in the production postprocessor on both +snapshots. The same path also passes `&middle_data`, a one-pointer stack object, +to the four-plane `sws_scale` API; an instrumented build reports that independent +stack-array contract violation first. The focused recorder compiles the +production postprocessor without ASan only to pass that earlier violation and +let instrumented libswscale expose the dimension-dependent heap access. + +The reproducer and recorder are +`evaluations/ffmpeg_dnn_output_shape_reproducer.c` and +`evaluations/run_ffmpeg_dnn_output_shape_reproducer.py`; structured vulnerable +and control records are under `results/sourcehunt-optimization/reproducers/`. +The sealed snapshots were built without the optional TensorFlow or OpenVINO +backend, so this is a production-function and backend-source proof rather than +an end-to-end model-loader run. The trigger is a local DNN model file, analogous +to the retained ARNNDN model-file issue, rather than ordinary media alone. + +The remaining terminal leads did not establish another violated invariant. +The encoder framework rejects audio frames larger than `avctx->frame_size` +before libopus channel remapping. FIC bounds slice offsets and sizes against its +packet allocation and zero-initializes rejected entries. APE falls back from +nonpositive final sizes and rejects aligned packet sizes above `INT_MAX`. +MediaCodec copies depend on platform buffer metadata and allocation contracts; +no media-controlled size bypass was established. Vulkan slice storage grows +before each copy and its shared context uses refstruct ownership. BFI and BMV +check destination spans, DCA supplies the required LFE history prefix, LRC +scans a NUL-terminated `AVBPrint`, and the reviewed HEVC/VVC, VLC multitable, +LPC, IMF CPL, APV, AMF, ATRAC, Xan, memory-utility, and encoder-side-data paths +resolved to producer bounds or API contracts. + +The all-ledger selector audit finds exactly 540 unique bounded paths, none +outside the 4,995-file deterministic corpus. Withholding ranks 517–540 +regenerates its sealed manifest exactly. Current totals are 18 confirmed root +causes and 16 dynamically reproduced issues; H.264 and Vulkan HEVC remain the +two source-only cases. Coverage is 540/4,995 (10.81%), with 4,455 ranked files +remaining. The next unchanged exact-path manifest seals ranks 541–564. + +### Blind wave 541–564 and MagicYUV prior-frame disclosure confirmation + +The frozen treatment again completed 24/24 exact-path targets with successful +source-bearing actions on its first pass. It used 4,876,442 tokens, made 145 +candidate-ledger updates, and submitted no formal findings. Peak estimated +context was 11,969 tokens, and the largest individual request was 17,061 +tokens. This is another case where terminal state materially understated the +blind hunter's useful discovery yield. + +Offline review confirmed a complete information-disclosure root cause in +`libavcodec/magicyuv.c`. For compressed slices, `READ_PLANE` decodes each row +only while `get_bits_left(&gb) > 0`, but neither the macro nor its caller +requires the decoded pixel count to reach the declared row width. The decoder +continues through prediction, ignores every slice worker's return value, sets +`got_frame`, and publishes the full frame. + +A public libavcodec harness first decodes a valid raw 16x16 gray MagicYUV frame, +unrefs it into FFmpeg's frame pool, and then supplies a 300-byte frame with a +valid Huffman table but a two-byte compressed slice containing zero coded +pixels. The second decode succeeds. Its left predictor transforms all 256 stale +pixels from the previous pooled frame in a reversible way; the harness verifies +the expected transformed value at every pixel. Thus an attacker-controlled +truncated frame discloses the complete prior decoded frame through the normal +output contract. The reproducer and recorder are +`evaluations/ffmpeg_magicyuv_truncated_slice_reproducer.c` and +`evaluations/run_ffmpeg_magicyuv_truncated_slice_reproducer.py`; structured +records are under `results/sourcehunt-optimization/reproducers/`. The behavior +is sanitizer-clean and reproduces on both byte-identical snapshots, as expected +for disclosure of valid pooled memory rather than an out-of-allocation access. + +The other strongest ledger candidates resolved to existing bounds or API +contracts. DPX's global stride/height check covers its aligned per-row unpack +reads. MagicYUV slice offsets themselves are monotonically bounded; the missing +decoded-width invariant is the retained issue. H.264 MP4-to-Annex-B performs a +counting pass, rejects output sizes above `INT_MAX`, then allocates exactly that +size before its copy pass. Huffyuv masks or shifts high-bit-depth symbols into +its 16,384-entry VLC domain. JPEG-LS caps the run index at 31. SMPTE 436M caps +payload and sample counts before fixed-array copies. IAMF's custom layout is +allocated to the declared channel count and its final count is checked. The +reviewed UDP, MMAL, vectorscope, MPEG motion, HTTP authentication, Ut Video, +showwaves, CDToons, graph printing, LATM, ffmpeg muxing, CUDA thumbnail, generic +encryption info, ProRes, IFF, buffer pools, and H.264 CABAC paths did not +establish another attacker-controlled violated allocation or lifetime +invariant. + +Coverage is now 564/4,995 (11.29%), with 4,431 ranked files remaining. Current +totals are 19 confirmed root causes and 17 dynamically reproduced issues; +H.264 slice sentinel collision and Vulkan HEVC RPS overflow remain source-only. + +### Blind wave 565–588 and three memory-safety confirmations + +The unchanged frozen treatment completed all 24 exact-path targets with +successful source-bearing actions on its first pass. It used 4,320,458 tokens, +made 166 candidate/finding tool calls, and submitted one formal finding. The +511 model calls had a peak estimated context of 11,912 tokens and a largest +individual input of 17,409 tokens. All 24 targets are byte-identical in the +independent control snapshot. + +The formal drawgraph report survived offline review and dynamic validation. The +filter resets its shared horizontal coordinate only inside the first metadata +series' successful parse path. If that primary metadata is missing while a +later configured series is present, the first iteration continues before the +reset, `s->x` still increments each frame, and the later series eventually +writes beyond the output width. A public two-pixel libavfilter graph produces +an ASan heap-buffer-overflow on its third frame on both snapshots. The harness, +recorder, and structured records are +`evaluations/ffmpeg_drawgraph_missing_primary_reproducer.c`, +`evaluations/run_ffmpeg_drawgraph_missing_primary_reproducer.py`, and the +corresponding `ffmpeg_drawgraph_missing_primary_*` JSON files under +`results/sourcehunt-optimization/reproducers/`. + +Offline terminal-ledger review confirmed two DNN defects. The shared output-name +parser allocates exactly four pointer slots, accepts four names, and then writes +a required NULL terminator into a fifth slot. TensorFlow detection explicitly +requires four outputs, so its intended configuration reaches the overflow during +`ff_dnn_init`, before backend lookup or model loading. The production-source +harness reproduces the ASan heap-buffer-overflow on both snapshots; it supplies +inert backend module stubs because the sealed builds omit optional TensorFlow. +The reproducer and recorder are +`evaluations/ffmpeg_dnn_output_names_reproducer.c` and +`evaluations/run_ffmpeg_dnn_output_names_reproducer.py`. + +TensorFlow request cleanup also computes the output count as +`sizeof(*output_tensors) / sizeof(output_tensors[0])`. Both operands describe one +pointer, so cleanup always deletes one tensor even though detection requires and +produces four. Sustained valid detection therefore leaks three complete output +tensors per frame. This remains source-confirmed because TensorFlow is absent +from the sealed builds. + +Focused review then recovered another dynamically confirmed root cause from the +NellyMoser terminal candidate. Its trellis search caps `idx_max` with +`OPT_SIZE` but rejects only `idx > idx_max`, admitting the invalid one-past index +35,768. A short public white-noise encode with `volume=10` and `-trellis 1` +reaches the edge: UBSan reports invalid `opt` and `path` indices, and ASan catches +a four-byte read exactly after the complete 3,290,656-byte `opt` allocation. +Both snapshots reproduce it. The durable recorder is +`evaluations/run_ffmpeg_nellymoser_trellis_reproducer.py`, with structured +`ffmpeg_nellymoser_trellis_*` records under the reproducer results directory. + +The remaining terminal candidates resolved to bounds or contracts. EATGV's +short token reads remain within the decoder API's required zero padding, and Ogg +buffers include explicit input padding. RTP QCELP limits interleave indexes to +its six groups and bounds every 315-byte store and frame extraction. PAF bounds +block indexes and audio/video offsets against its allocations. IFV guards every +index-entry dereference and aborts incomplete index reads. APV checks each scan +position before coefficient access. CBS AV1 calculates allocation and copy sizes +from the same unit lengths. Channel-layout parsing grows storage by actual +entries and validates declared counts. The Torch request initializes both +tensor pointers and transfers input ownership to the tensor deleter. ATRAC3+, +Indeo 4, VVC intra/SEI, H.263 encoding, JPEG XL parsing, Movie, amix, RM muxing, +VAAPI encoding, LAME, and IPFS gateway likewise did not establish another +attacker-controlled violated memory or lifetime invariant. + +The all-results selector audit finds exactly 588 unique bounded paths, none +outside the 4,995-file deterministic corpus. Withholding ranks 565–588 +reproduces its sealed manifest exactly. Current totals are 23 confirmed root +causes and 20 dynamically reproduced issues; H.264 slice sentinel collision, +Vulkan HEVC RPS overflow, and the TensorFlow tensor leak remain source-only. +Coverage is 588/4,995 (11.77%), with 4,407 ranked files remaining. The unchanged +next exact-path manifest seals ranks 589–612 with SHA-256 +`b3647d52c5674a82e58f8f95a82d74ee6f335dbef1319a1eeb0223bccb50b286`. + +### Blind wave 589–612 and RTP/AV1 ignored-OBU heap overflow + +The unchanged frozen treatment completed all 24 exact-path targets with +successful source-bearing actions. It used 3,993,950 tokens over 447 model +calls, made 122 candidate/finding tool calls, and submitted no formal findings. +Peak estimated context was 11,995 tokens and the largest individual input was +18,525 tokens. All 24 target files are byte-identical in the independent +control snapshot. + +Offline terminal-ledger review confirmed a heap-buffer-overflow in the RTP/AV1 +depacketizer. Temporal-delimiter and tile-list OBUs enter an ignore branch that +increments `pktpos` by the declared OBU size and decrements `rem_pkt_size`, but +does not advance `buf_ptr`. The next loop iteration consequently parses bytes +inside the ignored OBU as a new element while the output cursor retains the +entire ignored-size gap. `av_grow_packet` accounts only for that newly parsed +element; the following OBU-header or payload write can therefore begin beyond +the packet allocation. + +A direct production-handler harness supplies a 100-byte ignored temporal +delimiter followed by 17 trailing bytes. ASan reports a heap write 19 bytes +beyond an 81-byte allocation on both sealed snapshots. The reproducer, +recorder, and structured records are +`evaluations/ffmpeg_rtp_av1_ignored_obu_reproducer.c`, +`evaluations/run_ffmpeg_rtp_av1_ignored_obu_reproducer.py`, and the +corresponding `ffmpeg_rtp_av1_ignored_obu_*` JSON files under +`results/sourcehunt-optimization/reproducers/`. + +The remaining leads resolved to existing bounds or contracts. FastAudio +consumes exactly 320 bits from its ten 32-bit words. LUT2 input negotiation caps +both inputs at 12 bits per component, making its 24-bit table and index +consistent. Drawtext's real shifts produce only subpixel indices 0–15. +`fill_ones` has the allocation slack required by its stores. CENC's clear plus +protected arithmetic is signed-widened and monotonically bounded by the +remaining packet size. DPX allocation matches its format-specific writes. +SChannel grows buffers before reads and copies and clamps `SECBUFFER_EXTRA` to +the current input. MPC8's combinatorial traversal is limited by `k <= 16` and +`n <= 32`. Removegrain's SIMD span is `(width - 2) & ~15` with a scalar tail. +Swresample's initial-reflection allocation covers both extrema. The NAL +start-code vector scanner stays within its guarded loop and its relevant +callers provide the required padding. OH encoder output attributes remain +platform-owned, and the reviewed swscale vertical-ring candidates remained +producer-bounded. + +The deterministic all-results selector audit now finds exactly 612 unique +bounded paths across 68 event ledgers, none outside the 4,995-file corpus. +Withholding ranks 589–612 reproduces their sealed manifest and hash exactly. +Current totals are 24 confirmed root causes and 21 dynamically reproduced +issues; H.264 slice sentinel collision, Vulkan HEVC RPS overflow, and the +TensorFlow tensor leak remain source-only. Coverage is 612/4,995 (12.25%), with +4,383 ranked files remaining. The next unchanged exact-path manifest seals +ranks 613–636 with SHA-256 +`37bf25501605780e40c3f8efecdd3d8eb2fc048484269b230799e222f7aa41fd`. + +### Blind wave 613–636 and three memory-safety confirmations + +The unchanged frozen treatment completed all 24 exact-path targets with +successful source-bearing actions. It used 4,461,162 tokens over 499 model +calls, made 136 candidate/finding tool calls, and submitted two formal +findings. Peak estimated context was 11,888 tokens, the largest individual +input was 17,045 tokens, and six automatic context compactions occurred. All +24 target files are byte-identical in the independent control snapshot. + +Both formal reports survived source review and public-CLI dynamic validation. +In shufflepixels horizontal and vertical inverse modes, `nb_blocks` is the +ceiling of the plane dimension divided by the configured block size, but the +map has only one entry per plane pixel. The number of entries written at a +random destination is derived from the sequential input cursor. If an early +full-width input block is assigned to the final partial destination block, map +initialization writes beyond the allocation. Width 10, block width 4, and seed +1 produce an ASan four-byte write exactly after the 40-byte map allocation on +both snapshots; the vertical form reproduces as well. The durable recorder is +`evaluations/run_ffmpeg_shufflepixels_inverse_reproducer.py`, with structured +`ffmpeg_shufflepixels_inverse_*` records under the reproducer results directory. + +VIF's boundary mirroring similarly assumes that each image dimension can +support half the current filter width. A tap from its 17-wide scale-zero filter +can lie beyond twice a small dimension, so the one-step expression +`2 * dimension - index - 1` still yields an invalid index. No minimum size is +enforced before the vertical and horizontal float reads. Comparing two 2x2 +gray frames through the public VIF graph produces an ASan four-byte read exactly +after a 16-byte allocation in `vif_filter1d` on both snapshots. The recorder +and records are `evaluations/run_ffmpeg_vif_small_frame_reproducer.py` and +`ffmpeg_vif_small_frame_*`. + +Offline terminal-ledger review recovered a third issue from rate control. The +MPEG-family second-pass parser accepts `type:%d` from its passlog without +checking that the value is an `AVPictureType`, then `init_pass2` uses it to +index multiple five-entry statistics arrays. The recorder generates a normal +MPEG-2 first-pass log, changes the first record from `type:1` to `type:99`, and +runs the ordinary second-pass CLI path. Both snapshots report the index-99 +violation under UBSan followed by an ASan eight-byte heap-buffer-overflow read +in `ff_rate_control_init`. Artifacts are +`evaluations/run_ffmpeg_ratecontrol_stats_reproducer.py` and the structured +`ffmpeg_ratecontrol_stats_*` records. + +The remaining ledger candidates resolved to existing bounds or contracts. +WAV PEAK output permits only one- or two-byte PCM, sizes each growth from the +channel count and bytes per sample, and disables output before the monotonic +buffer count can exceed `INT_MAX`; negating `INT16_MIN` is well-defined after +integer promotion and conversion back to `int16_t`. RTP/AV1 encoder fragments +copy exactly the current packet remainder only while the element is larger, +then retain the bounded final remainder. AC-3 parser reads are protected by +packet padding. H.264 picture timing maps at most three clock timestamps into +its three-entry array. DV's largest fixed profile equals +`DV_MAX_FRAME_SIZE`. SPP has aligned row slack for its eight-wide stores, and +ANLMS receives zeroed offset storage and doubled delay/coefficient rings. +CBS AV1 bounds tile-group endpoints by the parsed tile count. The reviewed +NVDEC, Vulkan H.264, SIPR, sync queue, OAPV, MPEG-1/2 encode, Lead, D3D12, +extract-extradata, CUVID, audio-3D-scope, and MIPS prediction paths did not +establish another violated memory or lifetime invariant. The optional +libaribcaption CLUT writes lack local guards, but upstream restricts foreground, +background, and stroke colors to one fixed 8-by-16 palette, so the 256-entry +destination cannot be exhausted. + +The deterministic all-results selector audit now finds exactly 636 unique +bounded paths across 69 event ledgers, none outside the 4,995-file corpus. +Withholding ranks 613–636 reproduces their sealed manifest and hash exactly. +Current totals are 27 confirmed root causes and 24 dynamically reproduced +issues; H.264 slice sentinel collision, Vulkan HEVC RPS overflow, and the +TensorFlow tensor leak remain source-only. Coverage is 636/4,995 (12.73%), with +4,359 ranked files remaining. The next unchanged exact-path manifest seals +ranks 637–660 with SHA-256 +`56946773e050d03038312840b9add8fe280cd82fbf2648c728c75cd09463824b`. + +### Blind wave 637–660, decimate metric overflow, and Whisper queue overflow + +The unchanged frozen treatment completed all 24 exact-path targets with +successful source-bearing actions. It used 4,716,109 tokens over 533 model +calls, made 155 candidate/finding tool calls, and submitted no formal findings. +Peak estimated context was 11,916 tokens, the largest individual input was +16,550 tokens, and three automatic context compactions occurred. All 24 target +files are byte-identical in the independent control snapshot. + +Offline terminal-ledger review dynamically confirmed a heap-buffer-overflow in +decimate's chroma metric calculation. The filter accepts a minimum block width +of four and allocates its metric grid using a luma half-block width of two. For +YUV411P chroma, horizontal subsampling right-shifts that value by two, producing +a zero loop step. The chroma loop therefore leaves `x` at zero while incrementing +`xdest` on every iteration, eventually indexing beyond the luma-sized metric +grid. A public 16x16 YUV411P graph with `decimate=blockx=4` produces an ASan +eight-byte access exactly after the 64-byte metric allocation on both sealed +snapshots. The durable recorder and structured records are +`evaluations/run_ffmpeg_decimate_subsampled_block_reproducer.py` and the +corresponding `ffmpeg_decimate_subsampled_block_*` JSON files under the +reproducer results directory. Vertically subsampled YUV440P and YUV410P can +similarly erase the half-block height and reach division by zero; the retained +root cause uses the stronger memory-safety trigger. + +The same ledger contained a second genuine overflow in the optional Whisper +filter. Its minimum 20 ms queue allocates 320 float samples. Whisper consumes +complete audio frames, and FFmpeg imposes no matching frame-size ceiling: +`asetnsamples` may validly emit as many as `INT_MAX` samples. When a single frame +is larger than the queue, the capacity guard transcribes at most the existing +fill and then unconditionally copies the complete new frame into the fixed +queue. A normal 1,024-sample frame therefore exceeds the minimum allocation. +The sealed builds omit whisper.cpp, so this case is retained as source-confirmed +rather than dynamically reproduced. + +The strongest remaining candidates closed under their downstream contracts. +The FLIC video demuxer does ignore a failed full-size read after allocating its +packet, but the demux callback returns the EOF/error instead of releasing that +partially initialized packet to the decoder. NSV auxiliary sizes can underflow +the unsigned video size, but conversion to the signed packet API produces a +negative growth request that `av_grow_packet` rejects before allocation or +access. A crafted Opus code-3 CBR packet can similarly compute a negative frame +size, but range-decoder initialization rejects that size before reading, and no +unsafe parser consumer was established. The reviewed LUT3D, OpenVINO, GIF, +RTP, buffer-sink, MSMPEG4, SCPR, showspatial, RA288, H.264 picture, CBS SEI, +V4L2, YUV4MPEG, SMC encoder, ALAC, CDXL, HDR, Twofish, and IVI candidates did +not establish another distinct violated memory or lifetime invariant; the +OpenVINO output-shape lead overlaps the already retained generic NCHW output +shape root cause. + +The deterministic all-results selector audit now finds exactly 660 unique +bounded paths across 70 event ledgers, none outside the 4,995-file corpus. +Withholding ranks 637–660 reproduces their sealed manifest and hash exactly. +Current totals are 29 confirmed root causes and 25 dynamically reproduced +issues; H.264 slice sentinel collision, Vulkan HEVC RPS overflow, the TensorFlow +tensor leak, and the Whisper oversized-frame overflow remain source-only. +Coverage is 660/4,995 (13.21%), with 4,335 ranked files remaining. The next +unchanged exact-path manifest seals ranks 661–684 with SHA-256 +`86d85254f6934b18422d4cc950b5c2a76e9f6a87e1c1ae54abd44424cb8a03ac`. + +### Blind wave 661–684 and DNN classification-count overflow + +The unchanged frozen treatment completed all 24 exact-path targets with +successful source-bearing actions. It used 4,006,739 tokens over 461 model +calls, made 124 candidate/finding tool calls, and submitted no formal +findings. Peak estimated context was 11,956 tokens, the largest individual +input was 15,626 tokens, and three automatic context compactions occurred. +All 24 target files are byte-identical in the independent control snapshot. + +Offline terminal-ledger review dynamically confirmed a heap-buffer-overflow in +the DNN classification filter. Each production detection bounding box contains +exactly four classification-label and confidence slots, but +`dnn_classify_post_proc` indexes them with `bbox->classify_count` and increments +that count without enforcing the capacity. OpenVINO may derive its output count +directly from the loaded model, and its completion loop invokes the classifier +callback once for every output. A production-source harness invokes the same +callback five times on one production-allocated bounding-box side-data object. +The fifth callback produces an ASan eight-byte write exactly after the complete +660-byte allocation on both sealed snapshots. The reproducer, recorder, and +structured records are +`evaluations/ffmpeg_dnn_classify_count_reproducer.c`, +`evaluations/run_ffmpeg_dnn_classify_count_reproducer.py`, and the corresponding +`ffmpeg_dnn_classify_count_*` JSON files under the reproducer results directory. +The OpenVINO completion loop also passes `outputs` instead of +`&outputs[output_i]`; that independent indexing error is not needed for the +confirmed repeated-callback overflow. + +The remaining terminal candidates resolved to bounds or downstream contracts. +The X server does require each ZPixmap row to use its advertised scanline pad, +so xcbgrab's tightly packed SHM allocation is too small for some narrow +8-, 16-, and 24-bit captures. However, `ProcShmGetImage` computes that exact +padded length and checks it against the attached segment before calling +`GetImage`; an undersized segment receives `BadAccess` rather than an out-of- +bounds server write. The non-SHM reply carries the padded allocation, while +xcbgrab's tightly stepped cursor drawing remains inside the smaller logical +frame region. UTVideo's Huffman payload likewise cannot exceed its one-byte-per- +symbol buffer: a fixed eight-bit code is always a valid prefix code for the 256 +byte symbols, so an optimal Huffman tree has weighted length at most eight bits +per input byte, and the extra four bytes cover alignment. + +The `subfile,` option parser either selects the ordinary file protocol for the +short malformed forms or rejects an incomplete option sequence with its +pointers still inside the copied filename. Bink's unsigned remaining-size +underflow changes parsing, but `ffio_limit` bounds packet allocation and reads +to available input. IPMovie component sizes are 16-bit, their combined packet +allocation covers all three copies, and a negative short-audio size is rejected +by the packet API. SMC advances rows by the padded destination stride, keeping +previous-block references inside the allocated frame. Hap allocates the larger +of its complete texture and per-chunk Snappy worst case. PSX STR bounds its +16-bit sector index and count before the fixed-size sector copy; DXA caps its +frame size and allocates the header, palette, and frame together; and RL2's +validated index sizes remain representable by the index API. RTP Xiph and VP8, +SSIM, LADSPA, aphasemeter, DVB subtitle parsing, histeq, DShow, MPEG video DSP, +Bethsoft VID, random, sendcmd, and the reviewed codec utility paths likewise did +not establish another attacker-controlled violated memory or lifetime +invariant. + +The deterministic all-results selector audit now finds exactly 684 unique +bounded paths across 71 event ledgers, none outside the 4,995-file corpus. +Withholding ranks 661–684 reproduces their sealed manifest and hash exactly. +Current totals are 30 confirmed root causes and 26 dynamically reproduced +issues. Coverage is 684/4,995 (13.69%), with 4,311 ranked files remaining. The +next unchanged exact-path manifest seals ranks 685–708 with SHA-256 +`9e49f3ece93bb10739bda51586e545cda788ab4934910890dcb41159501ebe79`. + +### Blind wave 685–708 and three packet/filter indexing overflows + +The unchanged frozen treatment completed all 24 exact-path targets with +successful source-bearing actions. It used 4,472,243 tokens over 503 model +calls, made 134 candidate/finding tool calls, and submitted one formal finding. +Peak estimated context was 11,893 tokens, the largest individual input was +16,425 tokens, and one automatic context compaction occurred. All 24 target +files are byte-identical in the independent control snapshot. + +The formal finding is a dynamically confirmed out-of-bounds channel index in +the pan filter. `parse_channel_name` bounds the numbered `cN` syntax to +`MAX_CHANNELS`, but returns named `AVChannel` values without the same check. +Public named values include `UNK` at 768 and `AMBI0` at 1024, far above pan's +64-entry input-channel arrays. The public graph `pan=stereo|FL=AMBI0` first +reads `used_in_ch[1024]`; UBSan reports that exact index and ASan reports the +resulting invalid stack read on both sealed snapshots. If execution continues, +the following assignments write through the same invalid index in +`used_in_ch` and `pan->gain`. The durable CLI recorder and structured records +are `evaluations/run_ffmpeg_pan_named_channel_reproducer.py` and the +corresponding `ffmpeg_pan_named_channel_*` JSON files under the reproducer +results directory. + +Offline terminal-ledger review dynamically confirmed two additional RTP +packetizer roots. LATM represents an AAC access-unit size with one header byte +per 255 payload bytes, but writes that complete variable-length header into the +fixed RTP packet buffer before fragmentation and never compares its size with +the buffer. Through the public RTP muxer, a 382,500-byte AAC packet and an +ordinary 1,472-byte packet sink produce a 1,501-byte header; ASan observes the +1,500-byte `memset` crossing the complete RTP allocation on both snapshots. +The reproducer and recorder are +`evaluations/ffmpeg_rtp_latm_header_reproducer.c` and +`evaluations/run_ffmpeg_rtp_latm_header_reproducer.py`. + +RFC2190 H.263 has a separate small-packet failure. Common RTP initialization +accepts every packet size above its 12-byte header, while the RFC2190 +packetizer reserves another eight payload-header bytes. A valid 13-byte sink +therefore leaves one payload byte and derives a fragment size of negative +seven. That signed length reaches mode-B `memcpy`, where ASan reports +`negative-size-param` through the public muxer on both snapshots. The durable +artifacts are `evaluations/ffmpeg_rtp_h263_small_packet_reproducer.c`, +`evaluations/run_ffmpeg_rtp_h263_small_packet_reproducer.py`, and the matching +structured records under the reproducer results directory. + +The HEVC no-start-code candidate was based on a misread sentinel calculation. +`nal_find_startcode_internal` subtracts three from its local end pointer in +each of its first two loops and adds three back before the tail loop, so its +final `end + 3` is the caller's original end, not three bytes beyond it. +`nal_parse_units` consequently takes its normal end-of-input exit rather than +looping. JPEG-LS sizes its raw workspace at four bytes per component-pixel and +its escaped packet for the proven one-bit-per-15-input-bit maximum overhead. +vMix rounds frame storage to 16-pixel boundaries and validates both DC and AC +slice spans before its 8x8 writes. HQA similarly aligns its frame and confines +each of eight slice traversals to that storage. The GDI lead would require a +Windows desktop bitmap whose successful GDI allocation and reported geometry +already exceed signed packet-size representability; no realizable platform +configuration satisfying that chain was established. The reviewed FFV1, +ProRes RAW and Vulkan ProRes, VP9 probability, JPEG XL animation, V4L2, QSV, +FDK-AAC, Intrax8, AAP, amplify, OpenCL xfade, C93, movtext, and mptestsrc paths +likewise did not establish another violated memory or lifetime contract. + +The deterministic all-results selector audit now finds exactly 708 unique +bounded paths across 72 event ledgers, none outside the 4,995-file corpus. +Withholding ranks 685–708 reproduces their sealed manifest and hash exactly. +Current totals are 33 confirmed root causes and 29 dynamically reproduced +issues. Coverage is 708/4,995 (14.17%), with 4,287 ranked files remaining. The +next unchanged exact-path manifest seals ranks 709–732 with SHA-256 +`cfde32b2171a5b3e7f682090af7bce5fc770f6b7de30982193d02f0bf09635f5`. + +### Blind wave 709–732 and framepack alpha-plane invalid write + +The unchanged frozen treatment completed all 24 exact-path targets with +successful source-bearing actions. It used 4,321,772 tokens over 485 model +calls, made 133 candidate/finding tool calls, and submitted one formal +finding. Peak estimated context was 11,977 tokens, the largest individual +input was 17,047 tokens, and four automatic context compactions occurred. All +24 target files are byte-identical in the independent control snapshot. + +The formal MLZ finding was rejected during offline adjudication. Its bump +sequence reaches 32767 only after setting that value as the next bump code, +but 32767 is `MAX_CODE` and the earlier switch case flushes the dictionary; +the bump branch therefore cannot grow the accepted code range to 65536. +Independently, `decode_string` stores `match_len - 1` in an unsigned long, so a +zero-length dictionary entry becomes `ULONG_MAX` and is rejected by the +buffer-size comparison before the claimed preceding-byte write. + +Offline terminal-ledger review instead dynamically confirmed an invalid write +in the framepack filter. Framepack explicitly advertises four-plane YUVA +formats, but its side-by-side packing helper initializes only `dst[0]` through +`dst[2]` before passing the four-entry array to `av_image_copy2`. The generic +image copier derives four planes from YUVA's pixel descriptor and writes the +alpha plane through the indeterminate `dst[3]` pointer. A public graph joining +two 16x16 `yuva420p` sources with `framepack=sbs` produces an ASan invalid write +through `image_copy_plane` on both sealed snapshots. The vertical helper has +the same unset destination pointer and additionally leaves `linesizes[3]` +unset; `framepack=tab` also aborts under ASan. The durable recorder and +structured records are +`evaluations/run_ffmpeg_framepack_alpha_reproducer.py` and the corresponding +`ffmpeg_framepack_alpha_*` JSON files under the reproducer results directory. + +The strongest remaining terminal candidates resolved to bounds or unreachable +states. FITS can store `NAXIS1` through `NAXIS999`, but its eight-byte keyword +field truncates `NAXIS1000` to `NAXIS100`; the exact sequence check rejects it +before indexing beyond `naxisn[998]`. Identity and mid-equalizer negotiate one +common pixel format for both inputs, while their plane dimensions and +histogram indices remain within the matching allocations. Curves validates +coordinates to `[0,1]`, requires strictly increasing scaled x positions, and +sizes its PCHIP work arrays for the derivative endpoint accesses. Convolution +caps its odd row or column matrix at 49 entries, matching its fixed pointer +array. + +APAC's four-bit block length caps direct sample writes at 15 values in its +64-byte scratch block; oversized frame-sample arithmetic is rejected by the +audio buffer allocator. RoQ caps the chunk size before the packet-size +addition, and a wrapped negative allocation is rejected before its read. +XMV's unsigned video-size subtraction can weaken a container boundary, but its +frame-size field remains capped at 524,288 bytes and the packet API bounds the +actual read. MSCC's writer seek clamps to its allocation and its decoded-frame +buffers are sized from validated codec dimensions. OpenJPEG's packet writer +maintains `pos <= size`, and its component copy dimensions match the negotiated +frame layout. The swscale tail scratch plane holds 512 bytes, while a 32-pixel +block at its largest four-byte pixel type requires at most 128 bytes. The +reviewed libdav1d, D3D12 HEVC, H.264 motion prediction and metadata, MPJPEG, +SP5X, Targa, and generic format utility paths likewise did not establish +another violated memory or lifetime invariant. MPJPEG's `do`/`memcmp` shape +would be unsafe for a strict MIME boundary longer than its 2,048-byte scan +chunk, but FFmpeg's HTTP response parser truncates exported Content-Type lines +below that threshold; no public route to the oversized boundary was +established. + +The deterministic all-results selector audit now finds exactly 732 unique +bounded paths across 73 event ledgers, none outside the 4,995-file corpus. +Withholding ranks 709–732 reproduces their sealed manifest and hash exactly. +Current totals are 34 confirmed root causes and 30 dynamically reproduced +issues. Coverage is 732/4,995 (14.65%), with 4,263 ranked files remaining. The +next unchanged exact-path manifest seals ranks 733–756 with SHA-256 +`6883aabfd2be2a76c1ab5757c35c047e442415af67f3fd4c00cb225fbdaefe10`. + +### Blind wave 733–756 and FFV1 remap-table out-of-bounds read + +The unchanged frozen treatment completed all 24 exact-path targets with +successful source-bearing actions. It used 4,479,873 tokens over 503 model +calls and made 141 candidate/finding tool calls, with no formal findings. +Peak estimated context was 11,880 tokens, the largest individual input was +17,541 tokens, and six automatic context compactions occurred. All 24 target +files are byte-identical in the independent control snapshot. + +Offline terminal-ledger review dynamically confirmed a heap-buffer-overflow +read in FFV1 level-4 remapping. `decode_slice` allocates each 16-bit `fltmap` +for exactly `slice_width * slice_height` entries, and `decode_remap` records a +populated `remap_count` that need not be a power of two. `decode_plane` then +decodes symbols using `ceil(log2(remap_count))` bits and masks them to the next +power-of-two range without rejecting unused symbols. If `pixel_num` is below +that ceiling, an unused symbol is also beyond the physical allocation. + +A public FFmpeg encode of one 513x1 `yuv444p16le` frame with FFV1 level 4 and +`remap_mode=1` creates the required non-power-of-two table shape. The +unmodified sample decodes normally. Flipping bit zero of packet byte 18—one +bit of entropy data, with the valid remap syntax left intact—causes +`decode_line` to produce an unused masked symbol and `decode_plane` to read two +bytes outside the lookup table. ASan reports the same heap-buffer-overflow on +both sealed snapshots. The durable harness, recorder, and structured records +are `evaluations/ffmpeg_ffv1_remap_reproducer.c`, +`evaluations/run_ffmpeg_ffv1_remap_reproducer.py`, and the corresponding +`ffmpeg_ffv1_remap_*` artifacts under the reproducer results directory. + +The other terminal candidates resolved to established invariants or +unreachable states. The QCP data path enters its packet branch only while the +unsigned data size is nonzero; size one yields a zero-length packet rather +than a negative allocation. WebM DASH requires at least one decimal stream +index before leaving its `parsing_streams` state, bounds that index by +`nb_streams`, and validates all supported codec identifiers before output. +DeckLink's VANC ownership is balanced: creation returns one COM reference, +`SetAncillaryData` retains it, the caller releases its reference, and the +frame destructor releases the stored reference. Abitscope dispatches literal +8-, 16-, 32-, or 64-bit sample widths from the negotiated format; its coarse +stored depth does not control those reads. Corr's shared pixel-format list +negotiates one format across both inputs. AAC short windows expose at most 15 +scale-factor bands, so the eight 16-slot windows fit the 128-entry arrays. +FTR checks the accumulated channel offset before every plane copy. RTP HEVC +checks its two-byte payload header plus one byte before reading a FU header, +and its aggregate helper bounds every NAL length. WMA's fixed block length and +`BLOCK_MAX_SIZE` arrays cover its coefficient ranges. JPEG XL supplies libjxl +the actual FFmpeg buffer size and checks the external API result. The reviewed +VP5, VVC reference-list, SpeedHQ, RGB conversion, MIPS VP8, wavesynth, command +help, and device paths likewise did not establish another violated memory or +lifetime contract. + +The deterministic all-results selector audit now finds exactly 756 unique +bounded paths across 74 event ledgers, none outside the 4,995-file corpus. +Withholding ranks 733–756 reproduces their sealed manifest and hash exactly. +Current totals are 35 confirmed root causes and 31 dynamically reproduced +issues. Coverage is 756/4,995 (15.14%), with 4,239 ranked files remaining. +The next unchanged exact-path manifest seals ranks 757–780 with SHA-256 +`d73b8307b9ed87a96f6abffef992229da1b143bad4461f09d5065ec4e8a3c7aa`. + +### Blind wave 757–780, partial-block overflows, and rav1e FFI UB + +The unchanged frozen treatment completed all 24 exact-path targets with +successful source-bearing actions. It used 3,593,225 tokens over 420 model +calls, made 110 candidate/finding tool calls, and submitted three formal +findings. Peak estimated context was 11,876 tokens, the largest individual +input was 17,190 tokens, and four per-trajectory automatic context compactions +occurred. All 24 target files are byte-identical in the independent control +snapshot. + +Offline adjudication dynamically confirmed two memory-safety roots. First, +`tools/yuvcmp.c` sizes its macroblock-error bitmap using floor-rounded +`width / 16` and `height / 16`, while its luma and chroma comparison loops map +every active sample, including partial macroblocks, into that bitmap. Two valid +17x16 YUV420P inputs differing only at luma pixel `(16,0)` select index one in +a one-byte allocation. ASan reports the one-byte heap-buffer-overflow +read-modify-write on both sealed snapshots. The separate hunter claim about +`dump_blocks` is rejected: that loop enumerates only the allocated +`mb_x * mb_y` complete blocks, so it never independently reaches a partial +right or bottom block. Durable artifacts are +`evaluations/ffmpeg_yuvcmp_partial_mb_reproducer.c`, +`evaluations/run_ffmpeg_yuvcmp_partial_mb_reproducer.py`, and the matching +structured records under the reproducer results directory. + +Second, the DVD subtitle encoder checks a one-nibble-per-pixel RLE budget as +`floor(width * height / 2)`. It encodes the even and odd fields separately, +however, and pads every odd-width row to a complete byte. Its actual minimum +for this case is `ceil(width / 2) * height`. A public 1x200 bitmap subtitle and +a 142-byte output buffer pass the 100-byte RLE check but require 200 bytes; +ASan reports a stack-buffer-overflow in `dvd_encode_rle` on both snapshots. +The durable artifacts are `evaluations/ffmpeg_dvdsub_odd_width_reproducer.c`, +`evaluations/run_ffmpeg_dvdsub_odd_width_reproducer.py`, and their matching +structured records. + +The rav1e finding is retained with a narrower, source-accurate classification. +A disposable pass-one-only change was required to make this FFmpeg snapshot +emit real legacy stats with rav1e 0.7.1; the resulting complete stream was 156 +bytes: a 68-byte summary followed by 88 bytes of frame packets. The exact +unmodified FFmpeg wrapper consumed that stream successfully in pass two. +After partial consumption, `set_stats` advances `pass_data + pass_pos` but +continues passing the original total `pass_size`. Rav1e immediately creates a +Rust slice from that pointer and length, so the declared range exceeds the +FFmpeg allocation and violates the FFI precondition. Current rav1e consumed +only the next required eight bytes and Valgrind observed no physical +out-of-allocation access. The survivor is therefore recorded as real +source-and-API-confirmed FFI undefined behavior, not as an observed heap read. +The hunter's separate pass-one EOS `memcpy` overflow is rejected: rav1e's final +summary is designed to overwrite its placeholder header, the observed summary +was 68 bytes, and the existing fast-realloc capacity was 125 bytes, so the +tested write fit. No contract proving a larger summary was established. + +The remaining terminal leads resolved to producer or allocation contracts. +AVRn's interlaced last-row copy extends four bytes beyond the logical packet +length but remains inside FFmpeg's mandatory input-padding region. XPM +allocates the complete `NB_ELEMENTS`-to-the-power-`cpp` lookup space. Y41P's +packet check rounds width up and its image planes have aligned storage. IL can +leave an odd final row uninitialized but does not leave its allocations. LXF +PCM's five-byte blocks produce the two samples accounted for by the frame +size. Movtext bounds text, boxes, styles, and font-table traversal by packet +remaining sizes, while APNG checks both chunk and accumulated extradata +arithmetic against `INT_MAX`. + +QTRLE's two-times-base-material packet allocation covers the raw-pixel and +per-code worst case plus its explicit line and frame overhead. CBS H.264/5 +grows its destination on `ENOSPC`, and its source bit offset and size originate +inside the parsed unit. ALAC's 4,096-sample work buffers match the generic +encoder frame cap. VVC slice counts and cumulative explicit-slice counts are +parser-capped below `VVC_MAX_SLICES`. MIPS VP9 and LoongArch VP8 vector reads +operate under their reference-frame edge-padding contracts. DNN UV scaling +uses input/output chroma geometry matched to allocated frames; swscale's +converter byte counts are paired with the caller-derived destination strides; +and the reviewed top-level FFmpeg, WMV2, color-temperature, concat, ID3, and +public API header paths established no additional violated memory or lifetime +invariant. + +The deterministic all-results selector audit now finds exactly 780 unique +bounded paths across 75 event ledgers, none outside the 4,995-file corpus. +Withholding ranks 757–780 reproduces their sealed manifest and hash exactly. +Current totals are 38 confirmed root causes and 33 dynamically reproduced +issues; the five source/API-confirmed cases are H.264, Vulkan HEVC, TensorFlow's +tensor leak, Whisper, and rav1e. Coverage is 780/4,995 (15.62%), with 4,215 +ranked files remaining. The next unchanged exact-path manifest seals ranks +781–804 with SHA-256 +`5961e2cbbe97fb77e5b01f4bbf38058c8f383ef34a057b6960162f07620f4d8a`. diff --git a/evaluations/audit_sourcehunt_ffmpeg_selector.py b/evaluations/audit_sourcehunt_ffmpeg_selector.py new file mode 100644 index 00000000..108893ff --- /dev/null +++ b/evaluations/audit_sourcehunt_ffmpeg_selector.py @@ -0,0 +1,171 @@ +"""Audit all SourceHunt bounded-path ledgers and derive the next FFmpeg ranks.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +from clearwing.sourcehunt.preprocessor import Preprocessor +from clearwing.sourcehunt.ranker import Ranker + +SCHEMA_VERSION = "cw.sourcehunt.ffmpeg-selector-audit.v1" + + +def _arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--checkout", type=Path, required=True) + parser.add_argument("--results-root", type=Path, required=True) + parser.add_argument("--wave-size", type=int, default=24) + parser.add_argument("--withhold-paths", type=Path) + parser.add_argument("--expected-paths", type=Path) + parser.add_argument("--next-paths-output", type=Path) + parser.add_argument("--output", type=Path, required=True) + return parser.parse_args() + + +def _sha256_bytes(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def _sha256_file(path: Path) -> str: + return _sha256_bytes(path.read_bytes()) + + +def _load_paths(path: Path | None) -> list[str]: + if path is None: + return [] + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ValueError(f"invalid path manifest: {path}") from exc + if not isinstance(value, list) or any( + not isinstance(item, str) or not item for item in value + ): + raise ValueError(f"path manifest must be a JSON string array: {path}") + if len(value) != len(set(value)): + raise ValueError(f"path manifest contains duplicates: {path}") + return value + + +def _write_json(path: Path, payload: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text( + json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + temporary.replace(path) + + +def _bounded_paths(results_root: Path) -> tuple[set[str], list[str]]: + bounded: set[str] = set() + ledgers: list[str] = [] + for events in sorted(results_root.rglob("instrumentation/events.jsonl")): + ledger_used = False + for line in events.read_text(encoding="utf-8", errors="replace").splitlines(): + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + if not ( + event.get("event") == "stage" + and event.get("stage") == "rank" + and event.get("status") == "bounded" + ): + continue + paths = event.get("files") or [] + if not isinstance(paths, list): + raise ValueError(f"bounded rank event has invalid files: {events}") + bounded.update(str(path) for path in paths if path) + ledger_used = True + if ledger_used: + ledgers.append(str(events)) + return bounded, ledgers + + +def _ranked_paths(checkout: Path) -> list[str]: + result = Preprocessor( + repo_url="https://github.com/FFmpeg/FFmpeg.git", + local_path=str(checkout), + ).run() + files = Ranker(None).rank_heuristically(result.file_targets) + ordered = sorted( + files, + key=lambda item: ( + -float( + item.get( + "deterministic_rank_score", item.get("priority", 0.0) + ) + ), + str(item.get("path") or ""), + ), + ) + return [str(item.get("path") or "") for item in ordered] + + +def main() -> None: + args = _arguments() + checkout = args.checkout.expanduser().resolve() + results_root = args.results_root.expanduser().resolve() + output = args.output.expanduser().resolve() + if args.wave_size < 1: + raise ValueError("wave size must be positive") + if not checkout.is_dir() or not results_root.is_dir(): + raise ValueError("checkout and results root must exist") + + ranked = _ranked_paths(checkout) + corpus = set(ranked) + bounded, ledgers = _bounded_paths(results_root) + outside_corpus = sorted(bounded - corpus) + if outside_corpus: + raise ValueError("bounded paths outside deterministic corpus") + + withheld = _load_paths(args.withhold_paths) + absent_withheld = sorted(set(withheld) - bounded) + if absent_withheld: + raise ValueError("withheld paths are not present in the bounded ledger") + effective_bounded = bounded - set(withheld) + next_paths = [path for path in ranked if path not in effective_bounded][ + : args.wave_size + ] + + expected = _load_paths(args.expected_paths) + expected_matches = expected == next_paths if args.expected_paths else None + if expected_matches is False: + raise ValueError("derived next paths do not match the expected manifest") + + if args.next_paths_output: + _write_json(args.next_paths_output.expanduser().resolve(), next_paths) + payload = { + "schema_version": SCHEMA_VERSION, + "completed_at": datetime.now(UTC).isoformat(), + "checkout": str(checkout), + "results_root": str(results_root), + "corpus_count": len(ranked), + "bounded_unique_count": len(bounded), + "bounded_ledger_count": len(ledgers), + "bounded_ledgers": ledgers, + "outside_corpus": outside_corpus, + "withheld_paths": withheld, + "effective_bounded_count": len(effective_bounded), + "next_paths": next_paths, + "next_paths_sha256": _sha256_bytes( + (json.dumps(next_paths, indent=2) + "\n").encode() + ), + "expected_paths": str(args.expected_paths.resolve()) + if args.expected_paths + else None, + "expected_paths_sha256": _sha256_file(args.expected_paths.resolve()) + if args.expected_paths + else None, + "expected_matches": expected_matches, + } + _write_json(output, payload) + print(output) + + +if __name__ == "__main__": + main() diff --git a/evaluations/build_ffmpeg_arnndn_model.py b/evaluations/build_ffmpeg_arnndn_model.py new file mode 100644 index 00000000..b9431221 --- /dev/null +++ b/evaluations/build_ffmpeg_arnndn_model.py @@ -0,0 +1,59 @@ +"""Build a minimal arnndn model with an oversized denoise output layer.""" + +from __future__ import annotations + +import argparse +from pathlib import Path + + +def _arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("output", type=Path) + parser.add_argument("--denoise-outputs", type=int, default=23) + return parser.parse_args() + + +def _line(values: list[int]) -> str: + return " ".join(map(str, values)) + + +def _dense(nb_inputs: int, nb_neurons: int) -> list[str]: + return [ + _line([nb_inputs, nb_neurons, 0]), + _line([0] * (nb_inputs * nb_neurons)), + _line([0] * nb_neurons), + ] + + +def _gru(nb_inputs: int, nb_neurons: int) -> list[str]: + return [ + _line([nb_inputs, nb_neurons, 0]), + _line([0] * (nb_inputs * nb_neurons * 3)), + _line([0] * (nb_neurons * nb_neurons * 3)), + _line([0] * (nb_neurons * 3)), + ] + + +def build_model(output: Path, denoise_outputs: int) -> None: + """Write a syntactically valid minimal model with the requested output size.""" + + if not 1 <= denoise_outputs <= 128: + raise ValueError("denoise output count must be in [1, 128]") + + lines = ["rnnoise-nu model file version 1"] + lines.extend(_dense(42, 1)) + lines.extend(_gru(1, 1)) + lines.extend(_gru(44, 1)) + lines.extend(_gru(44, 1)) + lines.extend(_dense(1, denoise_outputs)) + lines.extend(_dense(1, 1)) + output.write_text("\n".join(lines) + "\n", encoding="utf-8") + + +def main() -> None: + args = _arguments() + build_model(args.output, args.denoise_outputs) + + +if __name__ == "__main__": + main() diff --git a/evaluations/build_sourcehunt_lair_dataset.py b/evaluations/build_sourcehunt_lair_dataset.py new file mode 100644 index 00000000..748d6495 --- /dev/null +++ b/evaluations/build_sourcehunt_lair_dataset.py @@ -0,0 +1,68 @@ +"""Build leakage-safe SourceHunt routing supervision from LAIR CVE goldens.""" + +from __future__ import annotations + +import argparse +import json + +from clearwing.eval.sourcehunt_lair import ( + LairSplitConfig, + adapt_lair_goldens, + load_lair_goldens, + write_lair_adapter_dataset, +) + + +def _arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Delexicalize LAIR GoldenChain files for offline SourceHunt routing", + ) + parser.add_argument("--goldens", required=True, help="LAIR output root, goldens dir, or JSON") + parser.add_argument("--output-dir", required=True) + parser.add_argument("--train", type=float, default=0.70) + parser.add_argument("--development", type=float, default=0.15) + parser.add_argument("--test", type=float, default=0.15) + parser.add_argument("--seed", default="lair-sourcehunt-v1") + parser.add_argument( + "--reserved-repository", + action="append", + default=["ffmpeg"], + help="Repository name excluded from optimization; repeatable (default: ffmpeg)", + ) + parser.add_argument("--overwrite", action="store_true") + return parser.parse_args() + + +def main() -> None: + args = _arguments() + goldens = load_lair_goldens(args.goldens) + dataset = adapt_lair_goldens( + goldens, + split_config=LairSplitConfig( + train=args.train, + development=args.development, + test=args.test, + seed=args.seed, + ), + reserved_repository_names=args.reserved_repository, + ) + manifest = write_lair_adapter_dataset( + dataset, + args.output_dir, + overwrite=args.overwrite, + ) + print(manifest) + print( + json.dumps( + { + "goldens": dataset.manifest.golden_count, + "excluded": dataset.manifest.excluded_golden_count, + "router_rows": dataset.manifest.router_row_count, + }, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/evaluations/ffmpeg_af_join_uaf_reproducer.c b/evaluations/ffmpeg_af_join_uaf_reproducer.c new file mode 100644 index 00000000..7ac183e5 --- /dev/null +++ b/evaluations/ffmpeg_af_join_uaf_reproducer.c @@ -0,0 +1,124 @@ +#include +#include + +#include "libavfilter/avfilter.h" +#include "libavfilter/buffersink.h" +#include "libavfilter/buffersrc.h" +#include "libavutil/channel_layout.h" +#include "libavutil/error.h" +#include "libavutil/frame.h" +#include "libavutil/samplefmt.h" + +#define SAMPLE_RATE 48000 +#define NB_SAMPLES 64 + +static int make_input(AVFrame **frame, float value) +{ + AVChannelLayout mono = AV_CHANNEL_LAYOUT_MONO; + AVFrame *result = av_frame_alloc(); + int ret; + + if (!result) + return AVERROR(ENOMEM); + result->format = AV_SAMPLE_FMT_FLTP; + result->sample_rate = SAMPLE_RATE; + result->nb_samples = NB_SAMPLES; + result->pts = 0; + if ((ret = av_channel_layout_copy(&result->ch_layout, &mono)) < 0 || + (ret = av_frame_get_buffer(result, 0)) < 0) { + av_frame_free(&result); + return ret; + } + for (int i = 0; i < NB_SAMPLES; i++) + ((float *)result->extended_data[0])[i] = value; + *frame = result; + return 0; +} + +static int create_filter(AVFilterContext **context, AVFilterGraph *graph, + const char *filter_name, const char *instance_name, + const char *arguments) +{ + const AVFilter *filter = avfilter_get_by_name(filter_name); + + if (!filter) + return AVERROR_FILTER_NOT_FOUND; + return avfilter_graph_create_filter(context, filter, instance_name, + arguments, NULL, graph); +} + +int main(void) +{ + const char *source_args = + "time_base=1/48000:sample_rate=48000:sample_fmt=fltp:channel_layout=mono"; + const char *join_args = + "inputs=2:channel_layout=3.0:map=0.0-FL|0.0-FR|1.0-FC"; + AVFilterGraph *graph = avfilter_graph_alloc(); + AVFilterContext *source0 = NULL; + AVFilterContext *source1 = NULL; + AVFilterContext *join = NULL; + AVFilterContext *sink = NULL; + AVFrame *input0 = NULL; + AVFrame *input1 = NULL; + AVFrame *output = NULL; + volatile float observed; + int ret = 1; + + if (!graph) + return 2; + if (create_filter(&source0, graph, "abuffer", "source0", source_args) < 0 || + create_filter(&source1, graph, "abuffer", "source1", source_args) < 0 || + create_filter(&join, graph, "join", "join", join_args) < 0 || + create_filter(&sink, graph, "abuffersink", "sink", NULL) < 0) { + fprintf(stderr, "filter_creation_failed=1\n"); + goto done; + } + if (avfilter_link(source0, 0, join, 0) < 0 || + avfilter_link(source1, 0, join, 1) < 0 || + avfilter_link(join, 0, sink, 0) < 0 || + avfilter_graph_config(graph, NULL) < 0) { + fprintf(stderr, "graph_configuration_failed=1\n"); + goto done; + } + if (make_input(&input0, 1.0f) < 0 || make_input(&input1, 2.0f) < 0) { + fprintf(stderr, "input_allocation_failed=1\n"); + goto done; + } + + /* These calls transfer both input buffer references into the graph. */ + if (av_buffersrc_add_frame(source0, input0) < 0 || + av_buffersrc_add_frame(source1, input1) < 0) { + fprintf(stderr, "input_submission_failed=1\n"); + goto done; + } + + output = av_frame_alloc(); + if (!output || av_buffersink_get_frame(sink, output) < 0) { + fprintf(stderr, "output_retrieval_failed=1\n"); + goto done; + } + + fprintf(stderr, + "output_channels=%d duplicate_first_planes=%d " + "missing_third_plane_owner=%d\n", + output->ch_layout.nb_channels, + output->extended_data[0] == output->extended_data[1], + av_frame_get_plane_buffer(output, 2) == NULL); + fflush(stderr); + + /* + * join has already freed the input frames. With the faulty deduplication + * condition, no AVBufferRef in output owns input 1's plane. This ordinary + * downstream read therefore accesses its freed allocation under ASan. + */ + observed = ((float *)output->extended_data[2])[0]; + fprintf(stderr, "third_channel_sample=%f\n", observed); + ret = 0; + +done: + av_frame_free(&output); + av_frame_free(&input0); + av_frame_free(&input1); + avfilter_graph_free(&graph); + return ret; +} diff --git a/evaluations/ffmpeg_caf_seek_reproducer.c b/evaluations/ffmpeg_caf_seek_reproducer.c new file mode 100644 index 00000000..aacb3b6b --- /dev/null +++ b/evaluations/ffmpeg_caf_seek_reproducer.c @@ -0,0 +1,71 @@ +#include +#include + +#include "libavformat/avformat.h" +#include "libavutil/intreadwrite.h" + +static int write_fixture(char *path) +{ + uint8_t input[107] = { 0 }; + uint8_t *cursor = input; + int fd; + ssize_t written; + + memcpy(cursor, "caff", 4); cursor += 4; + AV_WB16(cursor, 1); cursor += 2; + AV_WB16(cursor, 0); cursor += 2; + + memcpy(cursor, "desc", 4); cursor += 4; + AV_WB64(cursor, 32); cursor += 8; + AV_WB64(cursor, UINT64_C(0x40bf400000000000)); cursor += 8; /* 8000.0 */ + memcpy(cursor, "lpcm", 4); cursor += 4; + AV_WB32(cursor, 0); cursor += 4; /* format flags */ + AV_WB32(cursor, 0); cursor += 4; /* variable packet bytes */ + AV_WB32(cursor, 0); cursor += 4; /* variable packet frames */ + AV_WB32(cursor, 1); cursor += 4; /* channels */ + AV_WB32(cursor, 16); cursor += 4; /* bits per channel */ + + memcpy(cursor, "data", 4); cursor += 4; + AV_WB64(cursor, 5); cursor += 8; + AV_WB32(cursor, 0); cursor += 4; /* edit count */ + *cursor++ = 0; /* one-byte packet */ + + memcpy(cursor, "pakt", 4); cursor += 4; + AV_WB64(cursor, 26); cursor += 8; + AV_WB64(cursor, 1); cursor += 8; /* one packet */ + AV_WB64(cursor, 1); cursor += 8; /* one valid frame */ + AV_WB32(cursor, 0); cursor += 4; /* priming */ + AV_WB32(cursor, 0); cursor += 4; /* remainder */ + *cursor++ = 1; /* packet size */ + *cursor++ = 1; /* packet duration */ + + if (cursor != input + sizeof(input)) + return -1; + fd = mkstemp(path); + if (fd < 0) + return -1; + written = write(fd, input, sizeof(input)); + close(fd); + return written == sizeof(input) ? 0 : -1; +} + +int main(void) +{ + char path[] = "/tmp/ffmpeg-caf-seek-XXXXXX"; + AVFormatContext *format = NULL; + int ret; + + if (write_fixture(path) < 0) + return 2; + ret = avformat_open_input(&format, path, NULL, NULL); + unlink(path); + if (ret < 0) + return 3; + + /* The parsed packet table contains one entry at timestamp zero. A forward + * search beyond it returns -1, which the CAF callback uses as an index. */ + ret = av_seek_frame(format, 0, 1, 0); + + avformat_close_input(&format); + return ret < 0 ? 1 : 0; +} diff --git a/evaluations/ffmpeg_dnn_classify_count_reproducer.c b/evaluations/ffmpeg_dnn_classify_count_reproducer.c new file mode 100644 index 00000000..8c718f27 --- /dev/null +++ b/evaluations/ffmpeg_dnn_classify_count_reproducer.c @@ -0,0 +1,49 @@ +#include + +#include "libavfilter/vf_dnn_classify.c" + +int main(void) +{ + AVFrame *frame = av_frame_alloc(); + DnnClassifyContext classifier = { 0 }; + AVFilterContext filter = { 0 }; + AVDetectionBBoxHeader *header; + AVDetectionBBox *bbox; + float confidence = 0.9f; + DNNData output = { + .data = &confidence, + .dims = { 1, 1, 1, 1 }, + .dt = DNN_FLOAT, + .layout = DL_NCHW, + }; + + if (!frame) + return 2; + header = av_detection_bbox_create_side_data(frame, 1); + if (!header) { + av_frame_free(&frame); + return 3; + } + + bbox = av_get_detection_bbox(header, 0); + classifier.confidence = 0.0f; + classifier.dnnctx.model_filename = "five-output-model"; + filter.priv = &classifier; + + fprintf(stderr, + "classification_capacity=%d callbacks=5 bbox_count=%u\n", + AV_NUM_DETECTION_BBOX_CLASSIFY, header->nb_bboxes); + for (int callback = 0; callback < 5; callback++) { + fprintf(stderr, "callback=%d classify_count=%u\n", + callback + 1, bbox->classify_count); + fflush(stderr); + if (dnn_classify_post_proc(frame, &output, 0, &filter) < 0) { + av_frame_free(&frame); + return 4; + } + } + + fprintf(stderr, "unexpected_fifth_classification_success=1\n"); + av_frame_free(&frame); + return 0; +} diff --git a/evaluations/ffmpeg_dnn_output_names_reproducer.c b/evaluations/ffmpeg_dnn_output_names_reproducer.c new file mode 100644 index 00000000..8c1c7572 --- /dev/null +++ b/evaluations/ffmpeg_dnn_output_names_reproducer.c @@ -0,0 +1,53 @@ +#include + +#include "libavfilter/dnn_filter_common.h" +#include "libavfilter/dnn_interface.h" + +const DNNModule *ff_get_dnn_module(DNNBackendType backend_type, void *log_ctx) +{ + (void)backend_type; + (void)log_ctx; + return NULL; +} + +void *ff_dnn_child_next(DnnContext *ctx, void *prev) +{ + (void)ctx; + (void)prev; + return NULL; +} + +const AVClass *ff_dnn_child_class_iterate_with_mask(void **iter, + unsigned int backend_mask) +{ + (void)iter; + (void)backend_mask; + return NULL; +} + +void ff_dnn_init_child_class(DnnContext *ctx) +{ + (void)ctx; +} + +int main(void) +{ + DnnContext context = { 0 }; + + context.backend_type = DNN_TF; + context.model_filename = "unused-model.pb"; + context.model_inputname = "input"; + context.model_outputnames_string = "count&scores&classes&boxes"; + + fprintf(stderr, "tensorflow_backend=1 requested_outputs=4\n"); + fflush(stderr); + if (ff_dnn_init(&context, DFT_ANALYTICS_DETECT, NULL) >= 0) { + fprintf(stderr, "unexpected_initialization_success=1\n"); + ff_dnn_uninit(&context); + return 0; + } + + fprintf(stderr, "initialization_returned_without_overflow=1\n"); + ff_dnn_uninit(&context); + return 0; +} diff --git a/evaluations/ffmpeg_dnn_output_shape_reproducer.c b/evaluations/ffmpeg_dnn_output_shape_reproducer.c new file mode 100644 index 00000000..186a1e8c --- /dev/null +++ b/evaluations/ffmpeg_dnn_output_shape_reproducer.c @@ -0,0 +1,61 @@ +#include + +#include "libavfilter/dnn_interface.h" +#include "libavfilter/dnn/dnn_io_proc.h" +#include "libavutil/frame.h" +#include "libavutil/mem.h" +#include "libavutil/pixfmt.h" + +#define WIDTH 4 +#define HEIGHT 4 +#define CHANNELS 1 + +int main(void) +{ + AVFrame *frame = av_frame_alloc(); + float *tensor = NULL; + DNNData output = { + .dims = { 1, CHANNELS, HEIGHT, WIDTH }, + .dt = DNN_FLOAT, + .layout = DL_NCHW, + .scale = 255.0f, + .mean = 0.0f, + }; + int ret = 1; + + if (!frame) + return 2; + frame->format = AV_PIX_FMT_RGB24; + frame->width = WIDTH; + frame->height = HEIGHT; + if (av_frame_get_buffer(frame, 0) < 0) + goto done; + + /* A model may declare a one-channel output for an RGB input frame. */ + tensor = av_malloc_array(CHANNELS * WIDTH * HEIGHT, sizeof(*tensor)); + if (!tensor) + goto done; + for (int i = 0; i < CHANNELS * WIDTH * HEIGHT; i++) + tensor[i] = (float)i; + output.data = tensor; + + fprintf(stderr, + "tensor_channels=%d tensor_elements=%d output_format=rgb24\n", + output.dims[1], CHANNELS * WIDTH * HEIGHT); + fflush(stderr); + + /* + * The production NCHW postprocessor sizes middle_data from dims[1], but + * unconditionally converts frame->width * 3 elements per row for RGB24. + * The recorder leaves this one production function uninstrumented only to + * get past its independent one-pointer plane-array violation; instrumented + * libswscale then catches the undersized destination allocation. + */ + ret = ff_proc_from_dnn_to_frame(frame, &output, NULL); + fprintf(stderr, "postprocess_return=%d\n", ret); + +done: + av_free(tensor); + av_frame_free(&frame); + return ret < 0 ? 3 : ret; +} diff --git a/evaluations/ffmpeg_dovi_rpu_reproducer.c b/evaluations/ffmpeg_dovi_rpu_reproducer.c new file mode 100644 index 00000000..a6543d8c --- /dev/null +++ b/evaluations/ffmpeg_dovi_rpu_reproducer.c @@ -0,0 +1,126 @@ +#include +#include +#include + +#include "libavcodec/dovi_rpu.h" +#include "libavutil/dovi_meta.h" +#include "libavutil/mem.h" + +typedef struct BitWriter { + uint8_t data[16384]; + size_t bits; +} BitWriter; + +static void write_bits(BitWriter *writer, unsigned count, uint64_t value) +{ + for (unsigned i = count; i > 0; i--) { + size_t bit = writer->bits++; + if ((value >> (i - 1)) & 1) + writer->data[bit >> 3] |= 1U << (7 - (bit & 7)); + } +} + +static void write_ue(BitWriter *writer, uint32_t value) +{ + uint64_t code = (uint64_t)value + 1; + unsigned significant = 64 - __builtin_clzll(code); + + write_bits(writer, significant - 1, 0); + write_bits(writer, significant, code); +} + +static void write_se(BitWriter *writer, int32_t value) +{ + uint32_t code = value > 0 ? 2U * value - 1 : -2U * value; + write_ue(writer, code); +} + +static size_t build_parsed_rpu(BitWriter *writer) +{ + /* This exceeds set_se_golomb()'s documented 16-bit domain, but is + * accepted by get_se_golomb_long(). Repeating it also exceeds the + * generator's constant 177-byte allowance for an MMR segment. */ + const int32_t integer_coefficient = 100000000; + + write_bits(writer, 8, 25); /* Dolby Vision NAL prefix */ + write_bits(writer, 6, 2); /* rpu_type */ + write_bits(writer, 11, 0); /* rpu_format */ + write_bits(writer, 4, 1); /* vdr_rpu_profile */ + write_bits(writer, 4, 0); /* vdr_rpu_level */ + write_bits(writer, 1, 1); /* vdr_seq_info_present */ + write_bits(writer, 1, 0); /* chroma_resampling_explicit_filter_flag */ + write_bits(writer, 2, RPU_COEFF_FIXED); + write_ue(writer, 31); /* high parser-accepted denominator */ + write_bits(writer, 2, 0); /* vdr_rpu_normalized_idc */ + write_bits(writer, 1, 0); /* bl_video_full_range_flag */ + write_ue(writer, 2); /* bl_bit_depth = 10 */ + write_ue(writer, 2); /* ext_mapping_idc = 0, el_bit_depth = 10 */ + write_ue(writer, 2); /* vdr_bit_depth = 10 */ + write_bits(writer, 1, 0); /* spatial_resampling_filter_flag */ + write_bits(writer, 3, 0); /* dm_compression */ + write_bits(writer, 1, 0); /* el_spatial_resampling_filter_flag */ + write_bits(writer, 1, 1); /* disable_residual_flag */ + write_bits(writer, 1, 0); /* vdr_dm_metadata_present */ + write_bits(writer, 1, 0); /* use_prev_vdr_rpu */ + write_ue(writer, 0); /* vdr_rpu_id */ + write_ue(writer, 0); /* mapping_color_space */ + write_ue(writer, 0); /* mapping_chroma_format_idc */ + + for (int c = 0; c < 3; c++) { + write_ue(writer, AV_DOVI_MAX_PIECES - 1); /* nine pivots */ + for (int i = 0; i < AV_DOVI_MAX_PIECES + 1; i++) + write_bits(writer, 10, i ? 1 : 0); + } + + write_ue(writer, 0); /* num_x_partitions - 1 */ + write_ue(writer, 0); /* num_y_partitions - 1 */ + + for (int c = 0; c < 3; c++) { + for (int i = 0; i < AV_DOVI_MAX_PIECES; i++) { + write_ue(writer, AV_DOVI_MAPPING_MMR); + write_bits(writer, 2, 2); /* mmr_order = 3 */ + for (int coefficient = 0; coefficient < 22; coefficient++) { + write_se(writer, integer_coefficient); + write_bits(writer, 31, 0); /* fractional component */ + } + } + } + + while (writer->bits & 7) + write_bits(writer, 1, 0); + write_bits(writer, 32, 0); /* CRC is optional without AV_EF_CRCCHECK */ + write_bits(writer, 8, 0x80); /* terminator */ + return writer->bits / 8; +} + +int main(void) +{ + BitWriter input = { 0 }; + DOVIContext parser = { 0 }; + DOVIContext generator = { 0 }; + AVDOVIMetadata *metadata = NULL; + uint8_t *rpu = NULL; + int rpu_size = 0; + int ret; + + size_t input_size = build_parsed_rpu(&input); + fprintf(stderr, "parsed_rpu_bytes=%zu segments=%d coefficients_per_segment=%d\n", + input_size, 3 * AV_DOVI_MAX_PIECES, 22); + parser.cfg.dv_profile = 8; + ret = ff_dovi_rpu_parse(&parser, input.data, input_size, 0); + if (ret < 0) + return 2; + + ret = ff_dovi_get_metadata(&parser, &metadata); + if (ret <= 0) + return 3; + + generator.cfg.dv_profile = 8; + ret = ff_dovi_rpu_generate(&generator, metadata, 0, &rpu, &rpu_size); + + av_free(rpu); + av_free(metadata); + ff_dovi_ctx_unref(&parser); + ff_dovi_ctx_unref(&generator); + return ret < 0 ? 1 : 0; +} diff --git a/evaluations/ffmpeg_drawgraph_missing_primary_reproducer.c b/evaluations/ffmpeg_drawgraph_missing_primary_reproducer.c new file mode 100644 index 00000000..e146ca28 --- /dev/null +++ b/evaluations/ffmpeg_drawgraph_missing_primary_reproducer.c @@ -0,0 +1,109 @@ +#include +#include + +#include "libavfilter/avfilter.h" +#include "libavfilter/buffersink.h" +#include "libavfilter/buffersrc.h" +#include "libavutil/dict.h" +#include "libavutil/error.h" +#include "libavutil/frame.h" +#include "libavutil/pixfmt.h" + +#define INPUT_WIDTH 2 +#define INPUT_HEIGHT 2 +#define OUTPUT_WIDTH 2 +#define OUTPUT_HEIGHT 2 +#define FRAME_LIMIT 4096 + +static int create_filter(AVFilterContext **context, AVFilterGraph *graph, + const char *filter_name, const char *instance_name, + const char *arguments) +{ + const AVFilter *filter = avfilter_get_by_name(filter_name); + + if (!filter) + return AVERROR_FILTER_NOT_FOUND; + return avfilter_graph_create_filter(context, filter, instance_name, + arguments, NULL, graph); +} + +static AVFrame *make_input(int64_t pts) +{ + AVFrame *frame = av_frame_alloc(); + + if (!frame) + return NULL; + frame->format = AV_PIX_FMT_YUV420P; + frame->width = INPUT_WIDTH; + frame->height = INPUT_HEIGHT; + frame->pts = pts; + frame->duration = 1; + if (av_frame_get_buffer(frame, 0) < 0 || + av_dict_set(&frame->metadata, "secondary", "0", 0) < 0) { + av_frame_free(&frame); + return NULL; + } + return frame; +} + +int main(void) +{ + const char *source_args = + "video_size=2x2:pix_fmt=yuv420p:time_base=1/25:pixel_aspect=1/1"; + const char *drawgraph_args = + "m1=missing:m2=secondary:size=2x2:mode=dot:slide=frame"; + AVFilterGraph *graph = avfilter_graph_alloc(); + AVFilterContext *source = NULL; + AVFilterContext *drawgraph = NULL; + AVFilterContext *sink = NULL; + int ret = 1; + + if (!graph) + return 2; + if (create_filter(&source, graph, "buffer", "source", source_args) < 0 || + create_filter(&drawgraph, graph, "drawgraph", "drawgraph", + drawgraph_args) < 0 || + create_filter(&sink, graph, "buffersink", "sink", NULL) < 0) { + fprintf(stderr, "filter_creation_failed=1\n"); + goto done; + } + if (avfilter_link(source, 0, drawgraph, 0) < 0 || + avfilter_link(drawgraph, 0, sink, 0) < 0 || + avfilter_graph_config(graph, NULL) < 0) { + fprintf(stderr, "graph_configuration_failed=1\n"); + goto done; + } + + fprintf(stderr, + "missing_primary_metadata=1 secondary_metadata=1 output_width=%d\n", + OUTPUT_WIDTH); + fflush(stderr); + for (int64_t pts = 0; pts < FRAME_LIMIT; pts++) { + AVFrame *input = make_input(pts); + AVFrame *output = av_frame_alloc(); + + if (!input || !output) { + av_frame_free(&input); + av_frame_free(&output); + fprintf(stderr, "frame_allocation_failed=1\n"); + goto done; + } + if (av_buffersrc_add_frame(source, input) < 0 || + av_buffersink_get_frame(sink, output) < 0) { + av_frame_free(&input); + av_frame_free(&output); + fprintf(stderr, "frame_processing_failed=1 pts=%lld\n", + (long long)pts); + goto done; + } + av_frame_free(&input); + av_frame_free(&output); + } + + fprintf(stderr, "frame_limit_reached=1\n"); + ret = 0; + +done: + avfilter_graph_free(&graph); + return ret; +} diff --git a/evaluations/ffmpeg_dvdsub_odd_width_reproducer.c b/evaluations/ffmpeg_dvdsub_odd_width_reproducer.c new file mode 100644 index 00000000..43e1a650 --- /dev/null +++ b/evaluations/ffmpeg_dvdsub_odd_width_reproducer.c @@ -0,0 +1,52 @@ +#include +#include +#include + +#include "libavcodec/avcodec.h" + +int main(void) +{ + const AVCodec *codec = avcodec_find_encoder(AV_CODEC_ID_DVD_SUBTITLE); + AVCodecContext *context = avcodec_alloc_context3(codec); + AVSubtitleRect rect = { 0 }; + AVSubtitleRect *rects[] = { &rect }; + AVSubtitle subtitle = { 0 }; + uint32_t palette[256] = { 0 }; + uint8_t bitmap[200]; + uint8_t output[142]; + int ret = 2; + + if (!codec || !context) + goto done; + context->width = 720; + context->height = 576; + context->time_base = (AVRational) { 1, 1000 }; + if (avcodec_open2(context, codec, NULL) < 0) + goto done; + + memset(bitmap, 1, sizeof(bitmap)); + palette[1] = 0xffffffff; + rect.x = 0; + rect.y = 0; + rect.w = 1; + rect.h = 200; + rect.type = SUBTITLE_BITMAP; + rect.linesize[0] = 1; + rect.data[0] = bitmap; + rect.data[1] = (uint8_t *)palette; + subtitle.num_rects = 1; + subtitle.rects = rects; + subtitle.start_display_time = 0; + subtitle.end_display_time = 1000; + + fprintf(stderr, + "width=%d height=%d output_capacity=%zu checked_rle_budget=%d actual_rle_bytes=%d\n", + rect.w, rect.h, sizeof(output), rect.w * rect.h / 2, + rect.h); + ret = avcodec_encode_subtitle(context, output, sizeof(output), &subtitle); + fprintf(stderr, "encode_return=%d\n", ret); + +done: + avcodec_free_context(&context); + return ret < 0 ? 1 : 0; +} diff --git a/evaluations/ffmpeg_ffv1_remap_reproducer.c b/evaluations/ffmpeg_ffv1_remap_reproducer.c new file mode 100644 index 00000000..a3cfeaf2 --- /dev/null +++ b/evaluations/ffmpeg_ffv1_remap_reproducer.c @@ -0,0 +1,89 @@ +#include +#include +#include + +#include "libavcodec/avcodec.h" +#include "libavformat/avformat.h" + +static int decode_packet(const AVCodecParameters *parameters, + const AVPacket *source, int byte, int bit) +{ + const AVCodec *codec = avcodec_find_decoder(parameters->codec_id); + AVCodecContext *context = avcodec_alloc_context3(codec); + AVFrame *frame = av_frame_alloc(); + AVPacket *packet = av_packet_clone(source); + int ret = AVERROR(ENOMEM); + + if (!codec || !context || !frame || !packet) + goto done; + if ((ret = av_packet_make_writable(packet)) < 0) + goto done; + if ((ret = avcodec_parameters_to_context(context, parameters)) < 0 || + (ret = avcodec_open2(context, codec, NULL)) < 0) + goto done; + + packet->data[byte] ^= 1U << bit; + if ((ret = avcodec_send_packet(context, packet)) >= 0) + ret = avcodec_receive_frame(context, frame); + if (ret >= 0) + fprintf(stderr, "decoded_byte=%d decoded_bit=%d\n", byte, bit); + +done: + av_packet_free(&packet); + av_frame_free(&frame); + avcodec_free_context(&context); + return ret; +} + +int main(int argc, char **argv) +{ + AVFormatContext *format = NULL; + AVPacket *packet = av_packet_alloc(); + int stream_index; + int ret = 1; + + av_log_set_level(AV_LOG_QUIET); + if ((argc != 2 && argc != 4) || !packet) { + fprintf(stderr, "usage: %s sample.nut [byte bit]\n", argv[0]); + goto done; + } + if (avformat_open_input(&format, argv[1], NULL, NULL) < 0 || + avformat_find_stream_info(format, NULL) < 0) + goto done; + stream_index = av_find_best_stream(format, AVMEDIA_TYPE_VIDEO, + -1, -1, NULL, 0); + if (stream_index < 0) + goto done; + while (av_read_frame(format, packet) >= 0) { + if (packet->stream_index == stream_index) + break; + av_packet_unref(packet); + } + if (!packet->data) + goto done; + + fprintf(stderr, "packet_size=%d\n", packet->size); + if (argc == 4) { + const int byte = atoi(argv[2]); + const int bit = atoi(argv[3]); + if (byte < 0 || byte >= packet->size || bit < 0 || bit > 7) + goto done; + fprintf(stderr, "mutation_byte=%d mutation_bit=%d\n", byte, bit); + decode_packet(format->streams[stream_index]->codecpar, + packet, byte, bit); + } else { + for (int byte = 0; byte < packet->size; byte++) { + for (int bit = 0; bit < 8; bit++) { + fprintf(stderr, "mutation_byte=%d mutation_bit=%d\n", byte, bit); + decode_packet(format->streams[stream_index]->codecpar, + packet, byte, bit); + } + } + } + ret = 0; + +done: + av_packet_free(&packet); + avformat_close_input(&format); + return ret; +} diff --git a/evaluations/ffmpeg_hls_sample_aes_reproducer.c b/evaluations/ffmpeg_hls_sample_aes_reproducer.c new file mode 100644 index 00000000..4ce4b795 --- /dev/null +++ b/evaluations/ffmpeg_hls_sample_aes_reproducer.c @@ -0,0 +1,39 @@ +#include +#include + +#include "libavcodec/packet.h" +#include "libavformat/hls_sample_encryption.h" +#include "libavutil/aes.h" +#include "libavutil/mem.h" + +int main(void) +{ + HLSCryptoContext crypto = { 0 }; + AVPacket *packet = av_packet_alloc(); + const int packet_size = 64; + const int declared_frame_size = 8191; + uint8_t *adts; + int ret; + + if (!packet || av_new_packet(packet, packet_size) < 0) + return 2; + crypto.aes_ctx = av_aes_alloc(); + if (!crypto.aes_ctx) + return 3; + + memset(packet->data, 0, packet->size); + adts = packet->data; + adts[0] = 0xff; + adts[1] = 0xf1; /* sync, MPEG-4, no CRC */ + adts[2] = 0x50; /* AAC LC, 44.1 kHz */ + adts[3] = 0x80 | ((declared_frame_size >> 11) & 0x03); + adts[4] = (uint8_t)(declared_frame_size >> 3); + adts[5] = (declared_frame_size & 0x07) << 5; + adts[6] = 0xfc; + + ret = ff_hls_senc_decrypt_frame(AV_CODEC_ID_AAC, &crypto, packet); + + av_free(crypto.aes_ctx); + av_packet_free(&packet); + return ret < 0 ? 1 : 0; +} diff --git a/evaluations/ffmpeg_ismindex_reproducer.c b/evaluations/ffmpeg_ismindex_reproducer.c new file mode 100644 index 00000000..e39babef --- /dev/null +++ b/evaluations/ffmpeg_ismindex_reproducer.c @@ -0,0 +1,33 @@ +#include + +#include "libavutil/intreadwrite.h" + +/* Include the production tool so its private atom reader can be exercised + * directly without changing the vulnerable source. */ +#define main ffmpeg_ismindex_main +#include "tools/ismindex.c" +#undef main + +int main(void) +{ + uint8_t *input = av_mallocz(32); + AVIOContext *io; + struct Tracks tracks = { 0 }; + + if (!input) + return 2; + AV_WB32(input, 0); /* zero-sized atom */ + AV_WB32(input + 4, MKBETAG('t', 'f', 'r', 'a')); + AV_WB32(input + 12, 1); /* unknown track ID */ + + io = avio_alloc_context(input, 32, 0, NULL, NULL, NULL, NULL); + if (!io) + return 3; + + while (!read_tfra(&tracks, 0, io)) { + /* The production read_mfra loop has this same empty body. */ + } + + avio_context_free(&io); + return 0; +} diff --git a/evaluations/ffmpeg_jv_bitstream_reproducer.c b/evaluations/ffmpeg_jv_bitstream_reproducer.c new file mode 100644 index 00000000..be90cf91 --- /dev/null +++ b/evaluations/ffmpeg_jv_bitstream_reproducer.c @@ -0,0 +1,60 @@ +#include +#include +#include + +#include "libavcodec/avcodec.h" +#include "libavutil/frame.h" + +#define WIDTH 64 +#define HEIGHT 64 +#define VIDEO_SIZE 16 + +static void write_le32(uint8_t *dst, uint32_t value) +{ + dst[0] = value; + dst[1] = value >> 8; + dst[2] = value >> 16; + dst[3] = value >> 24; +} + +int main(void) +{ + const AVCodec *codec = avcodec_find_decoder(AV_CODEC_ID_JV); + AVCodecContext *context = NULL; + AVPacket *packet = NULL; + AVFrame *frame = NULL; + int ret = 1; + + if (!codec) + return 2; + context = avcodec_alloc_context3(codec); + packet = av_packet_alloc(); + frame = av_frame_alloc(); + if (!context || !packet || !frame) + goto done; + + context->width = WIDTH; + context->height = HEIGHT; + if (avcodec_open2(context, codec, NULL) < 0 || + av_new_packet(packet, 5 + VIDEO_SIZE) < 0) + goto done; + + write_le32(packet->data, VIDEO_SIZE); + packet->data[4] = 0; + memset(packet->data + 5, 0xff, VIDEO_SIZE); + + fprintf(stderr, "blocks=%d video_bytes=%d max_recursive_path=1\n", + WIDTH / 8 * (HEIGHT / 8), VIDEO_SIZE); + fflush(stderr); + if (avcodec_send_packet(context, packet) < 0) + goto done; + ret = avcodec_receive_frame(context, frame); + fprintf(stderr, "decoder_return=%d\n", ret); + ret = 0; + +done: + av_packet_free(&packet); + av_frame_free(&frame); + avcodec_free_context(&context); + return ret; +} diff --git a/evaluations/ffmpeg_lcl_multithread_reproducer.c b/evaluations/ffmpeg_lcl_multithread_reproducer.c new file mode 100644 index 00000000..6e929033 --- /dev/null +++ b/evaluations/ffmpeg_lcl_multithread_reproducer.c @@ -0,0 +1,138 @@ +#include +#include +#include + +#include + +#include "libavcodec/avcodec.h" +#include "libavutil/frame.h" +#include "libavutil/mem.h" + +#define WIDTH 16 +#define HEIGHT 16 +#define FRAME_BYTES (WIDTH * HEIGHT * 3) +#define HALF_BYTES (FRAME_BYTES / 2) + +static void write_le32(uint8_t *dst, uint32_t value) +{ + dst[0] = value; + dst[1] = value >> 8; + dst[2] = value >> 16; + dst[3] = value >> 24; +} + +static int make_packet(AVPacket *packet, const uint8_t *first, size_t first_size, + const uint8_t *second, size_t second_size, + uint32_t claimed_half_size) +{ + uLongf first_bound = compressBound(first_size); + uLongf second_bound = compressBound(second_size); + uint8_t *first_compressed = av_malloc(first_bound); + uint8_t *second_compressed = av_malloc(second_bound); + int ret = -1; + + if (!first_compressed || !second_compressed) + goto done; + if (compress2(first_compressed, &first_bound, first, first_size, + Z_BEST_SPEED) != Z_OK || + compress2(second_compressed, &second_bound, second, second_size, + Z_BEST_SPEED) != Z_OK) + goto done; + if (av_new_packet(packet, 8 + first_bound + second_bound) < 0) + goto done; + + write_le32(packet->data, first_bound); + write_le32(packet->data + 4, claimed_half_size); + memcpy(packet->data + 8, first_compressed, first_bound); + memcpy(packet->data + 8 + first_bound, second_compressed, second_bound); + ret = 0; + +done: + av_free(first_compressed); + av_free(second_compressed); + return ret; +} + +static int decode(AVCodecContext *context, AVPacket *packet, AVFrame *frame) +{ + int ret = avcodec_send_packet(context, packet); + if (ret < 0) + return ret; + return avcodec_receive_frame(context, frame); +} + +static int get_buffer(AVCodecContext *context, AVFrame *frame, int flags) +{ + int ret = avcodec_default_get_buffer2(context, frame, flags); + + if (ret < 0) + return ret; + for (int y = 0; y < context->height; y++) + memset(frame->data[0] + y * frame->linesize[0], 0xcc, + context->width * 3); + return 0; +} + +int main(void) +{ + const AVCodec *codec = avcodec_find_decoder(AV_CODEC_ID_ZLIB); + AVCodecContext *context = NULL; + AVPacket *packet = NULL; + AVFrame *frame = NULL; + const uint8_t short_first[] = { 0x11, 0x22, 0x33 }; + const uint8_t short_second[] = { 0x44, 0x55, 0x66 }; + int stale_bytes = 0; + int fresh_bytes = 0; + int ret = 1; + + if (!codec) + return 2; + context = avcodec_alloc_context3(codec); + packet = av_packet_alloc(); + frame = av_frame_alloc(); + if (!context || !packet || !frame) + goto done; + + context->width = WIDTH; + context->height = HEIGHT; + context->thread_count = 1; + context->get_buffer2 = get_buffer; + context->extradata = av_mallocz(8 + AV_INPUT_BUFFER_PADDING_SIZE); + if (!context->extradata) + goto done; + context->extradata_size = 8; + context->extradata[4] = 2; /* IMGTYPE_RGB24 */ + context->extradata[5] = 0xff; /* COMP_ZLIB_NORMAL */ + context->extradata[6] = 1; /* FLAG_MULTITHREAD */ + context->extradata[7] = 3; /* CODEC_ZLIB */ + if (avcodec_open2(context, codec, NULL) < 0) + goto done; + + if (make_packet(packet, short_first, sizeof(short_first), short_second, + sizeof(short_second), sizeof(short_first)) < 0 || + decode(context, packet, frame) < 0) + goto done; + + for (int y = 0; y < HEIGHT; y++) { + const uint8_t *row = frame->data[0] + y * frame->linesize[0]; + for (int x = 0; x < WIDTH * 3; x++) { + stale_bytes += row[x] == 0xa5; + fresh_bytes += row[x] == short_first[0] || + row[x] == short_first[1] || + row[x] == short_first[2] || + row[x] == short_second[0] || + row[x] == short_second[1] || + row[x] == short_second[2]; + } + } + + printf("fresh_bytes=%d leaked_malloc_fill_bytes=%d frame_bytes=%d\n", + fresh_bytes, stale_bytes, FRAME_BYTES); + ret = fresh_bytes == 6 && stale_bytes == FRAME_BYTES - 6 ? 0 : 1; + +done: + av_packet_free(&packet); + av_frame_free(&frame); + avcodec_free_context(&context); + return ret; +} diff --git a/evaluations/ffmpeg_magicyuv_truncated_slice_reproducer.c b/evaluations/ffmpeg_magicyuv_truncated_slice_reproducer.c new file mode 100644 index 00000000..2a39e19c --- /dev/null +++ b/evaluations/ffmpeg_magicyuv_truncated_slice_reproducer.c @@ -0,0 +1,111 @@ +#include +#include +#include + +#include "libavcodec/avcodec.h" +#include "libavutil/frame.h" + +#define TRUNCATED_PACKET_SIZE 300 +#define RAW_PACKET_SIZE 556 + +static void make_packet_data(uint8_t *packet, int raw) +{ + static const uint8_t header[] = { + 0x4d, 0x41, 0x47, 0x59, 0x20, 0x00, 0x00, 0x00, + 0x07, 0x6b, 0x0c, 0x00, 0x00, 0x00, 0x20, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0x0a, 0x01, 0x00, 0x00, /* table/slice area starts at 298 */ + 0x0a, 0x01, 0x00, 0x00, /* plane zero's only slice starts there */ + 0x01, 0x00, /* one plane and its slice index */ + }; + + memcpy(packet, header, sizeof(header)); + packet[42] = 1; + memset(packet + 43, 9, 254); + packet[297] = 8; + packet[298] = raw; + packet[299] = 1; /* supported left prediction */ + if (raw) + memset(packet + TRUNCATED_PACKET_SIZE, 0x41, 256); +} + +int main(void) +{ + const AVCodec *codec = avcodec_find_decoder(AV_CODEC_ID_MAGICYUV); + AVCodecContext *context = NULL; + AVPacket *packet = NULL; + AVFrame *frame = NULL; + uint8_t packet_data[RAW_PACKET_SIZE]; + unsigned transformed_prior_frame_bytes = 0; + int ret = 1; + + context = avcodec_alloc_context3(codec); + packet = av_packet_alloc(); + frame = av_frame_alloc(); + if (!codec || !context || !packet || !frame) { + fprintf(stderr, "allocation_failed=1\n"); + goto done; + } + context->width = context->coded_width = 16; + context->height = context->coded_height = 16; + ret = avcodec_open2(context, codec, NULL); + if (ret < 0) { + fprintf(stderr, "open_failed=%d\n", ret); + goto done; + } + ret = av_new_packet(packet, RAW_PACKET_SIZE); + if (ret < 0) { + fprintf(stderr, "packet_allocation_failed=%d\n", ret); + goto done; + } + make_packet_data(packet_data, 1); + memcpy(packet->data, packet_data, RAW_PACKET_SIZE); + + fprintf(stderr, "raw_packet_size=%d\n", RAW_PACKET_SIZE); + ret = avcodec_send_packet(context, packet); + if (ret < 0) { + fprintf(stderr, "send_failed=%d\n", ret); + goto done; + } + ret = avcodec_receive_frame(context, frame); + if (ret < 0) { + fprintf(stderr, "receive_failed=%d\n", ret); + goto done; + } + av_packet_unref(packet); + av_frame_unref(frame); + + ret = av_new_packet(packet, TRUNCATED_PACKET_SIZE); + if (ret < 0) + goto done; + make_packet_data(packet_data, 0); + memcpy(packet->data, packet_data, TRUNCATED_PACKET_SIZE); + ret = avcodec_send_packet(context, packet); + if (ret < 0) { + fprintf(stderr, "truncated_send_failed=%d\n", ret); + goto done; + } + ret = avcodec_receive_frame(context, frame); + if (ret < 0) { + fprintf(stderr, "truncated_receive_failed=%d\n", ret); + goto done; + } + for (int y = 0; y < frame->height; y++) + for (int x = 0; x < frame->width; x++) + transformed_prior_frame_bytes += + frame->data[0][y * frame->linesize[0] + x] == + (uint8_t)(0x41 * (x + y + 1) * (x + y + 2) / 2); + + fprintf(stderr, + "truncated_frame_decoded=1 transformed_prior_frame_bytes=%u " + "total_pixels=256\n", + transformed_prior_frame_bytes); + ret = transformed_prior_frame_bytes == 256 ? 0 : 4; + +done: + av_frame_free(&frame); + av_packet_free(&packet); + avcodec_free_context(&context); + return ret; +} diff --git a/evaluations/ffmpeg_mpc7_lastframelen_reproducer.c b/evaluations/ffmpeg_mpc7_lastframelen_reproducer.c new file mode 100644 index 00000000..c97b2c1f --- /dev/null +++ b/evaluations/ffmpeg_mpc7_lastframelen_reproducer.c @@ -0,0 +1,67 @@ +#include +#include + +#include "libavcodec/avcodec.h" +#include "libavutil/channel_layout.h" +#include "libavutil/mem.h" + +int main(void) +{ + const AVCodec *codec = avcodec_find_decoder(AV_CODEC_ID_MUSEPACK7); + AVCodecContext *context; + AVFrame *frame; + AVPacket *packet; + volatile uint32_t checksum = 0; + int ret; + + if (!codec) + return 2; + context = avcodec_alloc_context3(codec); + frame = av_frame_alloc(); + packet = av_packet_alloc(); + if (!context || !frame || !packet) + return 3; + + context->extradata = av_mallocz(16 + AV_INPUT_BUFFER_PADDING_SIZE); + if (!context->extradata) + return 4; + context->extradata_size = 16; + context->ch_layout = (AVChannelLayout)AV_CHANNEL_LAYOUT_STEREO; + context->sample_rate = 44100; + + /* mpc7_decode_init byte-swaps the four 32-bit extradata words. In the + * resulting bitstream, set gapless=1 and the 11-bit last-frame length to + * its maximum value, 2047. */ + context->extradata[14] = 0xf8; + context->extradata[15] = 0xff; + + ret = avcodec_open2(context, codec, NULL); + if (ret < 0) + return 5; + if (av_new_packet(packet, 1024) < 0) + return 6; + memset(packet->data, 0, packet->size); + packet->data[1] = 1; /* last_frame */ + + ret = avcodec_send_packet(context, packet); + if (ret < 0) + return 7; + ret = avcodec_receive_frame(context, frame); + if (ret < 0) + return 8; + if (frame->nb_samples != 2047 || frame->format != AV_SAMPLE_FMT_S16P) + return 9; + + /* A normal consumer trusts the public nb_samples contract. The decoder's + * backing planes contain only the 1152 samples requested from ff_get_buffer. */ + for (int channel = 0; channel < 2; channel++) { + const int16_t *samples = (const int16_t *)frame->extended_data[channel]; + for (int sample = 0; sample < frame->nb_samples; sample++) + checksum += samples[sample]; + } + + av_packet_free(&packet); + av_frame_free(&frame); + avcodec_free_context(&context); + return checksum == UINT32_MAX ? 1 : 0; +} diff --git a/evaluations/ffmpeg_qdm2_reproducer.c b/evaluations/ffmpeg_qdm2_reproducer.c new file mode 100644 index 00000000..5409454d --- /dev/null +++ b/evaluations/ffmpeg_qdm2_reproducer.c @@ -0,0 +1,44 @@ +#include + +#include "libavformat/avformat.h" +#include "libavformat/rtpdec.h" +#include "libavformat/rtpdec_formats.h" +#include "libavutil/mem.h" + +int main(void) +{ + AVFormatContext *format = avformat_alloc_context(); + AVPacket *packet = av_packet_alloc(); + AVStream *stream; + PayloadContext *payload; + uint32_t timestamp = 0; + uint8_t input[40] = { 0 }; + int ret; + + if (!format || !packet) + return 2; + stream = avformat_new_stream(format, NULL); + payload = av_mallocz(ff_qdm2_dynamic_handler.priv_data_size); + if (!stream || !payload) + return 3; + + input[0] = 0xFF; + input[1] = 30; /* configuration item length */ + input[2] = 4; /* stream configuration with extradata */ + input[30] = 1; /* big-endian block_size = 1 */ + input[31] = 2; /* end-item length */ + input[32] = 0; /* end-item type */ + input[33] = 0; /* subpacket ordering ID */ + input[34] = 0; /* one-byte subpacket length */ + input[35] = 1; /* one byte of data */ + input[36] = 0x41; + + ret = ff_qdm2_dynamic_handler.parse_packet( + format, payload, stream, packet, ×tamp, input, 37, 1, 0 + ); + + av_free(payload); + av_packet_free(&packet); + avformat_free_context(format); + return ret < 0 ? 1 : 0; +} diff --git a/evaluations/ffmpeg_rdt_aac_reproducer.c b/evaluations/ffmpeg_rdt_aac_reproducer.c new file mode 100644 index 00000000..dbbb9e60 --- /dev/null +++ b/evaluations/ffmpeg_rdt_aac_reproducer.c @@ -0,0 +1,75 @@ +#include +#include + +#include "libavformat/avformat.h" + +/* Private rmdec state consumed through the public RMStream pointer. */ +struct RMStream { + AVPacket pkt; + int videobufsize; + int videobufpos; + int curpic_num; + int cur_slice, slices; + int64_t pktpos; + int64_t audiotimestamp; + int sub_packet_cnt; + int sub_packet_size, sub_packet_h, coded_framesize; + int audio_framesize; + int sub_packet_lengths[16]; + int32_t deint_id; +}; + +/* Compile the production parser into this translation unit to reach its + * private payload context and packet callback without modifying FFmpeg. */ +#include "libavformat/rdt.c" + +int main(void) +{ + const int input_size = RTP_MAX_PACKET_LENGTH + 1024; + AVFormatContext *format = avformat_alloc_context(); + AVPacket *packet = av_packet_alloc(); + PayloadContext *payload = av_mallocz(sizeof(*payload)); + AVStream *stream; + uint8_t *input; + uint32_t timestamp = 0; + int ret; + + if (!format || !packet || !payload) + return 2; + stream = avformat_new_stream(format, NULL); + input = av_mallocz(input_size); + if (!stream || !input) + return 3; + if (rdt_init(format, stream->index, payload) < 0) + return 4; + + payload->rmst = av_calloc(1, sizeof(*payload->rmst)); + if (!payload->rmst) + return 5; + payload->nb_rmst = 1; + payload->rmst[0] = ff_rm_alloc_rmstream(); + if (!payload->rmst[0]) + return 6; + + stream->codecpar->codec_type = AVMEDIA_TYPE_AUDIO; + stream->codecpar->codec_id = AV_CODEC_ID_AAC; + payload->rmst[0]->deint_id = MKTAG('v', 'b', 'r', 'f'); + + /* One cached AAC subpacket with a one-byte length. Parsing consumes four + * bytes, then the RDT wrapper copies the entire remaining record. */ + input[0] = 0; + input[1] = 0x10; + input[2] = 0; + input[3] = 1; + + ret = rdt_parse_packet( + format, payload, stream, packet, ×tamp, input, input_size, 0, 0 + ); + + rdt_close_context(payload); + av_free(payload); + av_free(input); + av_packet_free(&packet); + avformat_free_context(format); + return ret < 0 ? 1 : 0; +} diff --git a/evaluations/ffmpeg_rdt_reproducer.c b/evaluations/ffmpeg_rdt_reproducer.c new file mode 100644 index 00000000..2075671e --- /dev/null +++ b/evaluations/ffmpeg_rdt_reproducer.c @@ -0,0 +1,28 @@ +#include + +#include "libavformat/rdt.h" + +int main(void) +{ + uint8_t input[16] = { 0 }; + int set_id; + int sequence_number; + int stream_id; + int is_keyframe; + uint32_t timestamp; + + input[0] = 0x80; /* status packet is followed by a data packet */ + input[1] = 0xFF; /* status packet */ + input[3] = 0; /* big-endian packet length = 0 */ + input[4] = 0; + + return ff_rdt_parse_header( + input, + sizeof(input), + &set_id, + &sequence_number, + &stream_id, + &is_keyframe, + ×tamp + ); +} diff --git a/evaluations/ffmpeg_rtp_av1_ignored_obu_reproducer.c b/evaluations/ffmpeg_rtp_av1_ignored_obu_reproducer.c new file mode 100644 index 00000000..829071be --- /dev/null +++ b/evaluations/ffmpeg_rtp_av1_ignored_obu_reproducer.c @@ -0,0 +1,61 @@ +#include +#include +#include +#include + +#include "libavformat/avformat.h" +#include "libavformat/rtpdec.h" +#include "libavformat/rtpdec_formats.h" +#include "libavutil/error.h" +#include "libavutil/mem.h" + +#define IGNORED_OBU_SIZE 100 +#define TRAILING_BYTES 17 + +int main(void) +{ + AVFormatContext *format = avformat_alloc_context(); + AVStream *stream; + AVPacket *packet = av_packet_alloc(); + PayloadContext *payload_context = NULL; + uint8_t payload[1 + 1 + IGNORED_OBU_SIZE + TRAILING_BYTES] = { 0 }; + uint32_t timestamp = 1; + int ret = 1; + + if (!format || !packet) + goto done; + stream = avformat_new_stream(format, NULL); + if (!stream) + goto done; + payload_context = av_mallocz(ff_av1_dynamic_handler.priv_data_size); + if (!payload_context) + goto done; + + /* N=1, W=0: first packet, with an explicit length before each OBU. */ + payload[0] = 0x08; + payload[1] = IGNORED_OBU_SIZE; + payload[2] = 0x10; /* AV1 temporal delimiter OBU, intentionally ignored. */ + + /* Bytes inside the ignored OBU become the next parser input if its input + * cursor is not advanced. The ignored size also moves the output cursor + * far beyond the packet space subsequently grown for these bytes. */ + payload[3] = 0x08; + + fprintf(stderr, + "ignored_obu_size=%d trailing_bytes=%d expected_output_gap=%d\n", + IGNORED_OBU_SIZE, TRAILING_BYTES, IGNORED_OBU_SIZE); + fflush(stderr); + ret = ff_av1_dynamic_handler.parse_packet( + format, payload_context, stream, packet, ×tamp, + payload, sizeof(payload), 1, RTP_FLAG_MARKER + ); + fprintf(stderr, "parse_return=%d packet_size=%d\n", ret, packet->size); + +done: + if (ff_av1_dynamic_handler.close && payload_context) + ff_av1_dynamic_handler.close(payload_context); + av_free(payload_context); + av_packet_free(&packet); + avformat_free_context(format); + return ret < 0 ? 2 : 0; +} diff --git a/evaluations/ffmpeg_rtp_h263_small_packet_reproducer.c b/evaluations/ffmpeg_rtp_h263_small_packet_reproducer.c new file mode 100644 index 00000000..b7ea8802 --- /dev/null +++ b/evaluations/ffmpeg_rtp_h263_small_packet_reproducer.c @@ -0,0 +1,87 @@ +#include +#include +#include + +#include "libavformat/avformat.h" +#include "libavutil/mem.h" +#include "libavutil/opt.h" + +#define RTP_PACKET_SIZE 13 +#define H263_PACKET_SIZE 16 + +static int discard_packet(void *opaque, const uint8_t *buf, int size) +{ + (void)opaque; + (void)buf; + return size; +} + +int main(void) +{ + AVFormatContext *format = NULL; + AVIOContext *io = NULL; + AVPacket *packet = NULL; + AVStream *stream; + uint8_t *io_buffer = NULL; + int ret = 1; + + if (avformat_alloc_output_context2(&format, NULL, "rtp", NULL) < 0) + goto done; + stream = avformat_new_stream(format, NULL); + if (!stream) + goto done; + + stream->codecpar->codec_type = AVMEDIA_TYPE_VIDEO; + stream->codecpar->codec_id = AV_CODEC_ID_H263; + stream->codecpar->width = 16; + stream->codecpar->height = 16; + stream->time_base = (AVRational){ 1, 25 }; + + if (av_opt_set(format->priv_data, "rtpflags", "rfc2190", 0) < 0) + goto done; + io_buffer = av_malloc(RTP_PACKET_SIZE); + if (!io_buffer) + goto done; + io = avio_alloc_context( + io_buffer, RTP_PACKET_SIZE, 1, NULL, NULL, discard_packet, NULL + ); + if (!io) + goto done; + io_buffer = NULL; + io->max_packet_size = RTP_PACKET_SIZE; + format->pb = io; + format->flags |= AVFMT_FLAG_CUSTOM_IO; + + if (avformat_write_header(format, NULL) < 0) + goto done; + packet = av_packet_alloc(); + if (!packet || av_new_packet(packet, H263_PACKET_SIZE) < 0) + goto done; + memset(packet->data, 0, packet->size); + packet->data[2] = 0x80; + packet->stream_index = stream->index; + packet->pts = packet->dts = 0; + packet->duration = 1; + + fprintf( + stderr, + "rtp_packet_size=%d max_payload_size=%d fragment_size=%d\n", + RTP_PACKET_SIZE, + RTP_PACKET_SIZE - 12, + RTP_PACKET_SIZE - 12 - 8 + ); + fflush(stderr); + ret = av_write_frame(format, packet); + fprintf(stderr, "write_return=%d\n", ret); + +done: + av_packet_free(&packet); + if (format) + av_write_trailer(format); + if (io) + avio_context_free(&io); + else + av_free(io_buffer); + avformat_free_context(format); + return ret < 0 ? 2 : 0; +} diff --git a/evaluations/ffmpeg_rtp_latm_header_reproducer.c b/evaluations/ffmpeg_rtp_latm_header_reproducer.c new file mode 100644 index 00000000..b72a5e6e --- /dev/null +++ b/evaluations/ffmpeg_rtp_latm_header_reproducer.c @@ -0,0 +1,93 @@ +#include +#include +#include + +#include "libavformat/avformat.h" +#include "libavutil/channel_layout.h" +#include "libavutil/mem.h" +#include "libavutil/opt.h" + +#define RTP_PACKET_SIZE 1472 +#define AAC_PACKET_SIZE (1500 * 0xFF) + +static int discard_packet(void *opaque, const uint8_t *buf, int size) +{ + (void)opaque; + (void)buf; + return size; +} + +int main(void) +{ + AVFormatContext *format = NULL; + AVIOContext *io = NULL; + AVPacket *packet = NULL; + AVStream *stream; + uint8_t *io_buffer = NULL; + int ret = 1; + + if (avformat_alloc_output_context2(&format, NULL, "rtp", NULL) < 0) + goto done; + stream = avformat_new_stream(format, NULL); + if (!stream) + goto done; + + stream->codecpar->codec_type = AVMEDIA_TYPE_AUDIO; + stream->codecpar->codec_id = AV_CODEC_ID_AAC; + stream->codecpar->sample_rate = 48000; + av_channel_layout_default(&stream->codecpar->ch_layout, 2); + stream->codecpar->extradata = av_mallocz(2 + AV_INPUT_BUFFER_PADDING_SIZE); + if (!stream->codecpar->extradata) + goto done; + stream->codecpar->extradata[0] = 0x11; + stream->codecpar->extradata[1] = 0x90; + stream->codecpar->extradata_size = 2; + stream->time_base = (AVRational){ 1, 48000 }; + + if (av_opt_set(format->priv_data, "rtpflags", "latm", 0) < 0) + goto done; + io_buffer = av_malloc(RTP_PACKET_SIZE); + if (!io_buffer) + goto done; + io = avio_alloc_context( + io_buffer, RTP_PACKET_SIZE, 1, NULL, NULL, discard_packet, NULL + ); + if (!io) + goto done; + io_buffer = NULL; + io->max_packet_size = RTP_PACKET_SIZE; + format->pb = io; + format->flags |= AVFMT_FLAG_CUSTOM_IO; + + if (avformat_write_header(format, NULL) < 0) + goto done; + packet = av_packet_alloc(); + if (!packet || av_new_packet(packet, AAC_PACKET_SIZE) < 0) + goto done; + memset(packet->data, 0, packet->size); + packet->stream_index = stream->index; + packet->pts = packet->dts = 0; + packet->duration = 1024; + + fprintf( + stderr, + "rtp_packet_size=%d aac_packet_size=%d latm_header_size=%d\n", + RTP_PACKET_SIZE, + AAC_PACKET_SIZE, + AAC_PACKET_SIZE / 0xFF + 1 + ); + fflush(stderr); + ret = av_write_frame(format, packet); + fprintf(stderr, "write_return=%d\n", ret); + +done: + av_packet_free(&packet); + if (format) + av_write_trailer(format); + if (io) + avio_context_free(&io); + else + av_free(io_buffer); + avformat_free_context(format); + return ret < 0 ? 2 : 0; +} diff --git a/evaluations/ffmpeg_yuvcmp_partial_mb_reproducer.c b/evaluations/ffmpeg_yuvcmp_partial_mb_reproducer.c new file mode 100644 index 00000000..480db9a8 --- /dev/null +++ b/evaluations/ffmpeg_yuvcmp_partial_mb_reproducer.c @@ -0,0 +1,20 @@ +#include + +/* Compile the production utility into a dedicated sanitizer binary without + * changing either sealed FFmpeg checkout. */ +#define main ffmpeg_yuvcmp_main +#include "tools/yuvcmp.c" +#undef main + +int main(int argc, char **argv) +{ + if (argc != 3) { + fprintf(stderr, "usage: %s first.yuv second.yuv\n", argv[0]); + return 2; + } + + char *arguments[] = { + argv[0], argv[1], argv[2], "17", "16", "pixelcmp", NULL, + }; + return ffmpeg_yuvcmp_main(6, arguments); +} diff --git a/evaluations/run_ffmpeg_af_join_uaf_reproducer.py b/evaluations/run_ffmpeg_af_join_uaf_reproducer.py new file mode 100644 index 00000000..ce3100ba --- /dev/null +++ b/evaluations/run_ffmpeg_af_join_uaf_reproducer.py @@ -0,0 +1,194 @@ +"""Build, run, and record the af_join missing-buffer-reference reproducer.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import platform +import subprocess +from datetime import datetime, timezone +from pathlib import Path + +SCHEMA_VERSION = "cw.ffmpeg.af-join-uaf-reproducer.v1" + + +def _arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--checkout", type=Path, required=True) + parser.add_argument( + "--harness", + type=Path, + default=Path(__file__).with_name("ffmpeg_af_join_uaf_reproducer.c"), + ) + parser.add_argument("--binary-output", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + return parser.parse_args() + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def _compile_command(harness: Path, binary: Path) -> list[str]: + command = [ + "clang", + "-I.", + "-D_ISOC11_SOURCE", + "-D_FILE_OFFSET_BITS=64", + "-D_LARGEFILE_SOURCE", + "-I./compat/dispatch_semaphore", + "-DPIC", + "-I./compat/stdbit", + "-DHAVE_AV_CONFIG_H", + "-fsanitize=address,undefined", + "-fno-omit-frame-pointer", + "-g", + "-O1", + "-std=c17", + "-fPIC", + "-o", + str(binary), + str(harness), + "-Llibavfilter", + "-Llibavformat", + "-Llibavcodec", + "-Llibswscale", + "-Llibswresample", + "-Llibavutil", + "-lavfilter", + "-lavformat", + "-lavcodec", + "-lswscale", + "-lswresample", + "-lavutil", + "-lm", + "-lbz2", + "-lz", + ] + if platform.system() == "Darwin": + command.extend( + [ + "-framework", + "Foundation", + "-framework", + "AudioToolbox", + "-framework", + "CoreAudio", + "-framework", + "AVFoundation", + "-framework", + "CoreGraphics", + "-framework", + "OpenGL", + "-framework", + "Metal", + "-framework", + "VideoToolbox", + "-framework", + "CoreImage", + "-framework", + "AppKit", + "-framework", + "CoreFoundation", + "-framework", + "CoreMedia", + "-framework", + "CoreVideo", + "-framework", + "CoreServices", + "-framework", + "Security", + "-liconv", + ] + ) + command.append("-pthread") + return command + + +def main() -> None: + args = _arguments() + checkout = args.checkout.expanduser().resolve() + harness = args.harness.expanduser().resolve() + binary = args.binary_output.expanduser().resolve() + output = args.output.expanduser().resolve() + if not harness.is_file() or not (checkout / "libavfilter/libavfilter.a").is_file(): + raise ValueError("harness and configured FFmpeg static libraries must exist") + + binary.parent.mkdir(parents=True, exist_ok=True) + output.parent.mkdir(parents=True, exist_ok=True) + compile_command = _compile_command(harness, binary) + compile_result = subprocess.run( + compile_command, cwd=checkout, check=False, capture_output=True, text=True + ) + environment: dict[str, str] | None = None + if compile_result.returncode == 0: + environment = os.environ.copy() + environment["ASAN_OPTIONS"] = ( + "halt_on_error=1:abort_on_error=1:detect_leaks=0" + ) + run_result = subprocess.run( + [str(binary)], + cwd=checkout, + env=environment, + check=False, + capture_output=True, + text=True, + ) + else: + run_result = subprocess.CompletedProcess( + [str(binary)], 127, "", "compile failed" + ) + + combined = run_result.stdout + run_result.stderr + indicators = { + "three_channel_output": "output_channels=3" in combined, + "duplicate_first_planes": "duplicate_first_planes=1" in combined, + "missing_third_plane_owner": "missing_third_plane_owner=1" in combined, + "asan_heap_use_after_free": "AddressSanitizer: heap-use-after-free" in combined, + "downstream_read_aborted": run_result.returncode != 0, + } + expected_observed = compile_result.returncode == 0 and all(indicators.values()) + commit_result = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=checkout, + check=False, + capture_output=True, + text=True, + ) + payload = { + "schema_version": SCHEMA_VERSION, + "completed_at": datetime.now(timezone.utc).isoformat(), + "checkout": str(checkout), + "checkout_commit": commit_result.stdout.strip() or None, + "harness": str(harness), + "harness_sha256": _sha256(harness), + "binary": str(binary), + "binary_sha256": _sha256(binary) if binary.is_file() else None, + "compile_command": compile_command, + "compile_returncode": compile_result.returncode, + "compile_stdout": compile_result.stdout, + "compile_stderr": compile_result.stderr, + "run_command": [str(binary)], + "asan_options": environment["ASAN_OPTIONS"] if environment else None, + "returncode": run_result.returncode, + "indicators": indicators, + "expected_observed": expected_observed, + "stdout": run_result.stdout, + "stderr": run_result.stderr, + } + temporary = output.with_suffix(output.suffix + ".tmp") + temporary.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + temporary.replace(output) + print(output) + if not expected_observed: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/evaluations/run_ffmpeg_ahistogram_reproducer.py b/evaluations/run_ffmpeg_ahistogram_reproducer.py new file mode 100644 index 00000000..c9cd204e --- /dev/null +++ b/evaluations/run_ffmpeg_ahistogram_reproducer.py @@ -0,0 +1,114 @@ +"""Run and record the ahistogram positive-full-scale ASan reproducer.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import subprocess +from datetime import UTC, datetime +from pathlib import Path + +SCHEMA_VERSION = "cw.ffmpeg.ahistogram-reproducer.v1" + + +def _arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--ffmpeg", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + return parser.parse_args() + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def _commit(binary: Path) -> str | None: + result = subprocess.run( + ["git", "-C", str(binary.parent), "rev-parse", "HEAD"], + check=False, + capture_output=True, + text=True, + ) + return result.stdout.strip() or None + + +def main() -> None: + args = _arguments() + ffmpeg = args.ffmpeg.expanduser().resolve() + output = args.output.expanduser().resolve() + if not ffmpeg.is_file(): + raise ValueError("ffmpeg binary does not exist") + + output.parent.mkdir(parents=True, exist_ok=True) + command = [ + str(ffmpeg), + "-hide_banner", + "-loglevel", + "verbose", + "-f", + "lavfi", + "-i", + "aevalsrc=1:s=48000:d=0.1", + "-filter_complex", + "[0:a]ahistogram=ascale=log:hmode=sign:size=1280x720[outv]", + "-map", + "[outv]", + "-frames:v", + "1", + "-f", + "null", + "-", + ] + environment = os.environ.copy() + environment["ASAN_OPTIONS"] = "halt_on_error=1:abort_on_error=1:detect_leaks=0" + result = subprocess.run( + command, + check=False, + capture_output=True, + text=True, + env=environment, + ) + combined = result.stdout + result.stderr + indicators = { + "asan_heap_buffer_overflow": "AddressSanitizer: heap-buffer-overflow" in combined, + "eight_byte_access": "of size 8" in combined, + "exact_allocation_end": "0 bytes after 10240-byte region" in combined, + "filter_frame_in_trace": "filter_frame" in combined, + "av_calloc_allocation_in_trace": "av_calloc" in combined, + "config_input_allocation_in_trace": "config_input" in combined, + } + expected_observed = all(indicators.values()) + payload = { + "schema_version": SCHEMA_VERSION, + "completed_at": datetime.now(UTC).isoformat(), + "checkout_commit": _commit(ffmpeg), + "ffmpeg": str(ffmpeg), + "ffmpeg_sha256": _sha256(ffmpeg), + "input_sample": 1.0, + "histogram_width": 1280, + "histogram_mode": "sign", + "amplitude_scale": "log", + "command": command, + "asan_options": environment["ASAN_OPTIONS"], + "returncode": result.returncode, + "indicators": indicators, + "expected_observed": expected_observed, + "stdout": result.stdout, + "stderr": result.stderr, + } + temporary = output.with_suffix(output.suffix + ".tmp") + temporary.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") + temporary.replace(output) + print(output) + if not expected_observed: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/evaluations/run_ffmpeg_arnndn_reproducer.py b/evaluations/run_ffmpeg_arnndn_reproducer.py new file mode 100644 index 00000000..f9ae0706 --- /dev/null +++ b/evaluations/run_ffmpeg_arnndn_reproducer.py @@ -0,0 +1,108 @@ +"""Run and record the arnndn oversized-output ASan reproducer.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import subprocess +from datetime import UTC, datetime +from pathlib import Path + +from build_ffmpeg_arnndn_model import build_model + +SCHEMA_VERSION = "cw.ffmpeg.arnndn-reproducer.v1" + + +def _arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--ffmpeg", type=Path, required=True) + parser.add_argument("--model-output", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + return parser.parse_args() + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def _commit(binary: Path) -> str | None: + result = subprocess.run( + ["git", "-C", str(binary.parent), "rev-parse", "HEAD"], + check=False, + capture_output=True, + text=True, + ) + return result.stdout.strip() or None + + +def main() -> None: + args = _arguments() + ffmpeg = args.ffmpeg.expanduser().resolve() + model = args.model_output.expanduser().resolve() + output = args.output.expanduser().resolve() + if not ffmpeg.is_file(): + raise ValueError("ffmpeg binary does not exist") + + model.parent.mkdir(parents=True, exist_ok=True) + output.parent.mkdir(parents=True, exist_ok=True) + build_model(model, 23) + command = [ + str(ffmpeg), + "-hide_banner", + "-loglevel", + "verbose", + "-f", + "lavfi", + "-i", + "anoisesrc=r=48000:d=0.1", + "-af", + f"arnndn=m={model}", + "-f", + "null", + "-", + ] + environment = os.environ.copy() + environment["ASAN_OPTIONS"] = "halt_on_error=1:abort_on_error=1" + result = subprocess.run(command, check=False, capture_output=True, text=True, env=environment) + combined = result.stdout + result.stderr + payload = { + "schema_version": SCHEMA_VERSION, + "completed_at": datetime.now(UTC).isoformat(), + "checkout_commit": _commit(ffmpeg), + "ffmpeg": str(ffmpeg), + "ffmpeg_sha256": _sha256(ffmpeg), + "model": str(model), + "model_sha256": _sha256(model), + "denoise_outputs": 23, + "expected_band_outputs": 22, + "command": command, + "returncode": result.returncode, + "asan_stack_buffer_overflow": "AddressSanitizer: stack-buffer-overflow" in combined, + "compute_dense_in_trace": "compute_dense" in combined, + "g_identified_as_overflowed_object": "'g.i'" in combined, + "stdout": result.stdout, + "stderr": result.stderr, + } + temporary = output.with_suffix(output.suffix + ".tmp") + temporary.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") + temporary.replace(output) + print(output) + + if not all( + ( + payload["asan_stack_buffer_overflow"], + payload["compute_dense_in_trace"], + payload["g_identified_as_overflowed_object"], + ) + ): + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/evaluations/run_ffmpeg_caf_seek_reproducer.py b/evaluations/run_ffmpeg_caf_seek_reproducer.py new file mode 100644 index 00000000..7c524d22 --- /dev/null +++ b/evaluations/run_ffmpeg_caf_seek_reproducer.py @@ -0,0 +1,191 @@ +"""Build, run, and record the CAF out-of-range seek ASan reproducer.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import platform +import subprocess +from datetime import UTC, datetime +from pathlib import Path + +SCHEMA_VERSION = "cw.ffmpeg.caf-seek-reproducer.v1" + + +def _arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--checkout", type=Path, required=True) + parser.add_argument( + "--harness", + type=Path, + default=Path(__file__).with_name("ffmpeg_caf_seek_reproducer.c"), + ) + parser.add_argument("--binary-output", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + return parser.parse_args() + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def _compile_command(harness: Path, binary: Path) -> list[str]: + command = [ + "clang", + "-I.", + "-D_ISOC11_SOURCE", + "-D_FILE_OFFSET_BITS=64", + "-D_LARGEFILE_SOURCE", + "-I./compat/dispatch_semaphore", + "-DPIC", + "-I./compat/stdbit", + "-DZLIB_CONST", + "-DHAVE_AV_CONFIG_H", + "-fsanitize=address,undefined", + "-fno-omit-frame-pointer", + "-g", + "-O1", + "-std=c17", + "-fPIC", + "-pthread", + "-o", + str(binary), + str(harness), + "-Llibavformat", + "-Llibavcodec", + "-Llibswresample", + "-Llibswscale", + "-Llibavutil", + "-lavformat", + "-lavcodec", + "-lswresample", + "-lswscale", + "-lavutil", + "-lm", + "-lbz2", + "-lz", + ] + if platform.system() == "Darwin": + command.extend( + [ + "-framework", + "CoreFoundation", + "-framework", + "Security", + "-liconv", + "-framework", + "AudioToolbox", + "-framework", + "VideoToolbox", + "-framework", + "CoreMedia", + "-framework", + "CoreVideo", + "-framework", + "CoreServices", + ] + ) + command.append("-pthread") + return command + + +def main() -> None: + args = _arguments() + checkout = args.checkout.expanduser().resolve() + harness = args.harness.expanduser().resolve() + binary = args.binary_output.expanduser().resolve() + output = args.output.expanduser().resolve() + if not harness.is_file() or not (checkout / "libavformat/libavformat.a").is_file(): + raise ValueError("harness and configured FFmpeg static libraries must exist") + + binary.parent.mkdir(parents=True, exist_ok=True) + output.parent.mkdir(parents=True, exist_ok=True) + compile_command = _compile_command(harness, binary) + compile_result = subprocess.run( + compile_command, + cwd=checkout, + check=False, + capture_output=True, + text=True, + ) + if compile_result.returncode == 0: + environment = os.environ.copy() + environment["ASAN_OPTIONS"] = ( + "halt_on_error=1:abort_on_error=1:detect_leaks=0" + ) + run_result = subprocess.run( + [str(binary)], + cwd=checkout, + env=environment, + check=False, + capture_output=True, + text=True, + ) + else: + run_result = subprocess.CompletedProcess( + [str(binary)], 127, "", "compile failed" + ) + + combined = run_result.stdout + run_result.stderr + indicators = { + "asan_heap_buffer_overflow": ( + "AddressSanitizer: heap-buffer-overflow" in combined + ), + "eight_byte_read": "READ of size 8" in combined, + "read_seek_in_trace": "read_seek" in combined, + } + expected_observed = all(indicators.values()) + commit_result = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=checkout, + check=False, + capture_output=True, + text=True, + ) + payload = { + "schema_version": SCHEMA_VERSION, + "completed_at": datetime.now(UTC).isoformat(), + "checkout": str(checkout), + "checkout_commit": commit_result.stdout.strip() or None, + "harness": str(harness), + "harness_sha256": _sha256(harness), + "binary": str(binary), + "binary_sha256": _sha256(binary) if binary.is_file() else None, + "index_entry_timestamp": 0, + "requested_timestamp": 1, + "seek_flags": 0, + "fixture_bytes": 107, + "fixture_packet_count": 1, + "fixture_variable_packet_bytes": True, + "fixture_variable_packet_frames": True, + "compile_command": compile_command, + "compile_returncode": compile_result.returncode, + "compile_stdout": compile_result.stdout, + "compile_stderr": compile_result.stderr, + "run_command": [str(binary)], + "asan_options": "halt_on_error=1:abort_on_error=1:detect_leaks=0", + "returncode": run_result.returncode, + "indicators": indicators, + "expected_observed": expected_observed, + "stdout": run_result.stdout, + "stderr": run_result.stderr, + } + temporary = output.with_suffix(output.suffix + ".tmp") + temporary.write_text( + json.dumps(payload, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + temporary.replace(output) + print(output) + if not expected_observed: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/evaluations/run_ffmpeg_d3d12va_upload_capacity_proof.py b/evaluations/run_ffmpeg_d3d12va_upload_capacity_proof.py new file mode 100644 index 00000000..e9c4cba7 --- /dev/null +++ b/evaluations/run_ffmpeg_d3d12va_upload_capacity_proof.py @@ -0,0 +1,221 @@ +"""Record a source/runtime proof for the D3D12VA H.264/HEVC upload overflow.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import subprocess +import tempfile +from datetime import datetime, timezone +from pathlib import Path + +SCHEMA_VERSION = "cw.ffmpeg.d3d12va-upload-capacity-proof.v1" +START_CODE = bytes.fromhex("000001") +LONG_START_CODE = bytes.fromhex("00000001") + + +def _arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--checkout", type=Path, required=True) + parser.add_argument("--ffmpeg", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + return parser.parse_args() + + +def _sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _run(command: list[str]) -> subprocess.CompletedProcess[str]: + return subprocess.run(command, check=False, capture_output=True, text=True) + + +def _nal_units(data: bytes, codec: str) -> list[dict[str, int]]: + starts: list[tuple[int, int]] = [] + cursor = 0 + while cursor + 3 < len(data): + if data[cursor : cursor + 4] == LONG_START_CODE: + starts.append((cursor, 4)) + cursor += 4 + elif data[cursor : cursor + 3] == START_CODE: + starts.append((cursor, 3)) + cursor += 3 + else: + cursor += 1 + units: list[dict[str, int]] = [] + for index, (start, start_size) in enumerate(starts): + payload = start + start_size + end = starts[index + 1][0] if index + 1 < len(starts) else len(data) + if payload >= end: + continue + nal_type = (data[payload] >> 1) & 63 if codec == "hevc" else data[payload] & 31 + units.append( + { + "type": nal_type, + "raw_size": end - payload, + "annex_b_size": end - start, + } + ) + return units + + +def _is_slice(codec: str, nal_type: int) -> bool: + return nal_type <= 31 if codec == "hevc" else nal_type in {1, 5} + + +def _codec_case(ffmpeg: Path, directory: Path, codec: str) -> dict[str, object]: + if codec == "h264": + width, height = 16, 5400 + encoder = "libx264" + suffix = "h264" + options = [ + "-preset", + "ultrafast", + "-tune", + "zerolatency", + "-qp", + "0", + "-x264-params", + "slices=1", + ] + else: + width = height = 16 + encoder = "libx265" + suffix = "hevc" + options = [ + "-preset", + "ultrafast", + "-x265-params", + "log-level=error:lossless=1:slices=1", + ] + + stream = directory / f"valid-{width}x{height}.{suffix}" + encode_command = [ + str(ffmpeg), + "-hide_banner", + "-loglevel", + "error", + "-f", + "lavfi", + "-i", + f"nullsrc=s={width}x{height}:r=1,format=yuv420p," + f"noise=alls=100:allf=t+u:all_seed={12345 + width + height}", + "-frames:v", + "1", + "-c:v", + encoder, + *options, + "-f", + suffix, + str(stream), + ] + encode = _run(encode_command) + decode_command = [ + str(ffmpeg), + "-hide_banner", + "-loglevel", + "error", + "-i", + str(stream), + "-frames:v", + "1", + "-f", + "null", + "-", + ] + decode = _run(decode_command) if encode.returncode == 0 else None + units = _nal_units(stream.read_bytes(), codec) if stream.is_file() else [] + slices = [unit for unit in units if _is_slice(codec, unit["type"])] + raw_image_capacity = width * height * 3 // 2 + physical_allocation_floor = (raw_image_capacity + 65535) // 65536 * 65536 + slice_bytes = sum(unit["raw_size"] for unit in slices) + d3d12_upload_bytes = slice_bytes + 3 * len(slices) + return { + "codec": codec, + "width": width, + "height": height, + "pixel_format": "yuv420p", + "raw_image_capacity": raw_image_capacity, + "d3d12_64k_aligned_allocation_floor": physical_allocation_floor, + "slice_count": len(slices), + "slice_bytes": slice_bytes, + "d3d12_start_code_bytes": 3 * len(slices), + "d3d12_upload_bytes": d3d12_upload_bytes, + "overflow_bytes": d3d12_upload_bytes - raw_image_capacity, + "bytes_beyond_64k_aligned_allocation_floor": ( + d3d12_upload_bytes - physical_allocation_floor + ), + "nal_units": units, + "stream_sha256": _sha256(stream) if stream.is_file() else None, + "encode_command": encode_command, + "encode_returncode": encode.returncode, + "encode_stdout": encode.stdout, + "encode_stderr": encode.stderr, + "decode_command": decode_command, + "decode_returncode": decode.returncode if decode else None, + "decode_stdout": decode.stdout if decode else "", + "decode_stderr": decode.stderr if decode else "encode failed", + "valid_stream_decoded": bool(decode and decode.returncode == 0), + "capacity_exceeded": d3d12_upload_bytes > raw_image_capacity, + "aligned_allocation_floor_exceeded": ( + d3d12_upload_bytes > physical_allocation_floor + ), + } + + +def main() -> None: + args = _arguments() + checkout = args.checkout.expanduser().resolve() + ffmpeg = args.ffmpeg.expanduser().resolve() + output = args.output.expanduser().resolve() + sources = [ + checkout / "libavcodec/d3d12va_decode.c", + checkout / "libavcodec/d3d12va_h264.c", + checkout / "libavcodec/d3d12va_hevc.c", + ] + if not ffmpeg.is_file() or any(not source.is_file() for source in sources): + raise ValueError("FFmpeg executable and D3D12VA sources must exist") + + version = _run([str(ffmpeg), "-version"]) + with tempfile.TemporaryDirectory(prefix="cw-d3d12va-proof-") as temporary: + directory = Path(temporary) + cases = [_codec_case(ffmpeg, directory, codec) for codec in ("h264", "hevc")] + + expected_observed = ( + all(case["valid_stream_decoded"] and case["capacity_exceeded"] for case in cases) + and cases[0]["aligned_allocation_floor_exceeded"] + ) + payload = { + "schema_version": SCHEMA_VERSION, + "completed_at": datetime.now(timezone.utc).isoformat(), + "checkout": str(checkout), + "checkout_commit": _run( + ["git", "-C", str(checkout), "rev-parse", "HEAD"] + ).stdout.strip(), + "ffmpeg": str(ffmpeg), + "ffmpeg_version": version.stdout, + "source_sha256": { + str(source.relative_to(checkout)): _sha256(source) for source in sources + }, + "cases": cases, + "expected_observed": expected_observed, + "scope": ( + "The D3D12VA helper allocates an upload resource using the raw-image " + "buffer size. H.264 and HEVC then copy every accepted VCL NAL into " + "that resource and prepend three bytes per slice without a capacity " + "check. The generated standards-valid streams demonstrate that the " + "bytes passed to those callbacks can exceed the allocation." + ), + } + output.parent.mkdir(parents=True, exist_ok=True) + temporary = output.with_suffix(output.suffix + ".tmp") + temporary.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + temporary.replace(output) + print(output) + if not expected_observed: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/evaluations/run_ffmpeg_decimate_subsampled_block_reproducer.py b/evaluations/run_ffmpeg_decimate_subsampled_block_reproducer.py new file mode 100644 index 00000000..8ab548fe --- /dev/null +++ b/evaluations/run_ffmpeg_decimate_subsampled_block_reproducer.py @@ -0,0 +1,113 @@ +"""Run and record decimate's subsampled-chroma block-size ASan reproducer.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import subprocess +from datetime import UTC, datetime +from pathlib import Path + +SCHEMA_VERSION = "cw.ffmpeg.decimate-subsampled-block-reproducer.v1" + + +def _arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--ffmpeg", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + return parser.parse_args() + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def main() -> None: + args = _arguments() + ffmpeg = args.ffmpeg.expanduser().resolve() + output = args.output.expanduser().resolve() + if not ffmpeg.is_file(): + raise ValueError("ffmpeg binary must exist") + + command = [ + str(ffmpeg), + "-hide_banner", + "-loglevel", + "error", + "-f", + "lavfi", + "-i", + "testsrc2=s=16x16:r=5:d=2,format=yuv411p", + "-vf", + "decimate=blockx=4", + "-frames:v", + "2", + "-f", + "null", + "-", + ] + environment = os.environ.copy() + environment["ASAN_OPTIONS"] = ( + "halt_on_error=1:abort_on_error=1:detect_leaks=0" + ) + result = subprocess.run( + command, + env=environment, + check=False, + capture_output=True, + text=True, + ) + + combined = result.stdout + result.stderr + indicators = { + "asan_heap_buffer_overflow": ( + "AddressSanitizer: heap-buffer-overflow" in combined + ), + "eight_byte_access": ( + "READ of size 8" in combined or "WRITE of size 8" in combined + ), + "access_after_64_byte_metric_buffer": ( + "0 bytes after 64-byte region" in combined + ), + "filter_frame_in_trace": "filter_frame" in combined, + "filter_aborted": result.returncode != 0, + } + expected_observed = all(indicators.values()) + commit_result = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=ffmpeg.parent, + check=False, + capture_output=True, + text=True, + ) + payload = { + "schema_version": SCHEMA_VERSION, + "completed_at": datetime.now(UTC).isoformat(), + "ffmpeg": str(ffmpeg), + "ffmpeg_sha256": _sha256(ffmpeg), + "checkout_commit": commit_result.stdout.strip() or None, + "command": command, + "asan_options": environment["ASAN_OPTIONS"], + "returncode": result.returncode, + "indicators": indicators, + "expected_observed": expected_observed, + "stdout": result.stdout, + "stderr": result.stderr, + } + output.parent.mkdir(parents=True, exist_ok=True) + temporary = output.with_suffix(output.suffix + ".tmp") + temporary.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + temporary.replace(output) + print(output) + if not expected_observed: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/evaluations/run_ffmpeg_dnn_classify_count_reproducer.py b/evaluations/run_ffmpeg_dnn_classify_count_reproducer.py new file mode 100644 index 00000000..bf7fb522 --- /dev/null +++ b/evaluations/run_ffmpeg_dnn_classify_count_reproducer.py @@ -0,0 +1,216 @@ +"""Build and record the DNN classification-count overflow reproducer.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import platform +import subprocess +from datetime import UTC, datetime +from pathlib import Path + +SCHEMA_VERSION = "cw.ffmpeg.dnn-classify-count-reproducer.v1" + + +def _arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--checkout", type=Path, required=True) + parser.add_argument( + "--harness", + type=Path, + default=Path(__file__).with_name( + "ffmpeg_dnn_classify_count_reproducer.c" + ), + ) + parser.add_argument("--binary-output", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + return parser.parse_args() + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def _compile_command( + checkout: Path, harness: Path, binary: Path +) -> list[str]: + command = [ + "clang", + "-I.", + "-D_ISOC11_SOURCE", + "-D_FILE_OFFSET_BITS=64", + "-D_LARGEFILE_SOURCE", + "-I./compat/dispatch_semaphore", + "-DPIC", + "-I./compat/stdbit", + "-DHAVE_AV_CONFIG_H", + "-fsanitize=address,undefined", + "-fno-omit-frame-pointer", + "-g", + "-O1", + "-std=c17", + "-fPIC", + "-ffunction-sections", + "-fdata-sections", + "-o", + str(binary), + str(harness), + "-Llibavfilter", + "-Llibavutil", + "-lavfilter", + "-lavutil", + "-lm", + "-lbz2", + "-lz", + ] + if platform.system() == "Darwin": + command.extend( + [ + "-Wl,-dead_strip", + "-framework", + "Foundation", + "-framework", + "AudioToolbox", + "-framework", + "CoreAudio", + "-framework", + "AVFoundation", + "-framework", + "CoreGraphics", + "-framework", + "OpenGL", + "-framework", + "Metal", + "-framework", + "VideoToolbox", + "-framework", + "CoreImage", + "-framework", + "AppKit", + "-framework", + "CoreFoundation", + "-framework", + "CoreMedia", + "-framework", + "CoreVideo", + "-framework", + "CoreServices", + "-framework", + "Security", + "-liconv", + ] + ) + command.append("-pthread") + return command + + +def main() -> None: + args = _arguments() + checkout = args.checkout.expanduser().resolve() + harness = args.harness.expanduser().resolve() + binary = args.binary_output.expanduser().resolve() + output = args.output.expanduser().resolve() + source = checkout / "libavfilter/vf_dnn_classify.c" + if not harness.is_file() or not source.is_file(): + raise ValueError("harness and DNN classifier source must exist") + + binary.parent.mkdir(parents=True, exist_ok=True) + output.parent.mkdir(parents=True, exist_ok=True) + compile_command = _compile_command(checkout, harness, binary) + compile_result = subprocess.run( + compile_command, + cwd=checkout, + check=False, + capture_output=True, + text=True, + ) + environment: dict[str, str] | None = None + if compile_result.returncode == 0: + environment = os.environ.copy() + environment["ASAN_OPTIONS"] = ( + "halt_on_error=1:abort_on_error=1:detect_leaks=0" + ) + run_result = subprocess.run( + [str(binary)], + cwd=checkout, + env=environment, + check=False, + capture_output=True, + text=True, + ) + else: + run_result = subprocess.CompletedProcess( + [str(binary)], 127, "", "compile failed" + ) + + combined = run_result.stdout + run_result.stderr + indicators = { + "four_entry_capacity": "classification_capacity=4" in combined, + "fifth_callback_reached": ( + "callback=5 classify_count=4" in combined + ), + "asan_heap_buffer_overflow": ( + "AddressSanitizer: heap-buffer-overflow" in combined + ), + "classifier_in_trace": "dnn_classify_post_proc" in combined, + "classification_aborted": run_result.returncode != 0, + } + expected_observed = compile_result.returncode == 0 and all( + indicators.values() + ) + commit_result = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=checkout, + check=False, + capture_output=True, + text=True, + ) + payload = { + "schema_version": SCHEMA_VERSION, + "completed_at": datetime.now(UTC).isoformat(), + "checkout": str(checkout), + "checkout_commit": commit_result.stdout.strip() or None, + "harness": str(harness), + "harness_sha256": _sha256(harness), + "source": str(source), + "source_sha256": _sha256(source), + "binary": str(binary), + "binary_sha256": _sha256(binary) if binary.is_file() else None, + "compile_command": compile_command, + "compile_returncode": compile_result.returncode, + "compile_stdout": compile_result.stdout, + "compile_stderr": compile_result.stderr, + "run_command": [str(binary)], + "asan_options": environment["ASAN_OPTIONS"] if environment else None, + "returncode": run_result.returncode, + "indicators": indicators, + "expected_observed": expected_observed, + "scope": ( + "The harness invokes the production static classifier callback " + "from vf_dnn_classify.c five times on one production-allocated " + "bounding-box side-data object. This mirrors the OpenVINO " + "completion loop when a model exposes five outputs, without " + "requiring the optional OpenVINO runtime in the sealed build." + ), + "stdout": run_result.stdout, + "stderr": run_result.stderr, + } + temporary = output.with_suffix(output.suffix + ".tmp") + temporary.write_text( + json.dumps(payload, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + temporary.replace(output) + print(output) + if not expected_observed: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/evaluations/run_ffmpeg_dnn_output_names_reproducer.py b/evaluations/run_ffmpeg_dnn_output_names_reproducer.py new file mode 100644 index 00000000..f8923df9 --- /dev/null +++ b/evaluations/run_ffmpeg_dnn_output_names_reproducer.py @@ -0,0 +1,199 @@ +"""Build and record the DNN output-name parser reproducer.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import platform +import subprocess +from datetime import datetime, timezone +from pathlib import Path + +SCHEMA_VERSION = "cw.ffmpeg.dnn-output-names-reproducer.v1" + + +def _arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--checkout", type=Path, required=True) + parser.add_argument( + "--harness", + type=Path, + default=Path(__file__).with_name("ffmpeg_dnn_output_names_reproducer.c"), + ) + parser.add_argument("--binary-output", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + return parser.parse_args() + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def _compile_command( + checkout: Path, harness: Path, binary: Path +) -> list[str]: + command = [ + "clang", + "-I.", + "-D_ISOC11_SOURCE", + "-D_FILE_OFFSET_BITS=64", + "-D_LARGEFILE_SOURCE", + "-I./compat/dispatch_semaphore", + "-DPIC", + "-I./compat/stdbit", + "-DHAVE_AV_CONFIG_H", + "-fsanitize=address,undefined", + "-fno-omit-frame-pointer", + "-g", + "-O1", + "-std=c17", + "-fPIC", + "-o", + str(binary), + str(harness), + str(checkout / "libavfilter/dnn_filter_common.c"), + "-Llibavfilter", + "-Llibavutil", + "-lavfilter", + "-lavutil", + "-lm", + "-lbz2", + "-lz", + ] + if platform.system() == "Darwin": + command.extend( + [ + "-framework", + "Foundation", + "-framework", + "AudioToolbox", + "-framework", + "CoreAudio", + "-framework", + "AVFoundation", + "-framework", + "CoreGraphics", + "-framework", + "OpenGL", + "-framework", + "Metal", + "-framework", + "VideoToolbox", + "-framework", + "CoreImage", + "-framework", + "AppKit", + "-framework", + "CoreFoundation", + "-framework", + "CoreMedia", + "-framework", + "CoreVideo", + "-framework", + "CoreServices", + "-framework", + "Security", + "-liconv", + ] + ) + command.append("-pthread") + return command + + +def main() -> None: + args = _arguments() + checkout = args.checkout.expanduser().resolve() + harness = args.harness.expanduser().resolve() + binary = args.binary_output.expanduser().resolve() + output = args.output.expanduser().resolve() + source = checkout / "libavfilter/dnn_filter_common.c" + if not harness.is_file() or not source.is_file(): + raise ValueError("harness and DNN filter-common source must exist") + + binary.parent.mkdir(parents=True, exist_ok=True) + output.parent.mkdir(parents=True, exist_ok=True) + compile_command = _compile_command(checkout, harness, binary) + compile_result = subprocess.run( + compile_command, cwd=checkout, check=False, capture_output=True, text=True + ) + environment: dict[str, str] | None = None + if compile_result.returncode == 0: + environment = os.environ.copy() + environment["ASAN_OPTIONS"] = ( + "halt_on_error=1:abort_on_error=1:detect_leaks=0" + ) + run_result = subprocess.run( + [str(binary)], + cwd=checkout, + env=environment, + check=False, + capture_output=True, + text=True, + ) + else: + run_result = subprocess.CompletedProcess( + [str(binary)], 127, "", "compile failed" + ) + + combined = run_result.stdout + run_result.stderr + indicators = { + "tensorflow_backend": "tensorflow_backend=1" in combined, + "four_output_names": "requested_outputs=4" in combined, + "asan_heap_buffer_overflow": ( + "AddressSanitizer: heap-buffer-overflow" in combined + ), + "initialization_aborted": run_result.returncode != 0, + } + expected_observed = compile_result.returncode == 0 and all(indicators.values()) + commit_result = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=checkout, + check=False, + capture_output=True, + text=True, + ) + payload = { + "schema_version": SCHEMA_VERSION, + "completed_at": datetime.now(timezone.utc).isoformat(), + "checkout": str(checkout), + "checkout_commit": commit_result.stdout.strip() or None, + "harness": str(harness), + "harness_sha256": _sha256(harness), + "source": str(source), + "source_sha256": _sha256(source), + "binary": str(binary), + "binary_sha256": _sha256(binary) if binary.is_file() else None, + "compile_command": compile_command, + "compile_returncode": compile_result.returncode, + "compile_stdout": compile_result.stdout, + "compile_stderr": compile_result.stderr, + "run_command": [str(binary)], + "asan_options": environment["ASAN_OPTIONS"] if environment else None, + "returncode": run_result.returncode, + "indicators": indicators, + "expected_observed": expected_observed, + "scope": ( + "Production DNN filter initialization. The sealed build omits the " + "optional TensorFlow backend, so the harness supplies inert module " + "stubs; the overflow occurs in backend-independent parsing before " + "module lookup or model loading." + ), + "stdout": run_result.stdout, + "stderr": run_result.stderr, + } + temporary = output.with_suffix(output.suffix + ".tmp") + temporary.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + temporary.replace(output) + print(output) + if not expected_observed: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/evaluations/run_ffmpeg_dnn_output_shape_reproducer.py b/evaluations/run_ffmpeg_dnn_output_shape_reproducer.py new file mode 100644 index 00000000..605af97e --- /dev/null +++ b/evaluations/run_ffmpeg_dnn_output_shape_reproducer.py @@ -0,0 +1,219 @@ +"""Build and record the DNN output-shape postprocessing diagnostic.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import platform +import subprocess +from datetime import datetime, timezone +from pathlib import Path + +SCHEMA_VERSION = "cw.ffmpeg.dnn-output-shape-reproducer.v1" + + +def _arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--checkout", type=Path, required=True) + parser.add_argument( + "--harness", + type=Path, + default=Path(__file__).with_name("ffmpeg_dnn_output_shape_reproducer.c"), + ) + parser.add_argument("--binary-output", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + return parser.parse_args() + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def _compile_commands( + checkout: Path, harness: Path, binary: Path +) -> list[list[str]]: + source_object = binary.with_name(binary.name + ".dnn_io_proc.o") + common = [ + "clang", + "-I.", + "-D_ISOC11_SOURCE", + "-D_FILE_OFFSET_BITS=64", + "-D_LARGEFILE_SOURCE", + "-I./compat/dispatch_semaphore", + "-DPIC", + "-I./compat/stdbit", + "-DHAVE_AV_CONFIG_H", + "-O1", + "-std=c17", + "-fPIC", + ] + source_command = [ + *common, + "-c", + str(checkout / "libavfilter/dnn/dnn_io_proc.c"), + "-o", + str(source_object), + ] + link_command = [ + *common, + "-fsanitize=address,undefined", + "-fno-omit-frame-pointer", + "-g", + "-o", + str(binary), + str(harness), + str(source_object), + "-Llibavfilter", + "-Llibswscale", + "-Llibavutil", + "-lavfilter", + "-lswscale", + "-lavutil", + "-lm", + "-lbz2", + "-lz", + ] + if platform.system() == "Darwin": + link_command.extend( + [ + "-framework", + "Foundation", + "-framework", + "AudioToolbox", + "-framework", + "CoreAudio", + "-framework", + "AVFoundation", + "-framework", + "CoreGraphics", + "-framework", + "OpenGL", + "-framework", + "Metal", + "-framework", + "VideoToolbox", + "-framework", + "CoreImage", + "-framework", + "AppKit", + "-framework", + "CoreFoundation", + "-framework", + "CoreMedia", + "-framework", + "CoreVideo", + "-framework", + "CoreServices", + "-framework", + "Security", + "-liconv", + ] + ) + link_command.append("-pthread") + return [source_command, link_command] + + +def main() -> None: + args = _arguments() + checkout = args.checkout.expanduser().resolve() + harness = args.harness.expanduser().resolve() + binary = args.binary_output.expanduser().resolve() + output = args.output.expanduser().resolve() + source = checkout / "libavfilter/dnn/dnn_io_proc.c" + if not harness.is_file() or not source.is_file(): + raise ValueError("harness and DNN postprocessor source must exist") + + binary.parent.mkdir(parents=True, exist_ok=True) + output.parent.mkdir(parents=True, exist_ok=True) + compile_commands = _compile_commands(checkout, harness, binary) + compile_results = [ + subprocess.run( + command, cwd=checkout, check=False, capture_output=True, text=True + ) + for command in compile_commands + ] + compile_returncode = next( + (result.returncode for result in compile_results if result.returncode), 0 + ) + environment: dict[str, str] | None = None + if compile_returncode == 0: + environment = os.environ.copy() + environment["ASAN_OPTIONS"] = ( + "halt_on_error=1:abort_on_error=1:detect_leaks=0" + ) + run_result = subprocess.run( + [str(binary)], + cwd=checkout, + env=environment, + check=False, + capture_output=True, + text=True, + ) + else: + run_result = subprocess.CompletedProcess( + [str(binary)], 127, "", "compile failed" + ) + + combined = run_result.stdout + run_result.stderr + indicators = { + "one_channel_nchw_tensor": "tensor_channels=1" in combined, + "rgb24_output_contract": "output_format=rgb24" in combined, + "asan_heap_buffer_overflow": ( + "AddressSanitizer: heap-buffer-overflow" in combined + ), + "postprocessing_aborted": run_result.returncode != 0, + } + expected_observed = compile_returncode == 0 and all(indicators.values()) + commit_result = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=checkout, + check=False, + capture_output=True, + text=True, + ) + payload = { + "schema_version": SCHEMA_VERSION, + "completed_at": datetime.now(timezone.utc).isoformat(), + "checkout": str(checkout), + "checkout_commit": commit_result.stdout.strip() or None, + "harness": str(harness), + "harness_sha256": _sha256(harness), + "source": str(source), + "source_sha256": _sha256(source), + "binary": str(binary), + "binary_sha256": _sha256(binary) if binary.is_file() else None, + "compile_commands": compile_commands, + "compile_returncode": compile_returncode, + "compile_stdout": "".join(result.stdout for result in compile_results), + "compile_stderr": "".join(result.stderr for result in compile_results), + "run_command": [str(binary)], + "asan_options": environment["ASAN_OPTIONS"] if environment else None, + "returncode": run_result.returncode, + "indicators": indicators, + "expected_observed": expected_observed, + "scope": ( + "Production postprocessor contract. dnn_io_proc.c is deliberately " + "compiled without ASan to pass its independent one-pointer source " + "plane-array read; the linked production libraries and harness " + "remain instrumented. The sealed build does not include an " + "optional DNN backend for end-to-end model loading." + ), + "stdout": run_result.stdout, + "stderr": run_result.stderr, + } + temporary = output.with_suffix(output.suffix + ".tmp") + temporary.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + temporary.replace(output) + print(output) + if not expected_observed: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/evaluations/run_ffmpeg_dovi_rpu_reproducer.py b/evaluations/run_ffmpeg_dovi_rpu_reproducer.py new file mode 100644 index 00000000..284dd443 --- /dev/null +++ b/evaluations/run_ffmpeg_dovi_rpu_reproducer.py @@ -0,0 +1,188 @@ +"""Build, run, and record the Dolby Vision RPU generator abort reproducer.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import platform +import subprocess +from datetime import UTC, datetime +from pathlib import Path + +SCHEMA_VERSION = "cw.ffmpeg.dovi-rpu-generator-reproducer.v1" + + +def _arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--checkout", type=Path, required=True) + parser.add_argument( + "--harness", + type=Path, + default=Path(__file__).with_name("ffmpeg_dovi_rpu_reproducer.c"), + ) + parser.add_argument("--binary-output", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + return parser.parse_args() + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def _compile_command(harness: Path, binary: Path) -> list[str]: + command = [ + "clang", + "-I.", + "-D_ISOC11_SOURCE", + "-D_FILE_OFFSET_BITS=64", + "-D_LARGEFILE_SOURCE", + "-I./compat/dispatch_semaphore", + "-DPIC", + "-I./compat/stdbit", + "-DZLIB_CONST", + "-DHAVE_AV_CONFIG_H", + "-fsanitize=address,undefined", + "-fno-omit-frame-pointer", + "-g", + "-O1", + "-std=c17", + "-fPIC", + "-pthread", + "-o", + str(binary), + str(harness), + "-Llibavcodec", + "-Llibswresample", + "-Llibswscale", + "-Llibavutil", + "-lavcodec", + "-lswresample", + "-lswscale", + "-lavutil", + "-lm", + "-lbz2", + "-lz", + ] + if platform.system() == "Darwin": + command.extend( + [ + "-framework", + "CoreFoundation", + "-framework", + "Security", + "-liconv", + "-framework", + "AudioToolbox", + "-framework", + "VideoToolbox", + "-framework", + "CoreMedia", + "-framework", + "CoreVideo", + "-framework", + "CoreServices", + ] + ) + command.append("-pthread") + return command + + +def main() -> None: + args = _arguments() + checkout = args.checkout.expanduser().resolve() + harness = args.harness.expanduser().resolve() + binary = args.binary_output.expanduser().resolve() + output = args.output.expanduser().resolve() + if not harness.is_file() or not (checkout / "libavcodec/libavcodec.a").is_file(): + raise ValueError("harness and configured FFmpeg static libraries must exist") + + binary.parent.mkdir(parents=True, exist_ok=True) + output.parent.mkdir(parents=True, exist_ok=True) + compile_command = _compile_command(harness, binary) + compile_result = subprocess.run( + compile_command, + cwd=checkout, + check=False, + capture_output=True, + text=True, + ) + if compile_result.returncode == 0: + environment = os.environ.copy() + environment["ASAN_OPTIONS"] = "halt_on_error=1:abort_on_error=1:detect_leaks=0" + run_result = subprocess.run( + [str(binary)], + cwd=checkout, + env=environment, + check=False, + capture_output=True, + text=True, + ) + else: + run_result = subprocess.CompletedProcess( + [str(binary)], 127, "", "compile failed" + ) + + combined = run_result.stdout + run_result.stderr + indicators = { + "production_parser_accepted_input": "parsed_rpu_bytes=" in combined, + "writer_undefined_shift": ( + "runtime error: shift exponent" in combined + ), + "terminating_writer_assertion": ( + "Assertion s->buf_ptr < s->buf_end failed" in combined + ), + "abnormal_termination": run_result.returncode < 0 or run_result.returncode >= 128, + } + expected_observed = all(indicators.values()) + commit_result = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=checkout, + check=False, + capture_output=True, + text=True, + ) + payload = { + "schema_version": SCHEMA_VERSION, + "completed_at": datetime.now(UTC).isoformat(), + "checkout": str(checkout), + "checkout_commit": commit_result.stdout.strip() or None, + "harness": str(harness), + "harness_sha256": _sha256(harness), + "binary": str(binary), + "binary_sha256": _sha256(binary) if binary.is_file() else None, + "parsed_coefficient_log2_denominator": 31, + "parsed_coefficient_integer_part": 100_000_000, + "mapping_segments": 24, + "coefficients_per_segment": 22, + "generator_estimated_bytes_per_segment": 177, + "compile_command": compile_command, + "compile_returncode": compile_result.returncode, + "compile_stdout": compile_result.stdout, + "compile_stderr": compile_result.stderr, + "run_command": [str(binary)], + "asan_options": "halt_on_error=1:abort_on_error=1:detect_leaks=0", + "returncode": run_result.returncode, + "indicators": indicators, + "expected_observed": expected_observed, + "stdout": run_result.stdout, + "stderr": run_result.stderr, + } + temporary = output.with_suffix(output.suffix + ".tmp") + temporary.write_text( + json.dumps(payload, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + temporary.replace(output) + print(output) + if not expected_observed: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/evaluations/run_ffmpeg_drawgraph_missing_primary_reproducer.py b/evaluations/run_ffmpeg_drawgraph_missing_primary_reproducer.py new file mode 100644 index 00000000..184f0a51 --- /dev/null +++ b/evaluations/run_ffmpeg_drawgraph_missing_primary_reproducer.py @@ -0,0 +1,198 @@ +"""Build and record the drawgraph missing-primary metadata reproducer.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import platform +import subprocess +from datetime import datetime, timezone +from pathlib import Path + +SCHEMA_VERSION = "cw.ffmpeg.drawgraph-missing-primary-reproducer.v1" + + +def _arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--checkout", type=Path, required=True) + parser.add_argument( + "--harness", + type=Path, + default=Path(__file__).with_name( + "ffmpeg_drawgraph_missing_primary_reproducer.c" + ), + ) + parser.add_argument("--binary-output", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + return parser.parse_args() + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def _compile_command(harness: Path, binary: Path) -> list[str]: + command = [ + "clang", + "-I.", + "-D_ISOC11_SOURCE", + "-D_FILE_OFFSET_BITS=64", + "-D_LARGEFILE_SOURCE", + "-I./compat/dispatch_semaphore", + "-DPIC", + "-I./compat/stdbit", + "-DHAVE_AV_CONFIG_H", + "-fsanitize=address,undefined", + "-fno-omit-frame-pointer", + "-g", + "-O1", + "-std=c17", + "-fPIC", + "-o", + str(binary), + str(harness), + "-Llibavfilter", + "-Llibavformat", + "-Llibavcodec", + "-Llibswscale", + "-Llibswresample", + "-Llibavutil", + "-lavfilter", + "-lavformat", + "-lavcodec", + "-lswscale", + "-lswresample", + "-lavutil", + "-lm", + "-lbz2", + "-lz", + ] + if platform.system() == "Darwin": + command.extend( + [ + "-framework", + "Foundation", + "-framework", + "AudioToolbox", + "-framework", + "CoreAudio", + "-framework", + "AVFoundation", + "-framework", + "CoreGraphics", + "-framework", + "OpenGL", + "-framework", + "Metal", + "-framework", + "VideoToolbox", + "-framework", + "CoreImage", + "-framework", + "AppKit", + "-framework", + "CoreFoundation", + "-framework", + "CoreMedia", + "-framework", + "CoreVideo", + "-framework", + "CoreServices", + "-framework", + "Security", + "-liconv", + ] + ) + command.append("-pthread") + return command + + +def main() -> None: + args = _arguments() + checkout = args.checkout.expanduser().resolve() + harness = args.harness.expanduser().resolve() + binary = args.binary_output.expanduser().resolve() + output = args.output.expanduser().resolve() + if not harness.is_file() or not (checkout / "libavfilter/libavfilter.a").is_file(): + raise ValueError("harness and configured FFmpeg static libraries must exist") + + binary.parent.mkdir(parents=True, exist_ok=True) + output.parent.mkdir(parents=True, exist_ok=True) + compile_command = _compile_command(harness, binary) + compile_result = subprocess.run( + compile_command, cwd=checkout, check=False, capture_output=True, text=True + ) + environment: dict[str, str] | None = None + if compile_result.returncode == 0: + environment = os.environ.copy() + environment["ASAN_OPTIONS"] = ( + "halt_on_error=1:abort_on_error=1:detect_leaks=0" + ) + run_result = subprocess.run( + [str(binary)], + cwd=checkout, + env=environment, + check=False, + capture_output=True, + text=True, + ) + else: + run_result = subprocess.CompletedProcess( + [str(binary)], 127, "", "compile failed" + ) + + combined = run_result.stdout + run_result.stderr + indicators = { + "missing_primary_metadata": "missing_primary_metadata=1" in combined, + "secondary_metadata_present": "secondary_metadata=1" in combined, + "two_pixel_output_width": "output_width=2" in combined, + "asan_heap_buffer_overflow": ( + "AddressSanitizer: heap-buffer-overflow" in combined + ), + "filter_aborted": run_result.returncode != 0, + } + expected_observed = compile_result.returncode == 0 and all(indicators.values()) + commit_result = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=checkout, + check=False, + capture_output=True, + text=True, + ) + payload = { + "schema_version": SCHEMA_VERSION, + "completed_at": datetime.now(timezone.utc).isoformat(), + "checkout": str(checkout), + "checkout_commit": commit_result.stdout.strip() or None, + "harness": str(harness), + "harness_sha256": _sha256(harness), + "binary": str(binary), + "binary_sha256": _sha256(binary) if binary.is_file() else None, + "compile_command": compile_command, + "compile_returncode": compile_result.returncode, + "compile_stdout": compile_result.stdout, + "compile_stderr": compile_result.stderr, + "run_command": [str(binary)], + "asan_options": environment["ASAN_OPTIONS"] if environment else None, + "returncode": run_result.returncode, + "indicators": indicators, + "expected_observed": expected_observed, + "stdout": run_result.stdout, + "stderr": run_result.stderr, + } + temporary = output.with_suffix(output.suffix + ".tmp") + temporary.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + temporary.replace(output) + print(output) + if not expected_observed: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/evaluations/run_ffmpeg_dvdsub_odd_width_reproducer.py b/evaluations/run_ffmpeg_dvdsub_odd_width_reproducer.py new file mode 100644 index 00000000..625e4f39 --- /dev/null +++ b/evaluations/run_ffmpeg_dvdsub_odd_width_reproducer.py @@ -0,0 +1,196 @@ +"""Build and record the DVD-subtitle odd-width RLE overflow reproducer.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import platform +import subprocess +from datetime import UTC, datetime +from pathlib import Path + +SCHEMA_VERSION = "cw.ffmpeg.dvdsub-odd-width-reproducer.v1" + + +def _arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--checkout", type=Path, required=True) + parser.add_argument( + "--harness", + type=Path, + default=Path(__file__).with_name("ffmpeg_dvdsub_odd_width_reproducer.c"), + ) + parser.add_argument("--binary-output", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + return parser.parse_args() + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def _compile_command(harness: Path, binary: Path) -> list[str]: + command = [ + "clang", + "-I.", + "-D_ISOC11_SOURCE", + "-D_FILE_OFFSET_BITS=64", + "-D_LARGEFILE_SOURCE", + "-I./compat/dispatch_semaphore", + "-DPIC", + "-I./compat/stdbit", + "-DHAVE_AV_CONFIG_H", + "-fsanitize=address,undefined", + "-fno-omit-frame-pointer", + "-g", + "-O1", + "-std=c17", + "-fPIC", + "-pthread", + "-o", + str(binary), + str(harness), + "-Llibavcodec", + "-Llibswresample", + "-Llibavutil", + "-lavcodec", + "-lswresample", + "-lavutil", + "-lm", + "-lbz2", + "-lz", + ] + if platform.system() == "Darwin": + command.extend( + [ + "-framework", + "CoreFoundation", + "-framework", + "Security", + "-liconv", + "-framework", + "AudioToolbox", + "-framework", + "VideoToolbox", + "-framework", + "CoreMedia", + "-framework", + "CoreVideo", + "-framework", + "CoreServices", + ] + ) + command.append("-pthread") + return command + + +def main() -> None: + args = _arguments() + checkout = args.checkout.expanduser().resolve() + harness = args.harness.expanduser().resolve() + binary = args.binary_output.expanduser().resolve() + output = args.output.expanduser().resolve() + source = checkout / "libavcodec/dvdsubenc.c" + if not harness.is_file() or not source.is_file(): + raise ValueError("harness and configured FFmpeg checkout must exist") + + binary.parent.mkdir(parents=True, exist_ok=True) + output.parent.mkdir(parents=True, exist_ok=True) + compile_command = _compile_command(harness, binary) + compile_result = subprocess.run( + compile_command, + cwd=checkout, + check=False, + capture_output=True, + text=True, + ) + environment = os.environ.copy() + environment["ASAN_OPTIONS"] = "halt_on_error=1:abort_on_error=1:detect_leaks=0" + run_command = [str(binary)] + if compile_result.returncode == 0: + run_result = subprocess.run( + run_command, + cwd=checkout, + env=environment, + check=False, + capture_output=True, + text=True, + ) + else: + run_result = subprocess.CompletedProcess( + run_command, 127, "", "compile failed" + ) + + combined = run_result.stdout + run_result.stderr + indicators = { + "odd_width_case": "width=1 height=200" in combined, + "underestimated_rle_budget": ( + "checked_rle_budget=100 actual_rle_bytes=200" in combined + ), + "asan_stack_buffer_overflow": ( + "AddressSanitizer: stack-buffer-overflow" in combined + ), + "production_encoder_in_trace": "dvdsub_encode" in combined, + "encoder_aborted": run_result.returncode != 0, + } + expected_observed = compile_result.returncode == 0 and all(indicators.values()) + commit_result = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=checkout, + check=False, + capture_output=True, + text=True, + ) + payload = { + "schema_version": SCHEMA_VERSION, + "completed_at": datetime.now(UTC).isoformat(), + "checkout": str(checkout), + "checkout_commit": commit_result.stdout.strip() or None, + "source": str(source), + "source_sha256": _sha256(source), + "harness": str(harness), + "harness_sha256": _sha256(harness), + "binary": str(binary), + "binary_sha256": _sha256(binary) if binary.is_file() else None, + "subtitle_width": 1, + "subtitle_height": 200, + "output_capacity": 142, + "checked_rle_budget": 100, + "actual_minimum_rle_bytes": 200, + "compile_command": compile_command, + "compile_returncode": compile_result.returncode, + "compile_stdout": compile_result.stdout, + "compile_stderr": compile_result.stderr, + "run_command": run_command, + "asan_options": environment["ASAN_OPTIONS"], + "returncode": run_result.returncode, + "indicators": indicators, + "expected_observed": expected_observed, + "scope": ( + "dvdsub_encode checks floor(width*height/2) bytes for RLE. " + "dvd_encode_rle encodes even and odd rows separately and pads " + "each odd-width row to a full byte, requiring " + "ceil(width/2)*height bytes even for a single-color bitmap." + ), + "stdout": run_result.stdout, + "stderr": run_result.stderr, + } + temporary = output.with_suffix(output.suffix + ".tmp") + temporary.write_text( + json.dumps(payload, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + temporary.replace(output) + print(output) + if not expected_observed: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/evaluations/run_ffmpeg_entropy_reproducer.py b/evaluations/run_ffmpeg_entropy_reproducer.py new file mode 100644 index 00000000..be18b5e6 --- /dev/null +++ b/evaluations/run_ffmpeg_entropy_reproducer.py @@ -0,0 +1,125 @@ +"""Run and record the high-bit-depth entropy histogram reproducer.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import subprocess +import tempfile +from datetime import UTC, datetime +from pathlib import Path + +SCHEMA_VERSION = "cw.ffmpeg.entropy-reproducer.v1" + + +def _arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--ffmpeg", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + return parser.parse_args() + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def main() -> None: + args = _arguments() + ffmpeg = args.ffmpeg.expanduser().resolve() + output = args.output.expanduser().resolve() + if not ffmpeg.is_file(): + raise ValueError("ffmpeg binary must exist") + + output.parent.mkdir(parents=True, exist_ok=True) + width = height = 4 + sample = 1 << 10 + raw_bytes = sample.to_bytes(2, "little") * width * height + with tempfile.TemporaryDirectory(prefix="ffmpeg-entropy-") as temporary: + raw = Path(temporary) / "invalid-gray10le.raw" + raw.write_bytes(raw_bytes) + command = [ + str(ffmpeg), + "-hide_banner", + "-loglevel", + "error", + "-f", + "rawvideo", + "-pixel_format", + "gray10le", + "-video_size", + f"{width}x{height}", + "-i", + str(raw), + "-vf", + "entropy", + "-frames:v", + "1", + "-f", + "null", + "-", + ] + environment = os.environ.copy() + environment["ASAN_OPTIONS"] = "halt_on_error=1:abort_on_error=1:detect_leaks=0" + result = subprocess.run( + command, + check=False, + capture_output=True, + text=True, + env=environment, + ) + + combined = result.stdout + result.stderr + indicators = { + "asan_heap_buffer_overflow": "AddressSanitizer: heap-buffer-overflow" in combined, + "eight_byte_histogram_access": ( + "READ of size 8" in combined or "WRITE of size 8" in combined + ), + "filter_frame_in_trace": "filter_frame" in combined, + } + expected_observed = all(indicators.values()) + commit_result = subprocess.run( + ["git", "-C", str(ffmpeg.parent), "rev-parse", "HEAD"], + check=False, + capture_output=True, + text=True, + ) + payload = { + "schema_version": SCHEMA_VERSION, + "completed_at": datetime.now(UTC).isoformat(), + "ffmpeg": str(ffmpeg), + "ffmpeg_sha256": _sha256(ffmpeg), + "checkout_commit": commit_result.stdout.strip() or None, + "pixel_format": "gray10le", + "width": width, + "height": height, + "declared_depth": 10, + "histogram_entries": 1 << 10, + "sample_value": sample, + "raw_input_hex": raw_bytes.hex(), + "command": command, + "asan_options": "halt_on_error=1:abort_on_error=1:detect_leaks=0", + "returncode": result.returncode, + "indicators": indicators, + "expected_observed": expected_observed, + "stdout": result.stdout, + "stderr": result.stderr, + } + temporary_output = output.with_suffix(output.suffix + ".tmp") + temporary_output.write_text( + json.dumps(payload, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + temporary_output.replace(output) + print(output) + if not expected_observed: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/evaluations/run_ffmpeg_ffv1_remap_reproducer.py b/evaluations/run_ffmpeg_ffv1_remap_reproducer.py new file mode 100644 index 00000000..b1b841ec --- /dev/null +++ b/evaluations/run_ffmpeg_ffv1_remap_reproducer.py @@ -0,0 +1,254 @@ +"""Build and record the FFV1 remap-table out-of-bounds-read reproducer.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import platform +import subprocess +from datetime import datetime, timezone +from pathlib import Path + +SCHEMA_VERSION = "cw.ffmpeg.ffv1-remap-reproducer.v1" + + +def _arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--checkout", type=Path, required=True) + parser.add_argument( + "--harness", + type=Path, + default=Path(__file__).with_name("ffmpeg_ffv1_remap_reproducer.c"), + ) + parser.add_argument("--sample-output", type=Path, required=True) + parser.add_argument("--binary-output", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + return parser.parse_args() + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def _compile_command(harness: Path, binary: Path) -> list[str]: + command = [ + "clang", + "-I.", + "-D_ISOC11_SOURCE", + "-D_FILE_OFFSET_BITS=64", + "-D_LARGEFILE_SOURCE", + "-I./compat/dispatch_semaphore", + "-DPIC", + "-I./compat/stdbit", + "-DZLIB_CONST", + "-DHAVE_AV_CONFIG_H", + "-fsanitize=address,undefined", + "-fno-omit-frame-pointer", + "-g", + "-O1", + "-std=c17", + "-fPIC", + "-pthread", + "-o", + str(binary), + str(harness), + "-Llibavformat", + "-Llibavcodec", + "-Llibswresample", + "-Llibavutil", + "-lavformat", + "-lavcodec", + "-lswresample", + "-lavutil", + "-lm", + "-lbz2", + "-lz", + ] + if platform.system() == "Darwin": + command.extend( + [ + "-framework", + "CoreFoundation", + "-framework", + "Security", + "-liconv", + "-framework", + "AudioToolbox", + "-framework", + "VideoToolbox", + "-framework", + "CoreMedia", + "-framework", + "CoreVideo", + "-framework", + "CoreServices", + ] + ) + command.append("-pthread") + return command + + +def main() -> None: + args = _arguments() + checkout = args.checkout.expanduser().resolve() + harness = args.harness.expanduser().resolve() + sample = args.sample_output.expanduser().resolve() + binary = args.binary_output.expanduser().resolve() + output = args.output.expanduser().resolve() + ffmpeg = checkout / "ffmpeg" + if not harness.is_file() or not ffmpeg.is_file(): + raise ValueError("harness and sanitizer-instrumented FFmpeg must exist") + + sample.parent.mkdir(parents=True, exist_ok=True) + binary.parent.mkdir(parents=True, exist_ok=True) + output.parent.mkdir(parents=True, exist_ok=True) + sample_command = [ + str(ffmpeg), + "-hide_banner", + "-loglevel", + "error", + "-f", + "lavfi", + "-i", + "nullsrc=s=513x1,format=yuv444p16le,geq=lum='X':cb=0:cr=0", + "-frames:v", + "1", + "-c:v", + "ffv1", + "-level", + "4", + "-strict", + "experimental", + "-remap_mode", + "1", + "-slicecrc", + "0", + "-coder", + "1", + "-f", + "nut", + "-y", + str(sample), + ] + sample_result = subprocess.run( + sample_command, check=False, capture_output=True, text=True + ) + baseline_command = [ + str(ffmpeg), + "-hide_banner", + "-loglevel", + "error", + "-i", + str(sample), + "-frames:v", + "1", + "-f", + "null", + "-", + ] + baseline_result = subprocess.run( + baseline_command, check=False, capture_output=True, text=True + ) + compile_command = _compile_command(harness, binary) + compile_result = subprocess.run( + compile_command, cwd=checkout, check=False, capture_output=True, text=True + ) + environment: dict[str, str] | None = None + if sample_result.returncode == 0 and compile_result.returncode == 0: + environment = os.environ.copy() + environment["ASAN_OPTIONS"] = ( + "halt_on_error=1:abort_on_error=1:detect_leaks=0" + ) + run_command = [str(binary), str(sample), "18", "0"] + run_result = subprocess.run( + run_command, + cwd=checkout, + env=environment, + check=False, + capture_output=True, + text=True, + ) + else: + run_command = [str(binary), str(sample), "18", "0"] + run_result = subprocess.CompletedProcess( + run_command, 127, "", "setup failed" + ) + + combined = run_result.stdout + run_result.stderr + indicators = { + "valid_sample_generated": sample_result.returncode == 0, + "valid_sample_decodes": baseline_result.returncode == 0, + "single_bit_mutation_applied": ( + "mutation_byte=18 mutation_bit=0" in combined + ), + "asan_heap_buffer_overflow": ( + "AddressSanitizer: heap-buffer-overflow" in combined + ), + "two_byte_out_of_bounds_read": "READ of size 2" in combined, + "decode_plane_sink": "in decode_plane" in combined, + } + expected_observed = compile_result.returncode == 0 and all(indicators.values()) + commit_result = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=checkout, + check=False, + capture_output=True, + text=True, + ) + payload = { + "schema_version": SCHEMA_VERSION, + "completed_at": datetime.now(timezone.utc).isoformat(), + "checkout": str(checkout), + "checkout_commit": commit_result.stdout.strip() or None, + "ffmpeg_sha256": _sha256(ffmpeg), + "harness": str(harness), + "harness_sha256": _sha256(harness), + "sample": str(sample), + "sample_sha256": _sha256(sample) if sample.is_file() else None, + "binary": str(binary), + "binary_sha256": _sha256(binary) if binary.is_file() else None, + "sample_command": sample_command, + "sample_returncode": sample_result.returncode, + "sample_stdout": sample_result.stdout, + "sample_stderr": sample_result.stderr, + "baseline_command": baseline_command, + "baseline_returncode": baseline_result.returncode, + "baseline_stdout": baseline_result.stdout, + "baseline_stderr": baseline_result.stderr, + "compile_command": compile_command, + "compile_returncode": compile_result.returncode, + "compile_stdout": compile_result.stdout, + "compile_stderr": compile_result.stderr, + "run_command": run_command, + "asan_options": environment["ASAN_OPTIONS"] if environment else None, + "returncode": run_result.returncode, + "indicators": indicators, + "expected_observed": expected_observed, + "scope": ( + "A valid 513x1 yuv444p16le FFV1 level-4 remap packet has a " + "non-power-of-two table count. Flipping bit zero of packet byte " + "18 changes attacker-controlled entropy data while preserving the " + "remap table. decode_line produces an unused masked symbol and " + "decode_plane reads outside the pixel_num-sized fltmap allocation." + ), + "stdout": run_result.stdout, + "stderr": run_result.stderr, + } + temporary = output.with_suffix(output.suffix + ".tmp") + temporary.write_text( + json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + temporary.replace(output) + print(output) + if not expected_observed: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/evaluations/run_ffmpeg_framepack_alpha_reproducer.py b/evaluations/run_ffmpeg_framepack_alpha_reproducer.py new file mode 100644 index 00000000..9ac5fd6b --- /dev/null +++ b/evaluations/run_ffmpeg_framepack_alpha_reproducer.py @@ -0,0 +1,125 @@ +"""Run and record the framepack alpha-plane invalid-write reproducer.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import subprocess +from datetime import UTC, datetime +from pathlib import Path + +SCHEMA_VERSION = "cw.ffmpeg.framepack-alpha-reproducer.v1" + + +def _arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--ffmpeg", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + return parser.parse_args() + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def main() -> None: + args = _arguments() + ffmpeg = args.ffmpeg.expanduser().resolve() + output = args.output.expanduser().resolve() + if not ffmpeg.is_file(): + raise ValueError("ffmpeg binary must exist") + + command = [ + str(ffmpeg), + "-hide_banner", + "-loglevel", + "error", + "-f", + "lavfi", + "-i", + "color=red:size=16x16,format=yuva420p", + "-f", + "lavfi", + "-i", + "color=blue:size=16x16,format=yuva420p", + "-filter_complex", + "[0:v][1:v]framepack=sbs", + "-frames:v", + "1", + "-f", + "null", + "-", + ] + environment = os.environ.copy() + environment["ASAN_OPTIONS"] = ( + "halt_on_error=1:abort_on_error=1:detect_leaks=0" + ) + environment["UBSAN_OPTIONS"] = "halt_on_error=0:print_stacktrace=0" + result = subprocess.run( + command, + env=environment, + check=False, + capture_output=True, + text=True, + ) + + combined = result.stdout + result.stderr + indicators = { + "asan_deadly_signal": "AddressSanitizer:DEADLYSIGNAL" in combined, + "invalid_write": "caused by a WRITE memory access" in combined, + "image_copy_plane_sink": "image_copy_plane" in combined, + "framepack_source_boundary": "horizontal_frame_pack" in combined, + "filter_aborted": result.returncode != 0, + } + expected_observed = all(indicators.values()) + checkout = ffmpeg.parent + commit_result = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=checkout, + check=False, + capture_output=True, + text=True, + ) + payload = { + "schema_version": SCHEMA_VERSION, + "completed_at": datetime.now(UTC).isoformat(), + "ffmpeg": str(ffmpeg), + "ffmpeg_sha256": _sha256(ffmpeg), + "checkout_commit": commit_result.stdout.strip() or None, + "command": command, + "asan_options": environment["ASAN_OPTIONS"], + "ubsan_options": environment["UBSAN_OPTIONS"], + "returncode": result.returncode, + "indicators": indicators, + "expected_observed": expected_observed, + "scope": ( + "The public framepack filter advertises alpha-bearing planar " + "formats. Its side-by-side helper initializes destination planes " + "zero through two but leaves dst[3] unset before av_image_copy2 " + "copies every plane in yuva420p. The alpha-plane copy therefore " + "writes through an indeterminate pointer. The vertical helper " + "independently leaves both dst[3] and linesizes[3] unset." + ), + "stdout": result.stdout, + "stderr": result.stderr, + } + output.parent.mkdir(parents=True, exist_ok=True) + temporary = output.with_suffix(output.suffix + ".tmp") + temporary.write_text( + json.dumps(payload, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + temporary.replace(output) + print(output) + if not expected_observed: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/evaluations/run_ffmpeg_hls_sample_aes_reproducer.py b/evaluations/run_ffmpeg_hls_sample_aes_reproducer.py new file mode 100644 index 00000000..7bd57778 --- /dev/null +++ b/evaluations/run_ffmpeg_hls_sample_aes_reproducer.py @@ -0,0 +1,172 @@ +"""Build, run, and record the HLS SAMPLE-AES ADTS length reproducer.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import platform +import subprocess +from datetime import UTC, datetime +from pathlib import Path + +SCHEMA_VERSION = "cw.ffmpeg.hls-sample-aes-reproducer.v1" + + +def _arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--checkout", type=Path, required=True) + parser.add_argument( + "--harness", + type=Path, + default=Path(__file__).with_name("ffmpeg_hls_sample_aes_reproducer.c"), + ) + parser.add_argument("--binary-output", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + return parser.parse_args() + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def _compile_command(harness: Path, binary: Path) -> list[str]: + command = [ + "clang", + "-I.", + "-D_ISOC11_SOURCE", + "-D_FILE_OFFSET_BITS=64", + "-D_LARGEFILE_SOURCE", + "-I./compat/dispatch_semaphore", + "-DPIC", + "-I./compat/stdbit", + "-DZLIB_CONST", + "-DHAVE_AV_CONFIG_H", + "-fsanitize=address,undefined", + "-fno-omit-frame-pointer", + "-g", + "-O1", + "-std=c17", + "-fPIC", + "-pthread", + "-o", + str(binary), + str(harness), + "-Llibavformat", + "-Llibavcodec", + "-Llibswresample", + "-Llibswscale", + "-Llibavutil", + "-lavformat", + "-lavcodec", + "-lswresample", + "-lswscale", + "-lavutil", + "-lm", + "-lbz2", + "-lz", + ] + if platform.system() == "Darwin": + command.extend( + [ + "-framework", + "CoreFoundation", + "-framework", + "Security", + "-liconv", + "-framework", + "AudioToolbox", + "-framework", + "VideoToolbox", + "-framework", + "CoreMedia", + "-framework", + "CoreVideo", + "-framework", + "CoreServices", + ] + ) + command.append("-pthread") + return command + + +def main() -> None: + args = _arguments() + checkout = args.checkout.expanduser().resolve() + harness = args.harness.expanduser().resolve() + binary = args.binary_output.expanduser().resolve() + output = args.output.expanduser().resolve() + if not harness.is_file() or not (checkout / "libavformat/libavformat.a").is_file(): + raise ValueError("harness and configured FFmpeg static libraries must exist") + + binary.parent.mkdir(parents=True, exist_ok=True) + output.parent.mkdir(parents=True, exist_ok=True) + compile_command = _compile_command(harness, binary) + compile_result = subprocess.run( + compile_command, cwd=checkout, check=False, capture_output=True, text=True + ) + if compile_result.returncode == 0: + environment = os.environ.copy() + environment["ASAN_OPTIONS"] = "halt_on_error=1:abort_on_error=1:detect_leaks=0" + run_result = subprocess.run( + [str(binary)], cwd=checkout, env=environment, + check=False, capture_output=True, text=True, + ) + else: + run_result = subprocess.CompletedProcess( + [str(binary)], 127, "", "compile failed" + ) + + combined = run_result.stdout + run_result.stderr + indicators = { + "asan_heap_buffer_overflow": "AddressSanitizer: heap-buffer-overflow" in combined, + "aes_decrypt_in_trace": ( + "aes_decrypt" in combined or "av_aes_crypt" in combined + ), + "out_of_bounds_access": ( + "READ of size" in combined or "WRITE of size" in combined + ), + } + expected_observed = all(indicators.values()) + commit_result = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=checkout, + check=False, capture_output=True, text=True, + ) + payload = { + "schema_version": SCHEMA_VERSION, + "completed_at": datetime.now(UTC).isoformat(), + "checkout": str(checkout), + "checkout_commit": commit_result.stdout.strip() or None, + "harness": str(harness), + "harness_sha256": _sha256(harness), + "binary": str(binary), + "binary_sha256": _sha256(binary) if binary.is_file() else None, + "packet_bytes": 64, + "declared_adts_frame_bytes": 8191, + "compile_command": compile_command, + "compile_returncode": compile_result.returncode, + "compile_stdout": compile_result.stdout, + "compile_stderr": compile_result.stderr, + "run_command": [str(binary)], + "asan_options": "halt_on_error=1:abort_on_error=1:detect_leaks=0", + "returncode": run_result.returncode, + "indicators": indicators, + "expected_observed": expected_observed, + "stdout": run_result.stdout, + "stderr": run_result.stderr, + } + temporary = output.with_suffix(output.suffix + ".tmp") + temporary.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + temporary.replace(output) + print(output) + if not expected_observed: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/evaluations/run_ffmpeg_ismindex_reproducer.py b/evaluations/run_ffmpeg_ismindex_reproducer.py new file mode 100644 index 00000000..395118d1 --- /dev/null +++ b/evaluations/run_ffmpeg_ismindex_reproducer.py @@ -0,0 +1,193 @@ +"""Build, run, and record the zero-sized ismindex tfra reproducer.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import platform +import subprocess +from datetime import UTC, datetime +from pathlib import Path + +SCHEMA_VERSION = "cw.ffmpeg.ismindex-reproducer.v1" + + +def _arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--checkout", type=Path, required=True) + parser.add_argument( + "--harness", + type=Path, + default=Path(__file__).with_name("ffmpeg_ismindex_reproducer.c"), + ) + parser.add_argument("--binary-output", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--timeout", type=float, default=1.0) + return parser.parse_args() + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def _compile_command(harness: Path, binary: Path) -> list[str]: + command = [ + "clang", + "-I.", + "-D_ISOC11_SOURCE", + "-D_FILE_OFFSET_BITS=64", + "-D_LARGEFILE_SOURCE", + "-I./compat/dispatch_semaphore", + "-DPIC", + "-I./compat/stdbit", + "-DZLIB_CONST", + "-DHAVE_AV_CONFIG_H", + "-fsanitize=address,undefined", + "-fno-omit-frame-pointer", + "-g", + "-O1", + "-std=c17", + "-fPIC", + "-pthread", + "-o", + str(binary), + str(harness), + "-Llibavformat", + "-Llibavcodec", + "-Llibswresample", + "-Llibswscale", + "-Llibavutil", + "-lavformat", + "-lavcodec", + "-lswresample", + "-lswscale", + "-lavutil", + "-lm", + "-lbz2", + "-lz", + ] + if platform.system() == "Darwin": + command.extend( + [ + "-framework", + "CoreFoundation", + "-framework", + "Security", + "-liconv", + "-framework", + "AudioToolbox", + "-framework", + "VideoToolbox", + "-framework", + "CoreMedia", + "-framework", + "CoreVideo", + "-framework", + "CoreServices", + ] + ) + command.append("-pthread") + return command + + +def main() -> None: + args = _arguments() + checkout = args.checkout.expanduser().resolve() + harness = args.harness.expanduser().resolve() + binary = args.binary_output.expanduser().resolve() + output = args.output.expanduser().resolve() + if args.timeout <= 0: + raise ValueError("timeout must be positive") + if not harness.is_file() or not (checkout / "libavformat/libavformat.a").is_file(): + raise ValueError("harness and configured FFmpeg static libraries must exist") + + binary.parent.mkdir(parents=True, exist_ok=True) + output.parent.mkdir(parents=True, exist_ok=True) + compile_command = _compile_command(harness, binary) + compile_result = subprocess.run( + compile_command, + cwd=checkout, + check=False, + capture_output=True, + text=True, + ) + timed_out = False + returncode: int | None = None + stdout = "" + stderr = "" + if compile_result.returncode == 0: + try: + run_result = subprocess.run( + [str(binary)], + cwd=checkout, + check=False, + capture_output=True, + text=True, + timeout=args.timeout, + ) + except subprocess.TimeoutExpired as exc: + timed_out = True + stdout = ( + (exc.stdout or b"").decode() + if isinstance(exc.stdout, bytes) + else exc.stdout or "" + ) + stderr = ( + (exc.stderr or b"").decode() + if isinstance(exc.stderr, bytes) + else exc.stderr or "" + ) + else: + returncode = run_result.returncode + stdout = run_result.stdout + stderr = run_result.stderr + + commit_result = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=checkout, + check=False, + capture_output=True, + text=True, + ) + payload = { + "schema_version": SCHEMA_VERSION, + "completed_at": datetime.now(UTC).isoformat(), + "checkout": str(checkout), + "checkout_commit": commit_result.stdout.strip() or None, + "harness": str(harness), + "harness_sha256": _sha256(harness), + "binary": str(binary), + "binary_sha256": _sha256(binary) if binary.is_file() else None, + "atom": "tfra", + "atom_size": 0, + "unknown_track_id": 1, + "timeout_seconds": args.timeout, + "compile_command": compile_command, + "compile_returncode": compile_result.returncode, + "compile_stdout": compile_result.stdout, + "compile_stderr": compile_result.stderr, + "run_command": [str(binary)], + "returncode": returncode, + "timed_out": timed_out, + "expected_observed": compile_result.returncode == 0 and timed_out, + "stdout": stdout, + "stderr": stderr, + } + temporary = output.with_suffix(output.suffix + ".tmp") + temporary.write_text( + json.dumps(payload, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + temporary.replace(output) + print(output) + if not payload["expected_observed"]: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/evaluations/run_ffmpeg_jv_bitstream_reproducer.py b/evaluations/run_ffmpeg_jv_bitstream_reproducer.py new file mode 100644 index 00000000..a6581a1f --- /dev/null +++ b/evaluations/run_ffmpeg_jv_bitstream_reproducer.py @@ -0,0 +1,169 @@ +"""Build, run, and record the JV recursive-bitstream reproducer.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import platform +import subprocess +from datetime import datetime, timezone +from pathlib import Path + +SCHEMA_VERSION = "cw.ffmpeg.jv-bitstream-reproducer.v1" + + +def _arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--checkout", type=Path, required=True) + parser.add_argument( + "--harness", + type=Path, + default=Path(__file__).with_name("ffmpeg_jv_bitstream_reproducer.c"), + ) + parser.add_argument("--binary-output", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + return parser.parse_args() + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def _compile_command(harness: Path, binary: Path) -> list[str]: + command = [ + "clang", + "-I.", + "-D_ISOC11_SOURCE", + "-D_FILE_OFFSET_BITS=64", + "-D_LARGEFILE_SOURCE", + "-I./compat/dispatch_semaphore", + "-DPIC", + "-I./compat/stdbit", + "-DHAVE_AV_CONFIG_H", + "-fsanitize=address,undefined", + "-fno-omit-frame-pointer", + "-g", + "-O1", + "-std=c17", + "-fPIC", + "-o", + str(binary), + str(harness), + "-Llibavcodec", + "-Llibswresample", + "-Llibavutil", + "-lavcodec", + "-lswresample", + "-lavutil", + "-lm", + "-lbz2", + "-lz", + ] + if platform.system() == "Darwin": + command.extend( + [ + "-framework", + "AudioToolbox", + "-framework", + "VideoToolbox", + "-framework", + "CoreFoundation", + "-framework", + "CoreMedia", + "-framework", + "CoreVideo", + "-framework", + "CoreServices", + "-liconv", + ] + ) + command.append("-pthread") + return command + + +def main() -> None: + args = _arguments() + checkout = args.checkout.expanduser().resolve() + harness = args.harness.expanduser().resolve() + binary = args.binary_output.expanduser().resolve() + output = args.output.expanduser().resolve() + if not harness.is_file() or not (checkout / "libavcodec/libavcodec.a").is_file(): + raise ValueError("harness and configured FFmpeg static libraries must exist") + + binary.parent.mkdir(parents=True, exist_ok=True) + output.parent.mkdir(parents=True, exist_ok=True) + compile_command = _compile_command(harness, binary) + compile_result = subprocess.run( + compile_command, cwd=checkout, check=False, capture_output=True, text=True + ) + environment: dict[str, str] | None = None + if compile_result.returncode == 0: + environment = os.environ.copy() + environment["ASAN_OPTIONS"] = ( + "halt_on_error=1:abort_on_error=1:detect_leaks=0" + ) + run_result = subprocess.run( + [str(binary)], + cwd=checkout, + env=environment, + check=False, + capture_output=True, + text=True, + ) + else: + run_result = subprocess.CompletedProcess( + [str(binary)], 127, "", "compile failed" + ) + + combined = run_result.stdout + run_result.stderr + indicators = { + "guard_minimum_satisfied": "blocks=64 video_bytes=16" in combined, + "recursive_path_selected": "max_recursive_path=1" in combined, + "decoder_returned_frame": "decoder_return=0" in combined, + "sanitizer_clean": "Sanitizer" not in combined, + } + expected_observed = compile_result.returncode == 0 and all(indicators.values()) + commit_result = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=checkout, + check=False, + capture_output=True, + text=True, + ) + payload = { + "schema_version": SCHEMA_VERSION, + "completed_at": datetime.now(timezone.utc).isoformat(), + "checkout": str(checkout), + "checkout_commit": commit_result.stdout.strip() or None, + "harness": str(harness), + "harness_sha256": _sha256(harness), + "binary": str(binary), + "binary_sha256": _sha256(binary) if binary.is_file() else None, + "compile_command": compile_command, + "compile_returncode": compile_result.returncode, + "compile_stdout": compile_result.stdout, + "compile_stderr": compile_result.stderr, + "run_command": [str(binary)], + "asan_options": environment["ASAN_OPTIONS"] if environment else None, + "returncode": run_result.returncode, + "indicators": indicators, + "expected_observed": expected_observed, + "stdout": run_result.stdout, + "stderr": run_result.stderr, + } + temporary = output.with_suffix(output.suffix + ".tmp") + temporary.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + temporary.replace(output) + print(output) + if not expected_observed: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/evaluations/run_ffmpeg_lcl_multithread_reproducer.py b/evaluations/run_ffmpeg_lcl_multithread_reproducer.py new file mode 100644 index 00000000..5aba2308 --- /dev/null +++ b/evaluations/run_ffmpeg_lcl_multithread_reproducer.py @@ -0,0 +1,166 @@ +"""Build, run, and record the LCL/ZLIB short-output reproducer.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import platform +import subprocess +from datetime import datetime, timezone +from pathlib import Path + +SCHEMA_VERSION = "cw.ffmpeg.lcl-multithread-reproducer.v1" + + +def _arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--checkout", type=Path, required=True) + parser.add_argument( + "--harness", + type=Path, + default=Path(__file__).with_name("ffmpeg_lcl_multithread_reproducer.c"), + ) + parser.add_argument("--binary-output", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + return parser.parse_args() + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def _compile_command(harness: Path, binary: Path) -> list[str]: + command = [ + "clang", + "-I.", + "-D_ISOC11_SOURCE", + "-D_FILE_OFFSET_BITS=64", + "-D_LARGEFILE_SOURCE", + "-I./compat/dispatch_semaphore", + "-DPIC", + "-I./compat/stdbit", + "-DZLIB_CONST", + "-DHAVE_AV_CONFIG_H", + "-fsanitize=address,undefined", + "-fno-omit-frame-pointer", + "-g", + "-O1", + "-std=c17", + "-fPIC", + "-pthread", + "-o", + str(binary), + str(harness), + "-Llibavcodec", + "-Llibswresample", + "-Llibavutil", + "-lavcodec", + "-lswresample", + "-lavutil", + "-lm", + "-lbz2", + "-lz", + ] + if platform.system() == "Darwin": + command.extend( + [ + "-framework", + "CoreFoundation", + "-framework", + "Security", + "-liconv", + "-framework", + "AudioToolbox", + "-framework", + "VideoToolbox", + "-framework", + "CoreMedia", + "-framework", + "CoreVideo", + "-framework", + "CoreServices", + ] + ) + command.append("-pthread") + return command + + +def main() -> None: + args = _arguments() + checkout = args.checkout.expanduser().resolve() + harness = args.harness.expanduser().resolve() + binary = args.binary_output.expanduser().resolve() + output = args.output.expanduser().resolve() + if not harness.is_file() or not (checkout / "libavcodec/libavcodec.a").is_file(): + raise ValueError("harness and configured FFmpeg static libraries must exist") + + binary.parent.mkdir(parents=True, exist_ok=True) + output.parent.mkdir(parents=True, exist_ok=True) + compile_command = _compile_command(harness, binary) + compile_result = subprocess.run( + compile_command, cwd=checkout, check=False, capture_output=True, text=True + ) + if compile_result.returncode == 0: + environment = os.environ.copy() + environment["ASAN_OPTIONS"] = ( + "halt_on_error=1:abort_on_error=1:detect_leaks=0:" + "malloc_fill_byte=165:max_malloc_fill_size=1048576" + ) + run_result = subprocess.run( + [str(binary)], cwd=checkout, env=environment, check=False, + capture_output=True, text=True + ) + else: + run_result = subprocess.CompletedProcess( + [str(binary)], 127, "", "compile failed" + ) + + combined = run_result.stdout + run_result.stderr + indicators = { + "decoder_accepted_short_output": run_result.returncode == 0, + "six_fresh_bytes": "fresh_bytes=6" in combined, + "762_malloc_fill_bytes": "leaked_malloc_fill_bytes=762" in combined, + "sanitizer_clean": "Sanitizer" not in combined, + } + expected_observed = all(indicators.values()) + commit_result = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=checkout, + check=False, capture_output=True, text=True, + ) + payload = { + "schema_version": SCHEMA_VERSION, + "completed_at": datetime.now(timezone.utc).isoformat(), + "checkout": str(checkout), + "checkout_commit": commit_result.stdout.strip() or None, + "harness": str(harness), + "harness_sha256": _sha256(harness), + "binary": str(binary), + "binary_sha256": _sha256(binary) if binary.is_file() else None, + "compile_command": compile_command, + "compile_returncode": compile_result.returncode, + "compile_stdout": compile_result.stdout, + "compile_stderr": compile_result.stderr, + "run_command": [str(binary)], + "asan_options": environment["ASAN_OPTIONS"] if compile_result.returncode == 0 else None, + "returncode": run_result.returncode, + "indicators": indicators, + "expected_observed": expected_observed, + "stdout": run_result.stdout, + "stderr": run_result.stderr, + } + temporary = output.with_suffix(output.suffix + ".tmp") + temporary.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + temporary.replace(output) + print(output) + if not expected_observed: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/evaluations/run_ffmpeg_magicyuv_truncated_slice_reproducer.py b/evaluations/run_ffmpeg_magicyuv_truncated_slice_reproducer.py new file mode 100644 index 00000000..11d6e37b --- /dev/null +++ b/evaluations/run_ffmpeg_magicyuv_truncated_slice_reproducer.py @@ -0,0 +1,180 @@ +"""Build and record the MagicYUV truncated-slice disclosure reproducer.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import platform +import subprocess +from datetime import datetime, timezone +from pathlib import Path + +SCHEMA_VERSION = "cw.ffmpeg.magicyuv-truncated-slice-reproducer.v1" + + +def _arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--checkout", type=Path, required=True) + parser.add_argument( + "--harness", + type=Path, + default=Path(__file__).with_name( + "ffmpeg_magicyuv_truncated_slice_reproducer.c" + ), + ) + parser.add_argument("--binary-output", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + return parser.parse_args() + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def _compile_command(harness: Path, binary: Path) -> list[str]: + command = [ + "clang", + "-I.", + "-D_ISOC11_SOURCE", + "-D_FILE_OFFSET_BITS=64", + "-D_LARGEFILE_SOURCE", + "-I./compat/dispatch_semaphore", + "-DPIC", + "-I./compat/stdbit", + "-DZLIB_CONST", + "-DHAVE_AV_CONFIG_H", + "-fsanitize=address,undefined", + "-fno-omit-frame-pointer", + "-g", + "-O1", + "-std=c17", + "-fPIC", + "-pthread", + "-o", + str(binary), + str(harness), + "-Llibavcodec", + "-Llibswresample", + "-Llibavutil", + "-lavcodec", + "-lswresample", + "-lavutil", + "-lm", + "-lbz2", + "-lz", + ] + if platform.system() == "Darwin": + command.extend( + [ + "-framework", + "CoreFoundation", + "-framework", + "Security", + "-liconv", + "-framework", + "AudioToolbox", + "-framework", + "VideoToolbox", + "-framework", + "CoreMedia", + "-framework", + "CoreVideo", + "-framework", + "CoreServices", + ] + ) + command.append("-pthread") + return command + + +def main() -> None: + args = _arguments() + checkout = args.checkout.expanduser().resolve() + harness = args.harness.expanduser().resolve() + binary = args.binary_output.expanduser().resolve() + output = args.output.expanduser().resolve() + if not harness.is_file() or not (checkout / "libavcodec/libavcodec.a").is_file(): + raise ValueError("harness and configured FFmpeg static libraries must exist") + + binary.parent.mkdir(parents=True, exist_ok=True) + output.parent.mkdir(parents=True, exist_ok=True) + compile_command = _compile_command(harness, binary) + compile_result = subprocess.run( + compile_command, cwd=checkout, check=False, capture_output=True, text=True + ) + environment: dict[str, str] | None = None + if compile_result.returncode == 0: + environment = os.environ.copy() + environment["ASAN_OPTIONS"] = ( + "halt_on_error=1:abort_on_error=1:detect_leaks=0" + ) + run_result = subprocess.run( + [str(binary)], + cwd=checkout, + env=environment, + check=False, + capture_output=True, + text=True, + ) + else: + run_result = subprocess.CompletedProcess( + [str(binary)], 127, "", "compile failed" + ) + + combined = run_result.stdout + run_result.stderr + indicators = { + "decoder_accepted_zero_sample_slice": run_result.returncode == 0, + "truncated_frame_decoded": "truncated_frame_decoded=1" in combined, + "entire_prior_frame_reversibly_disclosed": ( + "transformed_prior_frame_bytes=256" in combined + ), + "sanitizer_clean": "Sanitizer" not in combined, + } + expected_observed = compile_result.returncode == 0 and all(indicators.values()) + commit_result = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=checkout, + check=False, + capture_output=True, + text=True, + ) + payload = { + "schema_version": SCHEMA_VERSION, + "completed_at": datetime.now(timezone.utc).isoformat(), + "checkout": str(checkout), + "checkout_commit": commit_result.stdout.strip() or None, + "harness": str(harness), + "harness_sha256": _sha256(harness), + "binary": str(binary), + "binary_sha256": _sha256(binary) if binary.is_file() else None, + "raw_packet_size": 556, + "truncated_packet_size": 300, + "transformed_prior_frame_bytes": 256, + "compile_command": compile_command, + "compile_returncode": compile_result.returncode, + "compile_stdout": compile_result.stdout, + "compile_stderr": compile_result.stderr, + "run_command": [str(binary)], + "asan_options": environment["ASAN_OPTIONS"] if environment else None, + "returncode": run_result.returncode, + "indicators": indicators, + "expected_observed": expected_observed, + "stdout": run_result.stdout, + "stderr": run_result.stderr, + } + temporary = output.with_suffix(output.suffix + ".tmp") + temporary.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + temporary.replace(output) + print(output) + if not expected_observed: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/evaluations/run_ffmpeg_mestimate_mb_size_reproducer.py b/evaluations/run_ffmpeg_mestimate_mb_size_reproducer.py new file mode 100644 index 00000000..9364eaf1 --- /dev/null +++ b/evaluations/run_ffmpeg_mestimate_mb_size_reproducer.py @@ -0,0 +1,105 @@ +"""Record the vf_mestimate INT_MAX mb_size signed-shift undefined behavior.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import subprocess +from datetime import datetime, timezone +from pathlib import Path + +SCHEMA_VERSION = "cw.ffmpeg.mestimate-mb-size-reproducer.v1" + + +def _arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--checkout", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + return parser.parse_args() + + +def _sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def main() -> None: + args = _arguments() + checkout = args.checkout.expanduser().resolve() + output = args.output.expanduser().resolve() + ffmpeg = checkout / "ffmpeg" + source = checkout / "libavfilter/vf_mestimate.c" + if not ffmpeg.is_file() or not source.is_file(): + raise ValueError("configured FFmpeg executable and vf_mestimate.c must exist") + + command = [ + str(ffmpeg), + "-hide_banner", + "-loglevel", + "error", + "-f", + "lavfi", + "-i", + "testsrc2=s=64x64:r=1", + "-vf", + "mestimate=mb_size=2147483647", + "-frames:v", + "3", + "-f", + "null", + "-", + ] + environment = os.environ.copy() + environment["ASAN_OPTIONS"] = "halt_on_error=1:abort_on_error=1:detect_leaks=0" + environment["UBSAN_OPTIONS"] = "halt_on_error=1:print_stacktrace=1" + result = subprocess.run( + command, check=False, capture_output=True, text=True, env=environment + ) + combined = result.stdout + result.stderr + indicators = { + "public_int_max_option": "mestimate=mb_size=2147483647" in " ".join(command), + "signed_shift_ub": ( + "left shift of 1 by 31 places cannot be represented" in combined + ), + "production_source_line": "libavfilter/vf_mestimate.c:86" in combined, + "ubsan_aborted": result.returncode != 0, + } + expected_observed = all(indicators.values()) + payload = { + "schema_version": SCHEMA_VERSION, + "completed_at": datetime.now(timezone.utc).isoformat(), + "checkout": str(checkout), + "checkout_commit": subprocess.run( + ["git", "-C", str(checkout), "rev-parse", "HEAD"], + check=False, + capture_output=True, + text=True, + ).stdout.strip(), + "source": str(source), + "source_sha256": _sha256(source), + "command": command, + "asan_options": environment["ASAN_OPTIONS"], + "ubsan_options": environment["UBSAN_OPTIONS"], + "returncode": result.returncode, + "indicators": indicators, + "expected_observed": expected_observed, + "scope": ( + "The public mb_size option accepts INT_MAX. av_ceil_log2_c returns " + "31, after which config_input evaluates signed 1 << 31 before its " + "zero-block validation can reject the configuration." + ), + "stdout": result.stdout, + "stderr": result.stderr, + } + output.parent.mkdir(parents=True, exist_ok=True) + temporary = output.with_suffix(output.suffix + ".tmp") + temporary.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + temporary.replace(output) + print(output) + if not expected_observed: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/evaluations/run_ffmpeg_mpc7_lastframelen_reproducer.py b/evaluations/run_ffmpeg_mpc7_lastframelen_reproducer.py new file mode 100644 index 00000000..6b5287e3 --- /dev/null +++ b/evaluations/run_ffmpeg_mpc7_lastframelen_reproducer.py @@ -0,0 +1,170 @@ +"""Build, run, and record the Musepack SV7 last-frame ASan reproducer.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import platform +import subprocess +from datetime import UTC, datetime +from pathlib import Path + +SCHEMA_VERSION = "cw.ffmpeg.mpc7-lastframelen-reproducer.v1" + + +def _arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--checkout", type=Path, required=True) + parser.add_argument( + "--harness", + type=Path, + default=Path(__file__).with_name("ffmpeg_mpc7_lastframelen_reproducer.c"), + ) + parser.add_argument("--binary-output", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + return parser.parse_args() + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def _compile_command(harness: Path, binary: Path) -> list[str]: + command = [ + "clang", + "-I.", + "-D_ISOC11_SOURCE", + "-D_FILE_OFFSET_BITS=64", + "-D_LARGEFILE_SOURCE", + "-I./compat/dispatch_semaphore", + "-DPIC", + "-I./compat/stdbit", + "-DZLIB_CONST", + "-DHAVE_AV_CONFIG_H", + "-fsanitize=address,undefined", + "-fno-omit-frame-pointer", + "-g", + "-O1", + "-std=c17", + "-fPIC", + "-pthread", + "-o", + str(binary), + str(harness), + "-Llibavcodec", + "-Llibswresample", + "-Llibavutil", + "-lavcodec", + "-lswresample", + "-lavutil", + "-lm", + "-lbz2", + "-lz", + ] + if platform.system() == "Darwin": + command.extend( + [ + "-framework", + "CoreFoundation", + "-framework", + "Security", + "-liconv", + "-framework", + "AudioToolbox", + "-framework", + "VideoToolbox", + "-framework", + "CoreMedia", + "-framework", + "CoreVideo", + "-framework", + "CoreServices", + ] + ) + command.append("-pthread") + return command + + +def main() -> None: + args = _arguments() + checkout = args.checkout.expanduser().resolve() + harness = args.harness.expanduser().resolve() + binary = args.binary_output.expanduser().resolve() + output = args.output.expanduser().resolve() + if not harness.is_file() or not ( + checkout / "libavcodec/libavcodec.a" + ).is_file(): + raise ValueError("harness and configured FFmpeg static libraries must exist") + + binary.parent.mkdir(parents=True, exist_ok=True) + output.parent.mkdir(parents=True, exist_ok=True) + compile_command = _compile_command(harness, binary) + compile_result = subprocess.run( + compile_command, cwd=checkout, check=False, capture_output=True, text=True + ) + if compile_result.returncode == 0: + environment = os.environ.copy() + environment["ASAN_OPTIONS"] = ( + "halt_on_error=1:abort_on_error=1:detect_leaks=0" + ) + run_result = subprocess.run( + [str(binary)], cwd=checkout, env=environment, + check=False, capture_output=True, text=True, + ) + else: + run_result = subprocess.CompletedProcess( + [str(binary)], 127, "", "compile failed" + ) + + combined = run_result.stdout + run_result.stderr + indicators = { + "asan_heap_buffer_overflow": ( + "AddressSanitizer: heap-buffer-overflow" in combined + ), + "two_byte_read": "READ of size 2" in combined, + "main_in_trace": " in main" in combined, + } + expected_observed = all(indicators.values()) + commit_result = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=checkout, + check=False, capture_output=True, text=True, + ) + payload = { + "schema_version": SCHEMA_VERSION, + "completed_at": datetime.now(UTC).isoformat(), + "checkout": str(checkout), + "checkout_commit": commit_result.stdout.strip() or None, + "harness": str(harness), + "harness_sha256": _sha256(harness), + "binary": str(binary), + "binary_sha256": _sha256(binary) if binary.is_file() else None, + "allocated_samples_per_channel": 1152, + "reported_samples_per_channel": 2047, + "compile_command": compile_command, + "compile_returncode": compile_result.returncode, + "compile_stdout": compile_result.stdout, + "compile_stderr": compile_result.stderr, + "run_command": [str(binary)], + "asan_options": "halt_on_error=1:abort_on_error=1:detect_leaks=0", + "returncode": run_result.returncode, + "indicators": indicators, + "expected_observed": expected_observed, + "stdout": run_result.stdout, + "stderr": run_result.stderr, + } + temporary = output.with_suffix(output.suffix + ".tmp") + temporary.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + temporary.replace(output) + print(output) + if not expected_observed: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/evaluations/run_ffmpeg_nellymoser_trellis_reproducer.py b/evaluations/run_ffmpeg_nellymoser_trellis_reproducer.py new file mode 100644 index 00000000..9f158921 --- /dev/null +++ b/evaluations/run_ffmpeg_nellymoser_trellis_reproducer.py @@ -0,0 +1,119 @@ +"""Run and record the NellyMoser trellis off-by-one reproducer.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import subprocess +from datetime import UTC, datetime +from pathlib import Path + +SCHEMA_VERSION = "cw.ffmpeg.nellymoser-trellis-reproducer.v1" + + +def _arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--ffmpeg", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + return parser.parse_args() + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def main() -> None: + args = _arguments() + ffmpeg = args.ffmpeg.expanduser().resolve() + output = args.output.expanduser().resolve() + if not ffmpeg.is_file(): + raise ValueError("ffmpeg binary must exist") + + command = [ + str(ffmpeg), + "-hide_banner", + "-loglevel", + "error", + "-f", + "lavfi", + "-i", + "anoisesrc=color=white:sample_rate=44100:duration=0.05,volume=10", + "-c:a", + "nellymoser", + "-trellis", + "1", + "-f", + "null", + "-", + ] + environment = os.environ.copy() + environment["ASAN_OPTIONS"] = ( + "halt_on_error=0:abort_on_error=0:detect_leaks=0" + ) + environment["UBSAN_OPTIONS"] = "halt_on_error=0:print_stacktrace=0" + result = subprocess.run( + command, + env=environment, + check=False, + capture_output=True, + text=True, + ) + + combined = result.stdout + result.stderr + indicators = { + "trellis_float_index_at_opt_size": ( + "nellymoserenc.c:272:29: runtime error: index 35768 out of bounds" + in combined + ), + "trellis_path_index_at_opt_size": ( + "nellymoserenc.c:274:29: runtime error: index 35768 out of bounds" + in combined + ), + "asan_heap_buffer_overflow": ( + "AddressSanitizer: heap-buffer-overflow" in combined + ), + "read_after_trellis_allocation": ( + "READ of size 4" in combined + and "0 bytes after 3290656-byte region" in combined + ), + } + expected_observed = all(indicators.values()) + commit_result = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=ffmpeg.parent, + check=False, + capture_output=True, + text=True, + ) + payload = { + "schema_version": SCHEMA_VERSION, + "completed_at": datetime.now(UTC).isoformat(), + "ffmpeg": str(ffmpeg), + "ffmpeg_sha256": _sha256(ffmpeg), + "checkout_commit": commit_result.stdout.strip() or None, + "command": command, + "asan_options": environment["ASAN_OPTIONS"], + "ubsan_options": environment["UBSAN_OPTIONS"], + "returncode": result.returncode, + "indicators": indicators, + "expected_observed": expected_observed, + "stdout": result.stdout, + "stderr": result.stderr, + } + output.parent.mkdir(parents=True, exist_ok=True) + temporary = output.with_suffix(output.suffix + ".tmp") + temporary.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + temporary.replace(output) + print(output) + if not expected_observed: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/evaluations/run_ffmpeg_pan_named_channel_reproducer.py b/evaluations/run_ffmpeg_pan_named_channel_reproducer.py new file mode 100644 index 00000000..f7fa0416 --- /dev/null +++ b/evaluations/run_ffmpeg_pan_named_channel_reproducer.py @@ -0,0 +1,126 @@ +"""Run and record the pan named-channel out-of-bounds reproducer.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import subprocess +from datetime import UTC, datetime +from pathlib import Path + +SCHEMA_VERSION = "cw.ffmpeg.pan-named-channel-reproducer.v1" + + +def _arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--ffmpeg", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + return parser.parse_args() + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def main() -> None: + args = _arguments() + ffmpeg = args.ffmpeg.expanduser().resolve() + output = args.output.expanduser().resolve() + if not ffmpeg.is_file(): + raise ValueError("ffmpeg binary must exist") + + command = [ + str(ffmpeg), + "-hide_banner", + "-loglevel", + "error", + "-f", + "lavfi", + "-i", + "anullsrc=r=48000:cl=stereo", + "-af", + "pan=stereo|FL=AMBI0", + "-frames:a", + "1", + "-f", + "null", + "-", + ] + environment = os.environ.copy() + environment["ASAN_OPTIONS"] = ( + "halt_on_error=1:abort_on_error=1:detect_leaks=0" + ) + environment["UBSAN_OPTIONS"] = "halt_on_error=0:print_stacktrace=0" + result = subprocess.run( + command, + env=environment, + check=False, + capture_output=True, + text=True, + ) + + combined = result.stdout + result.stderr + indicators = { + "named_channel_index_1024": ( + "index 1024 out of bounds for type 'int[64]'" in combined + ), + "ubsan_out_of_bounds": ( + "UndefinedBehaviorSanitizer: undefined-behavior" in combined + ), + "pan_source_boundary": "libavfilter/af_pan.c" in combined, + "first_operation_is_read": "READ of size 4" in combined, + "asan_stack_violation": "AddressSanitizer: stack-" in combined, + "filter_aborted": result.returncode != 0, + } + expected_observed = all(indicators.values()) + checkout = ffmpeg.parent + commit_result = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=checkout, + check=False, + capture_output=True, + text=True, + ) + payload = { + "schema_version": SCHEMA_VERSION, + "completed_at": datetime.now(UTC).isoformat(), + "ffmpeg": str(ffmpeg), + "ffmpeg_sha256": _sha256(ffmpeg), + "checkout_commit": commit_result.stdout.strip() or None, + "command": command, + "asan_options": environment["ASAN_OPTIONS"], + "ubsan_options": environment["UBSAN_OPTIONS"], + "returncode": result.returncode, + "indicators": indicators, + "expected_observed": expected_observed, + "scope": ( + "AMBI0 is a public named AVChannel whose numeric value is 1024. " + "The pan parser accepts that value as an input-channel index even " + "though its used_in_ch and gain dimensions contain 64 entries. " + "The first observed invalid operation is the used_in_ch read; if " + "execution continues, the subsequent assignments also index the " + "same out-of-range slot." + ), + "stdout": result.stdout, + "stderr": result.stderr, + } + output.parent.mkdir(parents=True, exist_ok=True) + temporary = output.with_suffix(output.suffix + ".tmp") + temporary.write_text( + json.dumps(payload, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + temporary.replace(output) + print(output) + if not expected_observed: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/evaluations/run_ffmpeg_qdm2_reproducer.py b/evaluations/run_ffmpeg_qdm2_reproducer.py new file mode 100644 index 00000000..bc872d72 --- /dev/null +++ b/evaluations/run_ffmpeg_qdm2_reproducer.py @@ -0,0 +1,274 @@ +"""Build, run, and record the RTP/QDM2 small-block ASan reproducer.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import platform +import subprocess +import tempfile +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +SCHEMA_VERSION = "cw.ffmpeg.qdm2-reproducer.v1" + + +def _arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--checkout", type=Path, required=True) + parser.add_argument( + "--harness", + type=Path, + default=Path(__file__).with_name("ffmpeg_qdm2_reproducer.c"), + ) + parser.add_argument("--binary-output", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument( + "--expect", + choices=("vulnerable", "fixed"), + default="vulnerable", + ) + parser.add_argument( + "--rtpdec-source", + type=Path, + help="Optional offline replacement rtpdec_qdm2.c to link before libavformat", + ) + return parser.parse_args() + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def _run( + command: list[str], + *, + cwd: Path, + env: dict[str, str] | None = None, +) -> subprocess.CompletedProcess[str]: + return subprocess.run( + command, + cwd=cwd, + env=env, + check=False, + capture_output=True, + text=True, + ) + + +def _commit(checkout: Path) -> str | None: + result = _run(["git", "rev-parse", "HEAD"], cwd=checkout) + return result.stdout.strip() or None + + +def _compile_flags() -> list[str]: + return [ + "clang", + "-I.", + "-I./", + "-D_ISOC11_SOURCE", + "-D_FILE_OFFSET_BITS=64", + "-D_LARGEFILE_SOURCE", + "-I./compat/dispatch_semaphore", + "-DPIC", + "-I./compat/stdbit", + "-DZLIB_CONST", + "-DHAVE_AV_CONFIG_H", + "-DBUILDING_avformat", + "-fsanitize=address,undefined", + "-fno-omit-frame-pointer", + "-g", + "-O1", + "-std=c17", + "-fPIC", + "-pthread", + ] + + +def _link_flags() -> list[str]: + flags = [ + "-Llibavcodec", + "-Llibavdevice", + "-Llibavfilter", + "-Llibavformat", + "-Llibavutil", + "-Llibswscale", + "-Llibswresample", + "-lavformat", + "-lavcodec", + "-lswresample", + "-lswscale", + "-lavutil", + "-lm", + "-lbz2", + "-lz", + ] + if platform.system() == "Darwin": + flags[7:7] = [ + "-Wl,-dynamic,-search_paths_first", + "-Wl,-no_warn_duplicate_libraries", + ] + flags.extend( + [ + "-framework", + "CoreFoundation", + "-framework", + "Security", + "-liconv", + "-framework", + "AudioToolbox", + "-framework", + "VideoToolbox", + "-framework", + "CoreMedia", + "-framework", + "CoreVideo", + "-framework", + "CoreServices", + ] + ) + flags.append("-pthread") + return flags + + +def _write_json(path: Path, payload: Any) -> None: + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text( + json.dumps(payload, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + temporary.replace(path) + + +def main() -> None: + args = _arguments() + checkout = args.checkout.expanduser().resolve() + harness = args.harness.expanduser().resolve() + binary = args.binary_output.expanduser().resolve() + output = args.output.expanduser().resolve() + replacement = args.rtpdec_source.expanduser().resolve() if args.rtpdec_source else None + required = [ + checkout / "config.h", + checkout / "libavformat/libavformat.a", + checkout / "libavcodec/libavcodec.a", + checkout / "libswresample/libswresample.a", + checkout / "libswscale/libswscale.a", + checkout / "libavutil/libavutil.a", + harness, + ] + if any(not path.is_file() for path in required): + raise ValueError("checkout must contain the configured FFmpeg static libraries") + if replacement is not None and not replacement.is_file(): + raise ValueError("rtpdec replacement source does not exist") + + binary.parent.mkdir(parents=True, exist_ok=True) + output.parent.mkdir(parents=True, exist_ok=True) + compile_steps: list[dict[str, Any]] = [] + with tempfile.TemporaryDirectory(prefix="ffmpeg-qdm2-build-") as temporary: + replacement_object: Path | None = None + if replacement is not None: + replacement_object = Path(temporary) / "rtpdec_qdm2.o" + object_command = _compile_flags() + [ + "-Ilibavformat", + "-c", + str(replacement), + "-o", + str(replacement_object), + ] + object_result = _run(object_command, cwd=checkout) + compile_steps.append( + { + "command": object_command, + "returncode": object_result.returncode, + "stdout": object_result.stdout, + "stderr": object_result.stderr, + } + ) + if object_result.returncode != 0: + _write_json( + output, + { + "schema_version": SCHEMA_VERSION, + "completed_at": datetime.now(UTC).isoformat(), + "compile_steps": compile_steps, + }, + ) + raise SystemExit(1) + + link_command = _compile_flags() + ["-o", str(binary), str(harness)] + if replacement_object is not None: + link_command.append(str(replacement_object)) + link_command.extend(_link_flags()) + link_result = _run(link_command, cwd=checkout) + compile_steps.append( + { + "command": link_command, + "returncode": link_result.returncode, + "stdout": link_result.stdout, + "stderr": link_result.stderr, + } + ) + + if compile_steps[-1]["returncode"] == 0: + environment = os.environ.copy() + environment["ASAN_OPTIONS"] = "halt_on_error=1:abort_on_error=1:detect_leaks=0" + run_result = _run([str(binary)], cwd=checkout, env=environment) + else: + run_result = subprocess.CompletedProcess([str(binary)], 127, "", "compile failed") + combined = run_result.stdout + run_result.stderr + indicators = { + "asan_negative_size_param": "AddressSanitizer: negative-size-param" in combined, + "negative_size_is_minus_one": "size=-1" in combined, + "qdm2_parse_packet_in_trace": "qdm2_parse_packet" in combined, + "sanitizer_error": "ERROR: AddressSanitizer" in combined, + "clean_invalid_data_rejection": run_result.returncode == 1 + and "AddressSanitizer" not in combined, + } + expected_observed = ( + all( + indicators[key] + for key in ( + "asan_negative_size_param", + "negative_size_is_minus_one", + "qdm2_parse_packet_in_trace", + ) + ) + if args.expect == "vulnerable" + else indicators["clean_invalid_data_rejection"] + ) + payload = { + "schema_version": SCHEMA_VERSION, + "completed_at": datetime.now(UTC).isoformat(), + "checkout": str(checkout), + "checkout_commit": _commit(checkout), + "harness": str(harness), + "harness_sha256": _sha256(harness), + "binary": str(binary), + "binary_sha256": _sha256(binary) if binary.is_file() else None, + "expect": args.expect, + "replacement_source": str(replacement) if replacement else None, + "replacement_source_sha256": _sha256(replacement) if replacement else None, + "compile_steps": compile_steps, + "run_command": [str(binary)], + "asan_options": "halt_on_error=1:abort_on_error=1:detect_leaks=0", + "returncode": run_result.returncode, + "indicators": indicators, + "expected_observed": expected_observed, + "stdout": run_result.stdout, + "stderr": run_result.stderr, + } + _write_json(output, payload) + print(output) + if not expected_observed: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/evaluations/run_ffmpeg_ratecontrol_stats_reproducer.py b/evaluations/run_ffmpeg_ratecontrol_stats_reproducer.py new file mode 100644 index 00000000..bf356de2 --- /dev/null +++ b/evaluations/run_ffmpeg_ratecontrol_stats_reproducer.py @@ -0,0 +1,148 @@ +"""Run and record the two-pass rate-control picture-type ASan reproducer.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import subprocess +import tempfile +from datetime import UTC, datetime +from pathlib import Path + +SCHEMA_VERSION = "cw.ffmpeg.ratecontrol-stats-reproducer.v1" + + +def _arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--ffmpeg", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + return parser.parse_args() + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def main() -> None: + args = _arguments() + ffmpeg = args.ffmpeg.expanduser().resolve() + output = args.output.expanduser().resolve() + if not ffmpeg.is_file(): + raise ValueError("ffmpeg binary must exist") + + environment = os.environ.copy() + environment["ASAN_OPTIONS"] = ( + "halt_on_error=1:abort_on_error=1:detect_leaks=0" + ) + with tempfile.TemporaryDirectory(prefix="clearwing-ratecontrol-") as temp_dir: + passlog_prefix = Path(temp_dir) / "passlog" + common = [ + str(ffmpeg), + "-hide_banner", + "-loglevel", + "error", + "-f", + "lavfi", + "-i", + "color=c=black:s=16x16:r=1:d=2", + "-c:v", + "mpeg2video", + ] + pass1_command = [ + *common, + "-pass", + "1", + "-passlogfile", + str(passlog_prefix), + "-f", + "null", + "-", + ] + pass1_result = subprocess.run( + pass1_command, + env=environment, + check=False, + capture_output=True, + text=True, + ) + stats_path = Path(f"{passlog_prefix}-0.log") + original_stats = stats_path.read_text() if stats_path.is_file() else "" + malicious_stats = original_stats.replace("type:1", "type:99", 1) + stats_mutated = malicious_stats != original_stats + if stats_mutated: + stats_path.write_text(malicious_stats) + pass2_command = [ + *common, + "-pass", + "2", + "-passlogfile", + str(passlog_prefix), + "-f", + "null", + "-", + ] + pass2_result = subprocess.run( + pass2_command, + env=environment, + check=False, + capture_output=True, + text=True, + ) + + combined = pass2_result.stdout + pass2_result.stderr + indicators = { + "first_pass_succeeded": pass1_result.returncode == 0, + "stats_picture_type_mutated": stats_mutated, + "ubsan_type_99_index": "index 99 out of bounds" in combined, + "asan_heap_buffer_overflow": ( + "AddressSanitizer: heap-buffer-overflow" in combined + ), + "eight_byte_read": "READ of size 8" in combined, + "rate_control_init_in_trace": "ff_rate_control_init" in combined, + "encoder_aborted": pass2_result.returncode != 0, + } + expected_observed = all(indicators.values()) + commit_result = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=ffmpeg.parent, + check=False, + capture_output=True, + text=True, + ) + payload = { + "schema_version": SCHEMA_VERSION, + "completed_at": datetime.now(UTC).isoformat(), + "ffmpeg": str(ffmpeg), + "ffmpeg_sha256": _sha256(ffmpeg), + "checkout_commit": commit_result.stdout.strip() or None, + "pass1_command": pass1_command, + "pass1_returncode": pass1_result.returncode, + "pass1_stdout": pass1_result.stdout, + "pass1_stderr": pass1_result.stderr, + "original_stats": original_stats, + "malicious_stats": malicious_stats, + "pass2_command": pass2_command, + "asan_options": environment["ASAN_OPTIONS"], + "returncode": pass2_result.returncode, + "indicators": indicators, + "expected_observed": expected_observed, + "stdout": pass2_result.stdout, + "stderr": pass2_result.stderr, + } + output.parent.mkdir(parents=True, exist_ok=True) + temporary = output.with_suffix(output.suffix + ".tmp") + temporary.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + temporary.replace(output) + print(output) + if not expected_observed: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/evaluations/run_ffmpeg_rdt_aac_reproducer.py b/evaluations/run_ffmpeg_rdt_aac_reproducer.py new file mode 100644 index 00000000..e176a749 --- /dev/null +++ b/evaluations/run_ffmpeg_rdt_aac_reproducer.py @@ -0,0 +1,193 @@ +"""Build, run, and record the oversized RDT/AAC cache-copy reproducer.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import platform +import subprocess +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +SCHEMA_VERSION = "cw.ffmpeg.rdt-aac-reproducer.v1" + + +def _arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--checkout", type=Path, required=True) + parser.add_argument( + "--harness", + type=Path, + default=Path(__file__).with_name("ffmpeg_rdt_aac_reproducer.c"), + ) + parser.add_argument("--binary-output", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + return parser.parse_args() + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def _run( + command: list[str], + *, + cwd: Path, + env: dict[str, str] | None = None, +) -> subprocess.CompletedProcess[str]: + return subprocess.run( + command, + cwd=cwd, + env=env, + check=False, + capture_output=True, + text=True, + ) + + +def _compile_command(checkout: Path, harness: Path, binary: Path) -> list[str]: + command = [ + "clang", + "-I.", + "-D_ISOC11_SOURCE", + "-D_FILE_OFFSET_BITS=64", + "-D_LARGEFILE_SOURCE", + "-I./compat/dispatch_semaphore", + "-DPIC", + "-I./compat/stdbit", + "-DZLIB_CONST", + "-DHAVE_AV_CONFIG_H", + "-DBUILDING_avformat", + "-fsanitize=address,undefined", + "-fno-omit-frame-pointer", + "-g", + "-O1", + "-std=c17", + "-fPIC", + "-pthread", + "-o", + str(binary), + str(harness), + "-Llibavformat", + "-Llibavcodec", + "-Llibswresample", + "-Llibswscale", + "-Llibavutil", + "-lavformat", + "-lavcodec", + "-lswresample", + "-lswscale", + "-lavutil", + "-lm", + "-lbz2", + "-lz", + ] + if platform.system() == "Darwin": + command.extend( + [ + "-framework", + "CoreFoundation", + "-framework", + "Security", + "-liconv", + "-framework", + "AudioToolbox", + "-framework", + "VideoToolbox", + "-framework", + "CoreMedia", + "-framework", + "CoreVideo", + "-framework", + "CoreServices", + ] + ) + command.append("-pthread") + return command + + +def _write_json(path: Path, payload: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text( + json.dumps(payload, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + temporary.replace(path) + + +def main() -> None: + args = _arguments() + checkout = args.checkout.expanduser().resolve() + harness = args.harness.expanduser().resolve() + binary = args.binary_output.expanduser().resolve() + output = args.output.expanduser().resolve() + required = [ + checkout / "config.h", + checkout / "libavformat/libavformat.a", + checkout / "libavcodec/libavcodec.a", + checkout / "libswresample/libswresample.a", + checkout / "libswscale/libswscale.a", + checkout / "libavutil/libavutil.a", + harness, + ] + if any(not path.is_file() for path in required): + raise ValueError("checkout must contain the configured FFmpeg static libraries") + + binary.parent.mkdir(parents=True, exist_ok=True) + compile_command = _compile_command(checkout, harness, binary) + compile_result = _run(compile_command, cwd=checkout) + if compile_result.returncode == 0: + environment = os.environ.copy() + environment["ASAN_OPTIONS"] = "halt_on_error=1:abort_on_error=1:detect_leaks=0" + run_result = _run([str(binary)], cwd=checkout, env=environment) + else: + run_result = subprocess.CompletedProcess([str(binary)], 127, "", "compile failed") + + combined = run_result.stdout + run_result.stderr + indicators = { + "asan_heap_buffer_overflow": "AddressSanitizer: heap-buffer-overflow" in combined, + "write_overflow": "WRITE of size" in combined, + "memcpy_in_trace": "memcpy" in combined, + "rdt_parse_packet_in_trace": "rdt_parse_packet" in combined, + } + expected_observed = all(indicators.values()) + commit_result = _run(["git", "rev-parse", "HEAD"], cwd=checkout) + payload = { + "schema_version": SCHEMA_VERSION, + "completed_at": datetime.now(UTC).isoformat(), + "checkout": str(checkout), + "checkout_commit": commit_result.stdout.strip() or None, + "harness": str(harness), + "harness_sha256": _sha256(harness), + "binary": str(binary), + "binary_sha256": _sha256(binary) if binary.is_file() else None, + "input_bytes": 8192 + 1024, + "destination_bytes": 8192 + 64, + "compile_command": compile_command, + "compile_returncode": compile_result.returncode, + "compile_stdout": compile_result.stdout, + "compile_stderr": compile_result.stderr, + "run_command": [str(binary)], + "asan_options": "halt_on_error=1:abort_on_error=1:detect_leaks=0", + "returncode": run_result.returncode, + "indicators": indicators, + "expected_observed": expected_observed, + "stdout": run_result.stdout, + "stderr": run_result.stderr, + } + _write_json(output, payload) + print(output) + if not expected_observed: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/evaluations/run_ffmpeg_rdt_reproducer.py b/evaluations/run_ffmpeg_rdt_reproducer.py new file mode 100644 index 00000000..d047d3b5 --- /dev/null +++ b/evaluations/run_ffmpeg_rdt_reproducer.py @@ -0,0 +1,184 @@ +"""Build, run, and record the zero-length RDT status-packet reproducer.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import platform +import subprocess +from datetime import UTC, datetime +from pathlib import Path + +SCHEMA_VERSION = "cw.ffmpeg.rdt-reproducer.v1" + + +def _arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--checkout", type=Path, required=True) + parser.add_argument( + "--harness", + type=Path, + default=Path(__file__).with_name("ffmpeg_rdt_reproducer.c"), + ) + parser.add_argument("--binary-output", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--timeout", type=float, default=1.0) + return parser.parse_args() + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def _commit(checkout: Path) -> str | None: + result = subprocess.run( + ["git", "-C", str(checkout), "rev-parse", "HEAD"], + check=False, + capture_output=True, + text=True, + ) + return result.stdout.strip() or None + + +def _compile_command(checkout: Path, harness: Path, binary: Path) -> list[str]: + command = [ + "clang", + "-I.", + "-D_ISOC11_SOURCE", + "-D_FILE_OFFSET_BITS=64", + "-D_LARGEFILE_SOURCE", + "-I./compat/dispatch_semaphore", + "-DPIC", + "-I./compat/stdbit", + "-DZLIB_CONST", + "-DHAVE_AV_CONFIG_H", + "-fsanitize=address,undefined", + "-fno-omit-frame-pointer", + "-g", + "-O1", + "-std=c17", + "-fPIC", + "-pthread", + "-Llibavformat", + "-Llibavcodec", + "-Llibswresample", + "-Llibswscale", + "-Llibavutil", + "-o", + str(binary), + str(harness), + "-lavformat", + "-lavcodec", + "-lswresample", + "-lswscale", + "-lavutil", + "-lm", + "-lbz2", + "-lz", + ] + if platform.system() == "Darwin": + command.extend( + [ + "-framework", + "CoreFoundation", + "-framework", + "Security", + "-liconv", + "-framework", + "AudioToolbox", + "-framework", + "VideoToolbox", + "-framework", + "CoreMedia", + "-framework", + "CoreVideo", + "-framework", + "CoreServices", + ] + ) + command.append("-pthread") + return command + + +def main() -> None: + args = _arguments() + checkout = args.checkout.expanduser().resolve() + harness = args.harness.expanduser().resolve() + binary = args.binary_output.expanduser().resolve() + output = args.output.expanduser().resolve() + if args.timeout <= 0: + raise ValueError("timeout must be positive") + if not harness.is_file() or not (checkout / "libavformat/libavformat.a").is_file(): + raise ValueError("harness and configured FFmpeg static libraries must exist") + + binary.parent.mkdir(parents=True, exist_ok=True) + output.parent.mkdir(parents=True, exist_ok=True) + compile_command = _compile_command(checkout, harness, binary) + compile_result = subprocess.run( + compile_command, + cwd=checkout, + check=False, + capture_output=True, + text=True, + ) + timed_out = False + returncode: int | None = None + stdout = "" + stderr = "" + if compile_result.returncode == 0: + try: + run_result = subprocess.run( + [str(binary)], + cwd=checkout, + check=False, + capture_output=True, + text=True, + timeout=args.timeout, + ) + except subprocess.TimeoutExpired as exc: + timed_out = True + stdout = (exc.stdout or b"").decode() if isinstance(exc.stdout, bytes) else exc.stdout or "" + stderr = (exc.stderr or b"").decode() if isinstance(exc.stderr, bytes) else exc.stderr or "" + else: + returncode = run_result.returncode + stdout = run_result.stdout + stderr = run_result.stderr + + payload = { + "schema_version": SCHEMA_VERSION, + "completed_at": datetime.now(UTC).isoformat(), + "checkout": str(checkout), + "checkout_commit": _commit(checkout), + "harness": str(harness), + "harness_sha256": _sha256(harness), + "binary": str(binary), + "binary_sha256": _sha256(binary) if binary.is_file() else None, + "status_packet_length": 0, + "input_bytes": 16, + "timeout_seconds": args.timeout, + "compile_command": compile_command, + "compile_returncode": compile_result.returncode, + "compile_stdout": compile_result.stdout, + "compile_stderr": compile_result.stderr, + "run_command": [str(binary)], + "returncode": returncode, + "timed_out": timed_out, + "expected_observed": compile_result.returncode == 0 and timed_out, + "stdout": stdout, + "stderr": stderr, + } + temporary = output.with_suffix(output.suffix + ".tmp") + temporary.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") + temporary.replace(output) + print(output) + if not payload["expected_observed"]: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/evaluations/run_ffmpeg_rtp_av1_ignored_obu_reproducer.py b/evaluations/run_ffmpeg_rtp_av1_ignored_obu_reproducer.py new file mode 100644 index 00000000..17be5f05 --- /dev/null +++ b/evaluations/run_ffmpeg_rtp_av1_ignored_obu_reproducer.py @@ -0,0 +1,183 @@ +"""Build and record the RTP/AV1 ignored-OBU cursor reproducer.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import platform +import subprocess +from datetime import UTC, datetime +from pathlib import Path + +SCHEMA_VERSION = "cw.ffmpeg.rtp-av1-ignored-obu-reproducer.v1" + + +def _arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--checkout", type=Path, required=True) + parser.add_argument( + "--harness", + type=Path, + default=Path(__file__).with_name( + "ffmpeg_rtp_av1_ignored_obu_reproducer.c" + ), + ) + parser.add_argument("--binary-output", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + return parser.parse_args() + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def _compile_command(harness: Path, binary: Path) -> list[str]: + command = [ + "clang", + "-I.", + "-D_ISOC11_SOURCE", + "-D_FILE_OFFSET_BITS=64", + "-D_LARGEFILE_SOURCE", + "-I./compat/dispatch_semaphore", + "-DPIC", + "-I./compat/stdbit", + "-DZLIB_CONST", + "-DHAVE_AV_CONFIG_H", + "-fsanitize=address,undefined", + "-fno-omit-frame-pointer", + "-g", + "-O1", + "-std=c17", + "-fPIC", + "-pthread", + "-Llibavformat", + "-Llibavcodec", + "-Llibswresample", + "-Llibswscale", + "-Llibavutil", + "-o", + str(binary), + str(harness), + "-lavformat", + "-lavcodec", + "-lswresample", + "-lswscale", + "-lavutil", + "-lm", + "-lbz2", + "-lz", + ] + if platform.system() == "Darwin": + command.extend( + [ + "-framework", + "CoreFoundation", + "-framework", + "Security", + "-liconv", + "-framework", + "AudioToolbox", + "-framework", + "VideoToolbox", + "-framework", + "CoreMedia", + "-framework", + "CoreVideo", + "-framework", + "CoreServices", + ] + ) + command.append("-pthread") + return command + + +def main() -> None: + args = _arguments() + checkout = args.checkout.expanduser().resolve() + harness = args.harness.expanduser().resolve() + binary = args.binary_output.expanduser().resolve() + output = args.output.expanduser().resolve() + if not harness.is_file() or not (checkout / "libavformat/libavformat.a").is_file(): + raise ValueError("harness and configured FFmpeg static libraries must exist") + + binary.parent.mkdir(parents=True, exist_ok=True) + output.parent.mkdir(parents=True, exist_ok=True) + compile_command = _compile_command(harness, binary) + compile_result = subprocess.run( + compile_command, cwd=checkout, check=False, capture_output=True, text=True + ) + environment: dict[str, str] | None = None + if compile_result.returncode == 0: + environment = os.environ.copy() + environment["ASAN_OPTIONS"] = ( + "halt_on_error=1:abort_on_error=1:detect_leaks=0" + ) + run_result = subprocess.run( + [str(binary)], + cwd=checkout, + env=environment, + check=False, + capture_output=True, + text=True, + ) + else: + run_result = subprocess.CompletedProcess( + [str(binary)], 127, "", "compile failed" + ) + + combined = run_result.stdout + run_result.stderr + indicators = { + "ignored_temporal_delimiter": "ignored_obu_size=100" in combined, + "output_cursor_gap": "expected_output_gap=100" in combined, + "asan_heap_buffer_overflow": ( + "AddressSanitizer: heap-buffer-overflow" in combined + ), + "depacketizer_aborted": run_result.returncode != 0, + } + expected_observed = compile_result.returncode == 0 and all(indicators.values()) + commit_result = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=checkout, + check=False, + capture_output=True, + text=True, + ) + payload = { + "schema_version": SCHEMA_VERSION, + "completed_at": datetime.now(UTC).isoformat(), + "checkout": str(checkout), + "checkout_commit": commit_result.stdout.strip() or None, + "harness": str(harness), + "harness_sha256": _sha256(harness), + "binary": str(binary), + "binary_sha256": _sha256(binary) if binary.is_file() else None, + "ignored_obu_size": 100, + "trailing_bytes": 17, + "compile_command": compile_command, + "compile_returncode": compile_result.returncode, + "compile_stdout": compile_result.stdout, + "compile_stderr": compile_result.stderr, + "run_command": [str(binary)], + "asan_options": environment["ASAN_OPTIONS"] if environment else None, + "returncode": run_result.returncode, + "indicators": indicators, + "expected_observed": expected_observed, + "stdout": run_result.stdout, + "stderr": run_result.stderr, + } + temporary = output.with_suffix(output.suffix + ".tmp") + temporary.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + temporary.replace(output) + print(output) + if not expected_observed: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/evaluations/run_ffmpeg_rtp_h263_small_packet_reproducer.py b/evaluations/run_ffmpeg_rtp_h263_small_packet_reproducer.py new file mode 100644 index 00000000..7946ca0f --- /dev/null +++ b/evaluations/run_ffmpeg_rtp_h263_small_packet_reproducer.py @@ -0,0 +1,206 @@ +"""Build and record the RTP/RFC2190 undersized-packet reproducer.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import platform +import subprocess +from datetime import UTC, datetime +from pathlib import Path + +SCHEMA_VERSION = "cw.ffmpeg.rtp-h263-small-packet-reproducer.v1" + + +def _arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--checkout", type=Path, required=True) + parser.add_argument( + "--harness", + type=Path, + default=Path(__file__).with_name( + "ffmpeg_rtp_h263_small_packet_reproducer.c" + ), + ) + parser.add_argument("--binary-output", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + return parser.parse_args() + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def _compile_command(harness: Path, binary: Path) -> list[str]: + command = [ + "clang", + "-I.", + "-D_ISOC11_SOURCE", + "-D_FILE_OFFSET_BITS=64", + "-D_LARGEFILE_SOURCE", + "-I./compat/dispatch_semaphore", + "-DPIC", + "-I./compat/stdbit", + "-DZLIB_CONST", + "-DHAVE_AV_CONFIG_H", + "-fsanitize=address,undefined", + "-fno-omit-frame-pointer", + "-g", + "-O1", + "-std=c17", + "-fPIC", + "-pthread", + "-o", + str(binary), + str(harness), + "-Llibavformat", + "-Llibavcodec", + "-Llibswresample", + "-Llibswscale", + "-Llibavutil", + "-lavformat", + "-lavcodec", + "-lswresample", + "-lswscale", + "-lavutil", + "-lm", + "-lbz2", + "-lz", + ] + if platform.system() == "Darwin": + command.extend( + [ + "-framework", + "CoreFoundation", + "-framework", + "Security", + "-liconv", + "-framework", + "AudioToolbox", + "-framework", + "VideoToolbox", + "-framework", + "CoreMedia", + "-framework", + "CoreVideo", + "-framework", + "CoreServices", + ] + ) + command.append("-pthread") + return command + + +def main() -> None: + args = _arguments() + checkout = args.checkout.expanduser().resolve() + harness = args.harness.expanduser().resolve() + binary = args.binary_output.expanduser().resolve() + output = args.output.expanduser().resolve() + if not harness.is_file() or not ( + checkout / "libavformat/libavformat.a" + ).is_file(): + raise ValueError( + "harness and configured FFmpeg static libraries must exist" + ) + + binary.parent.mkdir(parents=True, exist_ok=True) + output.parent.mkdir(parents=True, exist_ok=True) + compile_command = _compile_command(harness, binary) + compile_result = subprocess.run( + compile_command, + cwd=checkout, + check=False, + capture_output=True, + text=True, + ) + environment: dict[str, str] | None = None + if compile_result.returncode == 0: + environment = os.environ.copy() + environment["ASAN_OPTIONS"] = ( + "halt_on_error=1:abort_on_error=1:detect_leaks=0" + ) + run_result = subprocess.run( + [str(binary)], + cwd=checkout, + env=environment, + check=False, + capture_output=True, + text=True, + ) + else: + run_result = subprocess.CompletedProcess( + [str(binary)], 127, "", "compile failed" + ) + + combined = run_result.stdout + run_result.stderr + indicators = { + "public_rtp_muxer_reached": "fragment_size=-7" in combined, + "negative_size_param": "AddressSanitizer: negative-size-param" in combined, + "negative_copy_size": "size=-7" in combined, + "rfc2190_packetizer_in_trace": ( + "ff_rtp_send_h263_rfc2190" in combined + ), + "muxer_aborted": run_result.returncode != 0, + } + expected_observed = compile_result.returncode == 0 and all( + indicators.values() + ) + commit_result = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=checkout, + check=False, + capture_output=True, + text=True, + ) + payload = { + "schema_version": SCHEMA_VERSION, + "completed_at": datetime.now(UTC).isoformat(), + "checkout": str(checkout), + "checkout_commit": commit_result.stdout.strip() or None, + "harness": str(harness), + "harness_sha256": _sha256(harness), + "binary": str(binary), + "binary_sha256": _sha256(binary) if binary.is_file() else None, + "rtp_packet_size": 13, + "max_payload_size": 1, + "rfc2190_header_size": 8, + "derived_fragment_size": -7, + "compile_command": compile_command, + "compile_returncode": compile_result.returncode, + "compile_stdout": compile_result.stdout, + "compile_stderr": compile_result.stderr, + "run_command": [str(binary)], + "asan_options": environment["ASAN_OPTIONS"] if environment else None, + "returncode": run_result.returncode, + "indicators": indicators, + "expected_observed": expected_observed, + "scope": ( + "The harness uses the public RTP muxer with RFC2190 H.263 " + "packetization and a 13-byte packet sink. RTP initialization " + "accepts every packet size above the 12-byte common header, but " + "the RFC2190 path needs another eight payload-header bytes and " + "subtracts them without validating the codec-specific minimum." + ), + "stdout": run_result.stdout, + "stderr": run_result.stderr, + } + temporary = output.with_suffix(output.suffix + ".tmp") + temporary.write_text( + json.dumps(payload, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + temporary.replace(output) + print(output) + if not expected_observed: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/evaluations/run_ffmpeg_rtp_latm_header_reproducer.py b/evaluations/run_ffmpeg_rtp_latm_header_reproducer.py new file mode 100644 index 00000000..58aa5eb4 --- /dev/null +++ b/evaluations/run_ffmpeg_rtp_latm_header_reproducer.py @@ -0,0 +1,206 @@ +"""Build and record the RTP/LATM length-header overflow reproducer.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import platform +import subprocess +from datetime import UTC, datetime +from pathlib import Path + +SCHEMA_VERSION = "cw.ffmpeg.rtp-latm-header-reproducer.v1" + + +def _arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--checkout", type=Path, required=True) + parser.add_argument( + "--harness", + type=Path, + default=Path(__file__).with_name( + "ffmpeg_rtp_latm_header_reproducer.c" + ), + ) + parser.add_argument("--binary-output", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + return parser.parse_args() + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def _compile_command(harness: Path, binary: Path) -> list[str]: + command = [ + "clang", + "-I.", + "-D_ISOC11_SOURCE", + "-D_FILE_OFFSET_BITS=64", + "-D_LARGEFILE_SOURCE", + "-I./compat/dispatch_semaphore", + "-DPIC", + "-I./compat/stdbit", + "-DZLIB_CONST", + "-DHAVE_AV_CONFIG_H", + "-fsanitize=address,undefined", + "-fno-omit-frame-pointer", + "-g", + "-O1", + "-std=c17", + "-fPIC", + "-pthread", + "-o", + str(binary), + str(harness), + "-Llibavformat", + "-Llibavcodec", + "-Llibswresample", + "-Llibswscale", + "-Llibavutil", + "-lavformat", + "-lavcodec", + "-lswresample", + "-lswscale", + "-lavutil", + "-lm", + "-lbz2", + "-lz", + ] + if platform.system() == "Darwin": + command.extend( + [ + "-framework", + "CoreFoundation", + "-framework", + "Security", + "-liconv", + "-framework", + "AudioToolbox", + "-framework", + "VideoToolbox", + "-framework", + "CoreMedia", + "-framework", + "CoreVideo", + "-framework", + "CoreServices", + ] + ) + command.append("-pthread") + return command + + +def main() -> None: + args = _arguments() + checkout = args.checkout.expanduser().resolve() + harness = args.harness.expanduser().resolve() + binary = args.binary_output.expanduser().resolve() + output = args.output.expanduser().resolve() + if not harness.is_file() or not ( + checkout / "libavformat/libavformat.a" + ).is_file(): + raise ValueError( + "harness and configured FFmpeg static libraries must exist" + ) + + binary.parent.mkdir(parents=True, exist_ok=True) + output.parent.mkdir(parents=True, exist_ok=True) + compile_command = _compile_command(harness, binary) + compile_result = subprocess.run( + compile_command, + cwd=checkout, + check=False, + capture_output=True, + text=True, + ) + environment: dict[str, str] | None = None + if compile_result.returncode == 0: + environment = os.environ.copy() + environment["ASAN_OPTIONS"] = ( + "halt_on_error=1:abort_on_error=1:detect_leaks=0" + ) + run_result = subprocess.run( + [str(binary)], + cwd=checkout, + env=environment, + check=False, + capture_output=True, + text=True, + ) + else: + run_result = subprocess.CompletedProcess( + [str(binary)], 127, "", "compile failed" + ) + + combined = run_result.stdout + run_result.stderr + indicators = { + "public_rtp_muxer_reached": "latm_header_size=1501" in combined, + "asan_heap_buffer_overflow": ( + "AddressSanitizer: heap-buffer-overflow" in combined + ), + "large_header_write": "WRITE of size 1500" in combined, + "overflow_after_rtp_buffer": "after 1472-byte region" in combined, + "muxer_aborted": run_result.returncode != 0, + } + expected_observed = compile_result.returncode == 0 and all( + indicators.values() + ) + commit_result = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=checkout, + check=False, + capture_output=True, + text=True, + ) + payload = { + "schema_version": SCHEMA_VERSION, + "completed_at": datetime.now(UTC).isoformat(), + "checkout": str(checkout), + "checkout_commit": commit_result.stdout.strip() or None, + "harness": str(harness), + "harness_sha256": _sha256(harness), + "binary": str(binary), + "binary_sha256": _sha256(binary) if binary.is_file() else None, + "rtp_packet_size": 1472, + "aac_packet_size": 1500 * 0xFF, + "latm_header_size": 1501, + "compile_command": compile_command, + "compile_returncode": compile_result.returncode, + "compile_stdout": compile_result.stdout, + "compile_stderr": compile_result.stderr, + "run_command": [str(binary)], + "asan_options": environment["ASAN_OPTIONS"] if environment else None, + "returncode": run_result.returncode, + "indicators": indicators, + "expected_observed": expected_observed, + "scope": ( + "The harness uses the public RTP muxer with its LATM option and a " + "normal 1,472-byte packet sink. No AVPacket size contract ties one " + "AAC access unit to the RTP payload size. A 382,500-byte packet " + "therefore produces a 1,501-byte PayloadLengthInfo header, which " + "the muxer writes into its 1,472-byte packet buffer before " + "fragmenting the payload." + ), + "stdout": run_result.stdout, + "stderr": run_result.stderr, + } + temporary = output.with_suffix(output.suffix + ".tmp") + temporary.write_text( + json.dumps(payload, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + temporary.replace(output) + print(output) + if not expected_observed: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/evaluations/run_ffmpeg_showfreqs_reproducer.py b/evaluations/run_ffmpeg_showfreqs_reproducer.py new file mode 100644 index 00000000..83868d1e --- /dev/null +++ b/evaluations/run_ffmpeg_showfreqs_reproducer.py @@ -0,0 +1,85 @@ +"""Run and record the showfreqs group-delay ASan reproducer.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import subprocess +from datetime import UTC, datetime +from pathlib import Path + +SCHEMA_VERSION = "cw.ffmpeg.showfreqs-delay-reproducer.v1" + + +def _arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--ffmpeg", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + return parser.parse_args() + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def main() -> None: + args = _arguments() + ffmpeg = args.ffmpeg.expanduser().resolve() + output = args.output.expanduser().resolve() + if not ffmpeg.is_file(): + raise ValueError("ffmpeg binary must exist") + command = [ + str(ffmpeg), "-hide_banner", "-loglevel", "error", + "-f", "lavfi", "-i", "anoisesrc=r=48000:d=0.2", + "-filter_complex", "[0:a]showfreqs=data=delay[outv]", + "-map", "[outv]", "-frames:v", "1", "-f", "null", "-", + ] + environment = os.environ.copy() + environment["ASAN_OPTIONS"] = "halt_on_error=1:abort_on_error=1:detect_leaks=0" + result = subprocess.run( + command, env=environment, check=False, capture_output=True, text=True + ) + combined = result.stdout + result.stderr + indicators = { + "asan_heap_buffer_overflow": "AddressSanitizer: heap-buffer-overflow" in combined, + "four_byte_read": "READ of size 4" in combined, + "eight_bytes_before_allocation": "8 bytes before" in combined, + "filter_frame_in_trace": "filter_frame" in combined, + } + expected_observed = all(indicators.values()) + checkout = ffmpeg.parent + commit_result = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=checkout, + check=False, capture_output=True, text=True, + ) + payload = { + "schema_version": SCHEMA_VERSION, + "completed_at": datetime.now(UTC).isoformat(), + "ffmpeg": str(ffmpeg), + "ffmpeg_sha256": _sha256(ffmpeg), + "checkout_commit": commit_result.stdout.strip() or None, + "command": command, + "asan_options": "halt_on_error=1:abort_on_error=1:detect_leaks=0", + "returncode": result.returncode, + "indicators": indicators, + "expected_observed": expected_observed, + "stdout": result.stdout, + "stderr": result.stderr, + } + output.parent.mkdir(parents=True, exist_ok=True) + temporary = output.with_suffix(output.suffix + ".tmp") + temporary.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + temporary.replace(output) + print(output) + if not expected_observed: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/evaluations/run_ffmpeg_shufflepixels_inverse_reproducer.py b/evaluations/run_ffmpeg_shufflepixels_inverse_reproducer.py new file mode 100644 index 00000000..ef830821 --- /dev/null +++ b/evaluations/run_ffmpeg_shufflepixels_inverse_reproducer.py @@ -0,0 +1,109 @@ +"""Run and record the shufflepixels inverse partial-block ASan reproducer.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import subprocess +from datetime import UTC, datetime +from pathlib import Path + +SCHEMA_VERSION = "cw.ffmpeg.shufflepixels-inverse-reproducer.v1" + + +def _arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--ffmpeg", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + return parser.parse_args() + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def main() -> None: + args = _arguments() + ffmpeg = args.ffmpeg.expanduser().resolve() + output = args.output.expanduser().resolve() + if not ffmpeg.is_file(): + raise ValueError("ffmpeg binary must exist") + + command = [ + str(ffmpeg), + "-hide_banner", + "-loglevel", + "error", + "-f", + "lavfi", + "-i", + "color=c=black:s=10x2,format=gray,crop=10:1", + "-vf", + "shufflepixels=width=4:mode=horizontal:direction=inverse:seed=1", + "-frames:v", + "1", + "-f", + "null", + "-", + ] + environment = os.environ.copy() + environment["ASAN_OPTIONS"] = ( + "halt_on_error=1:abort_on_error=1:detect_leaks=0" + ) + result = subprocess.run( + command, + env=environment, + check=False, + capture_output=True, + text=True, + ) + + combined = result.stdout + result.stderr + indicators = { + "asan_heap_buffer_overflow": ( + "AddressSanitizer: heap-buffer-overflow" in combined + ), + "four_byte_write": "WRITE of size 4" in combined, + "write_after_40_byte_map": "0 bytes after 40-byte region" in combined, + "config_output_in_trace": "config_output" in combined, + "filter_aborted": result.returncode != 0, + } + expected_observed = all(indicators.values()) + commit_result = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=ffmpeg.parent, + check=False, + capture_output=True, + text=True, + ) + payload = { + "schema_version": SCHEMA_VERSION, + "completed_at": datetime.now(UTC).isoformat(), + "ffmpeg": str(ffmpeg), + "ffmpeg_sha256": _sha256(ffmpeg), + "checkout_commit": commit_result.stdout.strip() or None, + "command": command, + "asan_options": environment["ASAN_OPTIONS"], + "returncode": result.returncode, + "indicators": indicators, + "expected_observed": expected_observed, + "stdout": result.stdout, + "stderr": result.stderr, + } + output.parent.mkdir(parents=True, exist_ok=True) + temporary = output.with_suffix(output.suffix + ".tmp") + temporary.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + temporary.replace(output) + print(output) + if not expected_observed: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/evaluations/run_ffmpeg_vif_small_frame_reproducer.py b/evaluations/run_ffmpeg_vif_small_frame_reproducer.py new file mode 100644 index 00000000..0b58280a --- /dev/null +++ b/evaluations/run_ffmpeg_vif_small_frame_reproducer.py @@ -0,0 +1,113 @@ +"""Run and record the VIF small-frame reflection ASan reproducer.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import subprocess +from datetime import UTC, datetime +from pathlib import Path + +SCHEMA_VERSION = "cw.ffmpeg.vif-small-frame-reproducer.v1" + + +def _arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--ffmpeg", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + return parser.parse_args() + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def main() -> None: + args = _arguments() + ffmpeg = args.ffmpeg.expanduser().resolve() + output = args.output.expanduser().resolve() + if not ffmpeg.is_file(): + raise ValueError("ffmpeg binary must exist") + + command = [ + str(ffmpeg), + "-hide_banner", + "-loglevel", + "error", + "-f", + "lavfi", + "-i", + "color=c=black:s=2x2,format=gray", + "-f", + "lavfi", + "-i", + "color=c=white:s=2x2,format=gray", + "-filter_complex", + "[0:v][1:v]vif", + "-frames:v", + "1", + "-f", + "null", + "-", + ] + environment = os.environ.copy() + environment["ASAN_OPTIONS"] = ( + "halt_on_error=1:abort_on_error=1:detect_leaks=0" + ) + result = subprocess.run( + command, + env=environment, + check=False, + capture_output=True, + text=True, + ) + + combined = result.stdout + result.stderr + indicators = { + "asan_heap_buffer_overflow": ( + "AddressSanitizer: heap-buffer-overflow" in combined + ), + "four_byte_read": "READ of size 4" in combined, + "read_after_16_byte_buffer": "0 bytes after 16-byte region" in combined, + "vif_filter1d_in_trace": "vif_filter1d" in combined, + "filter_aborted": result.returncode != 0, + } + expected_observed = all(indicators.values()) + commit_result = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=ffmpeg.parent, + check=False, + capture_output=True, + text=True, + ) + payload = { + "schema_version": SCHEMA_VERSION, + "completed_at": datetime.now(UTC).isoformat(), + "ffmpeg": str(ffmpeg), + "ffmpeg_sha256": _sha256(ffmpeg), + "checkout_commit": commit_result.stdout.strip() or None, + "command": command, + "asan_options": environment["ASAN_OPTIONS"], + "returncode": result.returncode, + "indicators": indicators, + "expected_observed": expected_observed, + "stdout": result.stdout, + "stderr": result.stderr, + } + output.parent.mkdir(parents=True, exist_ok=True) + temporary = output.with_suffix(output.suffix + ".tmp") + temporary.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + temporary.replace(output) + print(output) + if not expected_observed: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/evaluations/run_ffmpeg_xpsnr_reproducer.py b/evaluations/run_ffmpeg_xpsnr_reproducer.py new file mode 100644 index 00000000..f964b0b0 --- /dev/null +++ b/evaluations/run_ffmpeg_xpsnr_reproducer.py @@ -0,0 +1,122 @@ +"""Run and record the XPSNR odd-frame high-pass ASan reproducer.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import subprocess +from datetime import UTC, datetime +from pathlib import Path + +SCHEMA_VERSION = "cw.ffmpeg.xpsnr-reproducer.v1" +WIDTH = 2049 +HEIGHT = 1153 + + +def _arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--ffmpeg", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + return parser.parse_args() + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def _commit(binary: Path) -> str | None: + result = subprocess.run( + ["git", "-C", str(binary.parent), "rev-parse", "HEAD"], + check=False, + capture_output=True, + text=True, + ) + return result.stdout.strip() or None + + +def main() -> None: + args = _arguments() + ffmpeg = args.ffmpeg.expanduser().resolve() + output = args.output.expanduser().resolve() + if not ffmpeg.is_file(): + raise ValueError("ffmpeg binary does not exist") + + output.parent.mkdir(parents=True, exist_ok=True) + source = f"color=c=black:s={WIDTH}x{HEIGHT}:r=25:d=0.1,format=yuv444p" + command = [ + str(ffmpeg), + "-hide_banner", + "-loglevel", + "verbose", + "-f", + "lavfi", + "-i", + source, + "-f", + "lavfi", + "-i", + source, + "-filter_complex", + "[0:v][1:v]xpsnr[outv]", + "-map", + "[outv]", + "-frames:v", + "1", + "-f", + "null", + "-", + ] + environment = os.environ.copy() + environment["ASAN_OPTIONS"] = "halt_on_error=1:abort_on_error=1:detect_leaks=0" + result = subprocess.run( + command, + check=False, + capture_output=True, + text=True, + env=environment, + ) + combined = result.stdout + result.stderr + allocation_size = WIDTH * HEIGHT * 2 + indicators = { + "asan_heap_buffer_overflow": "AddressSanitizer: heap-buffer-overflow" in combined, + "two_byte_read": "READ of size 2" in combined, + "two_bytes_after_allocation": f"2 bytes after {allocation_size}-byte region" in combined, + "highds_in_trace": "highds" in combined, + "do_xpsnr_in_trace": "do_xpsnr" in combined, + "av_calloc_allocation_in_trace": "av_calloc" in combined, + } + expected_observed = all(indicators.values()) + payload = { + "schema_version": SCHEMA_VERSION, + "completed_at": datetime.now(UTC).isoformat(), + "checkout_commit": _commit(ffmpeg), + "ffmpeg": str(ffmpeg), + "ffmpeg_sha256": _sha256(ffmpeg), + "width": WIDTH, + "height": HEIGHT, + "pixel_format": "yuv444p", + "source_buffer_bytes": allocation_size, + "command": command, + "asan_options": environment["ASAN_OPTIONS"], + "returncode": result.returncode, + "indicators": indicators, + "expected_observed": expected_observed, + "stdout": result.stdout, + "stderr": result.stderr, + } + temporary = output.with_suffix(output.suffix + ".tmp") + temporary.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") + temporary.replace(output) + print(output) + if not expected_observed: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/evaluations/run_ffmpeg_yuvcmp_partial_mb_reproducer.py b/evaluations/run_ffmpeg_yuvcmp_partial_mb_reproducer.py new file mode 100644 index 00000000..b1ea914a --- /dev/null +++ b/evaluations/run_ffmpeg_yuvcmp_partial_mb_reproducer.py @@ -0,0 +1,177 @@ +"""Build and record the yuvcmp partial-macroblock overflow reproducer.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import subprocess +import tempfile +from datetime import UTC, datetime +from pathlib import Path + +SCHEMA_VERSION = "cw.ffmpeg.yuvcmp-partial-mb-reproducer.v1" + + +def _arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--checkout", type=Path, required=True) + parser.add_argument( + "--harness", + type=Path, + default=Path(__file__).with_name("ffmpeg_yuvcmp_partial_mb_reproducer.c"), + ) + parser.add_argument("--binary-output", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + return parser.parse_args() + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def main() -> None: + args = _arguments() + checkout = args.checkout.expanduser().resolve() + harness = args.harness.expanduser().resolve() + binary = args.binary_output.expanduser().resolve() + output = args.output.expanduser().resolve() + source = checkout / "tools/yuvcmp.c" + config = checkout / "config.h" + if not harness.is_file() or not source.is_file() or not config.is_file(): + raise ValueError("harness and configured FFmpeg checkout must exist") + + binary.parent.mkdir(parents=True, exist_ok=True) + output.parent.mkdir(parents=True, exist_ok=True) + compile_command = [ + "clang", + "-I.", + "-include", + "config.h", + "-fsanitize=address,undefined", + "-fno-omit-frame-pointer", + "-g", + "-O1", + "-std=c17", + "-o", + str(binary), + str(harness), + ] + compile_result = subprocess.run( + compile_command, + cwd=checkout, + check=False, + capture_output=True, + text=True, + ) + + width = 17 + height = 16 + luma_size = width * height + chroma_size = width * height // 4 + frame_size = luma_size + 2 * chroma_size + differing_offset = 16 + first_bytes = bytearray(frame_size) + second_bytes = bytearray(frame_size) + second_bytes[differing_offset] = 1 + environment = os.environ.copy() + environment["ASAN_OPTIONS"] = "halt_on_error=1:abort_on_error=1:detect_leaks=0" + + with tempfile.TemporaryDirectory(prefix="ffmpeg-yuvcmp-") as temporary: + first = Path(temporary) / "first.yuv" + second = Path(temporary) / "second.yuv" + first.write_bytes(first_bytes) + second.write_bytes(second_bytes) + run_command = [str(binary), str(first), str(second)] + if compile_result.returncode == 0: + run_result = subprocess.run( + run_command, + cwd=checkout, + env=environment, + check=False, + capture_output=True, + text=True, + ) + else: + run_result = subprocess.CompletedProcess( + run_command, 127, "", "compile failed" + ) + + combined = run_result.stdout + run_result.stderr + indicators = { + "asan_heap_buffer_overflow": ( + "AddressSanitizer: heap-buffer-overflow" in combined + ), + "one_byte_out_of_bounds_access": ( + "READ of size 1" in combined or "WRITE of size 1" in combined + ), + "production_function_in_trace": "ffmpeg_yuvcmp_main" in combined, + "partial_macroblock_pixel_reported": "pixel ( 16,0" in combined, + } + expected_observed = compile_result.returncode == 0 and all(indicators.values()) + commit_result = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=checkout, + check=False, + capture_output=True, + text=True, + ) + payload = { + "schema_version": SCHEMA_VERSION, + "completed_at": datetime.now(UTC).isoformat(), + "checkout": str(checkout), + "checkout_commit": commit_result.stdout.strip() or None, + "source": str(source), + "source_sha256": _sha256(source), + "harness": str(harness), + "harness_sha256": _sha256(harness), + "binary": str(binary), + "binary_sha256": _sha256(binary) if binary.is_file() else None, + "width": width, + "height": height, + "luma_size": luma_size, + "chroma_size": chroma_size, + "frame_size": frame_size, + "floor_macroblock_columns": width // 16, + "floor_macroblock_rows": height // 16, + "mberrors_allocation_bytes": (width // 16) * (height // 16), + "differing_luma_offset": differing_offset, + "differing_pixel": [16, 0], + "computed_macroblock_index": 1, + "compile_command": compile_command, + "compile_returncode": compile_result.returncode, + "compile_stdout": compile_result.stdout, + "compile_stderr": compile_result.stderr, + "run_command": run_command, + "asan_options": environment["ASAN_OPTIONS"], + "returncode": run_result.returncode, + "indicators": indicators, + "expected_observed": expected_observed, + "scope": ( + "For dimensions not divisible by 16, yuvcmp floor-divides the " + "mberrors allocation dimensions but compares every luma pixel. " + "Pixel (16,0) of a 17x16 frame therefore selects index one in a " + "one-byte allocation. The later blockdump loop cannot reach a " + "partial macroblock because it iterates only the floor-sized array." + ), + "stdout": run_result.stdout, + "stderr": run_result.stderr, + } + temporary_output = output.with_suffix(output.suffix + ".tmp") + temporary_output.write_text( + json.dumps(payload, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + temporary_output.replace(output) + print(output) + if not expected_observed: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/evaluations/run_sourcehunt_blind_campaign.py b/evaluations/run_sourcehunt_blind_campaign.py new file mode 100644 index 00000000..7294f701 --- /dev/null +++ b/evaluations/run_sourcehunt_blind_campaign.py @@ -0,0 +1,516 @@ +"""Run a sealed, resumable, vulnerable-only SourceHunt campaign in rank waves.""" + +from __future__ import annotations + +import argparse +import asyncio +import hashlib +import json +import subprocess +from dataclasses import asdict +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +from clearwing.providers import LLMEndpoint, ProviderManager +from clearwing.sourcehunt.optimization import ( + GENERIC_INSTRUCTIONS_COMPACT_V1, + require_generic_prompt, +) +from clearwing.sourcehunt.runner import SourceHuntRunner + +SCHEMA_VERSION = "cw.sourcehunt.blind-campaign.v1" +PROMPT_BUNDLE = "generic-security-v1" +DEFAULT_SCAFFOLD_PROFILE = "proof-refinement-ledger-v1" +SCAFFOLD_PROFILES = ( + "minimal-linear-v1", + "candidate-ledger-v1", + "candidate-ledger-closure-v1", + "candidate-ledger-source-retry-v1", + "candidate-ledger-source-retry-active-v1", + "proof-refinement-ledger-v1", +) +CONTEXT_PROFILE = "compact-small-model-v1" + + +def _arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--checkout", required=True) + parser.add_argument("--compile-commands", required=True) + parser.add_argument("--base-url", required=True) + parser.add_argument("--api-key", default="local") + parser.add_argument("--model", default="dsv4-flash-nvfp4") + parser.add_argument("--output-dir", required=True) + parser.add_argument( + "--scaffold-profile", + choices=SCAFFOLD_PROFILES, + default=DEFAULT_SCAFFOLD_PROFILE, + ) + parser.add_argument("--start-offset", type=int, default=24) + parser.add_argument( + "--offsets", + help="Comma-separated exact zero-based rank offsets; runs one sparse replay", + ) + parser.add_argument( + "--paths-file", + help="JSON array of exact repository paths; pins a sealed sparse replay", + ) + parser.add_argument("--wave-size", type=int, default=12) + parser.add_argument("--waves", type=int, default=6) + parser.add_argument("--max-hunter-steps", type=int, default=40) + parser.add_argument("--max-parallel", type=int, default=4) + parser.add_argument("--sandbox-cpus", type=float, default=2.0) + parser.add_argument("--temperature", type=float, default=0.0) + parser.add_argument("--max-output-tokens", type=int, default=4096) + return parser.parse_args() + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def _write_json(path: Path, payload: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text( + json.dumps(payload, indent=2, sort_keys=True, default=str) + "\n", + encoding="utf-8", + ) + temporary.replace(path) + + +def _head(checkout: Path) -> str: + return subprocess.run( + ["git", "-C", str(checkout), "rev-parse", "HEAD"], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + + +def _tracked_changes(checkout: Path) -> str: + return subprocess.run( + ["git", "-C", str(checkout), "status", "--porcelain", "--untracked-files=no"], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + + +def _campaign_config(args: argparse.Namespace, checkout: Path, compile_commands: Path) -> dict: + offsets = _parse_offsets(args.offsets) + paths = _load_paths(args.paths_file) + if offsets is not None and paths is not None: + raise ValueError("offsets and paths-file are mutually exclusive") + if args.start_offset < 0 or args.wave_size < 1 or args.waves < 1: + raise ValueError("campaign offsets and wave bounds must be positive") + if args.max_hunter_steps < 1 or args.max_parallel < 1: + raise ValueError("campaign execution bounds must be positive") + if not 0.0 <= args.temperature <= 2.0 or args.max_output_tokens < 1: + raise ValueError("invalid generation bounds") + if not checkout.is_dir() or not compile_commands.is_file(): + raise ValueError("checkout and compile_commands must exist") + if _tracked_changes(checkout): + raise ValueError("sealed checkout has tracked changes") + try: + compile_entries = len(json.loads(compile_commands.read_text(encoding="utf-8"))) + except (OSError, json.JSONDecodeError, TypeError) as exc: + raise ValueError("compile_commands must be a JSON array") from exc + + bounds = { + "start_offset": args.start_offset, + "wave_size": args.wave_size, + "waves": args.waves, + "max_hunter_steps": args.max_hunter_steps, + "max_parallel": args.max_parallel, + "sandbox_cpus": args.sandbox_cpus, + "temperature": args.temperature, + "max_output_tokens": args.max_output_tokens, + "no_rank": True, + "starting_band": "fast", + "redundancy": 1, + "agent_mode": "constrained", + } + if offsets is not None: + bounds["offsets"] = offsets + if paths is not None: + bounds["paths"] = paths + + return { + "schema_version": SCHEMA_VERSION, + "checkout": str(checkout), + "commit": _head(checkout), + "compile_commands": str(compile_commands), + "compile_commands_sha256": _sha256(compile_commands), + "compile_command_entries": compile_entries, + "base_url": args.base_url, + "model": args.model, + "prompt_bundle": PROMPT_BUNDLE, + "scaffold_profile": args.scaffold_profile, + "context_profile": CONTEXT_PROFILE, + "sealed_inputs": { + "campaign_hint": None, + "seed_corpus": None, + "learning_registry": None, + "mechanism_memory": False, + "patch_oracle": False, + "fixed_checkout": None, + "ground_truth": None, + }, + "bounds": bounds, + } + + +def _parse_offsets(raw: str | None) -> list[int] | None: + if raw is None: + return None + try: + offsets = [int(value.strip()) for value in raw.split(",") if value.strip()] + except ValueError as exc: + raise ValueError("offsets must be comma-separated integers") from exc + if not offsets: + raise ValueError("offsets cannot be empty") + if any(offset < 0 for offset in offsets): + raise ValueError("offsets cannot contain negative values") + if len(set(offsets)) != len(offsets): + raise ValueError("offsets cannot contain duplicates") + return sorted(offsets) + + +def _load_paths(raw: str | None) -> list[str] | None: + if raw is None: + return None + path = Path(raw).expanduser().resolve() + try: + paths = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ValueError("paths-file must be a readable JSON array") from exc + if not isinstance(paths, list) or not paths or any( + not isinstance(value, str) or not value for value in paths + ): + raise ValueError("paths-file must contain a non-empty JSON string array") + if len(set(paths)) != len(paths): + raise ValueError("paths-file cannot contain duplicate paths") + return paths + + +def _config_digest(config: dict) -> str: + return hashlib.sha256(json.dumps(config, sort_keys=True).encode()).hexdigest() + + +def _load_checkpoint(path: Path, config: dict) -> dict: + digest = _config_digest(config) + if not path.exists(): + return { + "schema_version": SCHEMA_VERSION, + "config_digest": digest, + "config": config, + "waves": {}, + } + checkpoint = json.loads(path.read_text(encoding="utf-8")) + if checkpoint.get("schema_version") != SCHEMA_VERSION: + raise ValueError("campaign checkpoint schema changed") + if checkpoint.get("config_digest") != digest or checkpoint.get("config") != config: + raise ValueError("campaign checkpoint does not match the requested sealed configuration") + return checkpoint + + +def _selected_files(session: Path) -> list[str]: + events = session / "instrumentation" / "events.jsonl" + if not events.is_file(): + return [] + for line in events.read_text(encoding="utf-8", errors="replace").splitlines(): + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + if event.get("stage") == "rank" and event.get("status") == "bounded": + return [str(path) for path in event.get("files", [])] + return [] + + +def _raw_candidate_calls(session: Path) -> list[dict[str, Any]]: + records: list[dict[str, Any]] = [] + for transcript in sorted(session.rglob("transcript.jsonl")): + for line in transcript.read_text(encoding="utf-8", errors="replace").splitlines(): + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + tool_call = event.get("tool_call") or {} + if event.get("event") != "tool_result" or tool_call.get("fn_name") not in { + "record_candidate", + "record_finding", + }: + continue + records.append( + { + "transcript": str(transcript.relative_to(session)), + "work_item_id": event.get("work_item_id"), + "step": event.get("step"), + "tool": tool_call.get("fn_name"), + "arguments": tool_call.get("fn_arguments"), + "result": event.get("tool_output"), + } + ) + return records + + +def _source_action_files(session: Path) -> list[str]: + """Return targets for which at least one source-bearing tool completed.""" + + source_tools = { + "execute", + "grep_source", + "read_file", + "read_source_file", + } + completed: set[str] = set() + for transcript in sorted(session.rglob("transcript.jsonl")): + target = "" + used_source = False + for line in transcript.read_text(encoding="utf-8", errors="replace").splitlines(): + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + if event.get("event") == "start": + target = str(event.get("file_path") or "") + tool_call = event.get("tool_call") or {} + tool_output = event.get("tool_output") + source_succeeded = not ( + isinstance(tool_output, dict) and tool_output.get("error") + ) + if ( + event.get("event") == "tool_result" + and tool_call.get("fn_name") in source_tools + and not event.get("repeated_skip") + and source_succeeded + ): + used_source = True + if target and used_source: + completed.add(target) + return sorted(completed) + + +async def _run_wave( + *, + args: argparse.Namespace, + checkout: Path, + output_root: Path, + provider: ProviderManager, + offset: int | None = None, + offsets: list[int] | None = None, + paths: list[str] | None = None, +) -> dict[str, Any]: + selection_modes = sum(value is not None for value in (offset, offsets, paths)) + if selection_modes != 1: + raise ValueError("provide exactly one contiguous offset, sparse offsets, or path list") + if paths is not None: + digest = hashlib.sha256(json.dumps(paths).encode()).hexdigest()[:8] + session_id = f"blind-pinned-files-n{len(paths):04d}-{digest}" + max_hunt_files = None + hunt_file_offset = 0 + hunt_file_offsets = None + hunt_file_paths = paths + result_offset = None + result_stop = None + elif offsets is not None: + digest = hashlib.sha256(json.dumps(offsets).encode()).hexdigest()[:8] + session_id = ( + f"blind-sparse-ranks-{offsets[0] + 1:04d}-{offsets[-1] + 1:04d}-" + f"n{len(offsets):04d}-{digest}" + ) + max_hunt_files = None + hunt_file_offset = 0 + hunt_file_offsets = offsets + hunt_file_paths = None + result_offset = None + result_stop = None + else: + assert offset is not None + stop = offset + args.wave_size + session_id = f"blind-ranks-{offset + 1:04d}-{stop:04d}" + max_hunt_files = args.wave_size + hunt_file_offset = offset + hunt_file_offsets = None + hunt_file_paths = None + result_offset = offset + result_stop = stop + sessions = output_root / "sessions" + runner = SourceHuntRunner( + repo_url="https://github.com/FFmpeg/FFmpeg.git", + local_path=str(checkout), + depth="standard", + budget_usd=1.0, + input_price_per_million=0.0, + output_price_per_million=0.0, + max_parallel=args.max_parallel, + output_dir=str(sessions), + output_formats=["sarif", "markdown", "json"], + no_verify=True, + no_exploit=True, + adversarial_verifier=False, + enable_calibration=False, + enable_mechanism_memory=False, + enable_patch_oracle=False, + enable_stability_verification=False, + enable_variant_loop=False, + enable_knowledge_graph=False, + enable_findings_pool=False, + enable_behavior_monitor=False, + model_override=args.model, + provider_manager=provider, + parent_session_id=session_id, + agent_mode="constrained", + prompt_mode="unconstrained", + prompt_bundle=PROMPT_BUNDLE, + scaffold_profile=args.scaffold_profile, + context_profile=CONTEXT_PROFILE, + campaign_hint=None, + starting_band="fast", + max_hunt_files=max_hunt_files, + hunt_file_offset=hunt_file_offset, + hunt_file_offsets=hunt_file_offsets, + hunt_file_paths=hunt_file_paths, + max_hunter_steps=args.max_hunter_steps, + hunter_temperature=args.temperature, + hunter_max_output_tokens=args.max_output_tokens, + redundancy_override=1, + no_rank=True, + sandbox_cpus=args.sandbox_cpus, + sandbox_factory=lambda: None, + preprocessing=True, + ) + started = datetime.now(UTC).isoformat() + result = await runner.arun() + session = sessions / session_id + candidates = _raw_candidate_calls(session) + source_action_files = _source_action_files(session) + selected_files = _selected_files(session) + requested_count = len(offsets) if offsets is not None else len(paths or []) + if (offsets is not None or paths is not None) and len(selected_files) != requested_count: + raise RuntimeError( + f"sparse selection resolved {len(selected_files)} of {requested_count} targets" + ) + if paths is not None and set(selected_files) != set(paths): + raise RuntimeError("pinned path selection drifted from the sealed manifest") + _write_json(session / "raw_candidate_calls.json", candidates) + return { + "status": result.status, + "started_at": started, + "completed_at": datetime.now(UTC).isoformat(), + "offset": result_offset, + "stop": result_stop, + "offsets": offsets, + "paths": paths, + "session_id": session_id, + "session_dir": str(session), + "selected_files": selected_files, + "files_ranked": result.files_ranked, + "files_hunted": result.files_hunted, + "files_examined": len(source_action_files), + "source_action_files": source_action_files, + "finding_count": len(result.findings), + "findings": [asdict(finding) for finding in result.findings], + "raw_candidate_call_count": len(candidates), + "tokens_used": result.tokens_used, + "cost_usd": result.cost_usd, + "output_paths": result.output_paths, + } + + +async def _main(args: argparse.Namespace) -> None: + checkout = Path(args.checkout).expanduser().resolve() + compile_commands = Path(args.compile_commands).expanduser().resolve() + output_root = Path(args.output_dir).expanduser().resolve() + output_root.mkdir(parents=True, exist_ok=True) + + # Fail closed if the selected production prompt picks up answer-bearing text. + require_generic_prompt(GENERIC_INSTRUCTIONS_COMPACT_V1) + config = _campaign_config(args, checkout, compile_commands) + checkpoint_path = output_root / "campaign.json" + checkpoint = _load_checkpoint(checkpoint_path, config) + _write_json(checkpoint_path, checkpoint) + + provider = ProviderManager.for_endpoint( + LLMEndpoint( + provider="openai_compat", + model=args.model, + base_url=args.base_url, + api_key=args.api_key, + source="blind_sourcehunt_campaign", + adapter="openai", + ) + ) + sparse_offsets = _parse_offsets(args.offsets) + pinned_paths = _load_paths(args.paths_file) + wave_specs: list[tuple[str, int | None, list[int] | None, list[str] | None]] + if sparse_offsets is not None and pinned_paths is not None: + raise ValueError("offsets and paths-file are mutually exclusive") + if pinned_paths is not None: + digest = hashlib.sha256(json.dumps(pinned_paths).encode()).hexdigest()[:8] + wave_specs = [(f"paths:{digest}", None, None, pinned_paths)] + elif sparse_offsets is not None: + key = "sparse:" + ",".join(str(offset) for offset in sparse_offsets) + wave_specs = [(key, None, sparse_offsets, None)] + else: + wave_specs = [ + (str(offset), offset, None, None) + for index in range(args.waves) + for offset in [args.start_offset + index * args.wave_size] + ] + + for key, offset, offsets, paths in wave_specs: + if checkpoint["waves"].get(key, {}).get("status") == "completed": + continue + try: + wave = await _run_wave( + args=args, + checkout=checkout, + output_root=output_root, + provider=provider, + offset=offset, + offsets=offsets, + paths=paths, + ) + except Exception as exc: + checkpoint["waves"][key] = { + "status": "failed", + "offset": offset, + "offsets": offsets, + "paths": paths, + "error": f"{type(exc).__name__}: {exc}", + "failed_at": datetime.now(UTC).isoformat(), + } + _write_json(checkpoint_path, checkpoint) + raise + checkpoint["waves"][key] = wave + _write_json(checkpoint_path, checkpoint) + print( + json.dumps( + { + "wave": key, + "status": wave["status"], + "files_hunted": wave["files_hunted"], + "files_examined": wave["files_examined"], + "findings": wave["finding_count"], + "raw_candidate_calls": wave["raw_candidate_call_count"], + "tokens": wave["tokens_used"], + }, + sort_keys=True, + ), + flush=True, + ) + + +def main() -> None: + asyncio.run(_main(_arguments())) + + +if __name__ == "__main__": + main() diff --git a/evaluations/run_sourcehunt_lair_validator.py b/evaluations/run_sourcehunt_lair_validator.py new file mode 100644 index 00000000..71c6521e --- /dev/null +++ b/evaluations/run_sourcehunt_lair_validator.py @@ -0,0 +1,96 @@ +"""Replay LAIR vulnerable/fixed source pairs through the Clearwing validator.""" + +from __future__ import annotations + +import argparse +import asyncio +import json +from pathlib import Path + +from clearwing.eval.sourcehunt_lair import load_lair_goldens +from clearwing.eval.sourcehunt_lair_validator import run_lair_validator_replay +from clearwing.providers import LLMEndpoint, ProviderManager +from clearwing.sourcehunt.validator import VALIDATOR_PROMPT_PROFILES, Validator + + +def _arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--campaign-root", required=True) + parser.add_argument("--output", required=True) + parser.add_argument("--base-url", required=True) + parser.add_argument("--model", default="dsv4-flash-nvfp4") + parser.add_argument( + "--prompt-profile", + choices=sorted(VALIDATOR_PROMPT_PROFILES), + default="legacy-v1", + ) + parser.add_argument("--api-key", default="local") + parser.add_argument("--max-parallel", type=int, default=2) + parser.add_argument("--max-output-tokens", type=int, default=8192) + parser.add_argument("--temperature", type=float, default=0.0) + parser.add_argument("--context-radius", type=int, default=18) + parser.add_argument("--max-context-chars", type=int, default=20_000) + return parser.parse_args() + + +async def _run(args: argparse.Namespace) -> None: + campaign_root = Path(args.campaign_root).expanduser().resolve() + goldens = load_lair_goldens(campaign_root) + provider = ProviderManager.for_endpoint( + LLMEndpoint( + provider="openai_compat", + model=args.model, + base_url=args.base_url, + api_key=args.api_key, + source="lair_validator_replay", + adapter="openai", + ) + ) + validator = Validator( + provider.get_native_client("verifier"), + gate_threshold=None, + enable_quick_pass=False, + prompt_profile=args.prompt_profile, + max_output_tokens=args.max_output_tokens, + temperature=args.temperature, + ) + summary = await run_lair_validator_replay( + goldens, + campaign_root, + validator, + model=args.model, + prompt_profile=args.prompt_profile, + max_output_tokens=args.max_output_tokens, + temperature=args.temperature, + max_parallel=args.max_parallel, + context_radius=args.context_radius, + max_context_chars=args.max_context_chars, + ) + output = Path(args.output).expanduser().resolve() + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(summary.model_dump_json(indent=2) + "\n", encoding="utf-8") + print(output) + print( + json.dumps( + { + "cases": summary.case_count, + "prompt_profile": summary.prompt_profile, + "context_profile": summary.context_profile, + "max_output_tokens": summary.max_output_tokens, + "temperature": summary.temperature, + "vulnerable_recall": summary.vulnerable_recall, + "fixed_rejection_rate": summary.fixed_rejection_rate, + "pair_accuracy": summary.pair_accuracy, + "model_errors": summary.model_errors, + }, + sort_keys=True, + ) + ) + + +def main() -> None: + asyncio.run(_run(_arguments())) + + +if __name__ == "__main__": + main() diff --git a/evaluations/run_sourcehunt_lair_validator_gepa.py b/evaluations/run_sourcehunt_lair_validator_gepa.py new file mode 100644 index 00000000..a9de0361 --- /dev/null +++ b/evaluations/run_sourcehunt_lair_validator_gepa.py @@ -0,0 +1,145 @@ +"""Run bounded leakage-safe GEPA over the opened LAIR validator fold.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path +from typing import Any + +from clearwing.eval.sourcehunt_gepa import ClearwingReflectionLM +from clearwing.eval.sourcehunt_lair import load_lair_goldens +from clearwing.eval.sourcehunt_lair_gepa import ( + VALIDATOR_PROMPT_COMPONENT, + LairValidatorGEPAAdapter, + optimize_lair_validator_prompt, + require_generic_validator_prompt, +) +from clearwing.providers import LLMEndpoint, ProviderManager +from clearwing.sourcehunt.validator import VALIDATOR_SOURCE_FIRST_COMPACT_PROMPT + + +def _arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--campaign-root", required=True) + parser.add_argument("--output-dir", required=True) + parser.add_argument("--base-url", required=True) + parser.add_argument("--model", default="dsv4-flash-nvfp4") + parser.add_argument("--api-key", default="local") + parser.add_argument("--max-metric-calls", type=int, default=72) + parser.add_argument("--reflection-max-tokens", type=int, default=4096) + parser.add_argument("--max-parallel", type=int, default=2) + parser.add_argument("--max-output-tokens", type=int, default=16_384) + parser.add_argument("--temperature", type=float, default=0.0) + parser.add_argument("--context-radius", type=int, default=18) + parser.add_argument("--max-context-chars", type=int, default=20_000) + parser.add_argument("--seed", type=int, default=0) + return parser.parse_args() + + +def main() -> None: + args = _arguments() + if args.max_metric_calls < 24: + raise ValueError("GEPA requires at least 24 metric calls for train and validation") + campaign_root = Path(args.campaign_root).expanduser().resolve() + output_root = Path(args.output_dir).expanduser().resolve() + output_root.mkdir(parents=True, exist_ok=True) + goldens = load_lair_goldens(campaign_root) + provider = ProviderManager.for_endpoint( + LLMEndpoint( + provider="openai_compat", + model=args.model, + base_url=args.base_url, + api_key=args.api_key, + source="lair_validator_gepa", + adapter="openai", + ) + ) + client = provider.get_native_client("verifier") + adapter = LairValidatorGEPAAdapter( + client, + goldens, + campaign_root, + model=args.model, + max_output_tokens=args.max_output_tokens, + temperature=args.temperature, + max_parallel=args.max_parallel, + context_radius=args.context_radius, + max_context_chars=args.max_context_chars, + max_metric_calls=args.max_metric_calls, + ) + trainset, valset = _opaque_split(list(adapter.examples)) + result = optimize_lair_validator_prompt( + adapter=adapter, + trainset=trainset, + valset=valset, + reflection_lm=ClearwingReflectionLM( + client, max_tokens=args.reflection_max_tokens + ), + seed_prompt=VALIDATOR_SOURCE_FIRST_COMPACT_PROMPT, + max_metric_calls=args.max_metric_calls, + run_dir=output_root / "gepa-state", + seed=args.seed, + ) + candidates = result.candidates + for candidate in candidates: + require_generic_validator_prompt( + candidate[VALIDATOR_PROMPT_COMPONENT], goldens + ) + payload = { + "schema_version": "cw.sourcehunt.lair-validator-gepa.v1", + "model": args.model, + "configuration": { + "max_metric_calls": args.max_metric_calls, + "reflection_max_tokens": args.reflection_max_tokens, + "max_parallel": args.max_parallel, + "max_output_tokens": args.max_output_tokens, + "temperature": args.temperature, + "context_radius": args.context_radius, + "max_context_chars": args.max_context_chars, + "seed": args.seed, + "train_cases": [example.case_id for example in trainset], + "validation_cases": [example.case_id for example in valset], + "actual_metric_calls": adapter.metric_calls, + }, + "result": result.to_dict(), + } + report = output_root / "result.json" + report.write_text( + json.dumps(payload, indent=2, sort_keys=True, default=_json_value) + "\n", + encoding="utf-8", + ) + print(report) + print( + json.dumps( + { + "best_idx": result.best_idx, + "best_score": result.val_aggregate_scores[result.best_idx], + "candidate_count": result.num_candidates, + "metric_calls": result.total_metric_calls, + }, + sort_keys=True, + ) + ) + + +def _opaque_split(examples: list[Any]) -> tuple[list[Any], list[Any]]: + ordered = sorted( + examples, + key=lambda example: hashlib.sha256( + f"lair-validator-gepa-v1\0{example.case_id}".encode() + ).digest(), + ) + midpoint = max(1, len(ordered) // 2) + return ordered[:midpoint], ordered[midpoint:] + + +def _json_value(value: Any) -> Any: + if isinstance(value, set): + return sorted(value) + return str(value) + + +if __name__ == "__main__": + main() diff --git a/evaluations/run_sourcehunt_lair_validator_replicates.py b/evaluations/run_sourcehunt_lair_validator_replicates.py new file mode 100644 index 00000000..1088eba2 --- /dev/null +++ b/evaluations/run_sourcehunt_lair_validator_replicates.py @@ -0,0 +1,232 @@ +"""Run and aggregate reproducible LAIR validator replay replicates.""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import os +import shutil +from pathlib import Path + +from clearwing.eval.sourcehunt_lair import load_lair_goldens +from clearwing.eval.sourcehunt_lair_replicates import ( + aggregate_replicates, + load_replay, +) +from clearwing.eval.sourcehunt_lair_validator import run_lair_validator_replay +from clearwing.providers import LLMEndpoint, ProviderManager +from clearwing.sourcehunt.validator import VALIDATOR_PROMPT_PROFILES, Validator + +DEFAULT_PROFILES = ("legacy-v1", "source-first-compact-v2") + + +def _arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--campaign-root", required=True) + parser.add_argument("--output-dir", required=True) + parser.add_argument("--base-url", required=True) + parser.add_argument("--model", default="dsv4-flash-nvfp4") + parser.add_argument("--api-key", default="local") + parser.add_argument( + "--profiles", + nargs="+", + choices=sorted(VALIDATOR_PROMPT_PROFILES), + default=list(DEFAULT_PROFILES), + ) + parser.add_argument("--replicates", type=int, default=5) + parser.add_argument( + "--seed-result", + action="append", + default=[], + metavar="PROFILE=PATH", + help="Copy an existing compatible replay into the next missing run slot.", + ) + parser.add_argument("--max-parallel", type=int, default=2) + parser.add_argument("--max-output-tokens", type=int, default=16_384) + parser.add_argument("--temperature", type=float, default=0.0) + parser.add_argument("--context-radius", type=int, default=18) + parser.add_argument("--max-context-chars", type=int, default=20_000) + return parser.parse_args() + + +async def _run(args: argparse.Namespace) -> None: + if args.replicates < 1: + raise ValueError("replicates must be positive") + if len(set(args.profiles)) != len(args.profiles): + raise ValueError("prompt profiles must be unique") + + campaign_root = Path(args.campaign_root).expanduser().resolve() + output_root = Path(args.output_dir).expanduser().resolve() + output_root.mkdir(parents=True, exist_ok=True) + seeds = _parse_seeds(args.seed_result, set(args.profiles)) + provenance_path = output_root / "provenance.json" + provenance = _load_provenance(provenance_path) + + for profile, sources in seeds.items(): + for source in sources: + slot = _next_missing_path(output_root, profile, args.replicates) + if slot is None: + break + replay = load_replay(source) + _require_configuration(replay, args, profile) + slot.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, slot) + provenance[str(slot.resolve())] = f"seed copy of {source.resolve()}" + _atomic_write( + provenance_path, + json.dumps(provenance, indent=2, sort_keys=True) + "\n", + ) + print(f"seeded {profile} {slot.name}", flush=True) + + goldens = load_lair_goldens(campaign_root) + provider = ProviderManager.for_endpoint( + LLMEndpoint( + provider="openai_compat", + model=args.model, + base_url=args.base_url, + api_key=args.api_key, + source="lair_validator_replication", + adapter="openai", + ) + ) + + pending = _interleaved_pending(output_root, args.profiles, args.replicates) + for profile, replicate, output in pending: + print( + f"starting {profile} replicate {replicate}/{args.replicates}", + flush=True, + ) + validator = Validator( + provider.get_native_client("verifier"), + gate_threshold=None, + enable_quick_pass=False, + prompt_profile=profile, + max_output_tokens=args.max_output_tokens, + temperature=args.temperature, + ) + summary = await run_lair_validator_replay( + goldens, + campaign_root, + validator, + model=args.model, + prompt_profile=profile, + max_output_tokens=args.max_output_tokens, + temperature=args.temperature, + max_parallel=args.max_parallel, + context_radius=args.context_radius, + max_context_chars=args.max_context_chars, + ) + _atomic_write(output, summary.model_dump_json(indent=2) + "\n") + print( + json.dumps( + { + "profile": profile, + "replicate": replicate, + "vulnerable_recall": summary.vulnerable_recall, + "fixed_rejection_rate": summary.fixed_rejection_rate, + "pair_accuracy": summary.pair_accuracy, + "model_errors": summary.model_errors, + }, + sort_keys=True, + ), + flush=True, + ) + + paths_by_profile = { + profile: [ + output_root / profile / f"run-{replicate:02d}.json" + for replicate in range(1, args.replicates + 1) + ] + for profile in args.profiles + } + result = aggregate_replicates( + paths_by_profile, + model=args.model, + max_output_tokens=args.max_output_tokens, + temperature=args.temperature, + context_radius=args.context_radius, + max_context_chars=args.max_context_chars, + max_parallel=args.max_parallel, + provenance=provenance, + ) + summary_path = output_root / "summary.json" + _atomic_write(summary_path, json.dumps(result, indent=2, sort_keys=True) + "\n") + print(summary_path, flush=True) + + +def _parse_seeds(values: list[str], profiles: set[str]) -> dict[str, list[Path]]: + parsed: dict[str, list[Path]] = {profile: [] for profile in profiles} + for value in values: + profile, separator, raw_path = value.partition("=") + if not separator or profile not in profiles or not raw_path: + raise ValueError(f"invalid --seed-result {value!r}; expected PROFILE=PATH") + path = Path(raw_path).expanduser().resolve() + if not path.is_file(): + raise ValueError(f"seed result does not exist: {path}") + parsed[profile].append(path) + return parsed + + +def _load_provenance(path: Path) -> dict[str, str]: + if not path.exists(): + return {} + payload = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(payload, dict) or not all( + isinstance(key, str) and isinstance(value, str) + for key, value in payload.items() + ): + raise ValueError(f"invalid replication provenance file: {path}") + return payload + + +def _require_configuration(replay, args: argparse.Namespace, profile: str) -> None: + expected = { + "model": args.model, + "prompt_profile": profile, + "max_output_tokens": args.max_output_tokens, + "temperature": args.temperature, + } + for field, value in expected.items(): + if getattr(replay, field) != value: + raise ValueError( + f"seed result {field} is {getattr(replay, field)!r}; expected {value!r}" + ) + + +def _next_missing_path(root: Path, profile: str, replicates: int) -> Path | None: + for replicate in range(1, replicates + 1): + path = root / profile / f"run-{replicate:02d}.json" + if not path.exists(): + return path + return None + + +def _interleaved_pending( + root: Path, + profiles: list[str], + replicates: int, +) -> list[tuple[str, int, Path]]: + pending: list[tuple[str, int, Path]] = [] + for replicate in range(1, replicates + 1): + ordered = profiles if replicate % 2 else list(reversed(profiles)) + for profile in ordered: + path = root / profile / f"run-{replicate:02d}.json" + if not path.exists(): + pending.append((profile, replicate, path)) + return pending + + +def _atomic_write(path: Path, content: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp") + temporary.write_text(content, encoding="utf-8") + temporary.replace(path) + + +def main() -> None: + asyncio.run(_run(_arguments())) + + +if __name__ == "__main__": + main() diff --git a/evaluations/run_sourcehunt_optimization.py b/evaluations/run_sourcehunt_optimization.py new file mode 100644 index 00000000..22f43eed --- /dev/null +++ b/evaluations/run_sourcehunt_optimization.py @@ -0,0 +1,181 @@ +"""Run one leakage-safe SourceHunt scaffold baseline against a snapshot pair.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any + +from clearwing.eval.sourcehunt import ( + AblationLevel, + AblationRunSpec, + GroundTruthManifest, + SourceHuntCase, + include_fixed_negative_cases, +) +from clearwing.eval.sourcehunt_gepa import ( + PROMPT_COMPONENT, + SourceHuntGEPAAdapter, + SourceHuntOptimizationExample, +) +from clearwing.providers import LLMEndpoint, ProviderManager +from clearwing.sourcehunt.optimization import ( + GENERIC_INSTRUCTIONS_COMPACT_V1, + GENERIC_INSTRUCTIONS_V1, +) + + +def _arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--manifest", default="evaluations/sourcehunt_ground_truth.yaml") + parser.add_argument("--case", default="ffmpeg-h264-slice-sentinel") + parser.add_argument("--vulnerable-checkout", required=True) + parser.add_argument("--fixed-checkout", required=True) + parser.add_argument("--base-url", required=True) + parser.add_argument("--api-key", default="local") + parser.add_argument("--model", default="dsv4-flash-nvfp4") + parser.add_argument("--scaffold", required=True) + parser.add_argument("--context-profile", default="compact-small-model-v1") + parser.add_argument("--output-dir", required=True) + parser.add_argument("--max-hunt-files", type=int, default=24) + parser.add_argument("--max-hunter-steps", type=int, default=40) + parser.add_argument("--max-parallel", type=int, default=4) + parser.add_argument("--sandbox-cpus", type=float, default=2.0) + parser.add_argument("--ranker-chunk-size", type=int, default=25) + parser.add_argument("--ranker-max-inflight", type=int, default=1) + parser.add_argument("--ranker-retries", type=int, default=1) + return parser.parse_args() + + +def _spec( + case: SourceHuntCase, + *, + model: str, + scaffold: str, + context_profile: str, +) -> AblationRunSpec: + return AblationRunSpec( + case_id=case.id, + repository=case.repository, + vulnerable_commit=case.vulnerable_commit, + case_digest=case.digest, + flow="legacy", + model_tier="local", + model=model, + prompt_bundle="generic-security-v1", + scaffold_profile=scaffold, + context_profile=context_profile, + level=AblationLevel.REPOSITORY, + ) + + +def _json_value(value: Any) -> Any: + if hasattr(value, "model_dump"): + return value.model_dump(mode="json") + if hasattr(value, "__dict__"): + return value.__dict__ + return str(value) + + +def main() -> None: + args = _arguments() + manifest_path = Path(args.manifest).expanduser().resolve() + manifest = include_fixed_negative_cases(GroundTruthManifest.load(manifest_path)) + positive = manifest.case(args.case) + negative = manifest.case(f"{args.case}-fixed-negative") + checkouts = { + positive.id: Path(args.vulnerable_checkout).expanduser().resolve(), + negative.id: Path(args.fixed_checkout).expanduser().resolve(), + } + provider = ProviderManager.for_endpoint( + LLMEndpoint( + provider="openai_compat", + model=args.model, + base_url=args.base_url, + api_key=args.api_key, + source="optimization_campaign", + adapter="openai", + ) + ) + output_root = Path(args.output_dir).expanduser().resolve() + session_root = output_root / "sessions" + examples = [ + SourceHuntOptimizationExample( + spec=_spec( + case, + model=args.model, + scaffold=args.scaffold, + context_profile=args.context_profile, + ), + case=case, + checkout=checkouts[case.id], + output_dir=session_root, + provider_manager=provider, + budget_usd=1.0, + input_price_per_million=0.0, + output_price_per_million=0.0, + max_hunt_files=args.max_hunt_files, + max_hunter_steps=args.max_hunter_steps, + max_parallel=args.max_parallel, + sandbox_cpus=args.sandbox_cpus, + ranker_chunk_size=args.ranker_chunk_size, + ranker_max_inflight_chunks=args.ranker_max_inflight, + ranker_chunk_max_retries=args.ranker_retries, + no_rank=True, + starting_band="fast", + redundancy_override=1, + depth="standard", + compile_commands=str(checkouts[case.id] / "compile_commands.json"), + ) + for case in (positive, negative) + ] + seed_prompt = ( + GENERIC_INSTRUCTIONS_COMPACT_V1 + if args.context_profile == "compact-small-model-v1" + else GENERIC_INSTRUCTIONS_V1 + ) + result = SourceHuntGEPAAdapter(manifest).evaluate( + examples, + {PROMPT_COMPONENT: seed_prompt}, + capture_traces=True, + ) + payload = { + "schema_version": 1, + "case": args.case, + "model": args.model, + "base_url": args.base_url, + "prompt_bundle": "generic-security-v1", + "scaffold_profile": args.scaffold, + "context_profile": args.context_profile, + "bounds": { + "max_hunt_files": args.max_hunt_files, + "max_hunter_steps": args.max_hunter_steps, + "max_parallel": args.max_parallel, + "sandbox_cpus": args.sandbox_cpus, + "ranker_chunk_size": args.ranker_chunk_size, + "ranker_max_inflight": args.ranker_max_inflight, + "ranker_retries": args.ranker_retries, + "no_rank": True, + "starting_band": "fast", + "redundancy_override": 1, + "depth": "standard", + }, + "scores": list(result.scores), + "objective_scores": result.objective_scores, + "outputs": result.outputs, + "trajectories": result.trajectories, + "num_metric_calls": result.num_metric_calls, + } + output_root.mkdir(parents=True, exist_ok=True) + report = output_root / f"{args.scaffold}--{args.context_profile}.json" + report.write_text( + json.dumps(payload, indent=2, sort_keys=True, default=_json_value) + "\n", + encoding="utf-8", + ) + print(report) + print(json.dumps({"scores": payload["scores"]}, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/evaluations/run_sourcehunt_survivor_validation.py b/evaluations/run_sourcehunt_survivor_validation.py new file mode 100644 index 00000000..35e38737 --- /dev/null +++ b/evaluations/run_sourcehunt_survivor_validation.py @@ -0,0 +1,137 @@ +"""Independently validate source-supported SourceHunt survivor reports.""" + +from __future__ import annotations + +import argparse +import asyncio +import hashlib +import json +from dataclasses import asdict +from pathlib import Path +from typing import Any + +from clearwing.findings.types import Finding +from clearwing.providers import LLMEndpoint, ProviderManager +from clearwing.sourcehunt.validator import Validator + +SCHEMA_VERSION = "cw.sourcehunt.survivor-validation.v1" + + +def _arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--checkout", required=True) + parser.add_argument("--cases", required=True) + parser.add_argument("--base-url", required=True) + parser.add_argument("--api-key", default="local") + parser.add_argument("--model", default="dsv4-flash-nvfp4") + parser.add_argument("--output", required=True) + parser.add_argument("--max-output-tokens", type=int, default=16_384) + parser.add_argument("--temperature", type=float, default=0.0) + return parser.parse_args() + + +def _numbered_window(source: Path, start: int, end: int) -> str: + lines = source.read_text(encoding="utf-8", errors="replace").splitlines() + lo = max(1, start) + hi = min(len(lines), end) + return "\n".join(f"{number:5d}: {lines[number - 1]}" for number in range(lo, hi + 1)) + + +def _source_context(checkout: Path, windows: list[dict[str, Any]]) -> str: + chunks: list[str] = [] + for window in windows: + relative = str(window["path"]) + source = checkout / relative + if not source.is_file(): + chunks.append(f"--- {relative}: file absent from current snapshot ---") + continue + chunks.append( + f"--- {relative}:{window['start']}-{window['end']} ---\n" + + _numbered_window( + source, + int(window["start"]), + int(window["end"]), + ) + ) + return "\n\n".join(chunks) + + +def _digest(value: Any) -> str: + return hashlib.sha256(json.dumps(value, sort_keys=True).encode()).hexdigest() + + +def _json_value(value: Any) -> Any: + if hasattr(value, "model_dump"): + return value.model_dump(mode="json") + if hasattr(value, "__dict__"): + return value.__dict__ + return str(value) + + +async def _main(args: argparse.Namespace) -> None: + if args.max_output_tokens < 1 or not 0 <= args.temperature <= 2: + raise ValueError("invalid generation bounds") + checkout = Path(args.checkout).expanduser().resolve() + case_path = Path(args.cases).expanduser().resolve() + cases = json.loads(case_path.read_text(encoding="utf-8")) + if not isinstance(cases, list) or not cases: + raise ValueError("cases must be a non-empty JSON array") + + provider = ProviderManager.for_endpoint( + LLMEndpoint( + provider="openai_compat", + model=args.model, + base_url=args.base_url, + api_key=args.api_key, + source="sourcehunt_survivor_validation", + adapter="openai", + ) + ) + validator = Validator( + provider.get_native_client("validator"), + enable_quick_pass=False, + prompt_profile="legacy-v1", + max_output_tokens=args.max_output_tokens, + temperature=args.temperature, + ) + + results: list[dict[str, Any]] = [] + for case in cases: + finding = Finding(**case["finding"]) + context = _source_context(checkout, case["windows"]) + verdict = await validator.avalidate(finding, source_context=context) + results.append( + { + "case_id": case["case_id"], + "finding_digest": _digest(asdict(finding)), + "source_context_digest": _digest(context), + "source_context_chars": len(context), + "verdict": asdict(verdict), + } + ) + + payload = { + "schema_version": SCHEMA_VERSION, + "checkout": str(checkout), + "cases": str(case_path), + "model": args.model, + "prompt_profile": "legacy-v1", + "temperature": args.temperature, + "max_output_tokens": args.max_output_tokens, + "results": results, + } + output = Path(args.output).expanduser().resolve() + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text( + json.dumps(payload, indent=2, sort_keys=True, default=_json_value) + "\n", + encoding="utf-8", + ) + print(output) + + +def main() -> None: + asyncio.run(_main(_arguments())) + + +if __name__ == "__main__": + main() diff --git a/evaluations/run_sourcehunt_target_diagnostic.py b/evaluations/run_sourcehunt_target_diagnostic.py new file mode 100644 index 00000000..10f965be --- /dev/null +++ b/evaluations/run_sourcehunt_target_diagnostic.py @@ -0,0 +1,147 @@ +"""Run a bounded, paired SourceHunt target-file scaffold diagnostic.""" + +from __future__ import annotations + +import argparse +import asyncio +import json +from pathlib import Path +from typing import Any + +from clearwing.providers import LLMEndpoint, ProviderManager +from clearwing.sandbox import HunterSandbox +from clearwing.sourcehunt.hunter import build_hunter_agent + + +def _arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--vulnerable-checkout", required=True) + parser.add_argument("--fixed-checkout", required=True) + parser.add_argument("--target-file", required=True) + parser.add_argument("--base-url", required=True) + parser.add_argument("--api-key", default="local") + parser.add_argument("--model", default="dsv4-flash-nvfp4") + parser.add_argument("--scaffold", default="state-interaction-ledger-v1") + parser.add_argument("--context-profile", default="compact-small-model-v1") + parser.add_argument("--output-dir", required=True) + parser.add_argument("--max-steps", type=int, default=24) + parser.add_argument("--sandbox-cpus", type=float, default=2.0) + return parser.parse_args() + + +def _json_value(value: Any) -> Any: + if hasattr(value, "model_dump"): + return value.model_dump(mode="json") + if hasattr(value, "__dict__"): + return value.__dict__ + return str(value) + + +async def _run_arm( + *, + label: str, + checkout: Path, + target_file: str, + output_dir: Path, + llm: Any, + scaffold: str, + context_profile: str, + max_steps: int, + sandbox_cpus: float, +) -> dict[str, Any]: + trajectory_dir = output_dir / label + manager = HunterSandbox( + repo_path=str(checkout), + languages=["c"], + deep_agent_mode=True, + default_cpus=sandbox_cpus, + ) + sandbox = None + try: + manager.build_image() + sandbox = manager.spawn(session_id=f"target-diagnostic-{label}") + hunter, ctx = build_hunter_agent( + file_target={ + "path": target_file, + "tier": "B", + "language": "c", + "loc": 0, + "tags": ["memory_unsafe"], + "imports_by": 0, + }, + repo_path=str(checkout), + sandbox=sandbox, + llm=llm, + session_id=f"target-diagnostic-{label}", + project_name="target", + prompt_bundle="generic-security-v1", + scaffold_profile=scaffold, + context_profile=context_profile, + agent_mode="deep", + max_steps_override=max_steps, + input_price_per_million=0.0, + output_price_per_million=0.0, + ) + ctx.trajectory_dir = trajectory_dir + result = await hunter.arun() + return { + "label": label, + "stop_reason": result.stop_reason, + "tokens": [result.input_tokens, result.output_tokens], + "model_calls": result.model_calls, + "compaction_count": result.compaction_count, + "peak_context_tokens": result.peak_context_tokens, + "candidates": ctx.candidates, + "domains": ctx.value_domains, + "consequences": ctx.domain_consequences, + "domain_candidates": ctx.domain_candidate_ids, + "findings": ctx.findings, + "transcript": str(trajectory_dir / "transcript.jsonl"), + } + finally: + manager.cleanup() + + +async def _main() -> None: + args = _arguments() + output_dir = Path(args.output_dir).expanduser().resolve() + output_dir.mkdir(parents=True, exist_ok=True) + provider = ProviderManager.for_endpoint( + LLMEndpoint( + provider="openai_compat", + model=args.model, + base_url=args.base_url, + api_key=args.api_key, + source="target_diagnostic", + adapter="openai", + ) + ) + llm = provider.get_native_client("hunter") + arms = [] + for label, checkout_arg in ( + ("vulnerable", args.vulnerable_checkout), + ("fixed", args.fixed_checkout), + ): + arms.append( + await _run_arm( + label=label, + checkout=Path(checkout_arg).expanduser().resolve(), + target_file=args.target_file, + output_dir=output_dir, + llm=llm, + scaffold=args.scaffold, + context_profile=args.context_profile, + max_steps=args.max_steps, + sandbox_cpus=args.sandbox_cpus, + ) + ) + summary = output_dir / "summary.json" + summary.write_text( + json.dumps(arms, indent=2, sort_keys=True, default=_json_value) + "\n", + encoding="utf-8", + ) + print(summary) + + +if __name__ == "__main__": + asyncio.run(_main()) diff --git a/evaluations/sourcehunt_ffmpeg_missed_rank_paths.json b/evaluations/sourcehunt_ffmpeg_missed_rank_paths.json new file mode 100644 index 00000000..92d1ba29 --- /dev/null +++ b/evaluations/sourcehunt_ffmpeg_missed_rank_paths.json @@ -0,0 +1,22 @@ +[ + "libavcodec/qtrle.c", + "libavcodec/atrac9dec.c", + "libavfilter/vf_histogram.c", + "libavfilter/vf_zscale.c", + "libavformat/hdsenc.c", + "libavcodec/g729postfilter.c", + "libavcodec/nuv.c", + "libavcodec/opus/silk.c", + "libavcodec/svq1enc.c", + "libavformat/dashdec.c", + "libavformat/ty.c", + "fftools/cmdutils.c", + "libavcodec/vc2enc.c", + "libavfilter/af_headphone.c", + "libavfilter/af_replaygain.c", + "libavfilter/f_graphmonitor.c", + "libavfilter/vf_pullup.c", + "libavfilter/vf_v360.c", + "libavformat/mpeg.c", + "libavformat/sga.c" +] diff --git a/evaluations/sourcehunt_ffmpeg_next_unseen_missing_paths.json b/evaluations/sourcehunt_ffmpeg_next_unseen_missing_paths.json new file mode 100644 index 00000000..4d817a5e --- /dev/null +++ b/evaluations/sourcehunt_ffmpeg_next_unseen_missing_paths.json @@ -0,0 +1,13 @@ +[ + "libavcodec/dxva2_h264.c", + "libavcodec/flashsv2enc.c", + "libavcodec/hnm4video.c", + "libavcodec/pngenc.c", + "libavcodec/vp9dsp_template.c", + "libavcodec/vulkan_encode_av1.c", + "libavfilter/af_afwtdn.c", + "libavfilter/avf_ahistogram.c", + "libavfilter/vf_xpsnr.c", + "libavformat/id3v2.c", + "libswscale/graph.c" +] diff --git a/evaluations/sourcehunt_ffmpeg_next_unseen_missing_paths_replay2.json b/evaluations/sourcehunt_ffmpeg_next_unseen_missing_paths_replay2.json new file mode 100644 index 00000000..19810a48 --- /dev/null +++ b/evaluations/sourcehunt_ffmpeg_next_unseen_missing_paths_replay2.json @@ -0,0 +1,6 @@ +[ + "libavcodec/hnm4video.c", + "libavcodec/pngenc.c", + "libavcodec/vulkan_encode_av1.c", + "libavformat/id3v2.c" +] diff --git a/evaluations/sourcehunt_ffmpeg_next_unseen_missing_paths_replay3.json b/evaluations/sourcehunt_ffmpeg_next_unseen_missing_paths_replay3.json new file mode 100644 index 00000000..e7add24e --- /dev/null +++ b/evaluations/sourcehunt_ffmpeg_next_unseen_missing_paths_replay3.json @@ -0,0 +1,4 @@ +[ + "libavcodec/hnm4video.c", + "libavformat/id3v2.c" +] diff --git a/evaluations/sourcehunt_ffmpeg_next_unseen_paths.json b/evaluations/sourcehunt_ffmpeg_next_unseen_paths.json new file mode 100644 index 00000000..5c812841 --- /dev/null +++ b/evaluations/sourcehunt_ffmpeg_next_unseen_paths.json @@ -0,0 +1,26 @@ +[ + "libavfilter/avf_ahistogram.c", + "libavcodec/mss12.c", + "libavformat/id3v2.c", + "fftools/ffmpeg_mux_init.c", + "libavcodec/hnm4video.c", + "libavcodec/pngenc.c", + "libavcodec/vulkan_encode_av1.c", + "libavformat/avidec.c", + "fftools/ffprobe.c", + "libavcodec/elbg.c", + "libavcodec/vp9dsp_template.c", + "libavfilter/vf_scale.c", + "libavfilter/af_adrc.c", + "libavutil/x86/tx_float_init.c", + "libavcodec/vp56.c", + "libavcodec/atrac3plus.c", + "libavformat/rtpdec_qdm2.c", + "libavcodec/dxva2_av1.c", + "libavcodec/flashsv2enc.c", + "libswscale/graph.c", + "libavcodec/dxva2_h264.c", + "libavfilter/vf_xpsnr.c", + "libavfilter/af_afwtdn.c", + "libavcodec/ffv1dec.c" +] diff --git a/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0205_0228.json b/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0205_0228.json new file mode 100644 index 00000000..7442ca66 --- /dev/null +++ b/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0205_0228.json @@ -0,0 +1,26 @@ +[ + "libavfilter/src_avsynctest.c", + "libavcodec/evrcdec.c", + "libavcodec/wmadec.c", + "libavcodec/qdmc.c", + "libavcodec/ansi.c", + "libavcodec/proresenc_kostya.c", + "libavfilter/vf_fspp.c", + "libavcodec/cfhd.c", + "libavformat/rdt.c", + "libavutil/opt.c", + "libavcodec/wavarc.c", + "libavcodec/dstdec.c", + "libavcodec/qsvenc.c", + "libavcodec/error_resilience.c", + "libavcodec/roqvideoenc.c", + "libavfilter/af_surround.c", + "libavdevice/iec61883.c", + "libavcodec/jpegxl_parser.c", + "libavcodec/dxva2_hevc.c", + "libavfilter/vf_minterpolate.c", + "libavutil/tx_template.c", + "libavcodec/aacps.c", + "libavcodec/eacmv.c", + "libavcodec/parser.c" +] diff --git a/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0229_0252.json b/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0229_0252.json new file mode 100644 index 00000000..ab5d7ced --- /dev/null +++ b/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0229_0252.json @@ -0,0 +1,26 @@ +[ + "libavcodec/vvc/ps.c", + "libavcodec/cdgraphics.c", + "libavdevice/decklink_dec.cpp", + "libavfilter/af_alimiter.c", + "libavfilter/af_adelay.c", + "libavcodec/4xm.c", + "libavformat/nutdec.c", + "libavcodec/ivi.c", + "libavcodec/vvc/thread.c", + "libavutil/tx.c", + "libavcodec/dolby_e.c", + "libavcodec/webp.c", + "libavcodec/ppc/blockdsp.c", + "libavcodec/dvdsubdec.c", + "libavformat/oggparsevorbis.c", + "libavcodec/flicvideo.c", + "libavcodec/opus/enc_psy.c", + "tools/ismindex.c", + "libavfilter/af_biquads.c", + "libavcodec/opus/enc.c", + "libavcodec/ylc.c", + "libavfilter/qsvvpp.c", + "libavcodec/atrac3.c", + "libavfilter/af_astats.c" +] diff --git a/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0253_0276.json b/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0253_0276.json new file mode 100644 index 00000000..b817ff2f --- /dev/null +++ b/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0253_0276.json @@ -0,0 +1,26 @@ +[ + "libavformat/aviobuf.c", + "libavcodec/cbs_av1_syntax_template.c", + "libswscale/swscale.c", + "libavcodec/aacsbr_fixed.c", + "libavcodec/dsicinvideo.c", + "libavcodec/libzvbi-teletextdec.c", + "libavformat/img2dec.c", + "libavfilter/vf_entropy.c", + "libavformat/cafdec.c", + "libavcodec/qpeg.c", + "libavcodec/smacker.c", + "libavfilter/vf_bilateral.c", + "libavformat/avienc.c", + "libavcodec/dovi_rpuenc.c", + "libavutil/hwcontext_vulkan.c", + "libavcodec/interplayvideo.c", + "fftools/ffmpeg_enc.c", + "libavfilter/vf_colorcorrect.c", + "libavcodec/pgssubdec.c", + "libavcodec/sbcdec.c", + "libavformat/flvdec.c", + "libavcodec/pictordec.c", + "libavfilter/af_atempo.c", + "libavformat/rtmppkt.c" +] diff --git a/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0277_0300.json b/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0277_0300.json new file mode 100644 index 00000000..121212ca --- /dev/null +++ b/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0277_0300.json @@ -0,0 +1,26 @@ +[ + "libavcodec/pthread_frame.c", + "libavcodec/cinepak.c", + "libavcodec/h264pred_template.c", + "libavcodec/scpr3.c", + "libavcodec/atrac3plusdec.c", + "libavformat/spdifenc.c", + "libavcodec/escape130.c", + "libavcodec/snow.c", + "libavcodec/mss2.c", + "libavfilter/af_loudnorm.c", + "libavcodec/ituh263dec.c", + "libavcodec/takdec.c", + "libavcodec/mpeg4videoenc.c", + "libavformat/wtvdec.c", + "libavcodec/av1dec.c", + "libavcodec/siren.c", + "libavcodec/videotoolboxenc.c", + "libavcodec/cbs_jpeg.c", + "libavfilter/vf_hysteresis.c", + "libavformat/movenchint.c", + "libavcodec/vorbisdec.c", + "libavcodec/shorten.c", + "libavcodec/mpegvideo.c", + "libavcodec/libvorbisdec.c" +] diff --git a/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0301_0324.json b/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0301_0324.json new file mode 100644 index 00000000..19fb4def --- /dev/null +++ b/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0301_0324.json @@ -0,0 +1,26 @@ +[ + "fftools/ffmpeg_sched.c", + "libavformat/asfenc.c", + "libavformat/fitsenc.c", + "libavfilter/avf_showfreqs.c", + "libavcodec/vp3dsp.c", + "libavfilter/vsrc_ddagrab.c", + "libavcodec/dxva2_vp9.c", + "libavcodec/twinvq.c", + "libavcodec/dxva2_vc1.c", + "libavformat/segment.c", + "libavcodec/vc1dec.c", + "libavcodec/ccaption_dec.c", + "libavfilter/vf_vaguedenoiser.c", + "libavcodec/h2645_sei.c", + "libavfilter/vf_nnedi.c", + "libavcodec/imx.c", + "fftools/textformat/avtextformat.c", + "libavformat/smacker.c", + "libavcodec/libx265.c", + "libavcodec/vvc/cabac.c", + "libavformat/hls_sample_encryption.c", + "libavformat/rtpdec_mpeg4.c", + "libavcodec/amfenc_av1.c", + "libavformat/crypto.c" +] diff --git a/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0325_0348.json b/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0325_0348.json new file mode 100644 index 00000000..1d06a608 --- /dev/null +++ b/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0325_0348.json @@ -0,0 +1,26 @@ +[ + "libavutil/hwcontext_qsv.c", + "libavcodec/mpc7.c", + "libavcodec/mediacodec_sw_buffer.c", + "libavformat/mxg.c", + "libavcodec/xxan.c", + "libavformat/oggenc.c", + "libavfilter/avf_showvolume.c", + "libavformat/rtpdec_h264.c", + "libavcodec/psd.c", + "libavcodec/aac/aacdec_usac_mps212.c", + "libavcodec/adpcm.c", + "libavcodec/cbs.c", + "libavfilter/f_ebur128.c", + "libavcodec/svq1dec.c", + "libavformat/rmdec.c", + "libavcodec/escape124.c", + "libavfilter/drawutils.c", + "libavcodec/flashsv.c", + "libavcodec/opus/dec.c", + "libavfilter/vf_mix.c", + "libavcodec/hevc/cabac.c", + "libavfilter/vf_deshake_opencl.c", + "libavcodec/aacsbr.c", + "libavcodec/rasc.c" +] diff --git a/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0349_0372.json b/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0349_0372.json new file mode 100644 index 00000000..c5c76245 --- /dev/null +++ b/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0349_0372.json @@ -0,0 +1,26 @@ +[ + "libavcodec/d3d12va_encode_av1.c", + "libavcodec/vaapi_encode_mpeg2.c", + "libavcodec/vp9mvs.c", + "libavcodec/mobiclip.c", + "libavcodec/dfa.c", + "libavfilter/dialoguenhance_template.c", + "libavcodec/vb.c", + "libavfilter/avfilter.c", + "libavfilter/vf_waveform.c", + "libavcodec/mjpegenc_common.c", + "libavcodec/omx.c", + "libavcodec/cinepakenc.c", + "libavcodec/videotoolbox.c", + "libavcodec/fmvc.c", + "libavdevice/v4l2.c", + "libavcodec/eamad.c", + "libavcodec/mss3.c", + "libavfilter/af_afftfilt.c", + "libavcodec/indeo3.c", + "libavfilter/formats.c", + "libavfilter/vf_ssim360.c", + "libavfilter/vf_bm3d.c", + "libavformat/seek.c", + "libavcodec/idcinvideo.c" +] diff --git a/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0373_0396.json b/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0373_0396.json new file mode 100644 index 00000000..809d5ee9 --- /dev/null +++ b/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0373_0396.json @@ -0,0 +1,26 @@ +[ + "libavformat/mmsh.c", + "libavfilter/vf_signature.c", + "libavfilter/vf_tmidequalizer.c", + "libavformat/swfdec.c", + "libavformat/mp3enc.c", + "libavformat/hashenc.c", + "libavfilter/af_earwax.c", + "libavutil/frame.c", + "libavcodec/ffv1enc_vulkan.c", + "libavfilter/af_afir.c", + "libavfilter/graphparser.c", + "libavcodec/h2645_parse.c", + "libswresample/rematrix.c", + "libavcodec/osq.c", + "libavformat/subtitles.c", + "libavfilter/vf_vpp_qsv.c", + "libavformat/vvc.c", + "libavcodec/bmp.c", + "libavcodec/bintext.c", + "libavcodec/bonk.c", + "libavcodec/pixlet.c", + "libavcodec/libx264.c", + "libavcodec/opus/celt.c", + "libavcodec/prosumer.c" +] diff --git a/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0397_0420.json b/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0397_0420.json new file mode 100644 index 00000000..0788cc6b --- /dev/null +++ b/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0397_0420.json @@ -0,0 +1,26 @@ +[ + "libavcodec/pafvideo.c", + "libavcodec/vulkan_ffv1.c", + "libavfilter/vf_xmedian.c", + "libavcodec/libaomenc.c", + "libavfilter/af_hdcd.c", + "libavcodec/ralf.c", + "libswscale/ops.c", + "libavcodec/cbs_h266_syntax_template.c", + "libavfilter/vf_palettegen.c", + "libavcodec/dovi_rpudec.c", + "libavformat/cdxl.c", + "libavcodec/gif.c", + "libavcodec/h264_cavlc.c", + "libavcodec/lcldec.c", + "libavformat/rtspdec.c", + "libavformat/dvdvideodec.c", + "libavfilter/vf_photosensitivity.c", + "libavcodec/v4l2_context.c", + "libavcodec/sonic.c", + "libavcodec/hevc/pred_template.c", + "libavcodec/d3d12va_encode.c", + "libavcodec/ra144.c", + "libavcodec/gdv.c", + "libavcodec/mjpegenc_huffman.c" +] diff --git a/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0421_0444.json b/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0421_0444.json new file mode 100644 index 00000000..5ea47e16 --- /dev/null +++ b/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0421_0444.json @@ -0,0 +1,26 @@ +[ + "libavcodec/magicyuvenc.c", + "libavcodec/truemotion2.c", + "libavcodec/tiertexseqv.c", + "libavcodec/aaccoder.c", + "libavcodec/audio_frame_queue.c", + "libavcodec/tiffenc.c", + "libavcodec/eac3dec.c", + "libavcodec/msmpeg4enc.c", + "libavcodec/aaccoder_twoloop.h", + "libavcodec/h264_direct.c", + "libavformat/prompeg.c", + "libavformat/mmst.c", + "libavformat/omadec.c", + "libavcodec/dcaadpcm.c", + "libavcodec/mediacodecenc.c", + "libavutil/avsscanf.c", + "libavcodec/d3d12va_decode.c", + "libavcodec/cbs_lcevc.c", + "libavformat/sierravmd.c", + "libavcodec/ass_split.c", + "libavcodec/dvdec.c", + "libavfilter/avfiltergraph.c", + "libavcodec/libvorbisenc.c", + "libavcodec/dxva2_mpeg2.c" +] diff --git a/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0445_0468.json b/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0445_0468.json new file mode 100644 index 00000000..8ae5898a --- /dev/null +++ b/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0445_0468.json @@ -0,0 +1,26 @@ +[ + "libavcodec/motionpixels.c", + "libavformat/vividas.c", + "libavcodec/nvenc.c", + "libavcodec/zmbvenc.c", + "libavcodec/g728dec.c", + "libavcodec/vvc/filter.c", + "libavcodec/rawdec.c", + "libavfilter/vsrc_life.c", + "libavformat/mpegenc.c", + "libavfilter/vsrc_mandelbrot.c", + "libavutil/hwcontext_vaapi.c", + "libavcodec/vc1.c", + "libavformat/gxfenc.c", + "libavformat/rtpenc_h264_hevc.c", + "libavcodec/clearvideo.c", + "libavcodec/msvideo1.c", + "libavfilter/median_template.c", + "libavcodec/rv40.c", + "libavcodec/amfenc.c", + "libavfilter/af_anequalizer.c", + "libavcodec/cri.c", + "libavfilter/vf_scale_npp.c", + "libavcodec/bsf/dts2pts.c", + "libavcodec/hqx.c" +] diff --git a/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0445_0468_replay1.json b/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0445_0468_replay1.json new file mode 100644 index 00000000..2e171ea9 --- /dev/null +++ b/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0445_0468_replay1.json @@ -0,0 +1,7 @@ +[ + "libavcodec/rawdec.c", + "libavcodec/vc1.c", + "libavcodec/msvideo1.c", + "libavfilter/af_anequalizer.c", + "libavcodec/bsf/dts2pts.c" +] diff --git a/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0445_0468_replay2.json b/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0445_0468_replay2.json new file mode 100644 index 00000000..28e5dd76 --- /dev/null +++ b/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0445_0468_replay2.json @@ -0,0 +1,4 @@ +[ + "libavcodec/rawdec.c", + "libavfilter/af_anequalizer.c" +] diff --git a/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0469_0492.json b/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0469_0492.json new file mode 100644 index 00000000..cbdf04df --- /dev/null +++ b/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0469_0492.json @@ -0,0 +1,26 @@ +[ + "libavcodec/sga.c", + "libavformat/wavdec.c", + "libavutil/parseutils.c", + "libavcodec/motion_est.c", + "tools/qt-faststart.c", + "libavfilter/vf_idet.c", + "libavfilter/vaf_spectrumsynth.c", + "libavfilter/vf_unsharp.c", + "libavformat/wvdec.c", + "libswscale/x86/ops.c", + "libavcodec/fraps.c", + "libavformat/asfdec_o.c", + "libavfilter/vf_pp7.c", + "libavcodec/h261enc.c", + "libavformat/rtmphttp.c", + "libavcodec/aac/aacdec_proc_template.c", + "libavfilter/vsrc_testsrc.c", + "libavcodec/mvha.c", + "libavcodec/hevc/refs.c", + "libavformat/4xm.c", + "libswscale/swscale_unscaled.c", + "libavcodec/vulkan_encode_h265.c", + "libavcodec/dxva2.c", + "libavformat/sdp.c" +] diff --git a/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0469_0492_replay1.json b/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0469_0492_replay1.json new file mode 100644 index 00000000..31732c4d --- /dev/null +++ b/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0469_0492_replay1.json @@ -0,0 +1,15 @@ +[ + "libavcodec/motion_est.c", + "libavfilter/vf_idet.c", + "libavfilter/vaf_spectrumsynth.c", + "libavfilter/vf_unsharp.c", + "libswscale/x86/ops.c", + "libavfilter/vf_pp7.c", + "libavcodec/h261enc.c", + "libavformat/rtmphttp.c", + "libavfilter/vsrc_testsrc.c", + "libavcodec/hevc/refs.c", + "libswscale/swscale_unscaled.c", + "libavcodec/vulkan_encode_h265.c", + "libavcodec/dxva2.c" +] diff --git a/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0493_0516.json b/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0493_0516.json new file mode 100644 index 00000000..88dd2469 --- /dev/null +++ b/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0493_0516.json @@ -0,0 +1,26 @@ +[ + "libavcodec/nvdec_h264.c", + "libavformat/network.c", + "libavformat/flvenc.c", + "libavformat/rtpenc.c", + "libavfilter/af_join.c", + "libavcodec/rl2.c", + "libavcodec/binkaudio.c", + "libavcodec/mv30.c", + "libavfilter/vf_thumbnail.c", + "libavfilter/af_lv2.c", + "libavcodec/opus/pvq.c", + "libavcodec/cbs_vp9.c", + "libavcodec/vqcdec.c", + "libavformat/srtp.c", + "libavcodec/libtheoraenc.c", + "libavcodec/packet.c", + "libavcodec/sipr.c", + "libavfilter/vf_stack.c", + "libavformat/rtpdec_vp9.c", + "libavcodec/ws-snd1.c", + "libavcodec/dirac_parser.c", + "libavfilter/af_aspectralstats.c", + "libavcodec/vulkan_encode.c", + "libavcodec/cbs_vp9_syntax_template.c" +] diff --git a/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0517_0540.json b/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0517_0540.json new file mode 100644 index 00000000..0c747fa9 --- /dev/null +++ b/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0517_0540.json @@ -0,0 +1,26 @@ +[ + "libavfilter/dnn/dnn_io_proc.c", + "libavformat/lrcdec.c", + "libavcodec/encode.c", + "libavcodec/vulkan_decode.c", + "libavcodec/bmvvideo.c", + "libavcodec/amfdec.c", + "libavcodec/dcadsp.c", + "libavcodec/vlc.c", + "libavutil/mem.c", + "libavcodec/vvc/intra_template.c", + "libavcodec/xan.c", + "libavformat/imf_cpl.c", + "libavcodec/kmvc.c", + "libavcodec/fic.c", + "libavcodec/hevc/filter.c", + "libavcodec/ra144enc.c", + "libavformat/apv.c", + "libavcodec/lpc.c", + "libavcodec/libopusenc.c", + "libavcodec/mediacodecdec_common.c", + "libavformat/ape.c", + "libavcodec/jvdec.c", + "libavcodec/atrac1.c", + "libavcodec/bfi.c" +] diff --git a/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0541_0564.json b/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0541_0564.json new file mode 100644 index 00000000..38322aee --- /dev/null +++ b/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0541_0564.json @@ -0,0 +1,26 @@ +[ + "libavformat/udp.c", + "libavcodec/mmaldec.c", + "libavcodec/bsf/h264_mp4toannexb.c", + "libavfilter/vf_vectorscope.c", + "libavcodec/mpegvideo_dec.c", + "libavformat/httpauth.c", + "libavcodec/utvideodec.c", + "libavfilter/avf_showwaves.c", + "libavcodec/cdtoons.c", + "libavcodec/magicyuv.c", + "libavcodec/dpx.c", + "fftools/graph/graphprint.c", + "libavformat/latmenc.c", + "fftools/ffmpeg_mux.c", + "libavformat/iamf_parse.c", + "libavcodec/smpte_436m.c", + "libavfilter/vf_thumbnail_cuda.c", + "libavutil/encryption_info.c", + "libavcodec/jpeglsdec.c", + "libavcodec/proresdec.c", + "libavcodec/huffyuvenc.c", + "libavformat/iff.c", + "libavutil/buffer.c", + "libavcodec/h264_cabac.c" +] diff --git a/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0565_0588.json b/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0565_0588.json new file mode 100644 index 00000000..1196a23a --- /dev/null +++ b/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0565_0588.json @@ -0,0 +1,26 @@ +[ + "libavcodec/atrac3plusdsp.c", + "libavfilter/dnn/dnn_backend_tf.c", + "libavformat/rmenc.c", + "libavfilter/src_movie.c", + "libavformat/rtpdec_qcelp.c", + "libavcodec/indeo4.c", + "libavcodec/vaapi_encode.c", + "libavformat/oggparsetheora.c", + "libavcodec/vvc/sei.c", + "libavformat/paf.c", + "libavformat/ipfsgateway.c", + "libavcodec/nellymoserenc.c", + "libavcodec/eatgv.c", + "libavcodec/jpegxl_parse.c", + "libavutil/channel_layout.c", + "libavcodec/vvc/intra.c", + "libavfilter/af_amix.c", + "libavcodec/ituh263enc.c", + "libavfilter/dnn/dnn_backend_torch.cpp", + "libavcodec/apv_entropy.c", + "libavfilter/f_drawgraph.c", + "libavcodec/libmp3lame.c", + "libavcodec/cbs_av1.c", + "libavformat/ifv.c" +] diff --git a/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0589_0612.json b/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0589_0612.json new file mode 100644 index 00000000..52e3baf7 --- /dev/null +++ b/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0589_0612.json @@ -0,0 +1,26 @@ +[ + "libavfilter/vf_removegrain.c", + "libswscale/slice.c", + "libavcodec/libsvtav1.c", + "libavcodec/mpc8.c", + "libavcodec/012v.c", + "libavcodec/ohenc.c", + "libavcodec/msp2dec.c", + "libavcodec/imm4.c", + "libavformat/rtpdec_av1.c", + "libswresample/resample.c", + "libavformat/tls_schannel.c", + "libavutil/aes_ctr.c", + "libavcodec/mediacodecdec.c", + "libavcodec/8bps.c", + "libavformat/avc.c", + "libavutil/imgutils.c", + "libavfilter/vf_lut2.c", + "libavfilter/vf_feedback.c", + "libavcodec/dpxenc.c", + "libavcodec/avcodec.c", + "libavfilter/vf_drawtext.c", + "libavcodec/fastaudio.c", + "libavfilter/vf_dctdnoiz.c", + "libswscale/ops_chain.c" +] diff --git a/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0613_0636.json b/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0613_0636.json new file mode 100644 index 00000000..84a2f547 --- /dev/null +++ b/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0613_0636.json @@ -0,0 +1,26 @@ +[ + "libavfilter/avf_a3dscope.c", + "libavcodec/nvdec_av1.c", + "libavutil/timecode.c", + "fftools/sync_queue.c", + "libavformat/dv.c", + "libavformat/dss.c", + "libavcodec/mips/hevcpred_msa.c", + "libavcodec/vulkan_encode_h264.c", + "libavcodec/sipr16k.c", + "libavcodec/ac3_parser.c", + "libavformat/rtpenc_av1.c", + "libavfilter/vf_spp.c", + "libavcodec/ratecontrol.c", + "libavformat/wavenc.c", + "libavcodec/liboapvenc.c", + "libavfilter/vf_shufflepixels.c", + "libavcodec/mpeg12enc.c", + "libavfilter/vf_vif.c", + "libavcodec/leaddec.c", + "libavcodec/libaribcaption.c", + "libavfilter/vf_deinterlace_d3d12.c", + "libavcodec/bsf/extract_extradata.c", + "libavfilter/anlms_template.c", + "libavcodec/cuviddec.c" +] diff --git a/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0637_0660.json b/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0637_0660.json new file mode 100644 index 00000000..ef701eee --- /dev/null +++ b/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0637_0660.json @@ -0,0 +1,26 @@ +[ + "libavcodec/ivi_dsp.c", + "libavcodec/hdrdec.c", + "libavutil/twofish.c", + "libavfilter/vf_lut3d.c", + "libavfilter/dnn/dnn_backend_openvino.c", + "libavformat/flic.c", + "libavcodec/opus/parse.c", + "libavcodec/gif_parser.c", + "libavformat/rtpdec.c", + "libavfilter/buffersink.c", + "libavcodec/msmpeg4dec.c", + "libavcodec/scpr.c", + "libavfilter/avf_showspatial.c", + "libavcodec/ra288.c", + "libavfilter/vf_decimate.c", + "libavcodec/h264_picture.c", + "libavcodec/cbs_sei.c", + "libavformat/nsvdec.c", + "libavcodec/v4l2_buffers.c", + "libavfilter/af_whisper.c", + "libavformat/yuv4mpegdec.c", + "libavcodec/smcenc.c", + "libavcodec/alac.c", + "libavcodec/cdxl.c" +] diff --git a/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0661_0684.json b/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0661_0684.json new file mode 100644 index 00000000..d186b5b6 --- /dev/null +++ b/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0661_0684.json @@ -0,0 +1,26 @@ +[ + "libavformat/rtpdec_xiph.c", + "libavfilter/vf_ssim.c", + "libavfilter/af_ladspa.c", + "libavcodec/smc.c", + "libavformat/bink.c", + "libavfilter/avf_aphasemeter.c", + "libavcodec/utils.c", + "libavfilter/vf_random.c", + "libavformat/bethsoftvid.c", + "libavformat/dxa.c", + "libavcodec/mpegvideoencdsp.c", + "libavcodec/utvideoenc.c", + "libavformat/avio.c", + "libavfilter/f_sendcmd.c", + "libavfilter/vf_dnn_classify.c", + "libavcodec/dvbsub_parser.c", + "libavdevice/xcbgrab.c", + "libavformat/rl2.c", + "libavcodec/hapenc.c", + "libavfilter/vf_histeq.c", + "libavformat/psxstr.c", + "libavformat/rtpdec_vp8.c", + "libavformat/ipmovie.c", + "libavdevice/dshow.c" +] diff --git a/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0685_0708.json b/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0685_0708.json new file mode 100644 index 00000000..7efe2bcf --- /dev/null +++ b/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0685_0708.json @@ -0,0 +1,26 @@ +[ + "libavcodec/prores_raw.c", + "libavfilter/aap_template.c", + "libavformat/rtpenc_latm.c", + "libavcodec/vp9prob.c", + "libavformat/jpegxl_anim_dec.c", + "libavcodec/v4l2_m2m.c", + "libavcodec/qsv.c", + "libavformat/nal.c", + "libavfilter/af_pan.c", + "libavfilter/vf_amplify.c", + "libavcodec/intrax8.c", + "libavcodec/jpeglsenc.c", + "libavcodec/c93.c", + "libavformat/rtpenc_h263_rfc2190.c", + "libavfilter/vf_xfade_opencl.c", + "libavcodec/vulkan_prores.c", + "libavcodec/ffv1.c", + "libavfilter/vsrc_mptestsrc.c", + "libavcodec/movtextenc.c", + "libavdevice/gdigrab.c", + "libavcodec/hq_hqa.c", + "libavcodec/vdpau_h264.c", + "libavcodec/libfdk-aacdec.c", + "libavcodec/vmixdec.c" +] diff --git a/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0709_0732.json b/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0709_0732.json new file mode 100644 index 00000000..ac4ad818 --- /dev/null +++ b/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0709_0732.json @@ -0,0 +1,26 @@ +[ + "libavcodec/libopenjpegenc.c", + "libavcodec/d3d12va_encode_hevc.c", + "libavformat/utils.c", + "libavcodec/sp5xdec.c", + "libavcodec/apac.c", + "libavcodec/xwd_parser.c", + "libavfilter/vf_midequalizer.c", + "libavformat/riffdec.c", + "libavcodec/avuienc.c", + "libavformat/idroqdec.c", + "libavcodec/fitsdec.c", + "libavformat/xmv.c", + "libavcodec/libdav1d.c", + "libavfilter/vf_framepack.c", + "libavcodec/h264_mvpred.h", + "libswscale/ops_dispatch.c", + "libavfilter/vf_identity.c", + "libavfilter/vf_curves.c", + "libavfilter/vf_convolution.c", + "libavcodec/mlz.c", + "libavformat/mpjpegdec.c", + "libavcodec/mscc.c", + "libavcodec/targaenc.c", + "libavcodec/bsf/h264_metadata.c" +] diff --git a/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0733_0756.json b/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0733_0756.json new file mode 100644 index 00000000..95b40217 --- /dev/null +++ b/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0733_0756.json @@ -0,0 +1,26 @@ +[ + "libavcodec/ffwavesynth.c", + "libavformat/webmdashenc.c", + "libavdevice/decklink_enc.cpp", + "libavfilter/avf_abitscope.c", + "libavcodec/libjxldec.c", + "libavfilter/vf_detelecine.c", + "libavfilter/vf_overlay_vaapi.c", + "libavcodec/ftr.c", + "libavcodec/vp5.c", + "libavcodec/vaapi_encode_mjpeg.c", + "tools/crypto_bench.c", + "libavfilter/vf_corr.c", + "libavfilter/af_amerge.c", + "libavformat/rtpdec_hevc.c", + "libavcodec/vvc/refs.c", + "libavcodec/ffv1dec_template.c", + "libavcodec/wmaenc.c", + "libavcodec/speedhqdec.c", + "libswscale/rgb2rgb_template.c", + "libavdevice/lavfi.c", + "libavcodec/aacpsy.c", + "fftools/opt_common.c", + "libavformat/qcp.c", + "libavcodec/mips/vp8_mc_msa.c" +] diff --git a/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0757_0780.json b/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0757_0780.json new file mode 100644 index 00000000..32293e4e --- /dev/null +++ b/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0757_0780.json @@ -0,0 +1,26 @@ +[ + "libavcodec/mips/vp9_mc_msa.c", + "libavcodec/dvdsubenc.c", + "libavcodec/pcm.c", + "libavcodec/movtextdec.c", + "libavfilter/vf_colortemperature.c", + "fftools/ffmpeg.c", + "libavformat/apngdec.c", + "libavfilter/vf_dnn_processing.c", + "libavcodec/loongarch/vp8_mc_lsx.c", + "libavcodec/librav1e.c", + "libavformat/id3v2enc.c", + "libavcodec/avrndec.c", + "libswscale/x86/rgb2rgb.c", + "libavcodec/alacenc.c", + "libavcodec/y41pdec.c", + "libavcodec/xpmdec.c", + "libavfilter/vf_il.c", + "libavcodec/wmv2dec.c", + "libavformat/concatdec.c", + "tools/yuvcmp.c", + "libavcodec/qtrleenc.c", + "libavcodec/cbs_h2645.c", + "libavcodec/vaapi_vvc.c", + "libavcodec/avcodec.h" +] diff --git a/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0781_0804.json b/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0781_0804.json new file mode 100644 index 00000000..a29f53ee --- /dev/null +++ b/evaluations/sourcehunt_ffmpeg_next_unseen_paths_0781_0804.json @@ -0,0 +1,26 @@ +[ + "libavcodec/mips/vp3dsp_idct_mmi.c", + "libavformat/rtpdec_qt.c", + "libavformat/cinedec.c", + "libavcodec/opus/rc.c", + "libavfilter/vf_scdet.c", + "libavcodec/g723_1.c", + "libavfilter/vf_zoompan.c", + "libavformat/lafdec.c", + "libavfilter/vf_xbr.c", + "libavformat/flacenc.c", + "libavfilter/vf_psnr.c", + "libavcodec/vdpau_av1.c", + "libavfilter/vf_libvmaf.c", + "libavcodec/proresenc_kostya_vulkan.c", + "libavformat/hevc.c", + "libavcodec/ppc/h264dsp.c", + "libavcodec/dds.c", + "libavfilter/vf_mestimate.c", + "libswscale/ops_optimizer.c", + "libavcodec/d3d12va_h264.c", + "libavcodec/intrax8dsp.c", + "libavfilter/vf_chromakey.c", + "libavformat/av1.c", + "libavdevice/android_camera.c" +] diff --git a/evaluations/sourcehunt_ffmpeg_remaining_rank_paths.json b/evaluations/sourcehunt_ffmpeg_remaining_rank_paths.json new file mode 100644 index 00000000..83f4b0f1 --- /dev/null +++ b/evaluations/sourcehunt_ffmpeg_remaining_rank_paths.json @@ -0,0 +1,3 @@ +[ + "libavcodec/nuv.c" +] diff --git a/evaluations/sourcehunt_ffmpeg_survivors.json b/evaluations/sourcehunt_ffmpeg_survivors.json new file mode 100644 index 00000000..1ee0bb14 --- /dev/null +++ b/evaluations/sourcehunt_ffmpeg_survivors.json @@ -0,0 +1,1325 @@ +[ + { + "case_id": "h264-slice-sentinel-collision", + "finding": { + "id": "ffmpeg-h264-slice-sentinel-collision", + "finding_type": "out_of_bounds_write", + "cwe": "CWE-787", + "file": "libavcodec/h264_slice.c", + "line_number": 1982, + "severity": "high", + "confidence": "high", + "evidence_level": "root_cause_explained", + "description": "The per-picture slice counter is an unbounded int, while decoded macroblock ownership is stored in a uint16_t slice table initialized to 0xFFFF. On slice 65535, storing slice_num aliases the unused-entry sentinel. Neighbor checks can then treat padded or uninitialized entries as members of the current slice, enabling the left-border exchange path at a row boundary to access top_borders at mb_x - 1.", + "code_snippet": "sl->slice_num = ++h->current_slice;", + "poc": "Decode one H.264 picture containing at least 65535 accepted slices so slice_num becomes 0xFFFF, then reach an intra macroblock at the left edge with the cross-slice deblock path active.", + "discovered_by": "sourcehunt:deepseek-v4-flash-0731", + "vulnerability_trace": { + "summary": "attacker-controlled slice count reaches a wider counter, truncates into the 16-bit ownership table as its 0xFFFF sentinel, and corrupts a neighbor-existence decision before a left-border exchange", + "steps": [ + {"file": "libavcodec/h264dec.c", "line": 210, "function": "ff_h264_alloc_tables", "code_snippet": "memset(h->slice_table_base, -1, st_size * sizeof(*h->slice_table_base));", "note": "The 16-bit table uses 0xFFFF for unavailable/padded entries."}, + {"file": "libavcodec/h264_slice.c", "line": 1982, "function": "h264_slice_init", "code_snippet": "sl->slice_num = ++h->current_slice;", "note": "No terminating bound prevents slice_num from reaching 65535."}, + {"file": "libavcodec/h264_cabac.c", "line": 2033, "function": "decode_mb_cabac", "code_snippet": "h->slice_table[mb_xy] = sl->slice_num;", "note": "The int counter is stored into the uint16_t table."}, + {"file": "libavcodec/h264_mb.c", "line": 532, "function": "xchg_mb_border", "code_snippet": "deblock_topleft = h->slice_table[sl->mb_xy - 1 - (h->mb_stride << MB_FIELD(sl))] == sl->slice_num;", "note": "At the collision, the unused sentinel compares equal to the current slice."}, + {"file": "libavcodec/h264_mb.c", "line": 543, "function": "xchg_mb_border", "code_snippet": "top_border_m1 = sl->top_borders[top_idx][sl->mb_x - 1];", "note": "A falsely present top-left neighbor enables exchange through the left-border pointer."} + ] + } + }, + "windows": [ + {"path": "libavcodec/h264dec.h", "start": 395, "end": 505}, + {"path": "libavcodec/h264dec.c", "start": 185, "end": 218}, + {"path": "libavcodec/h264_slice.c", "start": 1965, "end": 2005}, + {"path": "libavcodec/h264_cabac.c", "start": 2018, "end": 2040}, + {"path": "libavcodec/h264_mb.c", "start": 509, "end": 585} + ] + }, + { + "case_id": "vulkan-hevc-rps-list-overflow", + "finding": { + "id": "ffmpeg-vulkan-hevc-rps-list-overflow", + "finding_type": "out_of_bounds_write", + "cwe": "CWE-787", + "file": "libavcodec/vulkan_hevc.c", + "line_number": 775, + "severity": "high", + "confidence": "high", + "evidence_level": "root_cause_explained", + "description": "The Vulkan HEVC start-frame path copies each current short- or long-term RPS list into a Vulkan StdVideoDecodeH265PictureInfo array whose API-defined capacity is 8. FFmpeg's parser and RPS builder allow an individual list to contain up to 15 short-term references or up to 16 retained references. Unlike the NVDEC path, the Vulkan path does not reject a list count above the destination capacity, so i >= 8 writes beyond RefPicSetStCurrBefore, RefPicSetStCurrAfter, or RefPicSetLtCurr into adjacent picture-info fields.", + "code_snippet": "for (int i = 0; i < h->rps[ST_CURR_BEF].nb_refs; i++) { ... hp->h265pic.RefPicSetStCurrBefore[i] = j; }", + "poc": "Use Vulkan HEVC decoding on a valid stream whose current short-term-before, short-term-after, or long-term RPS contains more than eight used references that resolve in the DPB.", + "discovered_by": "sourcehunt:deepseek-v4-flash-0731", + "vulnerability_trace": { + "summary": "bitstream RPS counts survive parser checks and RPS construction with more than eight members, then directly bound writes into three eight-entry Vulkan API arrays", + "steps": [ + {"file": "libavcodec/hevc/ps.c", "line": 197, "function": "ff_hevc_decode_short_term_rps", "code_snippet": "if (rps->num_negative_pics >= HEVC_MAX_REFS || nb_positive_pics >= HEVC_MAX_REFS)", "note": "Each short-term side may have 15 entries; it is not capped at 8."}, + {"file": "libavcodec/hevc/refs.c", "line": 503, "function": "add_candidate_ref", "code_snippet": "if (ref == s->cur_frame || list->nb_refs >= HEVC_MAX_REFS)", "note": "RPS construction permits a list up to HEVC_MAX_REFS (16)."}, + {"file": "libavcodec/vulkan_hevc.c", "line": 770, "function": "vk_hevc_start_frame", "code_snippet": "for (int i = 0; i < h->rps[ST_CURR_BEF].nb_refs; i++)", "note": "The producer count directly controls the destination index."}, + {"file": "libavcodec/vulkan_hevc.c", "line": 775, "function": "vk_hevc_start_frame", "code_snippet": "hp->h265pic.RefPicSetStCurrBefore[i] = j;", "note": "The Khronos header defines this and the other two RefPicSet arrays with capacity 8."}, + {"file": "libavcodec/nvdec_hevc.c", "line": 222, "function": "nvdec_hevc_start_frame", "code_snippet": "if (s->rps[LT_CURR].nb_refs > FF_ARRAY_ELEMS(ppc->RefPicSetLtCurr) || ...)", "note": "An independent hardware path explicitly enforces the missing invariant."} + ] + } + }, + "windows": [ + {"path": "libavcodec/hevc/hevc.h", "start": 112, "end": 128}, + {"path": "libavcodec/hevc/ps.c", "start": 180, "end": 225}, + {"path": "libavcodec/hevc/refs.c", "start": 495, "end": 590}, + {"path": "libavcodec/vulkan_hevc.c", "start": 710, "end": 820}, + {"path": "libavcodec/nvdec_hevc.c", "start": 210, "end": 265} + ] + }, + { + "case_id": "arnndn-denoise-output-stack-overflow", + "finding": { + "id": "ffmpeg-arnndn-denoise-output-stack-overflow", + "finding_type": "out_of_bounds_write", + "cwe": "CWE-121", + "file": "libavfilter/af_arnndn.c", + "line_number": 1253, + "severity": "high", + "confidence": "high", + "evidence_level": "crash_reproduced", + "description": "The arnndn model parser accepts denoise_output->nb_neurons values from 0 through 128 but validates only that the separate VAD output has one neuron. During non-silent audio processing, rnnoise_channel supplies a fixed 22-float stack array as the gains destination. compute_dense writes nb_neurons floats to that pointer, so a model declaring 23 or more denoise outputs writes beyond the stack array. A generated 23-output model deterministically aborts under ASan at compute_dense with g in rnnoise_channel identified as the overflowed object.", + "code_snippet": "compute_dense(rnn->model->denoise_output, gains, rnn->denoise_gru_state);", + "poc": "Generate the model with `python evaluations/build_ffmpeg_arnndn_model.py /tmp/arnndn-23.model`, then process non-silent 48 kHz audio through `ffmpeg -f lavfi -i anoisesrc=r=48000:d=0.1 -af arnndn=m=/tmp/arnndn-23.model -f null -` using an ASan build.", + "discovered_by": "sourcehunt:deepseek-v4-flash-0731", + "vulnerability_trace": { + "summary": "a model-controlled denoise output count passes the generic 128-neuron parser bound, then controls a dense-layer loop writing into a fixed 22-float stack destination", + "steps": [ + {"file": "libavfilter/af_arnndn.c", "line": 227, "function": "rnnoise_model_from_file", "code_snippet": "if (fscanf(f, \"%d\", &in) != 1 || in < 0 || in > 128)", "note": "Every model dimension, including denoise output neurons, is accepted up to 128."}, + {"file": "libavfilter/af_arnndn.c", "line": 317, "function": "rnnoise_model_from_file", "code_snippet": "INPUT_DENSE(denoise_output);", "note": "The attacker-selected output count is persisted on the final denoise layer."}, + {"file": "libavfilter/af_arnndn.c", "line": 320, "function": "rnnoise_model_from_file", "code_snippet": "if (vad_output->nb_neurons != 1)", "note": "Only the VAD output shape is validated; denoise_output is not required to equal NB_BANDS."}, + {"file": "libavfilter/af_arnndn.c", "line": 1369, "function": "rnnoise_channel", "code_snippet": "float g[NB_BANDS];", "note": "The inference caller allocates exactly 22 float gains on the stack."}, + {"file": "libavfilter/af_arnndn.c", "line": 1357, "function": "compute_rnn", "code_snippet": "compute_dense(rnn->model->denoise_output, gains, rnn->denoise_gru_state);", "note": "The fixed stack array is passed as the destination for the model-sized output."}, + {"file": "libavfilter/af_arnndn.c", "line": 1255, "function": "compute_dense", "code_snippet": "for (int i = 0; i < N; i++)", "note": "N is the model's nb_neurons, and output[i] writes past g[21] when N exceeds 22."} + ] + } + }, + "windows": [ + {"path": "libavfilter/af_arnndn.c", "start": 187, "end": 327}, + {"path": "libavfilter/af_arnndn.c", "start": 1245, "end": 1378} + ] + }, + { + "case_id": "rtp-qdm2-small-block-heap-overflow", + "finding": { + "id": "ffmpeg-rtp-qdm2-small-block-heap-overflow", + "finding_type": "out_of_bounds_write", + "cwe": "CWE-122", + "file": "libavformat/rtpdec_qdm2.c", + "line_number": 211, + "severity": "high", + "confidence": "high", + "evidence_level": "crash_reproduced", + "description": "The RTP QDM2 configuration parser accepts an attacker-controlled 32-bit block_size without enforcing the minimum size of the reconstructed superblock header. qdm2_restore_block allocates exactly block_size bytes and unconditionally writes a header of at least two bytes. With block_size 1 and one cached subpacket, the header advances p beyond the packet, the remaining-size expression becomes -1, and memcpy receives that negative int as a huge size_t. The direct parse_packet harness deterministically aborts under ASan with negative-size-param (size=-1) in qdm2_parse_packet.", + "code_snippet": "to_copy = FFMIN(qdm->len[n], pkt->size - (p - pkt->data)); memcpy(p, qdm->buf[n], to_copy);", + "poc": "Build evaluations/ffmpeg_qdm2_reproducer.c against the sanitizer-instrumented FFmpeg static libraries and run it with ASAN_OPTIONS=halt_on_error=1:abort_on_error=1:detect_leaks=0.", + "discovered_by": "sourcehunt:deepseek-v4-flash-0731", + "vulnerability_trace": { + "summary": "an RTP configuration field controls the reconstructed packet allocation without a header-size invariant, producing an out-of-bounds header write and a negative copy length", + "steps": [ + {"file": "libavformat/rtpdec_qdm2.c", "line": 104, "function": "qdm2_parse_config", "code_snippet": "case 4: /* stream with extradata */", "note": "An RTP configuration item enters the stream-with-extradata path."}, + {"file": "libavformat/rtpdec_qdm2.c", "line": 121, "function": "qdm2_parse_config", "code_snippet": "qdm->block_size = AV_RB32(p + 26);", "note": "The packet-controlled 32-bit size is stored without a lower bound."}, + {"file": "libavformat/rtpdec_qdm2.c", "line": 199, "function": "qdm2_restore_block", "code_snippet": "if ((res = av_new_packet(pkt, qdm->block_size)) < 0)", "note": "The unchecked value becomes the exact destination allocation size."}, + {"file": "libavformat/rtpdec_qdm2.c", "line": 211, "function": "qdm2_restore_block", "code_snippet": "*p++ = qdm->block_type; *p++ = qdm->len[n];", "note": "At least two header bytes are written even when block_size is zero or one."}, + {"file": "libavformat/rtpdec_qdm2.c", "line": 220, "function": "qdm2_restore_block", "code_snippet": "to_copy = FFMIN(qdm->len[n], pkt->size - (p - pkt->data));", "note": "For block_size 1 after the two-byte header, the computed copy length is -1."}, + {"file": "libavformat/rtpdec_qdm2.c", "line": 221, "function": "qdm2_restore_block", "code_snippet": "memcpy(p, qdm->buf[n], to_copy);", "note": "The negative int is converted to a huge unsigned copy size; ASan aborts deterministically."} + ] + } + }, + "windows": [ + {"path": "libavformat/rtpdec_qdm2.c", "start": 76, "end": 129}, + {"path": "libavformat/rtpdec_qdm2.c", "start": 153, "end": 235}, + {"path": "libavformat/rtpdec_qdm2.c", "start": 238, "end": 308} + ] + }, + { + "case_id": "ahistogram-log-sign-positive-endpoint-overflow", + "finding": { + "id": "ffmpeg-ahistogram-log-sign-positive-endpoint-overflow", + "finding_type": "out_of_bounds_write", + "cwe": "CWE-787", + "file": "libavfilter/avf_ahistogram.c", + "line_number": 260, + "severity": "high", + "confidence": "high", + "evidence_level": "crash_reproduced", + "description": "In signed logarithmic mode, get_log_bin_sign maps a positive full-scale sample of exactly 1.0 to bin w rather than the last valid bin w - 1: log10(1.0) is zero, the clipped magnitude is one, and both half-width terms sum to w for even widths. filter_frame uses that result directly to increment a w-element uint64_t histogram. A constant 1.0 audio source with ahistogram=ascale=log:hmode=sign:size=1280x720 deterministically aborts under ASan on an 8-byte heap-buffer-overflow exactly past the 10,240-byte allocation.", + "code_snippet": "bin = s->get_bin(src[n], w); achistogram[bin]++;", + "poc": "Run an ASan FFmpeg build with `-f lavfi -i aevalsrc=1:s=48000:d=0.1 -filter_complex '[0:a]ahistogram=ascale=log:hmode=sign:size=1280x720[outv]' -map '[outv]' -frames:v 1 -f null -`.", + "discovered_by": "sourcehunt:deepseek-v4-flash-0731", + "vulnerability_trace": { + "summary": "a valid positive full-scale float sample reaches an endpoint calculation that returns the histogram width, then directly indexes one element beyond a heap allocation", + "steps": [ + {"file": "libavfilter/avf_ahistogram.c", "line": 129, "function": "config_input", "code_snippet": "s->achistogram = av_calloc(s->w, s->dchannels * sizeof(*s->achistogram));", "note": "Each histogram contains exactly w uint64_t elements, with valid indices 0 through w - 1."}, + {"file": "libavfilter/avf_ahistogram.c", "line": 150, "function": "get_log_bin_sign", "code_snippet": "return (w / 2) + FFSIGN(in) * lrintf(av_clipf(1.f + log10f(fabsf(in)) / 6.f, 0.f, 1.f) * (w / 2));", "note": "For in = +1.0 and even w, the expression returns w / 2 + w / 2 = w."}, + {"file": "libavfilter/avf_ahistogram.c", "line": 179, "function": "config_output", "code_snippet": "case SIGN: s->get_bin = get_log_bin_sign; break;", "note": "The vulnerable mapper is selected by the public ascale=log and hmode=sign options."}, + {"file": "libavfilter/avf_ahistogram.c", "line": 258, "function": "filter_frame", "code_snippet": "bin = s->get_bin(src[n], w);", "note": "A float audio sample controls the returned bin without a postcondition check."}, + {"file": "libavfilter/avf_ahistogram.c", "line": 260, "function": "filter_frame", "code_snippet": "achistogram[bin]++;", "note": "bin == w reads and writes one uint64_t beyond the allocation; ASan aborts on the read portion of the increment."} + ] + } + }, + "windows": [ + {"path": "libavfilter/avf_ahistogram.c", "start": 112, "end": 187}, + {"path": "libavfilter/avf_ahistogram.c", "start": 220, "end": 289} + ] + }, + { + "case_id": "xpsnr-odd-frame-highpass-oob-read", + "finding": { + "id": "ffmpeg-xpsnr-odd-frame-highpass-oob-read", + "finding_type": "out_of_bounds_read", + "cwe": "CWE-125", + "file": "libavfilter/vf_xpsnr.c", + "line_number": 105, + "severity": "medium", + "confidence": "high", + "evidence_level": "crash_reproduced", + "description": "For images above the high-pass threshold, XPSNR processes luma in an even-step 2x2 downsampled loop. On a truncated final block whose active width or height is odd, highds still runs its last iteration and reads through x + 3 and y + 3. A 2049x1153 yuv444p frame makes the final block active width odd and reaches one element beyond the tightly allocated 16-bit source plane. Two identical valid frames deterministically abort under ASan on a two-byte heap-buffer-overflow read two bytes beyond the 4,724,994-byte allocation.", + "code_snippet": "for (int y = y_act; y < h_act; y += 2) { for (int x = x_act; x < w_act; x += 2) { ... o_m0[(y+3)*o + x+3] ... } }", + "poc": "Pass two 2049x1153 yuv444p frames to an ASan FFmpeg build through `[0:v][1:v]xpsnr[outv]`; evaluations/run_ffmpeg_xpsnr_reproducer.py records the full command and sanitizer boundary.", + "discovered_by": "sourcehunt:deepseek-v4-flash-0731", + "vulnerability_trace": { + "summary": "odd active dimensions enter an even-step high-pass loop whose stencil reads three columns and rows ahead, escaping the tightly allocated source plane at the final block", + "steps": [ + {"file": "libavfilter/vf_xpsnr.c", "line": 96, "function": "highds", "code_snippet": "for (int y = y_act; y < h_act; y += 2)", "note": "The downsampled loop does not round an odd h_act down before its final iteration."}, + {"file": "libavfilter/vf_xpsnr.c", "line": 97, "function": "highds", "code_snippet": "for (int x = x_act; x < w_act; x += 2)", "note": "Likewise, an odd w_act allows a final iteration at w_act - 1."}, + {"file": "libavfilter/vf_xpsnr.c", "line": 105, "function": "highds", "code_snippet": "(int)o_m0[(y+3)*o + x+3]", "note": "The stencil reads beyond the active block; at the lower-right picture boundary this escapes the plane allocation."}, + {"file": "libavfilter/vf_xpsnr.c", "line": 181, "function": "calc_squared_error_and_weight", "code_snippet": "const int w_act = ... block_width - b_val;", "note": "The boundary margin is subtracted but its parity is not normalized for the step-two loop."}, + {"file": "libavfilter/vf_xpsnr.c", "line": 195, "function": "calc_squared_error_and_weight", "code_snippet": "sa_act = s->dsp.highds_func(x_act, y_act, w_act, h_act, o_m0, o);", "note": "Large frames dispatch the malformed active dimensions into the vulnerable stencil."}, + {"file": "libavfilter/vf_xpsnr.c", "line": 442, "function": "do_xpsnr", "code_snippet": "s->buf_org[c] = av_calloc(s->plane_width[c], s->plane_height[c] * sizeof(int16_t));", "note": "The 8-bit conversion plane is tightly allocated at width times height times two bytes, making the boundary read visible to ASan."} + ] + } + }, + "windows": [ + {"path": "libavfilter/vf_xpsnr.c", "start": 92, "end": 145}, + {"path": "libavfilter/vf_xpsnr.c", "start": 164, "end": 240}, + {"path": "libavfilter/vf_xpsnr.c", "start": 269, "end": 378}, + {"path": "libavfilter/vf_xpsnr.c", "start": 396, "end": 465} + ] + }, + { + "case_id": "rdt-zero-length-status-packet-loop", + "finding": { + "id": "ffmpeg-rdt-zero-length-status-packet-loop", + "finding_type": "infinite_loop", + "cwe": "CWE-835", + "file": "libavformat/rdt.c", + "line_number": 202, + "severity": "medium", + "confidence": "high", + "evidence_level": "crash_reproduced", + "description": "ff_rdt_parse_header skips leading RDT status packets while at least five bytes remain. It reads a packet-controlled 16-bit pkt_len and rejects only values greater than the remaining input. A zero length therefore passes validation, after which both buf and len are advanced by zero and the status-packet condition remains unchanged forever. A direct 16-byte header with the followed-by-data flag, status marker, and zero packet length deterministically consumes CPU until externally terminated.", + "code_snippet": "while (len >= 5 && buf[1] == 0xFF) { pkt_len = AV_RB16(buf + 3); if (pkt_len > len) return AVERROR_INVALIDDATA; buf += pkt_len; len -= pkt_len; }", + "poc": "Build evaluations/ffmpeg_rdt_reproducer.c against FFmpeg and run it with a one-second timeout; evaluations/run_ffmpeg_rdt_reproducer.py records the deterministic timeout result.", + "discovered_by": "sourcehunt:deepseek-v4-flash-0731", + "vulnerability_trace": { + "summary": "an RDT status-packet length of zero passes the sole upper-bound check and causes a packet-skipping loop to make no progress", + "steps": [ + {"file": "libavformat/rdt.c", "line": 202, "function": "ff_rdt_parse_header", "code_snippet": "while (len >= 5 && buf[1] == 0xFF /* status packet */)", "note": "An attacker-controlled status marker enters the packet-skipping loop while input remains."}, + {"file": "libavformat/rdt.c", "line": 205, "function": "ff_rdt_parse_header", "code_snippet": "if (!(buf[0] & 0x80)) return -1;", "note": "Setting the followed-by-data flag passes the only status-type gate."}, + {"file": "libavformat/rdt.c", "line": 208, "function": "ff_rdt_parse_header", "code_snippet": "pkt_len = AV_RB16(buf + 3);", "note": "The network packet supplies a zero 16-bit length."}, + {"file": "libavformat/rdt.c", "line": 209, "function": "ff_rdt_parse_header", "code_snippet": "if (pkt_len > len) return AVERROR_INVALIDDATA;", "note": "The check enforces only an upper bound; zero is accepted."}, + {"file": "libavformat/rdt.c", "line": 211, "function": "ff_rdt_parse_header", "code_snippet": "buf += pkt_len; len -= pkt_len; consumed += pkt_len;", "note": "All loop state changes by zero, so the next iteration observes the identical condition indefinitely."}, + {"file": "libavformat/rdt.c", "line": 361, "function": "ff_rdt_parse_packet", "code_snippet": "rv = ff_rdt_parse_header(buf, len, &set_id, &seq_no, &stream_id, &is_keyframe, ×tamp);", "note": "The public RDT packet parser calls the header routine on received packet bytes after only requiring len >= 12."} + ] + } + }, + "windows": [ + {"path": "libavformat/rdt.c", "start": 187, "end": 294}, + {"path": "libavformat/rdt.c", "start": 338, "end": 385} + ] + }, + { + "case_id": "rdt-aac-cache-buffer-overflow", + "finding": { + "id": "ffmpeg-rdt-aac-cache-buffer-overflow", + "finding_type": "out_of_bounds_write", + "cwe": "CWE-122", + "file": "libavformat/rdt.c", + "line_number": 317, + "severity": "high", + "confidence": "high", + "evidence_level": "crash_reproduced", + "description": "The RDT AAC cache path copies every unconsumed byte of a received record into a fixed RTP_MAX_PACKET_LENGTH plus padding buffer without checking the remaining length. RTSP allocates and passes a receive buffer ten times that nominal packet length, and TCP interleaving can carry a 16-bit record length. An AAC VBR packet declaring one cached subpacket consumes only its four-byte count and length table, then returns a positive cache count. A 9216-byte record therefore makes production rdt_parse_packet copy 9212 bytes into the 8256-byte payload buffer. The direct production-parser harness deterministically aborts under ASan on a heap-buffer-overflow write in that memcpy on both sealed snapshots.", + "code_snippet": "memcpy(rdt->buffer, buf + pos, len - pos);", + "poc": "Build evaluations/ffmpeg_rdt_aac_reproducer.c against the sanitizer-instrumented FFmpeg static libraries and run it with ASAN_OPTIONS=halt_on_error=1:abort_on_error=1:detect_leaks=0; evaluations/run_ffmpeg_rdt_aac_reproducer.py records the exact 9212-byte overflow copy.", + "discovered_by": "sourcehunt:deepseek-v4-flash-0731", + "vulnerability_trace": { + "summary": "an oversized RTSP/RDT record enters the AAC VBR cache path, whose small length table leaves nearly the entire record unconsumed before an unchecked copy into a fixed 8256-byte payload buffer", + "steps": [ + {"file": "libavformat/rtsp.c", "line": 63, "function": "file scope", "code_snippet": "#define RECVBUF_SIZE 10 * RTP_MAX_PACKET_LENGTH", "note": "The network receive allocation and read bound permit records substantially larger than the RDT payload cache."}, + {"file": "libavformat/rtsp.c", "line": 2462, "function": "ff_rtsp_fetch_packet", "code_snippet": "ret = ff_rdt_parse_packet(rtsp_st->transport_priv, pkt, &rt->recvbuf, len);", "note": "The received length is forwarded to the RDT parser without reducing it to RTP_MAX_PACKET_LENGTH."}, + {"file": "libavformat/rmdec.c", "line": 967, "function": "ff_rm_parse_packet", "code_snippet": "ast->sub_packet_cnt = (avio_rb16(pb) & 0xf0) >> 4;", "note": "A VBR AAC packet can declare one cached subpacket in its first two bytes."}, + {"file": "libavformat/rmdec.c", "line": 970, "function": "ff_rm_parse_packet", "code_snippet": "ast->sub_packet_lengths[x] = avio_rb16(pb);", "note": "With one subpacket, parsing consumes only one additional two-byte length before returning a positive cache count."}, + {"file": "libavformat/rdt.c", "line": 91, "function": "PayloadContext", "code_snippet": "char buffer[RTP_MAX_PACKET_LENGTH + AV_INPUT_BUFFER_PADDING_SIZE];", "note": "The persistent cache destination has only 8192 bytes plus 64 padding bytes."}, + {"file": "libavformat/rdt.c", "line": 317, "function": "rdt_parse_packet", "code_snippet": "memcpy(rdt->buffer, buf + pos, len - pos);", "note": "No destination bound is enforced; the reproduced record copies 9212 bytes and overwrites the heap beyond PayloadContext."} + ] + } + }, + "windows": [ + {"path": "libavformat/rdt.c", "start": 85, "end": 93}, + {"path": "libavformat/rdt.c", "start": 298, "end": 334}, + {"path": "libavformat/rmdec.c", "start": 909, "end": 1025}, + {"path": "libavformat/rtsp.c", "start": 61, "end": 64}, + {"path": "libavformat/rtsp.c", "start": 2347, "end": 2464} + ] + }, + { + "case_id": "ismindex-zero-sized-tfra-loop", + "finding": { + "id": "ffmpeg-ismindex-zero-sized-tfra-loop", + "finding_type": "infinite_loop", + "cwe": "CWE-835", + "file": "tools/ismindex.c", + "line_number": 436, + "severity": "low", + "confidence": "high", + "evidence_level": "crash_reproduced", + "description": "The ismindex MFRA parser repeatedly calls read_tfra until it returns a nonzero result. read_tfra trusts the atom's 32-bit size and, for a syntactically tagged tfra whose track ID is not present in the parsed track list, returns zero after seeking to pos + size. A zero-sized tfra therefore seeks back to the same input offset and makes the outer loop parse the identical atom forever. A direct harness around the production functions deterministically times out on both sealed snapshots.", + "code_snippet": "while (!read_tfra(tracks, start_index, f)) { }", + "poc": "Build evaluations/ffmpeg_ismindex_reproducer.c against FFmpeg and run it with a one-second timeout; evaluations/run_ffmpeg_ismindex_reproducer.py records the deterministic timeout.", + "discovered_by": "sourcehunt:deepseek-v4-flash-0731", + "vulnerability_trace": { + "summary": "a zero atom size combines with the unknown-track continue result so the MFRA parser's loop performs no forward progress", + "steps": [ + {"file": "tools/ismindex.c", "line": 325, "function": "read_tfra", "code_snippet": "int64_t pos = avio_tell(f);", "note": "The reader records the starting offset of the attacker-controlled atom."}, + {"file": "tools/ismindex.c", "line": 326, "function": "read_tfra", "code_snippet": "uint32_t size = avio_rb32(f);", "note": "The atom's 32-bit size is accepted without a minimum header-size check."}, + {"file": "tools/ismindex.c", "line": 337, "function": "read_tfra", "code_snippet": "if (!track) { ret = 0; goto fail; }", "note": "An unknown track is treated as a successful request to continue with the next atom."}, + {"file": "tools/ismindex.c", "line": 408, "function": "read_tfra", "code_snippet": "avio_seek(f, pos + size, SEEK_SET);", "note": "For size zero, the next position is exactly the starting offset."}, + {"file": "tools/ismindex.c", "line": 436, "function": "read_mfra", "code_snippet": "while (!read_tfra(tracks, start_index, f))", "note": "The zero return repeats the same read forever because no loop state advanced."} + ] + } + }, + "windows": [ + {"path": "tools/ismindex.c", "start": 321, "end": 410}, + {"path": "tools/ismindex.c", "start": 412, "end": 445} + ] + }, + { + "case_id": "entropy-high-bit-depth-histogram-oob", + "finding": { + "id": "ffmpeg-entropy-high-bit-depth-histogram-oob", + "finding_type": "out_of_bounds_write", + "cwe": "CWE-787", + "file": "libavfilter/vf_entropy.c", + "line_number": 130, + "severity": "high", + "confidence": "high", + "evidence_level": "crash_reproduced", + "description": "The entropy filter allocates exactly 1 << depth signed 64-bit histogram entries, then uses every raw 16-bit sample as an index without masking or range validation. A high-bit-depth frame can therefore carry a stored sample at or above its declared range and access beyond the heap allocation. A gray10le sample of 1024 addresses exactly one element beyond the 1024-entry allocation. The production ffmpeg filter graph deterministically aborts under ASan on an eight-byte heap-buffer-overflow access in filter_frame on both sealed snapshots.", + "code_snippet": "s->histogram[src16[x]]++;", + "poc": "Run evaluations/run_ffmpeg_entropy_reproducer.py against the sanitizer-instrumented ffmpeg binary; it supplies a 4x4 gray10le raw frame whose samples are 1024 and records the production filter's ASan heap-buffer-overflow.", + "discovered_by": "sourcehunt:deepseek-v4-flash-0731", + "vulnerability_trace": { + "summary": "a stored high-bit-depth sample is not constrained to its declared bit depth before indexing a histogram sized only for the declared range", + "steps": [ + {"file": "libavfilter/vf_entropy.c", "line": 73, "function": "config_input", "code_snippet": "s->depth = desc->comp[0].depth;", "note": "The negotiated pixel format supplies the declared component depth used to size the histogram."}, + {"file": "libavfilter/vf_entropy.c", "line": 94, "function": "config_input", "code_snippet": "s->histogram = av_malloc_array(1 << s->depth, sizeof(*s->histogram));", "note": "For gray10le, the heap allocation contains exactly 1024 eight-byte entries."}, + {"file": "libavfilter/vf_entropy.c", "line": 121, "function": "filter_frame", "code_snippet": "const uint16_t *src16 = (const uint16_t *)in->data[plane];", "note": "The stored sample is read as the full host-endian 16-bit value."}, + {"file": "libavfilter/vf_entropy.c", "line": 130, "function": "filter_frame", "code_snippet": "s->histogram[src16[x]]++;", "note": "No mask or upper-bound check precedes the index; sample 1024 accesses the first eight bytes beyond the allocation."} + ] + } + }, + "windows": [ + {"path": "libavfilter/vf_entropy.c", "start": 35, "end": 105}, + {"path": "libavfilter/vf_entropy.c", "start": 101, "end": 160} + ] + }, + { + "case_id": "caf-out-of-range-seek-index-underflow", + "finding": { + "id": "ffmpeg-caf-out-of-range-seek-index-underflow", + "finding_type": "out_of_bounds_read", + "cwe": "CWE-125", + "file": "libavformat/cafdec.c", + "line_number": 569, + "severity": "medium", + "confidence": "high", + "evidence_level": "crash_reproduced", + "description": "CAF's format-specific seek callback assumes that a nonempty variable-packet index guarantees a successful timestamp lookup. A forward seek beyond the last indexed timestamp instead makes av_index_search_timestamp return -1, which is used immediately as index_entries[-1] to load the frame timestamp and file position. A syntactically parsed one-packet CAF with a valid packet table followed by av_seek_frame past its last timestamp deterministically produces an ASan heap-buffer-overflow read in the production read_seek callback on both sealed snapshots.", + "code_snippet": "packet_cnt = av_index_search_timestamp(st, timestamp, flags); frame_cnt = sti->index_entries[packet_cnt].timestamp;", + "poc": "Build evaluations/ffmpeg_caf_seek_reproducer.c against the sanitizer-instrumented FFmpeg static libraries. The harness writes and parses a 107-byte variable-packet CAF, then calls public av_seek_frame for timestamp 1 after its only index entry at timestamp 0; evaluations/run_ffmpeg_caf_seek_reproducer.py records the ASan heap-buffer-overflow.", + "discovered_by": "sourcehunt:deepseek-v4-flash-0731", + "vulnerability_trace": { + "summary": "an out-of-range public seek returns the documented negative lookup result, but the CAF callback treats it as a valid packet-table index", + "steps": [ + {"file": "libavformat/cafdec.c", "line": 301, "function": "read_pakt_chunk", "code_snippet": "ret = av_add_index_entry(s->streams[0], pos, st->duration, 0, 0, AVINDEX_KEYFRAME);", "note": "A variable-packet CAF packet table creates attacker-shaped index entries during normal header parsing."}, + {"file": "libavformat/seek.c", "line": 161, "function": "ff_index_search_timestamp", "code_snippet": "if (m == nb_entries) return -1;", "note": "A forward lookup strictly beyond the final indexed timestamp has no matching entry and returns -1 by contract."}, + {"file": "libavformat/cafdec.c", "line": 568, "function": "read_seek", "code_snippet": "packet_cnt = av_index_search_timestamp(st, timestamp, flags);", "note": "The format-specific callback receives the negative lookup result through public av_seek_frame."}, + {"file": "libavformat/cafdec.c", "line": 569, "function": "read_seek", "code_snippet": "frame_cnt = sti->index_entries[packet_cnt].timestamp;", "note": "No packet_cnt < 0 check precedes the dereference; ASan observes an eight-byte read before the heap index allocation."}, + {"file": "libavformat/cafdec.c", "line": 570, "function": "read_seek", "code_snippet": "pos = sti->index_entries[packet_cnt].pos;", "note": "The same invalid entry would also supply the subsequent seek position."} + ] + } + }, + "windows": [ + {"path": "libavformat/cafdec.c", "start": 270, "end": 320}, + {"path": "libavformat/cafdec.c", "start": 550, "end": 585}, + {"path": "libavformat/seek.c", "start": 132, "end": 170}, + {"path": "libavformat/seek.c", "start": 606, "end": 640} + ] + }, + { + "case_id": "dovi-rpu-generator-coefficient-abort", + "finding": { + "id": "ffmpeg-dovi-rpu-generator-coefficient-abort", + "finding_type": "denial_of_service", + "cwe": "CWE-400", + "file": "libavcodec/dovi_rpuenc.c", + "line_number": 359, + "severity": "medium", + "confidence": "high", + "evidence_level": "crash_reproduced", + "description": "The Dolby Vision RPU parser accepts signed fixed-point coefficients through get_se_golomb_long with a fractional denominator up to 32 bits, but the generator passes each recovered integer component to set_se_golomb, whose documented domain is only 16 bits and whose range assertions are disabled in ordinary builds. A parsed integer component of 100,000,000 makes the writer request a code wider than put_bits supports and corrupts its bit_left state. Repeating the parser-valid value across 24 maximum-order MMR pieces also exceeds the generator's constant 177-byte-per-piece allocation estimate, after which the always-on flush assertion aborts. A production parser-to-metadata-to-generator harness deterministically reproduces the undefined shifts and abort on both sealed snapshots.", + "code_snippet": "set_se_golomb(pb, coef >> hdr->coef_log2_denom);", + "poc": "Build evaluations/ffmpeg_dovi_rpu_reproducer.c against the sanitizer-instrumented FFmpeg libraries and run it with ASAN_OPTIONS=halt_on_error=1:abort_on_error=1:detect_leaks=0; evaluations/run_ffmpeg_dovi_rpu_reproducer.py records the parser acceptance, UBSan writer failures, and terminating assertion.", + "discovered_by": "sourcehunt:deepseek-v4-flash-0731", + "vulnerability_trace": { + "summary": "parser-accepted long signed coefficients cross into a smaller signed-Golomb writer domain and a constant-size output estimate, corrupting bit-writer state until an always-on flush assertion aborts", + "steps": [ + {"file": "libavcodec/dovi_rpudec.c", "line": 435, "function": "ff_dovi_rpu_parse", "code_snippet": "hdr->coef_log2_denom = get_ue_golomb(gb);", "note": "The parser accepts fixed-point denominators from 13 through 32."}, + {"file": "libavcodec/dovi_rpudec.c", "line": 116, "function": "get_se_coef", "code_snippet": "ipart = get_se_golomb_long(gb);", "note": "The bitstream supplies a signed integer component through the long Exp-Golomb reader without the generator's 16-bit restriction."}, + {"file": "libavcodec/dovi_rpudec.c", "line": 619, "function": "ff_dovi_rpu_parse", "code_snippet": "curve->mmr_coef[i][j][k] = get_se_coef(gb, hdr);", "note": "A maximum-order MMR piece stores 22 attacker-selected coefficients, and three curves permit 24 pieces."}, + {"file": "libavcodec/dovi_rpuenc.c", "line": 359, "function": "put_se_coef", "code_snippet": "set_se_golomb(pb, coef >> hdr->coef_log2_denom);", "note": "The generator narrows the recovered integer component into the signed Golomb writer whose documented maximum is 16 bits."}, + {"file": "libavcodec/put_golomb.h", "line": 86, "function": "set_se_golomb", "code_snippet": "i = 2 * i - 1; ... set_ue_golomb(pb, i);", "note": "Large parser-accepted values exceed the helper's documented domain; its range protection is av_assert2 and absent in ordinary builds."}, + {"file": "libavcodec/dovi_rpuenc.c", "line": 694, "function": "ff_dovi_rpu_generate", "code_snippet": "case AV_DOVI_MAPPING_MMR: buffer_size += 177;", "note": "Allocation uses a fixed estimate although signed Exp-Golomb code length grows with coefficient magnitude."}, + {"file": "libavcodec/put_bits.h", "line": 160, "function": "flush_put_bits", "code_snippet": "av_assert0(s->buf_ptr < s->buf_end);", "note": "After writer-state corruption and exhausted capacity, the always-enabled boundary assertion terminates the process."} + ] + } + }, + "windows": [ + {"path": "libavcodec/dovi_rpudec.c", "start": 90, "end": 125}, + {"path": "libavcodec/dovi_rpudec.c", "start": 425, "end": 445}, + {"path": "libavcodec/dovi_rpudec.c", "start": 585, "end": 625}, + {"path": "libavcodec/dovi_rpuenc.c", "start": 334, "end": 368}, + {"path": "libavcodec/dovi_rpuenc.c", "start": 680, "end": 705}, + {"path": "libavcodec/dovi_rpuenc.c", "start": 755, "end": 785}, + {"path": "libavcodec/put_golomb.h", "start": 35, "end": 92}, + {"path": "libavcodec/put_bits.h", "start": 145, "end": 165} + ] + }, + { + "case_id": "showfreqs-delay-negative-bin-oob-read", + "finding": { + "id": "ffmpeg-showfreqs-delay-negative-bin-oob-read", + "finding_type": "out_of_bounds_read", + "cwe": "CWE-125", + "file": "libavfilter/avf_showfreqs.c", + "line_number": 455, + "severity": "medium", + "confidence": "high", + "evidence_level": "crash_reproduced", + "description": "The showfreqs filter's public data=delay mode computes group delay from the current and previous FFT bins, but its frequency loop starts at zero. On the first iteration, both RE(f-1, ch) and IM(f-1, ch) index fft_data[ch][-1]. Each per-channel FFT array starts at its heap allocation, so every produced frame in delay mode reads the AVComplexFloat immediately before that allocation. A production ffmpeg filter graph deterministically aborts under ASan on a four-byte heap-buffer-overflow read eight bytes before the FFT buffer on both sealed snapshots.", + "code_snippet": "for (f = 0; f < s->nb_freq; f++) { a = av_clipd((M_PI - P(IM(f, ch) * RE(f-1, ch) - IM(f-1, ch) * RE(f, ch), ...", + "poc": "Run `ffmpeg -f lavfi -i anoisesrc=r=48000:d=0.2 -filter_complex '[0:a]showfreqs=data=delay[outv]' -map '[outv]' -frames:v 1 -f null -` using the sanitizer-instrumented binary; evaluations/run_ffmpeg_showfreqs_reproducer.py records the ASan under-read.", + "discovered_by": "sourcehunt:deepseek-v4-flash-0731", + "vulnerability_trace": { + "summary": "the user-selectable delay visualization starts its previous-bin calculation at bin zero, turning f - 1 into a negative heap index", + "steps": [ + {"file": "libavfilter/avf_showfreqs.c", "line": 109, "function": "file scope", "code_snippet": "{ \"data\", \"set data mode\", OFFSET(data_mode), AV_OPT_TYPE_INT, {.i64=MAGNITUDE}, 0, NB_DATA-1, FLAGS, .unit = \"data\" },", "note": "The filter exposes data_mode as a public video-filter option."}, + {"file": "libavfilter/avf_showfreqs.c", "line": 112, "function": "file scope", "code_snippet": "{ \"delay\", \"show group delay\",0, AV_OPT_TYPE_CONST, {.i64=DELAY}, 0, 0, FLAGS, .unit = \"data\" },", "note": "Selecting data=delay reaches the vulnerable branch through ordinary filter configuration."}, + {"file": "libavfilter/avf_showfreqs.c", "line": 188, "function": "config_output", "code_snippet": "s->fft_data[i] = av_calloc(FFALIGN(s->win_size, 512), sizeof(**s->fft_data));", "note": "Each channel's FFT data begins at a separately allocated heap buffer with no valid preceding element."}, + {"file": "libavfilter/avf_showfreqs.c", "line": 408, "function": "plot_freqs", "code_snippet": "#define RE(x, ch) s->fft_data[ch][x].re\n#define IM(x, ch) s->fft_data[ch][x].im", "note": "The access macros apply the supplied signed index directly to the heap array."}, + {"file": "libavfilter/avf_showfreqs.c", "line": 454, "function": "plot_freqs", "code_snippet": "for (f = 0; f < s->nb_freq; f++)", "note": "The delay calculation begins with f equal to zero."}, + {"file": "libavfilter/avf_showfreqs.c", "line": 455, "function": "plot_freqs", "code_snippet": "a = av_clipd((M_PI - P(IM(f, ch) * RE(f-1, ch) - IM(f-1, ch) * RE(f, ch),", "note": "At f == 0, the previous-bin terms perform the reproduced reads from fft_data[ch][-1]."} + ] + } + }, + "windows": [ + {"path": "libavfilter/avf_showfreqs.c", "start": 79, "end": 115}, + {"path": "libavfilter/avf_showfreqs.c", "start": 141, "end": 193}, + {"path": "libavfilter/avf_showfreqs.c", "start": 395, "end": 462} + ] + }, + { + "case_id": "hls-sample-aes-adts-length-overflow", + "finding": { + "id": "ffmpeg-hls-sample-aes-adts-length-overflow", + "finding_type": "out_of_bounds_write", + "cwe": "CWE-787", + "file": "libavformat/hls_sample_encryption.c", + "line_number": 356, + "severity": "high", + "confidence": "high", + "evidence_level": "crash_reproduced", + "description": "The HLS SAMPLE-AES AAC path trusts the ADTS header's 13-bit frame_length after parsing only the bytes actually remaining in the packet. It never verifies that the declared frame fits those remaining bytes. decrypt_sync_frame consequently derives its AES block count from the attacker-declared length and decrypts in place through the AVPacket allocation. A 64-byte packet declaring an 8191-byte ADTS frame deterministically aborts under ASan when the production AES routine reads beyond the packet buffer on both sealed snapshots; the same loop also writes every decrypted block back through the out-of-range destination.", + "code_snippet": "num_of_encrypted_blocks = (frame->length - frame->header_length - 16)/16; av_aes_crypt(crypto_ctx->aes_ctx, data, data, num_of_encrypted_blocks, crypto_ctx->iv, 1);", + "poc": "Build evaluations/ffmpeg_hls_sample_aes_reproducer.c against the sanitizer-instrumented FFmpeg static libraries and run it with ASAN_OPTIONS=halt_on_error=1:abort_on_error=1:detect_leaks=0; evaluations/run_ffmpeg_hls_sample_aes_reproducer.py records the production AES out-of-bounds access.", + "discovered_by": "sourcehunt:deepseek-v4-flash-0731", + "vulnerability_trace": { + "summary": "an ADTS frame length that exceeds the remaining HLS packet becomes an in-place AES loop bound with no intervening packet-size validation", + "steps": [ + {"file": "libavformat/hls.c", "line": 2558, "function": "read_from_playlist", "code_snippet": "if (seg && seg->key_type == KEY_SAMPLE_AES && !strstr(pls->ctx->iformat->name, \"mov\"))", "note": "An HLS segment marked SAMPLE-AES enters the elementary-stream decryption path."}, + {"file": "libavformat/hls.c", "line": 2562, "function": "read_from_playlist", "code_snippet": "ff_hls_senc_decrypt_frame(codec_id, &c->crypto_ctx, pls->pkt);", "note": "The demuxed AAC AVPacket and its actual size are passed to the decrypter."}, + {"file": "libavcodec/adts_header.c", "line": 56, "function": "ff_adts_header_parse", "code_snippet": "size = get_bits(gbc, 13); /* aac_frame_length */", "note": "The AAC frame length is a 13-bit field controlled by the segment bytes."}, + {"file": "libavcodec/adts_header.c", "line": 57, "function": "ff_adts_header_parse", "code_snippet": "if (size < AV_AAC_ADTS_HEADER_SIZE)", "note": "The shared ADTS parser enforces a minimum but does not require the declared frame to fit the supplied buffer."}, + {"file": "libavformat/hls_sample_encryption.c", "line": 288, "function": "get_next_adts_frame", "code_snippet": "ret = avpriv_adts_header_parse (&adts_hdr, frame->data, ctx->buf_end - frame->data);", "note": "The HLS parser knows the exact number of remaining packet bytes when it parses the header."}, + {"file": "libavformat/hls_sample_encryption.c", "line": 293, "function": "get_next_adts_frame", "code_snippet": "frame->length = adts_hdr->frame_length;", "note": "The header's declared length is retained without comparing it to ctx->buf_end - frame->data."}, + {"file": "libavformat/hls_sample_encryption.c", "line": 354, "function": "decrypt_sync_frame", "code_snippet": "num_of_encrypted_blocks = (frame->length - frame->header_length - 16)/16;", "note": "The unbounded length controls how many 16-byte blocks the AES implementation processes."}, + {"file": "libavformat/hls_sample_encryption.c", "line": 356, "function": "decrypt_sync_frame", "code_snippet": "av_aes_crypt(crypto_ctx->aes_ctx, data, data, num_of_encrypted_blocks, crypto_ctx->iv, 1);", "note": "In-place decryption reads and writes beyond the AVPacket when the declared frame is larger than the remaining bytes; ASan catches the first out-of-range read."} + ] + } + }, + "windows": [ + {"path": "libavformat/hls.c", "start": 2540, "end": 2565}, + {"path": "libavcodec/adts_header.c", "start": 30, "end": 74}, + {"path": "libavformat/hls_sample_encryption.c", "start": 41, "end": 58}, + {"path": "libavformat/hls_sample_encryption.c", "start": 270, "end": 298}, + {"path": "libavformat/hls_sample_encryption.c", "start": 332, "end": 384}, + {"path": "libavutil/aes.c", "start": 155, "end": 175} + ] + }, + { + "case_id": "mpc7-last-frame-sample-count-overread", + "finding": { + "id": "ffmpeg-mpc7-last-frame-sample-count-overread", + "finding_type": "out_of_bounds_read", + "cwe": "CWE-125", + "file": "libavcodec/mpc7.c", + "line_number": 281, + "severity": "high", + "confidence": "high", + "evidence_level": "crash_reproduced", + "description": "The Musepack SV7 decoder reads an 11-bit last-frame sample count from demuxer-provided extradata without constraining it to the format's fixed 1,152 samples per frame. mpc7_decode_frame requests a planar S16 buffer with nb_samples equal to 1,152, synthesizes that fixed amount, and then, for a packet whose last-frame byte is nonzero, publishes the attacker-selected count of up to 2,047. Any consumer that processes the public AVFrame contract therefore reads beyond each 2,304-byte channel allocation. A public avcodec_send_packet/avcodec_receive_frame harness with lastframelen 2,047 deterministically aborts under ASan on the first two-byte read past the plane on both sealed snapshots.", + "code_snippet": "if(last_frame) frame->nb_samples = c->lastframelen;", + "poc": "Build evaluations/ffmpeg_mpc7_lastframelen_reproducer.c against the sanitizer-instrumented FFmpeg libraries and run it with ASAN_OPTIONS=halt_on_error=1:abort_on_error=1:detect_leaks=0; evaluations/run_ffmpeg_mpc7_lastframelen_reproducer.py records the public decoder returning 2,047 samples over a 1,152-sample allocation and the ensuing ASan read.", + "discovered_by": "sourcehunt:deepseek-v4-flash-0731", + "vulnerability_trace": { + "summary": "an 11-bit container field survives decoder initialization and replaces the public sample count after allocation and synthesis used the fixed 1,152-sample frame size", + "steps": [ + {"file": "libavformat/mpc.c", "line": 103, "function": "mpc_read_header", "code_snippet": "if ((ret = ff_get_extradata(s, st->codecpar, s->pb, 16)) < 0)", "note": "The Musepack file supplies the 16-byte codec extradata through the normal demuxer path."}, + {"file": "libavcodec/mpc7.c", "line": 111, "function": "mpc7_decode_init", "code_snippet": "c->lastframelen = get_bits(&gb, 11);", "note": "The decoder accepts the full 0 through 2,047 field range without comparing it to MPC_FRAME_SIZE."}, + {"file": "libavcodec/mpc.h", "line": 43, "function": "file scope", "code_snippet": "#define MPC_FRAME_SIZE (BANDS * SAMPLES_PER_BAND)", "note": "With 32 bands and 36 samples per band, every decoded Musepack frame contains 1,152 samples."}, + {"file": "libavcodec/mpc7.c", "line": 210, "function": "mpc7_decode_frame", "code_snippet": "frame->nb_samples = MPC_FRAME_SIZE;", "note": "The frame initially requests storage for exactly the fixed 1,152 samples per channel."}, + {"file": "libavcodec/mpc7.c", "line": 211, "function": "mpc7_decode_frame", "code_snippet": "if ((ret = ff_get_buffer(avctx, frame, 0)) < 0)", "note": "For planar S16 audio this produces a 2,304-byte allocation for each channel."}, + {"file": "libavcodec/mpc7.c", "line": 279, "function": "mpc7_decode_frame", "code_snippet": "ff_mpc_dequantize_and_synth(c, mb, (int16_t **)frame->extended_data, 2);", "note": "Synthesis fills the fixed frame before the published length is changed."}, + {"file": "libavcodec/mpc7.c", "line": 281, "function": "mpc7_decode_frame", "code_snippet": "frame->nb_samples = c->lastframelen;", "note": "A nonzero packet last-frame marker publishes up to 2,047 valid-looking samples, causing ordinary consumers to read past both channel planes."} + ] + } + }, + "windows": [ + {"path": "libavformat/mpc.c", "start": 58, "end": 110}, + {"path": "libavcodec/mpc.h", "start": 29, "end": 66}, + {"path": "libavcodec/mpc7.c", "start": 78, "end": 123}, + {"path": "libavcodec/mpc7.c", "start": 178, "end": 221}, + {"path": "libavcodec/mpc7.c", "start": 272, "end": 298} + ] + }, + { + "case_id": "lcl-zlib-multithread-short-output-disclosure", + "finding": { + "id": "ffmpeg-lcl-zlib-multithread-short-output-disclosure", + "finding_type": "information_disclosure", + "cwe": "CWE-200", + "file": "libavcodec/lcldec.c", + "line_number": 286, + "severity": "medium", + "confidence": "high", + "evidence_level": "root_cause_explained", + "description": "The LCL/ZLIB decoder's FLAG_MULTITHREAD path accepts each zlib stream when its actual output is shorter than the attacker-claimed half size. zlib_decomp returns the short count as success, both calls ignore that count, and decode_frame then sets len to the full image size before copying the whole decompression allocation into the output AVFrame. A public decoder harness whose two valid zlib streams expand to three bytes each is accepted as a 768-byte RGB24 frame and exposes the other 762 bytes of the uninitialized decompression buffer. ASan malloc-fill instrumentation deterministically marks and observes all 762 disclosed bytes on both sealed snapshots.", + "code_snippet": "ret = zlib_decomp(...); if (ret < 0) return ret; ... len = c->decomp_size;", + "poc": "Build evaluations/ffmpeg_lcl_multithread_reproducer.c against either sanitizer-instrumented FFmpeg snapshot and run with ASAN_OPTIONS=halt_on_error=1:abort_on_error=1:detect_leaks=0:malloc_fill_byte=165:max_malloc_fill_size=1048576; evaluations/run_ffmpeg_lcl_multithread_reproducer.py records six attacker-supplied bytes and 762 allocator-fill bytes in the returned frame.", + "discovered_by": "sourcehunt:deepseek-v4-flash-0731", + "vulnerability_trace": { + "summary": "two short but valid compressed streams are accepted against attacker-claimed expected lengths, their actual lengths are discarded, and the full decompression allocation is published as decoded pixels", + "steps": [ + {"file": "libavcodec/lcldec.c", "line": 179, "function": "decode_frame", "code_snippet": "unsigned int mthread_inlen, mthread_outlen;", "note": "The packet supplies both the compressed first-stream length and a claimed decompressed half length."}, + {"file": "libavcodec/lcldec.c", "line": 279, "function": "decode_frame", "code_snippet": "mthread_outlen = AV_RL32(buf + 4);", "note": "The attacker-controlled claimed half length becomes the expected count and the second stream's output offset."}, + {"file": "libavcodec/lcldec.c", "line": 155, "function": "zlib_decomp", "code_snippet": "if (expected > (unsigned int)zstream->total_out) return (unsigned int)zstream->total_out;", "note": "A valid stream that expands to fewer bytes than claimed returns its short positive count instead of invalid data."}, + {"file": "libavcodec/lcldec.c", "line": 281, "function": "decode_frame", "code_snippet": "ret = zlib_decomp(avctx, buf + 8, mthread_inlen, 0, mthread_outlen); if (ret < 0) return ret;", "note": "The first short positive count is accepted and discarded."}, + {"file": "libavcodec/lcldec.c", "line": 283, "function": "decode_frame", "code_snippet": "ret = zlib_decomp(avctx, buf + 8 + mthread_inlen, len - 8 - mthread_inlen, mthread_outlen, mthread_outlen);", "note": "The second short stream writes only a few bytes at the claimed offset; its short count is also discarded."}, + {"file": "libavcodec/lcldec.c", "line": 286, "function": "decode_frame", "code_snippet": "len = c->decomp_size;", "note": "The decoder falsely marks the entire decompression allocation as valid image data."}, + {"file": "libavcodec/lcldec.c", "line": 428, "function": "decode_frame", "code_snippet": "memcpy(outptr + pixel_ptr, encoded, 3 * width);", "note": "RGB conversion copies every claimed image byte into the public frame, including the reproduced uninitialized remainder."} + ] + } + }, + "windows": [ + {"path": "libavcodec/lcldec.c", "start": 125, "end": 160}, + {"path": "libavcodec/lcldec.c", "start": 164, "end": 185}, + {"path": "libavcodec/lcldec.c", "start": 263, "end": 293}, + {"path": "libavcodec/lcldec.c", "start": 418, "end": 433}, + {"path": "libavcodec/lcldec.c", "start": 540, "end": 625} + ] + }, + { + "case_id": "af-join-duplicate-map-use-after-free", + "finding": { + "id": "ffmpeg-af-join-duplicate-map-use-after-free", + "finding_type": "use_after_free", + "cwe": "CWE-416", + "file": "libavfilter/af_join.c", + "line_number": 472, + "severity": "high", + "confidence": "high", + "evidence_level": "crash_reproduced", + "description": "The join filter deduplicates plane buffers by scanning only nb_buffers entries but decides whether the buffer was new with j == i, where i is the output-channel index. A valid map that duplicates one input plane into two early output channels and maps a different input plane later makes i diverge from nb_buffers, so the later unique plane is omitted from the output frame's AVBufferRef list. try_push_frame then frees both input frames after delivering the output, leaving that plane dangling. A public two-input libavfilter graph returns a three-channel frame whose third plane has no owner; an ordinary downstream read deterministically produces an ASan heap-use-after-free on both sealed snapshots.", + "code_snippet": "for (j = 0; j < nb_buffers; j++) if (s->buffers[j]->buffer == buf->buffer) break; if (j == i) s->buffers[nb_buffers++] = buf;", + "poc": "Build and run evaluations/ffmpeg_af_join_uaf_reproducer.c with evaluations/run_ffmpeg_af_join_uaf_reproducer.py against either sealed snapshot.", + "discovered_by": "sourcehunt:deepseek-v4-flash-0731", + "vulnerability_trace": { + "summary": "a duplicate early output mapping desynchronizes the output-channel index from the unique-buffer count, causing a later plane to be published without an owning reference", + "steps": [ + {"file": "libavfilter/af_join.c", "line": 454, "function": "try_push_frame", "code_snippet": "for (i = 0; i < s->ch_layout.nb_channels; i++)", "note": "The loop visits output mappings, so i counts output channels rather than unique buffers."}, + {"file": "libavfilter/af_join.c", "line": 469, "function": "try_push_frame", "code_snippet": "for (j = 0; j < nb_buffers; j++)", "note": "The duplicate search correctly ranges over only the unique buffers accumulated so far."}, + {"file": "libavfilter/af_join.c", "line": 472, "function": "try_push_frame", "code_snippet": "if (j == i)", "note": "After an earlier duplicate mapping, i exceeds nb_buffers and a genuinely new later plane is not retained."}, + {"file": "libavfilter/af_join.c", "line": 487, "function": "try_push_frame", "code_snippet": "frame->buf[i] = av_buffer_ref(s->buffers[i]);", "note": "Only the incorrectly accumulated buffer list contributes ownership references to the output frame."}, + {"file": "libavfilter/af_join.c", "line": 522, "function": "try_push_frame", "code_snippet": "ret = ff_filter_frame(outlink, frame);", "note": "The output containing the unowned plane is delivered downstream."}, + {"file": "libavfilter/af_join.c", "line": 525, "function": "try_push_frame", "code_snippet": "av_frame_free(&s->input_frames[i]);", "note": "The input frame that owns the omitted plane is freed, leaving the published pointer dangling."} + ] + } + }, + "windows": [ + {"path": "libavfilter/af_join.c", "start": 419, "end": 525}, + {"path": "libavfilter/af_join.c", "start": 282, "end": 417} + ] + }, + { + "case_id": "drawgraph-missing-primary-metadata-heap-overflow", + "finding": { + "id": "ffmpeg-drawgraph-missing-primary-metadata-heap-overflow", + "finding_type": "out_of_bounds_write", + "cwe": "CWE-787", + "file": "libavfilter/f_drawgraph.c", + "line_number": 159, + "severity": "high", + "confidence": "high", + "evidence_level": "crash_reproduced", + "description": "The drawgraph filter checks and resets its horizontal output coordinate only inside the first metadata series' successful parse path. If m1 is absent or unparsable while any later configured series is present, the first loop iteration continues before that bound check, but s->x still increments after every frame. A later series then passes the unbounded x coordinate to draw_dot. A public libavfilter graph with a two-pixel-wide output, missing primary metadata, and present secondary metadata produces an ASan heap-buffer-overflow write in draw_dot on the third frame on both sealed snapshots.", + "code_snippet": "if (!e || !e->value) continue; ... if (i == 0 && s->x >= outlink->w) s->x = 0; ... draw_dot(fg, x, y, out); ... s->x++;", + "poc": "Build and run evaluations/ffmpeg_drawgraph_missing_primary_reproducer.c with evaluations/run_ffmpeg_drawgraph_missing_primary_reproducer.py against either sealed snapshot.", + "discovered_by": "sourcehunt:deepseek-v4-flash-0731", + "vulnerability_trace": { + "summary": "missing metadata for the first graph series skips the only horizontal bound reset, while a later series writes at the unbounded per-frame coordinate", + "steps": [ + {"file": "libavfilter/f_drawgraph.c", "line": 225, "function": "filter_frame", "code_snippet": "e = av_dict_get(metadata, s->key[i], NULL, 0); if (!e || !e->value) continue;", "note": "Missing primary metadata exits the i == 0 iteration before coordinate maintenance."}, + {"file": "libavfilter/f_drawgraph.c", "line": 246, "function": "filter_frame", "code_snippet": "if (i == 0 && (s->x >= outlink->w || s->slide == 3))", "note": "The only reset or scrolling bound for s->x is nested inside the metadata-present path for the first series."}, + {"file": "libavfilter/f_drawgraph.c", "line": 269, "function": "filter_frame", "code_snippet": "x = s->x;", "note": "A present later series receives the unbounded shared coordinate."}, + {"file": "libavfilter/f_drawgraph.c", "line": 293, "function": "filter_frame", "code_snippet": "draw_dot(fg, x, y, out);", "note": "Dot mode reaches the raw four-byte output write with x at or beyond the output width."}, + {"file": "libavfilter/f_drawgraph.c", "line": 320, "function": "filter_frame", "code_snippet": "s->x++;", "note": "Every input frame advances the coordinate even when the first metadata series was absent."}, + {"file": "libavfilter/f_drawgraph.c", "line": 159, "function": "draw_dot", "code_snippet": "AV_WN32(out->data[0] + y * out->linesize[0] + x * 4, fg);", "note": "ASan observes the four-byte heap-buffer-overflow at this unchecked pixel address."} + ] + } + }, + "windows": [ + {"path": "libavfilter/f_drawgraph.c", "start": 147, "end": 160}, + {"path": "libavfilter/f_drawgraph.c", "start": 162, "end": 341} + ] + }, + { + "case_id": "dnn-four-output-name-parser-heap-overflow", + "finding": { + "id": "ffmpeg-dnn-four-output-name-parser-heap-overflow", + "finding_type": "out_of_bounds_write", + "cwe": "CWE-787", + "file": "libavfilter/dnn_filter_common.c", + "line_number": 50, + "severity": "high", + "confidence": "high", + "evidence_level": "crash_reproduced", + "description": "The shared DNN output-name parser allocates MAX_SUPPORTED_OUTPUTS_NB, exactly four, pointer slots and writes every parsed token into that array without reserving a terminator slot or enforcing the maximum. It then unconditionally stores a NULL terminator at parsed_vals[val_num]. TensorFlow detection explicitly requires four output names, so its intended count writes the terminator into a fifth pointer slot. A production ff_dnn_init harness with four generic output names produces an ASan heap-buffer-overflow before backend lookup or model loading on both snapshots.", + "code_snippet": "parsed_vals = av_calloc(MAX_SUPPORTED_OUTPUTS_NB, sizeof(*parsed_vals)); ... parsed_vals[val_num++] = val; ... parsed_vals[val_num] = NULL;", + "poc": "Build and run evaluations/ffmpeg_dnn_output_names_reproducer.c with evaluations/run_ffmpeg_dnn_output_names_reproducer.py against either sealed snapshot.", + "discovered_by": "sourcehunt:deepseek-v4-flash-0731", + "vulnerability_trace": { + "summary": "the intended four-output TensorFlow detector configuration fills a four-pointer allocation and then writes its required NULL terminator into a fifth slot", + "steps": [ + {"file": "libavfilter/vf_dnn_detect.c", "line": 630, "function": "check_output_nb", "code_snippet": "if (output_nb != 4) return AVERROR(EINVAL);", "note": "The TensorFlow detection filter requires exactly four named outputs."}, + {"file": "libavfilter/dnn_filter_common.c", "line": 24, "function": "file scope", "code_snippet": "#define MAX_SUPPORTED_OUTPUTS_NB 4", "note": "Four is also the complete pointer capacity allocated by the parser."}, + {"file": "libavfilter/dnn_filter_common.c", "line": 34, "function": "separate_output_names", "code_snippet": "parsed_vals = av_calloc(MAX_SUPPORTED_OUTPUTS_NB, sizeof(*parsed_vals));", "note": "No fifth slot is reserved for a NULL terminator."}, + {"file": "libavfilter/dnn_filter_common.c", "line": 42, "function": "separate_output_names", "code_snippet": "parsed_vals[val_num] = val; val_num++;", "note": "Four valid names consume all four allocated pointer slots; the loop also lacks a maximum-count check."}, + {"file": "libavfilter/dnn_filter_common.c", "line": 50, "function": "separate_output_names", "code_snippet": "parsed_vals[val_num] = NULL;", "note": "ASan observes an eight-byte heap-buffer-overflow when val_num is four."}, + {"file": "libavfilter/dnn_filter_common.c", "line": 95, "function": "ff_dnn_init", "code_snippet": "ctx->model_outputnames = separate_output_names(...);", "note": "The overflow occurs during normal filter initialization before backend lookup or model loading."} + ] + } + }, + "windows": [ + {"path": "libavfilter/dnn_filter_common.c", "start": 24, "end": 53}, + {"path": "libavfilter/dnn_filter_common.c", "start": 73, "end": 102}, + {"path": "libavfilter/vf_dnn_detect.c", "start": 630, "end": 672} + ] + }, + { + "case_id": "tensorflow-multi-output-tensor-leak", + "finding": { + "id": "ffmpeg-tensorflow-multi-output-tensor-leak", + "finding_type": "resource_leak", + "cwe": "CWE-772", + "file": "libavfilter/dnn/dnn_backend_tf.c", + "line_number": 102, + "severity": "medium", + "confidence": "high", + "evidence_level": "root_cause_explained", + "description": "TensorFlow inference allocates and returns task->nb_output tensors, and the registered TensorFlow detection path requires four outputs. Cleanup attempts to derive that runtime count with sizeof(*output_tensors) divided by sizeof(output_tensors[0]); both operands describe one pointer, so the result is always one. Only the first TensorFlow output tensor is deleted before the pointer array is freed. Every processed detection frame therefore leaks three complete model output tensors, allowing sustained valid use to exhaust memory. The sealed builds omit TensorFlow, so this remains source-confirmed rather than dynamically reproduced.", + "code_snippet": "int nb_output = sizeof(*request->output_tensors)/sizeof(request->output_tensors[0]); for (i = 0; i < nb_output; ++i) TF_DeleteTensor(...);", + "poc": "Configure vf_dnn_detect with its required four-output TensorFlow model and process frames while monitoring output-tensor allocations; three of four tensors per request are not passed to TF_DeleteTensor.", + "discovered_by": "sourcehunt:deepseek-v4-flash-0731", + "vulnerability_trace": { + "summary": "four TensorFlow outputs are produced per detection frame, but pointer-type sizeof arithmetic hard-codes cleanup to one tensor", + "steps": [ + {"file": "libavfilter/vf_dnn_detect.c", "line": 634, "function": "check_output_nb", "code_snippet": "if (output_nb != 4)", "note": "The registered TensorFlow detector accepts only four-output configurations."}, + {"file": "libavfilter/dnn/dnn_backend_tf.c", "line": 669, "function": "fill_model_input_tf", "code_snippet": "output_tensors = av_calloc(task->nb_output, sizeof(*output_tensors));", "note": "The request allocates pointer storage for all four outputs."}, + {"file": "libavfilter/dnn/dnn_backend_tf.c", "line": 153, "function": "tf_start_inference", "code_snippet": "TF_SessionRun(... output_tensors, task->nb_output, ...);", "note": "TensorFlow populates all runtime output tensors."}, + {"file": "libavfilter/dnn/dnn_backend_tf.c", "line": 102, "function": "tf_free_request", "code_snippet": "int nb_output = sizeof(*request->output_tensors)/sizeof(request->output_tensors[0]);", "note": "Both sizeof operands are one pointer, making nb_output equal one regardless of the task."}, + {"file": "libavfilter/dnn/dnn_backend_tf.c", "line": 103, "function": "tf_free_request", "code_snippet": "for (uint32_t i = 0; i < nb_output; ++i)", "note": "Only output_tensors[0] is deleted; the other three TensorFlow tensor allocations are leaked each frame."} + ] + } + }, + "windows": [ + {"path": "libavfilter/dnn/dnn_backend_tf.c", "start": 84, "end": 111}, + {"path": "libavfilter/dnn/dnn_backend_tf.c", "start": 140, "end": 162}, + {"path": "libavfilter/dnn/dnn_backend_tf.c", "start": 662, "end": 717}, + {"path": "libavfilter/vf_dnn_detect.c", "start": 630, "end": 672} + ] + }, + { + "case_id": "dnn-nchw-output-shape-heap-overflow", + "finding": { + "id": "ffmpeg-dnn-nchw-output-shape-heap-overflow", + "finding_type": "out_of_bounds_read", + "cwe": "CWE-125", + "file": "libavfilter/dnn/dnn_io_proc.c", + "line_number": 93, + "severity": "high", + "confidence": "high", + "evidence_level": "crash_reproduced", + "description": "The generic DNN RGB postprocessor allocates its NCHW intermediary as frame width times height times the model-declared output channel count, but unconditionally asks swscale to process frame width times three elements per row and later addresses three planes. The TensorFlow and OpenVINO backends pass model-declared output shapes here without validating the output channel count against RGB24/BGR24. A one-channel NCHW tensor for a 4x4 RGB24 output produces an ASan heap-buffer-overflow in the production postprocessor on both sealed snapshots. The same path also passes a one-pointer object to swscale's four-plane array API, an independent stack-object contract violation that an entirely instrumented build observes first.", + "code_snippet": "middle_data = av_malloc(plane_size * output->dims[1]); ... sws_scale(... frame->width * 3 * src_datatype_size ...)", + "poc": "Build and run evaluations/ffmpeg_dnn_output_shape_reproducer.c with evaluations/run_ffmpeg_dnn_output_shape_reproducer.py against either sealed snapshot.", + "discovered_by": "sourcehunt:deepseek-v4-flash-0731", + "vulnerability_trace": { + "summary": "a model-declared NCHW output channel count sizes an intermediary allocation, while RGB conversion ignores that shape and processes and addresses three channels", + "steps": [ + {"file": "libavfilter/dnn/dnn_backend_openvino.c", "line": 387, "function": "infer_completion_callback", "code_snippet": "outputs[i].dims[1] = output_shape.rank > 2 ? dims[output_shape.rank - 3] : 1;", "note": "The loaded model's output shape populates DNNData."}, + {"file": "libavfilter/dnn/dnn_backend_openvino.c", "line": 448, "function": "infer_completion_callback", "code_snippet": "ff_proc_from_dnn_to_frame(task->out_frame, outputs, ctx);", "note": "No output-channel/format validation intervenes before generic postprocessing."}, + {"file": "libavfilter/dnn/dnn_io_proc.c", "line": 74, "function": "ff_proc_from_dnn_to_frame", "code_snippet": "middle_data = av_malloc(plane_size * output->dims[1]);", "note": "The model-controlled channel count sizes the NCHW byte intermediary."}, + {"file": "libavfilter/dnn/dnn_io_proc.c", "line": 93, "function": "ff_proc_from_dnn_to_frame", "code_snippet": "sws_scale(... frame->width * 3 * src_datatype_size ...)", "note": "RGB conversion always consumes and writes three channels per row; ASan catches the undersized intermediary access."}, + {"file": "libavfilter/dnn/dnn_io_proc.c", "line": 119, "function": "ff_proc_from_dnn_to_frame", "code_snippet": "planar_data[1] = (uint8_t *)middle_data + plane_size * 2;", "note": "The second conversion also unconditionally constructs three plane pointers."} + ] + } + }, + "windows": [ + {"path": "libavfilter/dnn/dnn_io_proc.c", "start": 38, "end": 145}, + {"path": "libavfilter/dnn/dnn_backend_openvino.c", "start": 370, "end": 460}, + {"path": "libavfilter/dnn/dnn_backend_tf.c", "start": 697, "end": 735}, + {"path": "libavfilter/vf_dnn_processing.c", "start": 83, "end": 135} + ] + }, + { + "case_id": "nellymoser-trellis-off-by-one-heap-overflow", + "finding": { + "id": "ffmpeg-nellymoser-trellis-off-by-one-heap-overflow", + "finding_type": "out_of_bounds_read_write", + "cwe": "CWE-193", + "file": "libavcodec/nellymoserenc.c", + "line_number": 272, + "severity": "high", + "confidence": "high", + "evidence_level": "crash_reproduced", + "description": "The NellyMoser encoder's optional trellis search caps idx_max with OPT_SIZE but rejects a candidate only when idx is greater than that inclusive cap. An exactly-OPT_SIZE transition therefore indexes one element beyond a trellis row. Those transitions are reachable from ordinary audio samples, and repeated row-edge writes can corrupt the following row; at the final band the corresponding read crosses the end of the complete allocation. A short public white-noise encode with volume 10 and trellis enabled produces UBSan reports for opt and path at index 35768 followed by an ASan heap-buffer-overflow read exactly after the 3,290,656-byte opt allocation on both sealed snapshots.", + "code_snippet": "idx_max = FFMIN(OPT_SIZE, cand[band - 1] + q); ... if (idx > idx_max) break; ... opt[band][idx]", + "poc": "Run evaluations/run_ffmpeg_nellymoser_trellis_reproducer.py against either sealed sanitizer-instrumented FFmpeg binary.", + "discovered_by": "sourcehunt:deepseek-v4-flash-0731", + "vulnerability_trace": { + "summary": "an inclusive upper bound admits trellis state OPT_SIZE even though every row's final legal state is OPT_SIZE minus one", + "steps": [ + {"file": "libavcodec/nellymoserenc.c", "line": 57, "function": "file scope", "code_snippet": "#define OPT_SIZE ((1<<15) + 3000)", "note": "Each trellis row has exactly 35,768 elements indexed from zero through 35,767."}, + {"file": "libavcodec/nellymoserenc.c", "line": 262, "function": "get_exponent_dynamic", "code_snippet": "idx_max = FFMIN(OPT_SIZE, cand[band - 1] + q);", "note": "The computed maximum may be the element count rather than the final valid index."}, + {"file": "libavcodec/nellymoserenc.c", "line": 267, "function": "get_exponent_dynamic", "code_snippet": "idx = i + ff_nelly_delta_table[j];", "note": "Reachable predecessor states and valid delta-table values produce idx equal to exactly 35,768."}, + {"file": "libavcodec/nellymoserenc.c", "line": 268, "function": "get_exponent_dynamic", "code_snippet": "if (idx > idx_max) break;", "note": "Equality is accepted, so the invalid one-past state reaches both trellis arrays."}, + {"file": "libavcodec/nellymoserenc.c", "line": 272, "function": "get_exponent_dynamic", "code_snippet": "if (opt[band][idx] > tmp)", "note": "UBSan identifies index 35,768, and ASan observes a four-byte read exactly after the complete opt allocation at the final row."}, + {"file": "libavcodec/nellymoserenc.c", "line": 274, "function": "get_exponent_dynamic", "code_snippet": "path[band][idx] = j;", "note": "The same off-by-one also writes through the one-past path index; earlier row-edge accesses corrupt the first element of the following row."} + ] + } + }, + "windows": [ + {"path": "libavcodec/nellymoserenc.c", "start": 55, "end": 71}, + {"path": "libavcodec/nellymoserenc.c", "start": 238, "end": 299}, + {"path": "libavcodec/nellymoserenc.c", "start": 307, "end": 375} + ] + }, + { + "case_id": "magicyuv-truncated-slice-prior-frame-disclosure", + "finding": { + "id": "ffmpeg-magicyuv-truncated-slice-prior-frame-disclosure", + "finding_type": "information_disclosure", + "cwe": "CWE-200", + "file": "libavcodec/magicyuv.c", + "line_number": 123, + "severity": "medium", + "confidence": "high", + "evidence_level": "root_cause_explained", + "description": "MagicYUV's compressed-slice macro stops each output row when the slice bitstream has no bits left but never requires the decoded pixel count to equal the declared row width. The slice callback then applies prediction to the untouched frame-pool bytes, the decoder ignores slice callback returns, and it publishes the whole frame. A public libavcodec harness decodes and unreferences a raw 16x16 gray frame, then supplies a 300-byte frame with a valid table and a two-byte compressed slice containing no samples. FFmpeg returns a second frame whose 256 bytes are a reversible transformation of the entire prior pooled frame on both sealed snapshots.", + "code_snippet": "for (; x < width && get_bits_left(&gb) > 0; x++) dst[x] = get_vlc2(...); dst += stride;", + "poc": "Build and run evaluations/ffmpeg_magicyuv_truncated_slice_reproducer.c with evaluations/run_ffmpeg_magicyuv_truncated_slice_reproducer.py against either sealed snapshot.", + "discovered_by": "sourcehunt:deepseek-v4-flash-0731", + "vulnerability_trace": { + "summary": "an exhausted compressed slice is accepted without filling its declared rows, after which prediction and the public decoded-frame contract expose stale frame-pool contents", + "steps": [ + {"file": "libavcodec/magicyuv.c", "line": 123, "function": "READ_PLANE", "code_snippet": "for (; x < width && get_bits_left(&gb) > 0; x++)", "note": "Bitstream exhaustion ends decoding without requiring x to reach width."}, + {"file": "libavcodec/magicyuv.c", "line": 303, "function": "magy_decode_slice", "code_snippet": "for (k = 0; k < height; k++) READ_PLANE(dst, i, 1, 7)", "note": "Every declared row advances even when zero pixels were decoded."}, + {"file": "libavcodec/magicyuv.c", "line": 308, "function": "magy_decode_slice", "code_snippet": "s->llviddsp.add_left_pred(dst, dst, width, 0);", "note": "Prediction transforms the untouched pooled bytes reversibly rather than initializing them."}, + {"file": "libavcodec/magicyuv.c", "line": 632, "function": "magy_decode_frame", "code_snippet": "avctx->execute2(avctx, s->magy_decode_slice, NULL, NULL, s->nb_slices);", "note": "Slice return values are not collected or checked."}, + {"file": "libavcodec/magicyuv.c", "line": 658, "function": "magy_decode_frame", "code_snippet": "*got_frame = 1;", "note": "The incomplete frame is returned as fully decoded."} + ] + } + }, + "windows": [ + {"path": "libavcodec/magicyuv.c", "start": 118, "end": 145}, + {"path": "libavcodec/magicyuv.c", "start": 263, "end": 360}, + {"path": "libavcodec/magicyuv.c", "start": 540, "end": 665}, + {"path": "libavcodec/lossless_videodsp.c", "start": 60, "end": 85} + ] + }, + { + "case_id": "rtp-av1-ignored-obu-output-cursor-heap-overflow", + "finding": { + "id": "ffmpeg-rtp-av1-ignored-obu-output-cursor-heap-overflow", + "finding_type": "out_of_bounds_write", + "cwe": "CWE-787", + "file": "libavformat/rtpdec_av1.c", + "line_number": 252, + "severity": "high", + "confidence": "high", + "evidence_level": "crash_reproduced", + "description": "The RTP/AV1 depacketizer intentionally discards temporal-delimiter and tile-list OBUs, but that branch advances the output packet cursor and remaining input length without advancing the input cursor. The next loop iteration therefore parses bytes inside the ignored OBU as a new element while retaining an output cursor displaced by the complete ignored size. Packet growth accounts only for the newly parsed element, so its header or payload write begins beyond the allocated packet. A direct production-handler harness with a 100-byte temporal delimiter followed by 17 bytes produces an ASan heap-buffer-overflow write 19 bytes beyond an 81-byte allocation on both sealed snapshots.", + "code_snippet": "if (obu_type == AV1_OBU_TEMPORAL_DELIMITER || obu_type == AV1_OBU_TILE_LIST) { pktpos += obu_size; rem_pkt_size -= obu_size; ... continue; }", + "poc": "Build and run evaluations/ffmpeg_rtp_av1_ignored_obu_reproducer.c with evaluations/run_ffmpeg_rtp_av1_ignored_obu_reproducer.py against either sealed snapshot.", + "discovered_by": "sourcehunt:deepseek-v4-flash-0731", + "vulnerability_trace": { + "summary": "discarding an ignored OBU advances only the logical output cursor and remaining length, so a following parser iteration grows a small packet and writes through a large unallocated gap", + "steps": [ + {"file": "libavformat/rtpdec_av1.c", "line": 224, "function": "av1_handle_packet", "code_snippet": "num_lebs = parse_leb(ctx, buf_ptr, rem_pkt_size, &obu_size);", "note": "An RTP-controlled LEB declares the size of a temporal-delimiter or tile-list OBU."}, + {"file": "libavformat/rtpdec_av1.c", "line": 246, "function": "av1_handle_packet", "code_snippet": "if ((obu_type == AV1_OBU_TEMPORAL_DELIMITER) || (obu_type == AV1_OBU_TILE_LIST))", "note": "These two OBU types enter the discard path."}, + {"file": "libavformat/rtpdec_av1.c", "line": 249, "function": "av1_handle_packet", "code_snippet": "pktpos += obu_size; rem_pkt_size -= obu_size;", "note": "The branch moves the output cursor and consumes the declared remaining length but never performs buf_ptr += obu_size."}, + {"file": "libavformat/rtpdec_av1.c", "line": 289, "function": "av1_handle_packet", "code_snippet": "av_grow_packet(pkt, output_size)", "note": "The next iteration grows storage only for the bytes parsed from inside the ignored OBU, not for the gap already added to pktpos."}, + {"file": "libavformat/rtpdec_av1.c", "line": 305, "function": "av1_handle_packet", "code_snippet": "pkt->data[pktpos++] = *buf_ptr++ | AV1F_OBU_HAS_SIZE_FIELD;", "note": "ASan observes the resulting write beyond the heap packet allocation before payload copying."} + ] + } + }, + "windows": [ + {"path": "libavformat/rtpdec_av1.c", "start": 120, "end": 210}, + {"path": "libavformat/rtpdec_av1.c", "start": 211, "end": 375}, + {"path": "libavformat/rtp_av1.h", "start": 55, "end": 125} + ] + }, + { + "case_id": "shufflepixels-inverse-partial-block-map-overflow", + "finding": { + "id": "ffmpeg-shufflepixels-inverse-partial-block-map-overflow", + "finding_type": "out_of_bounds_write", + "cwe": "CWE-787", + "file": "libavfilter/vf_shufflepixels.c", + "line_number": 95, + "severity": "high", + "confidence": "high", + "evidence_level": "crash_reproduced", + "description": "Horizontal and vertical inverse shuffling divide the plane into a ceiling number of blocks but allocate the map for only the exact plane area. The number of entries written to a randomly selected destination block is derived from the remaining input position, not the selected destination block. When a full-width input chunk is assigned to the final partial destination block, the map write crosses the allocation. A public filter invocation with width 10, block width 4, and seed 1 produces an ASan four-byte write exactly after the 40-byte map allocation on both sealed snapshots; the analogous vertical configuration also reproduces.", + "code_snippet": "width = FFMIN(s->block_w, s->planewidth[0] - x); map[rand * s->block_w + i] = map[rand * s->block_w] + i;", + "poc": "Run evaluations/run_ffmpeg_shufflepixels_inverse_reproducer.py against either sealed ASan ffmpeg binary.", + "discovered_by": "sourcehunt:deepseek-v4-flash-0731", + "vulnerability_trace": { + "summary": "ceiling block count admits a partial final destination, but inverse-map initialization writes a full input-chunk width at that destination", + "steps": [ + {"file": "libavfilter/vf_shufflepixels.c", "line": 322, "function": "config_output", "code_snippet": "s->map = av_calloc(inlink->w * inlink->h, sizeof(*s->map));", "note": "Horizontal map storage contains exactly one int32 entry per luma pixel."}, + {"file": "libavfilter/vf_shufflepixels.c", "line": 329, "function": "config_output", "code_snippet": "s->nb_blocks = (s->planewidth[0] + s->block_w - 1) / s->block_w;", "note": "A non-divisible width creates a ceiling-counted final partial block."}, + {"file": "libavfilter/vf_shufflepixels.c", "line": 85, "function": "make_horizontal_map", "code_snippet": "width = FFMIN(s->block_w, s->planewidth[0] - x);", "note": "Inverse mode sizes the copy from the sequential input cursor rather than from the randomly selected destination block."}, + {"file": "libavfilter/vf_shufflepixels.c", "line": 95, "function": "make_horizontal_map", "code_snippet": "map[rand * s->block_w + i] = map[rand * s->block_w] + i;", "note": "Selecting the last partial destination for an earlier full chunk writes beyond the exact-width map allocation."} + ] + } + }, + "windows": [ + {"path": "libavfilter/vf_shufflepixels.c", "start": 35, "end": 145}, + {"path": "libavfilter/vf_shufflepixels.c", "start": 300, "end": 365}, + {"path": "libavfilter/vf_shufflepixels.c", "start": 400, "end": 430} + ] + }, + { + "case_id": "vif-small-frame-reflection-oob-read", + "finding": { + "id": "ffmpeg-vif-small-frame-reflection-oob-read", + "finding_type": "out_of_bounds_read", + "cwe": "CWE-125", + "file": "libavfilter/vf_vif.c", + "line_number": 241, + "severity": "high", + "confidence": "high", + "evidence_level": "crash_reproduced", + "description": "VIF's one-step symmetric boundary reflection assumes each plane dimension is large enough for half of the active filter. For a small frame, a tap can lie beyond twice the dimension; applying 2 * dimension - index - 1 once then produces a negative index. The horizontal and vertical passes dereference that result without a minimum-dimension guard. A public VIF graph comparing two 2x2 gray frames produces an ASan four-byte read exactly after a 16-byte allocation in vif_filter1d on both sealed snapshots.", + "code_snippet": "jj = jj < 0 ? -jj : (jj >= w ? 2 * w - jj - 1 : jj); img_coeff = temp[jj];", + "poc": "Run evaluations/run_ffmpeg_vif_small_frame_reproducer.py against either sealed ASan ffmpeg binary.", + "discovered_by": "sourcehunt:deepseek-v4-flash-0731", + "vulnerability_trace": { + "summary": "a 17-tap filter reaches beyond twice a small image dimension, and single reflection turns the tap into an out-of-range negative or positive buffer index", + "steps": [ + {"file": "libavfilter/vf_vif.c", "line": 47, "function": "file scope", "code_snippet": "static const uint8_t vif_filter1d_width1[4] = { 17, 9, 5, 3 };", "note": "Scale zero applies a 17-tap filter with radius eight."}, + {"file": "libavfilter/vf_vif.c", "line": 503, "function": "config_input_ref", "code_snippet": "s->data_buf[i] = av_calloc(s->width, s->height * sizeof(float));", "note": "Internal source and result storage contains exactly width times height floats, with no minimum frame-size validation."}, + {"file": "libavfilter/vf_vif.c", "line": 241, "function": "vif_filter1d", "code_snippet": "ii = ii < 0 ? -ii : (ii >= h ? 2 * h - ii - 1 : ii);", "note": "One reflection is insufficient when a filter tap lies more than one image extent beyond the boundary."}, + {"file": "libavfilter/vf_vif.c", "line": 270, "function": "vif_filter1d", "code_snippet": "jj = jj < 0 ? -jj : (jj >= w ? 2 * w - jj - 1 : jj);", "note": "The horizontal path produces the same invalid index."}, + {"file": "libavfilter/vf_vif.c", "line": 272, "function": "vif_filter1d", "code_snippet": "img_coeff = temp[jj];", "note": "ASan observes the resulting out-of-bounds float read through the public filter graph."} + ] + } + }, + "windows": [ + {"path": "libavfilter/vf_vif.c", "start": 35, "end": 105}, + {"path": "libavfilter/vf_vif.c", "start": 205, "end": 270}, + {"path": "libavfilter/vf_vif.c", "start": 284, "end": 380}, + {"path": "libavfilter/vf_vif.c", "start": 475, "end": 545} + ] + }, + { + "case_id": "ratecontrol-pass2-picture-type-oob-read", + "finding": { + "id": "ffmpeg-ratecontrol-pass2-picture-type-oob-read", + "finding_type": "out_of_bounds_read", + "cwe": "CWE-125", + "file": "libavcodec/ratecontrol.c", + "line_number": 357, + "severity": "high", + "confidence": "high", + "evidence_level": "crash_reproduced", + "description": "The MPEG-family two-pass encoder parses picture type as an unrestricted signed integer from the passlog and stores it in RateControlEntry. Initialization then uses it directly as the index for multiple five-entry statistics arrays. Changing a valid first-pass record from type 1 to type 99 and running the ordinary second-pass CLI path produces a UBSan index-99 report followed by an ASan eight-byte heap-buffer-overflow read in ff_rate_control_init on both sealed snapshots.", + "code_snippet": "sscanf(..., \"type:%d ...\", &rce->pict_type, ...); rcc->i_cplx_sum[rce->pict_type] += ...;", + "poc": "Run evaluations/run_ffmpeg_ratecontrol_stats_reproducer.py against either sealed ASan ffmpeg binary.", + "discovered_by": "sourcehunt:deepseek-v4-flash-0731", + "vulnerability_trace": { + "summary": "a local passlog controls an unchecked picture-type integer that immediately indexes fixed five-entry rate-control arrays", + "steps": [ + {"file": "libavcodec/ratecontrol.c", "line": 552, "function": "ff_rate_control_init", "code_snippet": "for (i = 0; i < 5; i++)", "note": "The affected predictor and accumulated-statistic arrays have five initialized entries."}, + {"file": "libavcodec/ratecontrol.c", "line": 615, "function": "ff_rate_control_init", "code_snippet": "sscanf(p, \" in:%*d out:%*d type:%d ...\", &rce->pict_type, ...)", "note": "The passlog parser validates field count but imposes no AVPictureType range."}, + {"file": "libavcodec/ratecontrol.c", "line": 356, "function": "init_pass2", "code_snippet": "rce->new_pict_type = rce->pict_type;", "note": "The unchecked value is propagated unchanged during second-pass initialization."}, + {"file": "libavcodec/ratecontrol.c", "line": 357, "function": "init_pass2", "code_snippet": "rcc->i_cplx_sum[rce->pict_type] += rce->i_tex_bits * rce->qscale;", "note": "UBSan reports index 99 and ASan reports the subsequent out-of-allocation read."} + ] + } + }, + "windows": [ + {"path": "libavcodec/ratecontrol.c", "start": 330, "end": 375}, + {"path": "libavcodec/ratecontrol.c", "start": 540, "end": 640}, + {"path": "libavcodec/ratecontrol.h", "start": 35, "end": 130} + ] + }, + { + "case_id": "decimate-subsampled-chroma-block-metric-overflow", + "finding": { + "id": "ffmpeg-decimate-subsampled-chroma-block-metric-overflow", + "finding_type": "out_of_bounds_read_write", + "cwe": "CWE-787", + "file": "libavfilter/vf_decimate.c", + "line_number": 123, + "severity": "high", + "confidence": "high", + "evidence_level": "crash_reproduced", + "description": "Decimate allocates its block-difference grid from luma dimensions and half of the configured block size, then further right-shifts those half-block dimensions for chroma. The public minimum block size is four, so YUV411P's horizontal subsampling reduces blockx/2 from two to zero. The chroma metric loop consequently never advances x while xdest increases without bound, indexing beyond the luma-sized bdiffs allocation. A public 16x16 YUV411P graph with blockx=4 produces an ASan eight-byte access exactly after the 64-byte metric allocation on both sealed snapshots.", + "code_snippet": "hblockx = dm->blockx / 2; if (plane) hblockx >>= dm->hsub; for (x = 0; x < width; x += hblockx) bdiffs[ydest * dm->nxblocks + xdest++] += acc;", + "poc": "Run evaluations/run_ffmpeg_decimate_subsampled_block_reproducer.py against either sealed ASan ffmpeg binary.", + "discovered_by": "sourcehunt:deepseek-v4-flash-0731", + "vulnerability_trace": { + "summary": "subsampling reduces a permitted half-block dimension to zero, so the chroma metric loop repeatedly indexes beyond its luma-sized destination grid", + "steps": [ + {"file": "libavfilter/vf_decimate.c", "line": 77, "function": "file scope", "code_snippet": "{ \"blockx\", ... {.i64 = 32}, 4, 1<<9, FLAGS }", "note": "The public option permits a four-pixel horizontal block."}, + {"file": "libavfilter/vf_decimate.c", "line": 373, "function": "file scope", "code_snippet": "AV_PIX_FMT_YUV440P, AV_PIX_FMT_YUV411P, AV_PIX_FMT_YUV410P", "note": "The advertised input formats include chroma subsampling factors large enough to erase a half-block dimension."}, + {"file": "libavfilter/vf_decimate.c", "line": 400, "function": "config_output", "code_snippet": "dm->nxblocks = (w + dm->blockx/2 - 1) / (dm->blockx/2);", "note": "The metric allocation is sized from valid luma half-block dimensions."}, + {"file": "libavfilter/vf_decimate.c", "line": 104, "function": "calc_diffs", "code_snippet": "int hblockx = dm->blockx / 2; ... hblockx >>= dm->hsub;", "note": "For blockx=4 and YUV411P hsub=2, the chroma horizontal step becomes zero."}, + {"file": "libavfilter/vf_decimate.c", "line": 117, "function": "calc_diffs", "code_snippet": "for (x = 0; x < width; x += hblockx)", "note": "x remains zero indefinitely while each iteration advances xdest."}, + {"file": "libavfilter/vf_decimate.c", "line": 123, "function": "calc_diffs", "code_snippet": "bdiffs[ydest * dm->nxblocks + xdest] += acc;", "note": "ASan observes the resulting eight-byte access immediately beyond the 64-byte metric allocation."} + ] + } + }, + "windows": [ + {"path": "libavfilter/vf_decimate.c", "start": 70, "end": 145}, + {"path": "libavfilter/vf_decimate.c", "start": 320, "end": 430} + ] + }, + { + "case_id": "whisper-oversized-audio-frame-heap-overflow", + "finding": { + "id": "ffmpeg-whisper-oversized-audio-frame-heap-overflow", + "finding_type": "out_of_bounds_write", + "cwe": "CWE-787", + "file": "libavfilter/af_whisper.c", + "line_number": 315, + "severity": "high", + "confidence": "high", + "evidence_level": "root_cause_explained", + "description": "Whisper allocates its audio queue from the queue duration, whose public minimum is 20 ms or 320 samples. It consumes complete upstream AVFrames without imposing a sample maximum. When one frame contains more samples than the queue, the capacity guard first transcribes at most the existing fill, then unconditionally copies the entire oversized frame into the fixed queue. FFmpeg's public asetnsamples filter can produce frames up to INT_MAX samples, establishing that no framework contract restricts an audio frame to the Whisper queue capacity. The sealed builds omit the optional Whisper dependency, so this remains source-confirmed rather than dynamically reproduced.", + "code_snippet": "if (fill + samples > queue_size) run_transcription(ctx, frame, fill); memcpy(audio_buffer + fill, input_data, samples * sizeof(*audio_buffer));", + "poc": "Build FFmpeg with whisper.cpp, then pass asetnsamples=n=1024 into whisper=model=:queue=20000; the filter allocates 320 floats and copies a 1024-sample frame.", + "discovered_by": "sourcehunt:deepseek-v4-flash-0731", + "vulnerability_trace": { + "summary": "the filter's duration-sized queue can be smaller than one valid upstream audio frame, but its overflow guard drains only prior contents before copying that whole frame", + "steps": [ + {"file": "libavfilter/af_whisper.c", "line": 446, "function": "file scope", "code_snippet": "{ \"queue\", ... AV_OPT_TYPE_DURATION, ... 20000, HOURS, ... }", "note": "A valid queue may be only 20 ms."}, + {"file": "libavfilter/af_whisper.c", "line": 115, "function": "init", "code_snippet": "audio_buffer_queue_size = av_rescale(queue, 16000, AV_TIME_BASE);", "note": "The minimum queue allocates exactly 320 float samples."}, + {"file": "libavfilter/af_asetnsamples.c", "line": 43, "function": "file scope", "code_snippet": "{ \"nb_out_samples\", ... 1, INT_MAX, FLAGS }", "note": "A standard upstream FFmpeg filter may validly emit a frame much larger than that queue."}, + {"file": "libavfilter/af_whisper.c", "line": 397, "function": "activate", "code_snippet": "ret = ff_inlink_consume_frame(inlink, &frame);", "note": "Whisper consumes the entire frame rather than requesting a queue-bounded sample slice."}, + {"file": "libavfilter/af_whisper.c", "line": 307, "function": "filter_frame", "code_snippet": "if (audio_buffer_fill_size + samples > audio_buffer_queue_size) run_transcription(... audio_buffer_fill_size);", "note": "The guard can remove only existing queued samples; it neither splits nor rejects an individually oversized frame."}, + {"file": "libavfilter/af_whisper.c", "line": 315, "function": "filter_frame", "code_snippet": "memcpy(audio_buffer + audio_buffer_fill_size, input_data, samples * sizeof(*audio_buffer));", "note": "The complete oversized frame is copied into the smaller fixed allocation."} + ] + } + }, + "windows": [ + {"path": "libavfilter/af_whisper.c", "start": 95, "end": 135}, + {"path": "libavfilter/af_whisper.c", "start": 186, "end": 320}, + {"path": "libavfilter/af_whisper.c", "start": 383, "end": 475}, + {"path": "libavfilter/af_asetnsamples.c", "start": 35, "end": 75}, + {"path": "libavfilter/avfilter.c", "start": 1510, "end": 1535} + ] + }, + { + "case_id": "dnn-classify-output-count-heap-overflow", + "finding": { + "id": "ffmpeg-dnn-classify-output-count-heap-overflow", + "finding_type": "out_of_bounds_write", + "cwe": "CWE-787", + "file": "libavfilter/vf_dnn_classify.c", + "line_number": 102, + "severity": "high", + "confidence": "high", + "evidence_level": "crash_reproduced", + "description": "Each detection bounding box has fixed storage for four classification labels and confidences, but dnn_classify_post_proc writes at bbox->classify_count and increments that count without enforcing the four-entry limit. OpenVINO may derive its output count directly from the loaded model without a corresponding cap, and its inference-completion loop invokes the classifier callback once for every output. A production-source harness invokes that same callback five times on one production-allocated bounding-box side-data object; the fifth callback produces an ASan eight-byte write exactly after the 660-byte allocation on both sealed snapshots.", + "code_snippet": "bbox->classify_confidences[bbox->classify_count] = ...; ... bbox->classify_count++;", + "poc": "Build and run evaluations/ffmpeg_dnn_classify_count_reproducer.c with evaluations/run_ffmpeg_dnn_classify_count_reproducer.py against either sealed snapshot.", + "discovered_by": "sourcehunt:deepseek-v4-flash-0731", + "vulnerability_trace": { + "summary": "a model-derived OpenVINO output count drives repeated classification callbacks, while each bounding box contains only four classification slots and the callback never checks its monotonically increasing index", + "steps": [ + {"file": "libavutil/detection_bbox.h", "line": 50, "function": "file scope", "code_snippet": "#define AV_NUM_DETECTION_BBOX_CLASSIFY 4", "note": "Each bounding box has exactly four label and confidence entries."}, + {"file": "libavfilter/dnn/dnn_backend_openvino.c", "line": 683, "function": "init_model_ov", "code_snippet": "nb_outputs = output_size;", "note": "When no names are supplied, the loaded OpenVINO model's output count is accepted without the bounding-box classification cap."}, + {"file": "libavfilter/dnn/dnn_backend_openvino.c", "line": 473, "function": "infer_completion_callback", "code_snippet": "for (int output_i = 0; output_i < ov_model->nb_outputs; output_i++)", "note": "Inference completion invokes the classification postprocessor once per model output."}, + {"file": "libavfilter/vf_dnn_classify.c", "line": 102, "function": "dnn_classify_post_proc", "code_snippet": "bbox->classify_confidences[bbox->classify_count] = av_make_q(...);", "note": "The current count indexes the fixed array without checking it against four; ASan catches the fifth write immediately after the complete side-data allocation."}, + {"file": "libavfilter/vf_dnn_classify.c", "line": 110, "function": "dnn_classify_post_proc", "code_snippet": "bbox->classify_count++;", "note": "Every accepted callback advances the unbounded index."} + ] + } + }, + "windows": [ + {"path": "libavfilter/vf_dnn_classify.c", "start": 35, "end": 115}, + {"path": "libavfilter/dnn/dnn_backend_openvino.c", "start": 334, "end": 490}, + {"path": "libavfilter/dnn/dnn_backend_openvino.c", "start": 575, "end": 705}, + {"path": "libavutil/detection_bbox.h", "start": 25, "end": 90}, + {"path": "libavutil/detection_bbox.c", "start": 20, "end": 75} + ] + }, + { + "case_id": "pan-named-input-channel-oob", + "finding": { + "id": "ffmpeg-pan-named-input-channel-oob", + "finding_type": "out_of_bounds_read_write", + "cwe": "CWE-129", + "file": "libavfilter/af_pan.c", + "line_number": 214, + "severity": "high", + "confidence": "high", + "evidence_level": "crash_reproduced", + "description": "The pan expression parser bounds numbered channel references to its 64-column matrix, but returns named AVChannel identifiers without the same bound. Public names include UNK at 768 and AMBI0 at 1024. During filter initialization, AMBI0 first indexes the 64-entry used_in_ch stack array and then indexes the corresponding gain-matrix column. A public ffmpeg filter graph with FL=AMBI0 produces a UBSan index-1024 violation on the used_in_ch read followed by an ASan invalid stack read on both sealed snapshots; if execution continues, the assignments use the same out-of-range index for writes.", + "code_snippet": "if (used_in_ch[in_ch_id]) ... used_in_ch[in_ch_id] = 1; pan->gain[out_ch_id][in_ch_id] = sign * gain;", + "poc": "Run evaluations/run_ffmpeg_pan_named_channel_reproducer.py against either sealed sanitizer-instrumented ffmpeg binary.", + "discovered_by": "sourcehunt:deepseek-v4-flash-0731", + "vulnerability_trace": { + "summary": "a public named channel is represented by an enum value far above the pan filter's 64-channel storage limit, but only the numbered parser branch enforces that limit before the value is used as an array index", + "steps": [ + {"file": "libavutil/channel_layout.h", "line": 108, "function": "file scope", "code_snippet": "AV_CHAN_AMBISONIC_BASE = 0x400", "note": "AMBI0 is the public name for numeric channel identifier 1024."}, + {"file": "libavfilter/af_pan.c", "line": 74, "function": "parse_channel_name", "code_snippet": "channel_id = av_channel_from_string(buf); ... *rchannel = channel_id;", "note": "The named branch accepts the library channel identifier without checking it against MAX_CHANNELS."}, + {"file": "libavfilter/af_pan.c", "line": 84, "function": "parse_channel_name", "code_snippet": "channel_id >= 0 && channel_id < MAX_CHANNELS", "note": "The corresponding numbered cN branch does enforce the 64-channel storage bound."}, + {"file": "libavfilter/af_pan.c", "line": 214, "function": "init", "code_snippet": "if (used_in_ch[in_ch_id])", "note": "With FL=AMBI0, UBSan observes the first invalid operation as a read of used_in_ch[1024]."}, + {"file": "libavfilter/af_pan.c", "line": 220, "function": "init", "code_snippet": "used_in_ch[in_ch_id] = 1;", "note": "If execution continues after the invalid read, this assignment writes through the same out-of-range stack index."}, + {"file": "libavfilter/af_pan.c", "line": 221, "function": "init", "code_snippet": "pan->gain[out_ch_id][in_ch_id] = sign * gain;", "note": "The persistent gain matrix is also indexed by the unbounded named channel identifier."} + ] + } + }, + "windows": [ + {"path": "libavfilter/af_pan.c", "start": 35, "end": 95}, + {"path": "libavfilter/af_pan.c", "start": 116, "end": 230}, + {"path": "libavutil/channel_layout.h", "start": 65, "end": 120} + ] + }, + { + "case_id": "rtp-latm-payload-length-header-overflow", + "finding": { + "id": "ffmpeg-rtp-latm-payload-length-header-overflow", + "finding_type": "out_of_bounds_write", + "cwe": "CWE-787", + "file": "libavformat/rtpenc_latm.c", + "line_number": 44, + "severity": "high", + "confidence": "high", + "evidence_level": "crash_reproduced", + "description": "RTP/LATM encodes an AAC access unit's size as one length byte per 255 payload bytes, but never ensures that this variable-length header fits the fixed RTP packet buffer before writing it. The public RTP muxer accepts AVPackets independently of the output packet size. With a normal 1,472-byte packet sink, a 382,500-byte AAC packet creates a 1,501-byte header; memset writes 1,500 bytes into the 1,472-byte allocation and ASan reports a heap-buffer-overflow on both sealed snapshots. The following fragmentation arithmetic would also become negative once the header exceeds max_payload_size.", + "code_snippet": "header_size = size/0xFF + 1; memset(s->buf, 0xFF, header_size - 1);", + "poc": "Build and run evaluations/ffmpeg_rtp_latm_header_reproducer.c with evaluations/run_ffmpeg_rtp_latm_header_reproducer.py against either sealed snapshot.", + "discovered_by": "sourcehunt:deepseek-v4-flash-0731", + "vulnerability_trace": { + "summary": "an unbounded AAC packet size expands into a LATM length header larger than the fixed RTP packet buffer, which is written in full before payload fragmentation", + "steps": [ + {"file": "libavformat/rtpenc.c", "line": 159, "function": "rtp_write_header", "code_snippet": "s->buf = av_malloc(s1->packet_size);", "note": "The RTP staging allocation is fixed by the packet sink; the public reproducer uses 1,472 bytes."}, + {"file": "libavformat/rtpenc.c", "line": 599, "function": "rtp_write_packet", "code_snippet": "case AV_CODEC_ID_AAC: if (s->flags & FF_RTP_FLAG_MP4A_LATM) ff_rtp_send_latm(s1, pkt->data, size);", "note": "LATM dispatch passes the complete AVPacket size without capping it to the RTP payload size."}, + {"file": "libavformat/rtpenc_latm.c", "line": 43, "function": "ff_rtp_send_latm", "code_snippet": "header_size = size/0xFF + 1;", "note": "A 382,500-byte access unit produces a 1,501-byte PayloadLengthInfo header."}, + {"file": "libavformat/rtpenc_latm.c", "line": 44, "function": "ff_rtp_send_latm", "code_snippet": "memset(s->buf, 0xFF, header_size - 1);", "note": "ASan observes the 1,500-byte write crossing the complete 1,472-byte RTP allocation on both snapshots."}, + {"file": "libavformat/rtpenc_latm.c", "line": 51, "function": "ff_rtp_send_latm", "code_snippet": "len = FFMIN(size, s->max_payload_size - (!offset ? header_size : 0));", "note": "Once the header exceeds max_payload_size, the first fragment length is negative and later copy arithmetic is invalid as well."} + ] + } + }, + "windows": [ + {"path": "libavformat/rtpenc_latm.c", "start": 20, "end": 62}, + {"path": "libavformat/rtpenc.c", "start": 135, "end": 170}, + {"path": "libavformat/rtpenc.c", "start": 550, "end": 610}, + {"path": "libavformat/rtpenc.h", "start": 25, "end": 95} + ] + }, + { + "case_id": "rtp-h263-rfc2190-small-packet-negative-copy", + "finding": { + "id": "ffmpeg-rtp-h263-rfc2190-small-packet-negative-copy", + "finding_type": "out_of_bounds_read_write", + "cwe": "CWE-195", + "file": "libavformat/rtpenc_h263_rfc2190.c", + "line_number": 96, + "severity": "high", + "confidence": "high", + "evidence_level": "crash_reproduced", + "description": "The RTP muxer accepts every packet sink larger than its 12-byte common header, but RFC2190 H.263 reserves another eight bytes and subtracts them from the remaining payload capacity without checking a codec-specific minimum. A public muxer configured with a valid 13-byte packet sink therefore has one payload byte and derives a fragment length of -7. The resynchronization search preserves that negative length, and mode-B packetization passes it to memcpy, where ASan reports negative-size-param on both sealed snapshots.", + "code_snippet": "len = FFMIN(s->max_payload_size - 8, size); ... memcpy(s->buf + 8, buf, len);", + "poc": "Build and run evaluations/ffmpeg_rtp_h263_small_packet_reproducer.c with evaluations/run_ffmpeg_rtp_h263_small_packet_reproducer.py against either sealed snapshot.", + "discovered_by": "sourcehunt:deepseek-v4-flash-0731", + "vulnerability_trace": { + "summary": "a packet size valid under RTP's common-header check is smaller than the RFC2190 payload header, so subtracting the codec header creates a negative fragment length that reaches memcpy", + "steps": [ + {"file": "libavformat/rtpenc.c", "line": 155, "function": "rtp_write_header", "code_snippet": "if (s1->packet_size <= 12) return AVERROR(EIO);", "note": "A 13-byte packet sink is explicitly accepted by the common RTP initialization check."}, + {"file": "libavformat/rtpenc.c", "line": 163, "function": "rtp_write_header", "code_snippet": "s->max_payload_size = s1->packet_size - 12;", "note": "The accepted sink leaves max_payload_size equal to one."}, + {"file": "libavformat/rtpenc_h263_rfc2190.c", "line": 129, "function": "ff_rtp_send_h263_rfc2190", "code_snippet": "len = FFMIN(s->max_payload_size - 8, size);", "note": "Subtracting the eight-byte mode-B header produces a fragment length of -7."}, + {"file": "libavformat/rtpenc_h263.c", "line": 28, "function": "ff_h263_find_resync_marker_reverse", "code_snippet": "const uint8_t *p = end - 1; ... return end;", "note": "With a negative end pointer, the resynchronization loop does not recover a nonnegative fragment length."}, + {"file": "libavformat/rtpenc_h263_rfc2190.c", "line": 96, "function": "send_mode_b", "code_snippet": "memcpy(s->buf + 8, buf, len);", "note": "ASan reports negative-size-param with size -7 through the public muxer on both snapshots."} + ] + } + }, + "windows": [ + {"path": "libavformat/rtpenc_h263_rfc2190.c", "start": 40, "end": 195}, + {"path": "libavformat/rtpenc_h263.c", "start": 20, "end": 42}, + {"path": "libavformat/rtpenc.c", "start": 135, "end": 170}, + {"path": "libavformat/rtpenc.c", "start": 615, "end": 640} + ] + }, + { + "case_id": "ffv1-remap-table-symbol-oob-read", + "finding": { + "id": "ffmpeg-ffv1-remap-table-symbol-oob-read", + "finding_type": "out_of_bounds_read", + "cwe": "CWE-125", + "file": "libavcodec/ffv1dec.c", + "line_number": 150, + "severity": "high", + "confidence": "high", + "evidence_level": "crash_reproduced", + "description": "FFV1 level-4 remapping allocates each decoded lookup table for exactly slice_width times slice_height entries and records the number of populated entries, but entropy symbols are decoded in a ceil(log2(remap_count))-bit space. For a non-power-of-two remap count, the mask therefore admits unused indices; when pixel_num is below that power-of-two ceiling, those indices are also outside the physical table. A valid 513x1 yuv444p16le remapped FFV1 packet has this shape. Flipping one entropy bit at packet byte 18 makes decode_line produce an out-of-range masked symbol, and decode_plane performs a two-byte heap-buffer-overflow read on both sealed snapshots.", + "code_snippet": "((uint16_t*)(src + stride*y))[x*pixel_stride] = sc->fltmap[remap_index][sample[1][x] & mask];", + "poc": "Run evaluations/run_ffmpeg_ffv1_remap_reproducer.py against either sealed sanitizer-instrumented FFmpeg checkout. The recorder creates a valid remapped sample, confirms the unmodified sample decodes, and applies only the deterministic byte-18 bit-0 mutation.", + "discovered_by": "sourcehunt:deepseek-v4-flash-0731", + "vulnerability_trace": { + "summary": "a bitstream-defined non-power-of-two remap count is represented by a wider power-of-two symbol alphabet, and an unused entropy symbol indexes beyond the pixel-count-sized lookup table", + "steps": [ + {"file": "libavcodec/ffv1dec.c", "line": 402, "function": "decode_slice", "code_snippet": "const int pixel_num = sc->slice_width * sc->slice_height;", "note": "The decoder derives the physical remap-table capacity from the number of pixels in the slice."}, + {"file": "libavcodec/ffv1dec.c", "line": 411, "function": "decode_slice", "code_snippet": "av_fast_malloc(&sc->fltmap[p], &sc->fltmap_size[p], pixel_num * sizeof(*sc->fltmap[p]));", "note": "Each 16-bit lookup table is allocated for exactly pixel_num entries."}, + {"file": "libavcodec/ffv1dec.c", "line": 355, "function": "decode_remap", "code_snippet": "sc->remap_count[p] = j;", "note": "The attacker-controlled remap syntax records a populated count that need not be a power of two."}, + {"file": "libavcodec/ffv1dec.c", "line": 111, "function": "decode_plane", "code_snippet": "bits = av_ceil_log2(sc->remap_count[remap_index]); mask = (1<fltmap[remap_index][sample[1][x] & mask]", "note": "The unused symbol is used directly as a lookup index; ASan reports a two-byte heap-buffer-overflow read in decode_plane on both snapshots."} + ] + } + }, + "windows": [ + {"path": "libavcodec/ffv1.c", "start": 218, "end": 257}, + {"path": "libavcodec/ffv1dec.c", "start": 96, "end": 162}, + {"path": "libavcodec/ffv1dec.c", "start": 238, "end": 273}, + {"path": "libavcodec/ffv1dec.c", "start": 298, "end": 424}, + {"path": "libavcodec/ffv1dec_template.c", "start": 34, "end": 130} + ] + }, + { + "case_id": "framepack-alpha-plane-uninitialized-destination", + "finding": { + "id": "ffmpeg-framepack-alpha-plane-uninitialized-destination", + "finding_type": "invalid_pointer_write", + "cwe": "CWE-457", + "file": "libavfilter/vf_framepack.c", + "line_number": 248, + "severity": "high", + "confidence": "high", + "evidence_level": "crash_reproduced", + "description": "The framepack filter explicitly negotiates alpha-bearing planar formats, but its side-by-side helper initializes only destination planes zero through two before passing the four-entry destination array to av_image_copy2. For YUVA input, the generic copy helper visits plane three and writes the alpha plane through the uninitialized dst[3] pointer. The top/bottom helper independently leaves both dst[3] and linesizes[3] uninitialized. A public two-source yuva420p framepack graph produces an AddressSanitizer invalid write through image_copy_plane on both sealed snapshots.", + "code_snippet": "uint8_t *dst[4]; ... dst[0] = ...; dst[1] = ...; dst[2] = ...; av_image_copy2(dst, ...);", + "poc": "Run evaluations/run_ffmpeg_framepack_alpha_reproducer.py against either sealed sanitizer-instrumented ffmpeg binary.", + "discovered_by": "sourcehunt:deepseek-v4-flash-0731", + "vulnerability_trace": { + "summary": "an advertised four-plane pixel format reaches a packing helper that initializes only three destination-plane entries, after which the generic image copier writes the alpha plane through the unset fourth pointer", + "steps": [ + {"file": "libavfilter/vf_framepack.c", "line": 67, "function": "file scope", "code_snippet": "AV_PIX_FMT_YUVA420P, AV_PIX_FMT_YUVA422P, AV_PIX_FMT_YUVA444P", "note": "The filter explicitly advertises alpha-bearing four-plane formats."}, + {"file": "libavfilter/vf_framepack.c", "line": 241, "function": "horizontal_frame_pack", "code_snippet": "uint8_t *dst[4];", "note": "The destination-plane array has automatic storage and no initializer."}, + {"file": "libavfilter/vf_framepack.c", "line": 244, "function": "horizontal_frame_pack", "code_snippet": "dst[0] = ...; dst[1] = ...; dst[2] = ...;", "note": "Only the luma and two chroma entries are assigned; dst[3] remains indeterminate."}, + {"file": "libavutil/imgutils.c", "line": 402, "function": "image_copy", "code_snippet": "planes_nb = FFMAX(planes_nb, desc->comp[i].plane + 1);", "note": "For YUVA420P, the generic copier derives four planes from the pixel descriptor."}, + {"file": "libavutil/imgutils.c", "line": 415, "function": "image_copy", "code_snippet": "copy_plane(dst_data[i], dst_linesizes[i], src_data[i], src_linesizes[i], bwidth, h);", "note": "The plane-three copy writes through dst[3]; ASan observes the resulting invalid write on both snapshots."}, + {"file": "libavfilter/vf_framepack.c", "line": 267, "function": "vertical_frame_pack", "code_snippet": "uint8_t *dst[4]; int linesizes[4];", "note": "The vertical helper has the same missing destination pointer and also omits the fourth stride."} + ] + } + }, + "windows": [ + {"path": "libavfilter/vf_framepack.c", "start": 35, "end": 80}, + {"path": "libavfilter/vf_framepack.c", "start": 230, "end": 295}, + {"path": "libavutil/imgutils.c", "start": 375, "end": 425} + ] + }, + { + "case_id": "yuvcmp-partial-macroblock-overflow", + "finding": { + "id": "ffmpeg-yuvcmp-partial-macroblock-overflow", + "finding_type": "out_of_bounds_read_write", + "cwe": "CWE-122", + "file": "tools/yuvcmp.c", + "line_number": 106, + "severity": "high", + "confidence": "high", + "evidence_level": "crash_reproduced", + "description": "yuvcmp allocates its macroblock-error bitmap for floor(width / 16) times floor(height / 16) entries, but maps every active luma and chroma sample to a macroblock. For a partial right or bottom macroblock, x / 16 or y / 16 reaches the excluded rounded-up coordinate and indexes outside that allocation. Two valid 17x16 YUV420P inputs that differ only at luma pixel (16,0) select index one in a one-byte allocation. ASan reports a one-byte heap-buffer-overflow read-modify-write on both sealed snapshots.", + "code_snippet": "mb = x / 16 + (y / 16) * mb_x; ... mberrors[mb] |= 1;", + "poc": "Run evaluations/run_ffmpeg_yuvcmp_partial_mb_reproducer.py against either sealed sanitizer-instrumented FFmpeg checkout.", + "discovered_by": "sourcehunt:deepseek-v4-flash-0731", + "vulnerability_trace": { + "summary": "floor-rounded macroblock dimensions size the error bitmap, while full active-plane coordinates produce rounded-up partial-block indices that directly control a read-modify-write", + "steps": [ + {"file": "tools/yuvcmp.c", "line": 71, "function": "main", "code_snippet": "mb_x = width / 16; mb_y = height / 16;", "note": "User-supplied dimensions are rounded down when computing the macroblock grid."}, + {"file": "tools/yuvcmp.c", "line": 74, "function": "main", "code_snippet": "mberrors = malloc(mb_x * mb_y);", "note": "The bitmap has no entries for partial macroblocks."}, + {"file": "tools/yuvcmp.c", "line": 90, "function": "main", "code_snippet": "for(c = 0; c < lsiz; c++)", "note": "The comparison still visits every active luma sample, including partial right and bottom blocks."}, + {"file": "tools/yuvcmp.c", "line": 95, "function": "main", "code_snippet": "mb = x / 16 + (y / 16) * mb_x;", "note": "For width 17, sample x=16 maps to macroblock column one although mb_x is one."}, + {"file": "tools/yuvcmp.c", "line": 106, "function": "main", "code_snippet": "mberrors[mb] |= 1;", "note": "The read-modify-write accesses index one of a one-byte allocation; ASan confirms the overflow on both snapshots."} + ] + } + }, + "windows": [ + {"path": "tools/yuvcmp.c", "start": 45, "end": 145} + ] + }, + { + "case_id": "dvdsub-odd-width-rle-overflow", + "finding": { + "id": "ffmpeg-dvdsub-odd-width-rle-overflow", + "finding_type": "out_of_bounds_write", + "cwe": "CWE-787", + "file": "libavcodec/dvdsubenc.c", + "line_number": 42, + "severity": "high", + "confidence": "high", + "evidence_level": "crash_reproduced", + "description": "The DVD subtitle encoder budgets floor(width times height / 2) output bytes for a one-nibble-per-pixel RLE worst case. It encodes even and odd fields separately, however, and pads every odd-width row to a complete byte. The required minimum for that case is ceil(width / 2) times height. A public 1x200 bitmap subtitle and a 142-byte caller buffer pass the 100-byte RLE budget check but require 200 RLE bytes; ASan reports a stack-buffer-overflow in dvd_encode_rle on both sealed snapshots.", + "code_snippet": "if ((q - outbuf) + vrect.w * vrect.h / 2 + 17 + 21 > outbuf_size) ... dvd_encode_rle(&q, ...);", + "poc": "Run evaluations/run_ffmpeg_dvdsub_odd_width_reproducer.py against either sealed sanitizer-instrumented FFmpeg checkout.", + "discovered_by": "sourcehunt:deepseek-v4-flash-0731", + "vulnerability_trace": { + "summary": "a floor-rounded whole-image RLE budget omits per-row nibble padding, after which the field-separated encoder writes the missing byte for every odd-width row", + "steps": [ + {"file": "libavcodec/dvdsubenc.c", "line": 344, "function": "dvdsub_encode", "code_snippet": "vrect.w * vrect.h / 2", "note": "The output-capacity check rounds the aggregate pixel count down to bytes."}, + {"file": "libavcodec/dvdsubenc.c", "line": 349, "function": "dvdsub_encode", "code_snippet": "dvd_encode_rle(&q, vrect.data[0], vrect.w * 2, vrect.w, (vrect.h + 1) >> 1, cmap);", "note": "Even rows are encoded as an independent field."}, + {"file": "libavcodec/dvdsubenc.c", "line": 352, "function": "dvdsub_encode", "code_snippet": "dvd_encode_rle(&q, vrect.data[0] + vrect.w, vrect.w * 2, vrect.w, vrect.h >> 1, cmap);", "note": "Odd rows form the second field, preserving one row-padding boundary per source row."}, + {"file": "libavcodec/dvdsubenc.c", "line": 92, "function": "dvd_encode_rle", "code_snippet": "if (ncnt & 1) PUTNIBBLE(0);", "note": "Every odd-width row is padded to a full byte, so width one consumes one byte rather than one half-byte."}, + {"file": "libavcodec/dvdsubenc.c", "line": 42, "function": "dvd_encode_rle", "code_snippet": "*q++ = bitbuf | ((val) & 0x0f);", "note": "The omitted padding bytes cross the caller's buffer; ASan confirms the write on both snapshots."} + ] + } + }, + "windows": [ + {"path": "libavcodec/dvdsubenc.c", "start": 35, "end": 100}, + {"path": "libavcodec/dvdsubenc.c", "start": 270, "end": 405} + ] + }, + { + "case_id": "rav1e-twopass-remaining-length-ffi-ub", + "finding": { + "id": "ffmpeg-rav1e-twopass-remaining-length-ffi-ub", + "finding_type": "ffi_undefined_behavior", + "cwe": "CWE-125", + "file": "libavcodec/librav1e.c", + "line_number": 164, + "severity": "medium", + "confidence": "high", + "evidence_level": "root_cause_explained", + "description": "FFmpeg's rav1e pass-two feeder advances pass_data by pass_pos after each partial consumption but continues declaring the original total pass_size to rav1e_twopass_in. The decoded allocation contains only pass_size bytes, so every call after pass_pos becomes nonzero describes a byte range extending pass_pos bytes beyond the allocation. A real 156-byte legacy two-pass stream generated with rav1e 0.7.1 was accepted by the exact unmodified FFmpeg pass-two wrapper, and rav1e's C API immediately constructs a Rust slice from the supplied pointer and length, violating from_raw_parts' allocation-bound precondition. Current rav1e consumed only the next required bytes, so this is retained as real FFI undefined behavior, not as a sanitizer- or Valgrind-observed physical out-of-bounds read.", + "code_snippet": "rav1e_twopass_in(ctx->ctx, ctx->pass_data + ctx->pass_pos, ctx->pass_size);", + "poc": "Generate a complete legacy two-pass stats stream with rav1e 0.7.1 and feed it through unmodified FFmpeg pass two; after a partial consumption, inspect the advanced pointer and unchanged total length passed to rav1e_twopass_in.", + "discovered_by": "sourcehunt:deepseek-v4-flash-0731", + "vulnerability_trace": { + "summary": "a correctly allocated decoded stats buffer is partially consumed, but an advanced pointer is paired with the original total length and crosses the Rust FFI boundary as an invalid slice range", + "steps": [ + {"file": "libavcodec/librav1e.c", "line": 351, "function": "librav1e_encode_init", "code_snippet": "ctx->pass_size = av_base64_decode(ctx->pass_data, avctx->stats_in, ctx->pass_size);", "note": "Pass-two stats are decoded into an allocation sized for the complete decoded stream."}, + {"file": "libavcodec/librav1e.c", "line": 163, "function": "set_stats", "code_snippet": "while (ret > 0 && ctx->pass_size - ctx->pass_pos > 0)", "note": "A positive return advances pass_pos and permits another partial-feed call."}, + {"file": "libavcodec/librav1e.c", "line": 164, "function": "set_stats", "code_snippet": "rav1e_twopass_in(ctx->ctx, ctx->pass_data + ctx->pass_pos, ctx->pass_size)", "note": "The pointer advances, but the length remains the original total instead of pass_size minus pass_pos."}, + {"file": "libavcodec/librav1e.c", "line": 167, "function": "set_stats", "code_snippet": "ctx->pass_pos += ret;", "note": "Real rav1e stats produce positive partial consumption, making the invalid pointer-length pair reachable on a subsequent call."} + ] + } + }, + "windows": [ + {"path": "libavcodec/librav1e.c", "start": 119, "end": 171}, + {"path": "libavcodec/librav1e.c", "start": 330, "end": 365}, + {"path": "libavcodec/librav1e.c", "start": 525, "end": 565} + ] + }, + { + "case_id": "d3d12va-h264-hevc-upload-capacity-overflow", + "finding": { + "id": "ffmpeg-d3d12va-h264-hevc-upload-capacity-overflow", + "finding_type": "out_of_bounds_write", + "cwe": "CWE-787", + "file": "libavcodec/d3d12va_h264.c", + "line_number": 133, + "severity": "high", + "confidence": "high", + "evidence_level": "root_cause_explained", + "description": "The common D3D12VA decoder allocates its fixed upload resource from the decoded frame's raw-image buffer size, but the H.264 and HEVC backends copy every accepted VCL NAL into that mapped resource and prepend three bytes per slice without checking the resulting byte count. Compressed-picture size is not bounded by decoded-image size. The durable proof generates a standards-valid, normally decodable 16x5400 yuv420p H.264 Baseline stream whose slice bytes plus inserted prefixes exceed both the resource's 129,600-byte declared width and its 131,072-byte 64 KiB-aligned allocation floor. A valid 16x16 HEVC Main stream independently exceeds its 384-byte declared resource width. The exact byte counts are deterministically recorded by the proof script and can vary slightly with the installed encoder build. The duplicated write paths and resource-size policy are byte-identical in both sealed snapshots; no Windows D3D12 runtime was available, so the overflow is retained as source- and producer-confirmed rather than sanitizer-observed.", + "code_snippet": "*(uint32_t *)mapped_ptr = START_CODE; mapped_ptr += START_CODE_SIZE; memcpy(mapped_ptr, &ctx_pic->bitstream[position], size);", + "poc": "Run evaluations/run_ffmpeg_d3d12va_upload_capacity_proof.py for either sealed checkout. It creates valid H.264 and HEVC streams, confirms software decoding, and records the exact VCL bytes that the D3D12 callbacks copy versus the common upload-resource capacity.", + "discovered_by": "sourcehunt:deepseek-v4-flash-0731", + "vulnerability_trace": { + "summary": "valid H.264 or HEVC slice NALs can exceed a raw-image-sized upload allocation, after which the D3D12 backend adds per-slice start codes and performs unchecked mapped writes", + "steps": [ + {"file": "libavcodec/d3d12va_decode.c", "line": 141, "function": "ff_d3d12va_get_suitable_max_bitstream_size", "code_snippet": "return av_image_get_buffer_size(frames_ctx->sw_format, avctx->coded_width, avctx->coded_height, 1);", "note": "Upload capacity is derived from decoded pixel storage, not from the actual compressed access-unit size."}, + {"file": "libavcodec/d3d12va_decode.c", "line": 192, "function": "d3d12va_get_valid_helper_objects", "code_snippet": ".Width = ctx->bitstream_size", "note": "Every helper upload resource has exactly that fixed capacity."}, + {"file": "libavcodec/h264dec.c", "line": 673, "function": "decode_nal_units", "code_snippet": "FF_HW_CALL(avctx, decode_slice, nal->raw_data, nal->raw_size)", "note": "Accepted packet-controlled H.264 VCL NAL bytes reach the hardware callback unchanged; the valid proof stream exceeds the 129,600-byte resource width and the 131,072-byte allocation-alignment floor."}, + {"file": "libavcodec/hevc/hevcdec.c", "line": 3049, "function": "decode_slice_data", "code_snippet": "FF_HW_CALL(s->avctx, decode_slice, nal->raw_data, nal->raw_size)", "note": "The HEVC hardware path has the same producer chain; the valid 16x16 proof stream exceeds the 384-byte declared resource width."}, + {"file": "libavcodec/d3d12va_h264.c", "line": 133, "function": "update_input_arguments", "code_snippet": "*(uint32_t *)mapped_ptr = START_CODE; mapped_ptr += 3; memcpy(mapped_ptr, &ctx_pic->bitstream[position], size);", "note": "H.264 writes three new prefix bytes per slice plus every slice byte without comparing mapped_ptr against resource capacity; the durable record preserves the exact excess for the local encoder build."}, + {"file": "libavcodec/d3d12va_hevc.c", "line": 129, "function": "update_input_arguments", "code_snippet": "*(uint32_t *)mapped_ptr = START_CODE; mapped_ptr += 3; memcpy(mapped_ptr, &ctx_pic->bitstream[position], size);", "note": "The independently duplicated HEVC loop performs the same unchecked mapped write."} + ] + } + }, + "windows": [ + {"path": "libavcodec/d3d12va_decode.c", "start": 132, "end": 205}, + {"path": "libavcodec/d3d12va_decode.c", "start": 405, "end": 432}, + {"path": "libavcodec/d3d12va_h264.c", "start": 75, "end": 155}, + {"path": "libavcodec/d3d12va_hevc.c", "start": 74, "end": 150}, + {"path": "libavcodec/h264dec.c", "start": 645, "end": 680}, + {"path": "libavcodec/hevc/hevcdec.c", "start": 3032, "end": 3053} + ] + }, + { + "case_id": "mestimate-int-max-mb-size-signed-shift-ub", + "finding": { + "id": "ffmpeg-mestimate-int-max-mb-size-signed-shift-ub", + "finding_type": "denial_of_service", + "cwe": "CWE-190", + "file": "libavfilter/vf_mestimate.c", + "line_number": 86, + "severity": "low", + "confidence": "high", + "evidence_level": "crash_reproduced", + "description": "The public mestimate mb_size option accepts values through INT_MAX. For INT_MAX, av_ceil_log2_c returns 31 and config_input evaluates signed 1 << 31 before its later zero-block check rejects the impossible geometry. That shift is undefined behavior in C. Running the production filter with mb_size=2147483647 makes UBSan report the signed-shift violation at vf_mestimate.c:86 and abort on both sealed snapshots. This requires an explicitly supplied extreme filter option and is therefore classified as a low-severity configuration-triggered denial of service rather than a media-only vulnerability.", + "code_snippet": "s->log2_mb_size = av_ceil_log2_c(s->mb_size); s->mb_size = 1 << s->log2_mb_size;", + "poc": "Run evaluations/run_ffmpeg_mestimate_mb_size_reproducer.py against either sealed UBSan-instrumented checkout.", + "discovered_by": "sourcehunt:deepseek-v4-flash-0731", + "vulnerability_trace": { + "summary": "a public integer option admits INT_MAX, whose rounded power-of-two conversion evaluates an undefined signed shift before validation can return an error", + "steps": [ + {"file": "libavfilter/vf_mestimate.c", "line": 61, "function": "mestimate_options", "code_snippet": "{ \"mb_size\", ... 8, INT_MAX, FLAGS }", "note": "The public filtering option explicitly admits INT_MAX."}, + {"file": "libavutil/common.h", "line": 436, "function": "av_ceil_log2_c", "code_snippet": "return av_log2((x - 1U) << 1);", "note": "For INT_MAX, the helper returns 31."}, + {"file": "libavfilter/vf_mestimate.c", "line": 86, "function": "config_input", "code_snippet": "s->mb_size = 1 << s->log2_mb_size;", "note": "Signed one shifted into bit 31 is undefined; UBSan reports and aborts here on both snapshots."}, + {"file": "libavfilter/vf_mestimate.c", "line": 92, "function": "config_input", "code_snippet": "if (s->b_count == 0) return AVERROR(EINVAL);", "note": "The intended invalid-geometry rejection occurs only after the undefined operation."} + ] + } + }, + "windows": [ + {"path": "libavfilter/vf_mestimate.c", "start": 45, "end": 105}, + {"path": "libavutil/common.h", "start": 425, "end": 445} + ] + } +] diff --git a/handoff.md b/handoff.md new file mode 100644 index 00000000..e618d14d --- /dev/null +++ b/handoff.md @@ -0,0 +1,157 @@ +# PR #140 SourceHunt resume handoff + +Date: 2026-08-13 +Branch: `feat/sourcehunt-resume` +PR: https://github.com/Lazarus-AI/clearwing/pull/140 +Base: `origin/main` at `d0376498b896a34e32fd19b9ae4364208ce70d32` + +## Pause point + +The end-to-end resume rewrite is implemented, organized into the four requested +logical commits, and ready for review. The branch was force-pushed to the +existing draft PR after this handoff was written. No exact coroutine, provider +request, sandbox, or mid-agent transcript restoration is claimed. + +The recovery invariant is: + +> If a valid atomic work result exists, reuse it; otherwise run the work again. + +## State authorities + +- `session.json` owns schema version, effective behavior-affecting config, + repository metadata, and exact selected-source identity. It contains no + provider credentials. +- `rank-plan.json` is written atomically only after the entire ranking pass + completes successfully. Missing, malformed, interrupted, or degraded ranking + reruns from pristine preprocessed inputs. +- `work-results/.json` is the immutable completion proof + for one hunt item, including successful zero-finding work and bounded context + needed to reconstruct promotion work. +- `spend-ledger.jsonl` remains the sole authority for lifetime LLM spend. +- One advisory session lock prevents concurrent standalone session writers. +- Campaign `findings_pool.jsonl` remains campaign-owned and is not used as a + second standalone recovery authority. + +The prototype mutable checkpoint/journal implementation was replaced rather +than layered over. There are no persisted pending, in-progress, retryable, +cancelled, or failed work transitions. + +## Implemented behavior + +- Completed and completed-zero-finding work is restored and skipped. +- Missing, interrupted, malformed, truncated, or ID-mismatched work reruns. +- Unstarted work runs normally. +- Completed findings and cluster descriptors are restored into the live + `FindingsPool` before unfinished hunters dispatch. +- Promotions are reconstructed recursively from completed parent results using + persisted bounded transcript context; completed promoted work is not + duplicated. +- A complete rank plan is restored exactly. Partial or degraded ranking is + usable only for the current invocation and is never committed as complete. +- Preprocessing is rerun. Source identity hashes the exact selected relative + paths and complete file bytes. The session output directory is excluded from + all selected-input analyzers, including imports-by, callgraph, Semgrep, and + taint. Git commit is metadata only. +- Verification, exploitation, reporting, and later enrichment may rerun. +- Settled spend restores once. Orphaned reservations settle conservatively once + according to whether a cap was active when reserved. The saved lifetime cap + is fixed on resume. +- Provider quota exhaustion is distinct from authentication failures, rate + limits, and budget exhaustion. A run-shared stop state blocks new dispatch, + sibling tasks unwind safely, committed work remains valid, and the runner + returns `status="provider_exhausted"` with exit code 3. +- Fresh provider credentials, endpoint, or model may be supplied on resume. +- Lock ownership lives in `arun()`; `run()` does not duplicate it. +- Parent-owned campaign/evaluation runs do not create standalone resume state. +- CLI surface is intentionally narrow: `clearwing sourcehunt --resume SESSION_ID` + plus runtime provider/model, output-root, live, and logging options. + +## Commit structure + +Run `git log --oneline --reverse origin/main..HEAD` for the current hashes. The +four commits are: + +1. `Stop sourcehunt on provider exhaustion` +2. `Add immutable sourcehunt resume store` +3. `Integrate sourcehunt completion recovery` +4. `Expose sourcehunt resume in the CLI` + +## Verification completed + +Before the final bounded-cluster simplification: + +- Full SourceHunt suite with writable Clearwing home and localhost socket + permission: `895 passed in 83.62s` +- Affected resume/runner/pool/ranker/spend/preprocessor suite: `226 passed` +- Exact-source analyzer suite: `94 passed` +- Campaign suite: `22 passed` +- Live webhook suite: `22 passed` +- Ruff on every changed Python file: passed +- `git diff --check origin/main`: passed +- Persistence benchmark: 100 atomic result writes and reloads passed in the + store suite; isolated pytest wall time was `0.37s` + +After the final bounded-cluster simplification: + +- Resume store, resume pool, and FindingsPool suites: `52 passed` +- Ruff for the touched files: passed +- `git diff --check`: passed + +Recommended first command on the next machine: + +```bash +git fetch fork feat/sourcehunt-resume +git switch feat/sourcehunt-resume +git reset --hard fork/feat/sourcehunt-resume + +pytest -q \ + tests/test_sourcehunt_resume_store.py \ + tests/test_sourcehunt_resume_pool.py \ + tests/test_sourcehunt_resume_runner.py \ + tests/test_sourcehunt_provider_exhaustion.py \ + tests/test_sourcehunt_resume_spend.py \ + tests/test_sourcehunt_resume_cli.py \ + tests/test_sourcehunt_pool_budget.py \ + tests/test_sourcehunt_runner.py \ + tests/test_sourcehunt_ranker.py \ + tests/test_llm_spend_budget.py \ + tests/test_sourcehunt_preprocessor.py \ + tests/test_findings_pool.py \ + tests/test_sourcehunt_subsystem.py +``` + +For the full suite, use a writable home. Live webhook tests also need permission +to bind localhost sockets: + +```bash +CLEARWING_HOME=/tmp/clearwing-test-home pytest -q 'tests/test_sourcehunt*.py' +``` + +## Remaining review concern + +The architecture is materially narrower than the prototype, and the mutable +611-line checkpoint/journal plus 1,101-line grab-bag test were removed. However, +the final diff against `origin/main` is approximately `+3233/-144`, compared +with the prototype's `+3207/-134`. The focused replacement tests and strict +immutable-result validation account for much of the size, but the requested +"materially smaller" raw line-count target is not met. Do not claim otherwise. + +`clearwing/sourcehunt/resume.py` is about 658 lines and remains the largest new +production component. Its strict validation is intentional, but it is the +clearest place for a future review-driven simplification if line count must be +reduced without weakening recovery guarantees. + +## Important implementation notes + +- `ProviderExhaustedError` deliberately subclasses `BaseException`. This keeps + ordinary broad `except Exception` fallbacks from silently scheduling more paid + work. Explicit orchestration boundaries catch it; cancellation/telemetry + boundaries catch `BaseException` only to clean up and re-raise. +- `SourceHuntResumeStore.load_completed_work()` scans result files once per + store instance and caches them. Each successful work item writes one small + atomic file; no growing journal is rescanned on the hot path. +- Cluster membership is reconstructed from canonical completed findings. Work + files persist bounded cluster descriptors only, avoiding O(n²) growth. +- The branch history was rewritten from the actual PR base, so updating the + existing PR required `git push --force-with-lease fork + feat/sourcehunt-resume:feat/sourcehunt-resume`. diff --git a/pyproject.toml b/pyproject.toml index 721baa9c..75666a56 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -95,8 +95,11 @@ docs = [ inspect-ai = [ "inspect-ai>=0.3.0", ] +optimization = [ + "gepa>=0.1.4,<0.2", +] all = [ - "clearwing[metasploit,browser,vector,dev,docs,inspect-ai]", + "clearwing[metasploit,browser,vector,dev,docs,inspect-ai,optimization]", ] [project.urls] diff --git a/tests/test_deep_agent_loop.py b/tests/test_deep_agent_loop.py index 61284735..2473b96b 100644 --- a/tests/test_deep_agent_loop.py +++ b/tests/test_deep_agent_loop.py @@ -88,6 +88,121 @@ async def test_constrained_mode_stops_at_max_steps(): assert result.stop_reason == "max_steps" +@pytest.mark.asyncio +async def test_closure_countdown_is_only_added_near_step_cap(): + hunter, llm = _make_hunter(agent_mode="constrained", max_steps=4) + hunter.closing_steps = 3 + llm.achat.return_value = FakeResponse( + tool_calls_list=[_make_tool_call("think", {"notes": "thinking"})], + ) + + with patch("clearwing.sourcehunt.hunter.HunterTrajectoryLogger") as mock_traj: + mock_traj.for_hunter.return_value = MagicMock() + result = await hunter.arun() + + systems = [call.kwargs["system"] for call in llm.achat.call_args_list] + assert systems[0] == "test prompt" + assert "3 model call(s) remain" in systems[1] + assert "2 model call(s) remain" in systems[2] + assert "1 model call(s) remain" in systems[3] + assert all("Do not start broad exploration" in system for system in systems[1:]) + assert result.stop_reason == "max_steps" + + +@pytest.mark.asyncio +async def test_initial_source_action_gets_one_bounded_retry(): + llm = AsyncMock() + source_calls = 0 + + def read_source_file(**_kwargs): + nonlocal source_calls + source_calls += 1 + return "source" + + tool = NativeToolSpec( + name="read_source_file", + description="read", + schema={"type": "object", "properties": {}}, + handler=read_source_file, + ) + responses = iter( + [ + FakeResponse(text="I'll read it now."), + FakeResponse(tool_calls_list=[_make_tool_call("read_source_file")]), + FakeResponse(text="done"), + ] + ) + message_snapshots = [] + + async def respond(**kwargs): + message_snapshots.append([(message.role, message.content) for message in kwargs["messages"]]) + return next(responses) + + llm.achat.side_effect = respond + hunter = NativeHunter( + llm=llm, + prompt="test", + tools=[tool], + ctx=HunterContext(repo_path="/tmp/repo"), + max_steps=4, + initial_source_action_retries=1, + ) + + with patch("clearwing.sourcehunt.hunter.HunterTrajectoryLogger") as mock_traj: + mock_traj.for_hunter.return_value = MagicMock() + result = await hunter.arun() + + assert message_snapshots[1][-1][0] == "user" + assert "Do not write or simulate" in message_snapshots[1][-1][1] + assert source_calls == 1 + assert result.stop_reason == "completed" + + +@pytest.mark.asyncio +async def test_initial_source_action_retry_exhaustion_is_not_completed_coverage(): + hunter, llm = _make_hunter(agent_mode="constrained", max_steps=4) + hunter.initial_source_action_retries = 1 + llm.achat.return_value = FakeResponse(text="I will call the tool now.") + + with patch("clearwing.sourcehunt.hunter.HunterTrajectoryLogger") as mock_traj: + mock_traj.for_hunter.return_value = MagicMock() + result = await hunter.arun() + + assert llm.achat.call_count == 2 + assert result.stop_reason == "no_source_action" + + +@pytest.mark.asyncio +async def test_failed_source_tool_does_not_satisfy_initial_source_action(): + llm = AsyncMock() + tool = NativeToolSpec( + name="read_source_file", + description="read", + schema={"type": "object", "properties": {}}, + handler=lambda **_kwargs: {"error": "file not found"}, + ) + llm.achat.side_effect = [ + FakeResponse(tool_calls_list=[_make_tool_call("read_source_file")]), + FakeResponse(text="I will retry with the tool."), + FakeResponse(text="Still trying."), + ] + hunter = NativeHunter( + llm=llm, + prompt="test", + tools=[tool], + ctx=HunterContext(repo_path="/tmp/repo"), + max_steps=4, + initial_source_action_retries=1, + ) + + with patch("clearwing.sourcehunt.hunter.HunterTrajectoryLogger") as mock_traj: + mock_traj.for_hunter.return_value = MagicMock() + result = await hunter.arun() + + assert llm.achat.call_count == 3 + assert result.stop_reason == "no_source_action" + + @pytest.mark.asyncio async def test_deep_mode_terminates_on_budget(): hunter, llm = _make_hunter(agent_mode="deep", max_steps=500, budget_usd=0.01) @@ -124,6 +239,336 @@ async def test_deep_mode_safety_cap(): assert result.stop_reason == "max_steps" +@pytest.mark.asyncio +async def test_zero_price_override_uses_step_cap_instead_of_nominal_cost(): + hunter, llm = _make_hunter(agent_mode="deep", max_steps=3, budget_usd=0.01) + hunter.input_price_per_million = 0.0 + hunter.output_price_per_million = 0.0 + llm.achat.return_value = FakeResponse( + tool_calls_list=[_make_tool_call("think", {"notes": "thinking"})], + ) + + with patch("clearwing.sourcehunt.hunter.HunterTrajectoryLogger") as mock_traj: + mock_traj.for_hunter.return_value = MagicMock() + with patch("clearwing.sourcehunt.hunter._estimate_cost_usd", return_value=10.0): + result = await hunter.arun() + + assert llm.achat.call_count == 3 + assert result.stop_reason == "max_steps" + assert result.cost_usd == 0.0 + + +@pytest.mark.asyncio +async def test_candidate_gate_blocks_source_sweep_until_ledger_update(): + llm = AsyncMock() + ctx = HunterContext(repo_path="/tmp/repo") + source_calls = 0 + + def read_source_file(**_kwargs): + nonlocal source_calls + source_calls += 1 + return "source" + + def record_candidate(candidate_id, **_kwargs): + ctx.candidates[candidate_id] = {"status": "investigating"} + ctx.candidate_revision += 1 + return "candidate saved" + + tools = [ + NativeToolSpec( + name="read_source_file", + description="read", + schema={ + "type": "object", + "properties": {"path": {"type": "string"}}, + "required": ["path"], + }, + handler=read_source_file, + ), + NativeToolSpec( + name="record_candidate", + description="candidate", + schema={ + "type": "object", + "properties": {"candidate_id": {"type": "string"}}, + "required": ["candidate_id"], + }, + handler=record_candidate, + ), + ] + llm.achat.side_effect = [ + FakeResponse(tool_calls_list=[_make_tool_call("read_source_file", {"path": "a.c"})]), + FakeResponse(tool_calls_list=[_make_tool_call("read_source_file", {"path": "b.c"})]), + FakeResponse(tool_calls_list=[_make_tool_call("read_source_file", {"path": "c.c"})]), + FakeResponse( + tool_calls_list=[_make_tool_call("record_candidate", {"candidate_id": "C1"})] + ), + FakeResponse(text="done"), + ] + hunter = NativeHunter( + llm=llm, + prompt="test", + tools=tools, + ctx=ctx, + max_steps=6, + candidate_gate_after_source_actions=2, + ) + + with patch("clearwing.sourcehunt.hunter.HunterTrajectoryLogger") as mock_traj: + mock_traj.for_hunter.return_value = MagicMock() + result = await hunter.arun() + + assert result.stop_reason == "completed" + assert source_calls == 2 + assert ctx.candidates == {"C1": {"status": "investigating"}} + + +@pytest.mark.asyncio +async def test_domain_candidate_checkpoint_requires_structured_proof(): + llm = AsyncMock() + ctx = HunterContext(repo_path="/tmp/repo") + ctx.value_domains["D1"] = { + "assessment": "overlap_possible", + "blocking_guard_locations": [], + } + ctx.domain_consequences["D1"] = {"assessment": "unresolved"} + ctx.domain_consequence_plans["D1"] = { + "boundary_facts": [{"line": 9, "token": "i", "expression": "i - 1"}] + } + ctx.domain_candidate_ids["D1"] = "C1" + ctx.candidates["C1"] = {"status": "investigating"} + source_calls = 0 + candidate_calls = 0 + proof_calls = 0 + + def read_source_file(**_kwargs): + nonlocal source_calls + source_calls += 1 + return "source" + + def record_candidate(**_kwargs): + nonlocal candidate_calls + candidate_calls += 1 + return "candidate saved" + + def record_domain_proof(**_kwargs): + nonlocal proof_calls + proof_calls += 1 + ctx.candidate_revision += 1 + return "proof narrowed" + + def tool(name, handler): + return NativeToolSpec( + name=name, + description=name, + schema={"type": "object", "properties": {}}, + handler=handler, + ) + + tools = [ + tool("read_source_file", read_source_file), + tool("record_candidate", record_candidate), + tool("record_domain_proof", record_domain_proof), + ] + llm.achat.side_effect = [ + FakeResponse(tool_calls_list=[_make_tool_call("read_source_file")]), + FakeResponse(tool_calls_list=[_make_tool_call("read_source_file")]), + FakeResponse(tool_calls_list=[_make_tool_call("read_source_file")]), + FakeResponse(tool_calls_list=[_make_tool_call("record_candidate")]), + FakeResponse(tool_calls_list=[_make_tool_call("record_domain_proof")]), + FakeResponse(text="done"), + ] + hunter = NativeHunter( + llm=llm, + prompt="test", + tools=tools, + ctx=ctx, + max_steps=7, + candidate_gate_after_source_actions=2, + ) + + with patch("clearwing.sourcehunt.hunter.HunterTrajectoryLogger") as mock_traj: + logger = MagicMock() + mock_traj.for_hunter.return_value = logger + result = await hunter.arun() + + proof_gates = [ + call + for call in logger.log.call_args_list + if len(call.args) > 1 + and isinstance(call.args[1], dict) + and call.args[1].get("domain_proof_gate") + ] + assert result.stop_reason == "completed" + assert source_calls == 2 + assert candidate_calls == 0 + assert proof_calls == 1 + assert len(proof_gates) == 2 + + +@pytest.mark.asyncio +async def test_unresolved_domain_proof_allows_one_refinement_then_requires_proof(): + llm = AsyncMock() + ctx = HunterContext(repo_path="/tmp/repo") + ctx.value_domains["D1"] = { + "assessment": "overlap_possible", + "blocking_guard_locations": [], + } + ctx.domain_consequences["D1"] = {"assessment": "unresolved"} + ctx.domain_consequence_plans["D1"] = { + "boundary_facts": [{"line": 9, "token": "i", "expression": "i - 1"}] + } + ctx.domain_candidate_ids["D1"] = "C1" + ctx.candidates["C1"] = {"status": "investigating"} + source_calls = 0 + proof_calls = 0 + refinement_calls = 0 + + def read_source_file(**_kwargs): + nonlocal source_calls + source_calls += 1 + return "source" + + def record_domain_proof(**_kwargs): + nonlocal proof_calls + proof_calls += 1 + ctx.candidate_revision += 1 + ctx.domain_refinement_pending_proof.discard("D1") + if proof_calls == 1: + ctx.domain_proof_obligations["D1"] = ["attacker_reaches_producer"] + return "proof narrowed" + ctx.domain_proof_obligations.pop("D1", None) + ctx.candidates["C1"]["status"] = "validated" + return "proof validated" + + def read_domain_proof_refinement(**_kwargs): + nonlocal refinement_calls + refinement_calls += 1 + ctx.domain_refinements_read.add(("D1", "attacker_reaches_producer")) + ctx.domain_refinement_pending_proof.add("D1") + return "bounded refinement" + + def tool(name, handler): + return NativeToolSpec( + name=name, + description=name, + schema={"type": "object", "properties": {}}, + handler=handler, + ) + + tools = [ + tool("read_source_file", read_source_file), + tool("record_domain_proof", record_domain_proof), + tool("read_domain_proof_refinement", read_domain_proof_refinement), + ] + llm.achat.side_effect = [ + FakeResponse(tool_calls_list=[_make_tool_call("read_source_file", {"path": "a.c"})]), + FakeResponse(tool_calls_list=[_make_tool_call("read_source_file", {"path": "b.c"})]), + FakeResponse(tool_calls_list=[_make_tool_call("read_source_file", {"path": "c.c"})]), + FakeResponse(tool_calls_list=[_make_tool_call("record_domain_proof")]), + FakeResponse(tool_calls_list=[_make_tool_call("read_source_file", {"path": "d.c"})]), + FakeResponse(tool_calls_list=[_make_tool_call("read_domain_proof_refinement")]), + FakeResponse(tool_calls_list=[_make_tool_call("read_source_file", {"path": "e.c"})]), + FakeResponse(tool_calls_list=[_make_tool_call("record_domain_proof")]), + FakeResponse(text="done"), + ] + hunter = NativeHunter( + llm=llm, + prompt="test", + tools=tools, + ctx=ctx, + max_steps=10, + candidate_gate_after_source_actions=2, + enable_domain_proof_refinement=True, + ) + + with patch("clearwing.sourcehunt.hunter.HunterTrajectoryLogger") as mock_traj: + logger = MagicMock() + mock_traj.for_hunter.return_value = logger + result = await hunter.arun() + + logged_payloads = [ + call.args[1] + for call in logger.log.call_args_list + if len(call.args) > 1 and isinstance(call.args[1], dict) + ] + gate_keys = sorted( + key + for payload in logged_payloads + for key in payload + if key.endswith("_gate") + ) + assert result.stop_reason == "completed" + assert source_calls == 2 + assert proof_calls == 2 + assert refinement_calls == 1 + assert any( + payload.get("domain_proof_refinement_gate") for payload in logged_payloads + ), gate_keys + assert any(payload.get("domain_refinement_proof_gate") for payload in logged_payloads) + + +@pytest.mark.asyncio +async def test_window_gate_blocks_reads_until_generic_ranking_runs(): + llm = AsyncMock() + ctx = HunterContext(repo_path="/tmp/repo") + source_calls = 0 + + def read_source_file(**_kwargs): + nonlocal source_calls + source_calls += 1 + return "source" + + def rank_source_windows(**_kwargs): + ctx.source_windows_ranked = True + return {"windows": []} + + tools = [ + NativeToolSpec( + name="read_source_file", + description="read", + schema={ + "type": "object", + "properties": {"path": {"type": "string"}}, + "required": ["path"], + }, + handler=read_source_file, + ), + NativeToolSpec( + name="rank_source_windows", + description="rank", + schema={ + "type": "object", + "properties": {"path": {"type": "string"}}, + "required": ["path"], + }, + handler=rank_source_windows, + ), + ] + llm.achat.side_effect = [ + FakeResponse(tool_calls_list=[_make_tool_call("read_source_file", {"path": "a.c"})]), + FakeResponse(tool_calls_list=[_make_tool_call("rank_source_windows", {"path": "a.c"})]), + FakeResponse(tool_calls_list=[_make_tool_call("read_source_file", {"path": "a.c"})]), + FakeResponse(text="done"), + ] + hunter = NativeHunter( + llm=llm, + prompt="test", + tools=tools, + ctx=ctx, + max_steps=5, + require_source_windows=True, + ) + + with patch("clearwing.sourcehunt.hunter.HunterTrajectoryLogger") as mock_traj: + mock_traj.for_hunter.return_value = MagicMock() + result = await hunter.arun() + + assert result.stop_reason == "completed" + assert source_calls == 1 + assert ctx.source_windows_ranked is True + + @pytest.mark.asyncio async def test_deep_mode_stops_on_degenerate_loop(): # Real failure mode observed against crAPI with a local devstral model @@ -197,7 +642,7 @@ async def achat_side_effect(**kwargs): with patch("clearwing.sourcehunt.hunter.HunterTrajectoryLogger") as mock_traj: mock_logger = MagicMock() mock_traj.for_hunter.return_value = mock_logger - result = await hunter.arun() + await hunter.arun() # In constrained mode, after 3 identical calls the 4th+ should be skipped logged = mock_logger.log.call_args_list diff --git a/tests/test_hunt_reporting_tools.py b/tests/test_hunt_reporting_tools.py index 9a22a723..8ec3b219 100644 --- a/tests/test_hunt_reporting_tools.py +++ b/tests/test_hunt_reporting_tools.py @@ -67,3 +67,38 @@ def test_record_finding_allows_different_lines(tools, ctx): assert "Finding recorded" in result assert len(ctx.findings) == 2 + + +def test_scaffold_finding_requires_validated_candidate_and_corroboration(tools, ctx): + ctx.require_validated_candidate_before_finding = True + ctx.candidates["C1"] = {"status": "investigating"} + tools["record_trace_step"](file="app.py", line=42, note="entry to sink") + + unresolved = _record_finding(tools, candidate_id="C1") + ctx.candidates["C1"]["status"] = "validated" + suspicion = _record_finding(tools, candidate_id="C1") + accepted = _record_finding( + tools, + candidate_id="C1", + evidence_level="static_corroboration", + ) + + assert "already marked validated" in unresolved + assert "static_corroboration or stronger" in suspicion + assert "Finding recorded" in accepted + assert ctx.findings[0].extra["candidate_id"] == "C1" + + +def test_scaffold_finding_requires_an_active_candidate(tools, ctx): + ctx.require_active_candidate_before_finding = True + ctx.candidates["C1"] = {"status": "rejected"} + tools["record_trace_step"](file="app.py", line=42, note="entry to sink") + + absent = _record_finding(tools) + rejected = _record_finding(tools, candidate_id="C1") + ctx.candidates["C1"]["status"] = "investigating" + accepted = _record_finding(tools, candidate_id="C1") + + assert "requires candidate_id for an active candidate" in absent + assert "requires candidate_id for an active candidate" in rejected + assert "Finding recorded" in accepted diff --git a/tests/test_kimi_compat.py b/tests/test_kimi_compat.py new file mode 100644 index 00000000..75d2f930 --- /dev/null +++ b/tests/test_kimi_compat.py @@ -0,0 +1,71 @@ +"""Tests for the Kimi Code provider preset and endpoint resolution.""" + +from __future__ import annotations + +from clearwing.providers.catalog import preset_by_key +from clearwing.providers.env import _default_openai_compat_model, resolve_llm_endpoint + + +class TestKimiCodeCatalog: + def test_preset_uses_official_coding_api_defaults(self): + preset = preset_by_key("kimi-code") + + assert preset is not None + assert preset.display_name == "Kimi Code (membership)" + assert preset.default_base_url == "https://api.kimi.com/coding/v1" + assert preset.default_model == "k3-256k" + assert preset.api_key_env_var == "KIMI_CODE_API_KEY" + assert preset.provider_adapter == "openai" + assert preset.is_openai_compat + assert preset.docs_url == "https://www.kimi.com/code/console" + + def test_preset_offers_current_coding_model_ids(self): + preset = preset_by_key("kimi-code") + + assert preset is not None + assert preset.alt_models == ( + "k3", + "kimi-for-coding", + "kimi-for-coding-highspeed", + ) + + def test_underscore_alias_is_supported(self): + preset = preset_by_key("kimi_code") + + assert preset is not None + assert preset.key == "kimi-code" + + +class TestKimiEndpointResolution: + def test_kimi_code_url_has_membership_default(self): + assert _default_openai_compat_model("https://api.kimi.com/coding/v1") == "k3-256k" + + def test_cli_flags_route_to_openai_adapter(self): + endpoint = resolve_llm_endpoint( + cli_base_url="https://api.kimi.com/coding/v1", + cli_api_key="test-key", + config_provider={}, + ) + + assert endpoint.provider == "openai_compat" + assert endpoint.base_url == "https://api.kimi.com/coding/v1" + assert endpoint.model == "k3-256k" + assert endpoint.api_key == "test-key" + assert endpoint.is_openai_compat + + def test_setup_style_config_preserves_explicit_adapter(self, monkeypatch): + for name in ("CLEARWING_BASE_URL", "CLEARWING_API_KEY", "CLEARWING_MODEL"): + monkeypatch.delenv(name, raising=False) + + endpoint = resolve_llm_endpoint( + config_provider={ + "base_url": "https://api.kimi.com/coding/v1", + "api_key": "test-key", + "model": "kimi-for-coding", + "adapter": "openai", + }, + ) + + assert endpoint.provider == "openai_compat" + assert endpoint.model == "kimi-for-coding" + assert endpoint.adapter == "openai" diff --git a/tests/test_native_reasoning_effort.py b/tests/test_native_reasoning_effort.py index 48f0a263..9825697c 100644 --- a/tests/test_native_reasoning_effort.py +++ b/tests/test_native_reasoning_effort.py @@ -40,6 +40,26 @@ def test_deepseek_r1_keeps_medium(self): result = AsyncLLMClient._auto_resolve_reasoning_effort("deepseek-r1") assert result == "medium" + def test_local_deepseek_v4_flash_omits_reasoning_effort(self): + result = AsyncLLMClient._auto_resolve_reasoning_effort("dsv4-flash-nvfp4") + assert result is None + + @pytest.mark.parametrize("model", ["k3", "k3-256k"]) + def test_kimi_code_k3_uses_supported_high_effort(self, model): + result = AsyncLLMClient._auto_resolve_reasoning_effort(model) + assert result == "high" + + @pytest.mark.parametrize( + "model", + [ + "kimi-for-coding", + "kimi-for-coding-highspeed", + ], + ) + def test_kimi_k2_7_models_omit_reasoning_effort(self, model): + result = AsyncLLMClient._auto_resolve_reasoning_effort(model) + assert result is None + def test_mistral_resolves_to_none(self): result = AsyncLLMClient._auto_resolve_reasoning_effort("mistral-large-2407") assert result is None @@ -273,6 +293,7 @@ def test_streaming_path_retries_once(self): "Status: 400 Bad Request. " 'Body: {"error":{"message":"`reasoning_effort` is not supported"}}' ) + # A minimal stand-in for genai's StreamEnd (captured_* fields); # achat_stream rebuilds a ChatResponse from it via # _chat_response_from_stream_end. diff --git a/tests/test_setup_and_doctor.py b/tests/test_setup_and_doctor.py index fc0c3808..bb3eda8c 100644 --- a/tests/test_setup_and_doctor.py +++ b/tests/test_setup_and_doctor.py @@ -20,8 +20,8 @@ from unittest.mock import patch import pytest -from rich.console import Console import yaml +from rich.console import Console from clearwing.providers import KNOWN_PROVIDERS, preset_by_key from clearwing.ui.commands import doctor, setup @@ -32,7 +32,7 @@ DoctorCheck, DoctorSection, ) -from clearwing.ui.commands.setup import _mask_secret, _write_config +from clearwing.ui.commands.setup import _mask_secret, _run_test_invoke, _write_config # --- Provider catalog ------------------------------------------------------ @@ -205,6 +205,7 @@ def test_writes_minimal_provider_section(self, tmp_cli): ) path = tmp_cli.config.DEFAULT_CONFIG_PATH assert path.exists() + assert path.stat().st_mode & 0o777 == 0o600 data = yaml.safe_load(path.read_text()) assert data == { "provider": { @@ -283,6 +284,26 @@ def test_anthropic_without_base_url(self, tmp_cli): assert data["provider"]["model"] == "claude-sonnet-4-6" assert data["provider"]["api_key"] == "sk-ant-test" + def test_writes_kimi_code_provider_section(self, tmp_cli): + preset = preset_by_key("kimi-code") + _write_config( + tmp_cli, + preset, + base_url="https://api.kimi.com/coding/v1", + api_key_literal="${KIMI_CODE_API_KEY}", + model="k3-256k", + ) + + data = yaml.safe_load(tmp_cli.config.DEFAULT_CONFIG_PATH.read_text()) + assert data == { + "provider": { + "base_url": "https://api.kimi.com/coding/v1", + "api_key": "${KIMI_CODE_API_KEY}", + "model": "k3-256k", + "adapter": "openai", + } + } + def test_openai_oauth_writes_auth_marker_without_api_key(self, tmp_cli): preset = preset_by_key("openai-oauth") _write_config( @@ -303,6 +324,30 @@ def test_openai_oauth_writes_auth_marker_without_api_key(self, tmp_cli): } +class TestRunTestInvoke: + def test_kimi_code_401_explains_membership_key_isolation(self): + stream = io.StringIO() + console = Console(file=stream, force_terminal=False) + preset = preset_by_key("kimi-code") + + with patch( + "clearwing.providers.ProviderManager.for_endpoint", + side_effect=RuntimeError("HTTP 401: Invalid Authentication"), + ): + _run_test_invoke( + console, + preset, + base_url="https://api.kimi.com/coding/v1", + api_key_literal="sk-test", + model="k3-256k", + ) + + output = stream.getvalue() + assert "separate from Open Platform keys" in output + assert "www.kimi.com/code/console" in output + assert "membership tier" in output + + # --- Doctor: DoctorCheck + DoctorSection aggregation --------------------- diff --git a/tests/test_sourcehunt_gepa.py b/tests/test_sourcehunt_gepa.py new file mode 100644 index 00000000..1a4d5c46 --- /dev/null +++ b/tests/test_sourcehunt_gepa.py @@ -0,0 +1,222 @@ +"""Core-GEPA adapter tests without requiring the optional GEPA package.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from clearwing.eval.sourcehunt import ( + AblationLevel, + AblationRunSpec, + GroundTruthManifest, + RunObservation, + StageFunnel, + include_fixed_negative_cases, +) +from clearwing.eval.sourcehunt_gepa import ( + PROMPT_COMPONENT, + SourceHuntGEPAAdapter, + SourceHuntOptimizationExample, + score_sourcehunt_observation, +) +from clearwing.sourcehunt.optimization import GENERIC_INSTRUCTIONS_V1 + + +def _spec(case) -> AblationRunSpec: + return AblationRunSpec( + case_id=case.id, + repository=case.repository, + vulnerable_commit=case.vulnerable_commit, + case_digest=case.digest, + flow="legacy", + model_tier="local", + model="deepseek-local", + prompt_bundle="generic-security-v1", + scaffold_profile="minimal-linear-v1", + level=AblationLevel.REPOSITORY, + ) + + +def _observation(spec, *, positive: bool, session_dir: Path) -> RunObservation: + return RunObservation( + run_id=spec.id, + context_id=spec.context_id, + case_id=spec.case_id, + flow=spec.flow, + model_tier=spec.model_tier, + model=spec.model, + prompt_bundle=spec.prompt_bundle, + scaffold_profile=spec.scaffold_profile, + context_profile=spec.context_profile, + level=spec.level, + replicate=spec.replicate, + session_dir=str(session_dir), + status="completed", + funnel=StageFunnel( + target_in_working_set=True, + true_candidate_generated=positive, + correct_certificate_compiled=positive, + ), + true_positives=int(positive), + false_negatives=int(not positive), + finding_count=int(positive), + report_claim_count=int(positive), + ) + + +def test_adapter_requires_negative_controls() -> None: + manifest = GroundTruthManifest.load("evaluations/sourcehunt_ground_truth.yaml") + + with pytest.raises(ValueError, match="fixed/clean negatives"): + SourceHuntGEPAAdapter(manifest) + + +def test_adapter_evaluates_generic_candidate_and_emits_reflection_data(tmp_path) -> None: + manifest = include_fixed_negative_cases( + GroundTruthManifest.load("evaluations/sourcehunt_ground_truth.yaml") + ) + positive_case = manifest.cases[0] + negative_case = manifest.case(f"{positive_case.id}-fixed-negative") + calls: list[dict] = [] + + async def execute(spec, case, **kwargs): + calls.append(kwargs) + expected_positive = case.ground_truth.expected_decision == "confirmed" + return _observation( + spec, + positive=expected_positive, + session_dir=tmp_path / kwargs["session_id"], + ) + + adapter = SourceHuntGEPAAdapter(manifest, executor=execute) + examples = [ + SourceHuntOptimizationExample( + spec=_spec(case), + case=case, + checkout=tmp_path, + output_dir=tmp_path, + provider_manager=object(), + input_price_per_million=0.0, + output_price_per_million=0.0, + max_hunt_files=24, + max_hunter_steps=40, + ) + for case in (positive_case, negative_case) + ] + + result = adapter.evaluate( + examples, + {PROMPT_COMPONENT: GENERIC_INSTRUCTIONS_V1}, + capture_traces=True, + ) + + assert result.scores == [1.0, 1.0] + assert result.num_metric_calls == 2 + assert len(result.trajectories) == 2 + assert all(call["prompt_candidate"] == GENERIC_INSTRUCTIONS_V1 for call in calls) + assert all(call["input_price_per_million"] == 0.0 for call in calls) + assert all(call["output_price_per_million"] == 0.0 for call in calls) + assert all(call["max_hunt_files"] == 24 for call in calls) + assert all(call["max_hunter_steps"] == 40 for call in calls) + assert all(call["ranker_chunk_size"] == 25 for call in calls) + assert all(call["ranker_max_inflight_chunks"] == 1 for call in calls) + assert all(call["ranker_chunk_max_retries"] == 1 for call in calls) + assert all(call["max_parallel"] == 4 for call in calls) + assert all(call["starting_band"] == "fast" for call in calls) + assert all(call["redundancy_override"] == 1 for call in calls) + assert all(call["depth"] == "standard" for call in calls) + assert all(call["no_rank"] is True for call in calls) + assert len({call["session_id"] for call in calls}) == 2 + reflective = adapter.make_reflective_dataset( + {PROMPT_COMPONENT: GENERIC_INSTRUCTIONS_V1}, + result, + [PROMPT_COMPONENT], + ) + assert len(reflective[PROMPT_COMPONENT]) == 2 + assert "Do not add repository names" in reflective[PROMPT_COMPONENT][0]["Feedback"] + + +def test_adapter_rejects_solution_leaking_candidate(tmp_path) -> None: + manifest = include_fixed_negative_cases( + GroundTruthManifest.load("evaluations/sourcehunt_ground_truth.yaml") + ) + case = manifest.cases[0] + adapter = SourceHuntGEPAAdapter(manifest, executor=None) # type: ignore[arg-type] + example = SourceHuntOptimizationExample( + spec=_spec(case), + case=case, + checkout=tmp_path, + output_dir=tmp_path, + provider_manager=object(), + ) + + with pytest.raises(ValueError, match="leaks benchmark answers"): + adapter.evaluate( + [example], + {PROMPT_COMPONENT: f"Start in {case.ground_truth.target_files[0]}"}, + ) + + +def test_optimization_examples_reject_assisted_ablation_levels(tmp_path) -> None: + case = GroundTruthManifest.load("evaluations/sourcehunt_ground_truth.yaml").cases[0] + spec = AblationRunSpec( + case_id=case.id, + repository=case.repository, + vulnerable_commit=case.vulnerable_commit, + case_digest=case.digest, + flow="legacy", + model_tier="local", + model="deepseek-local", + prompt_bundle="generic-security-v1", + scaffold_profile="minimal-linear-v1", + level=AblationLevel.TARGET_FILE, + hints={"target_files": case.ground_truth.target_files}, + ) + + with pytest.raises(ValueError, match="blind repository-level"): + SourceHuntOptimizationExample( + spec=spec, + case=case, + checkout=tmp_path, + output_dir=tmp_path, + provider_manager=object(), + ) + + +def test_optimization_examples_reject_proof_flow_until_prompt_is_wired(tmp_path) -> None: + case = GroundTruthManifest.load("evaluations/sourcehunt_ground_truth.yaml").cases[0] + spec = AblationRunSpec( + case_id=case.id, + repository=case.repository, + vulnerable_commit=case.vulnerable_commit, + case_digest=case.digest, + flow="proof", + model_tier="local", + model="deepseek-local", + prompt_bundle="generic-security-v1", + scaffold_profile="minimal-linear-v1", + level=AblationLevel.REPOSITORY, + ) + + with pytest.raises(ValueError, match="legacy discovery flow"): + SourceHuntOptimizationExample( + spec=spec, + case=case, + checkout=tmp_path, + output_dir=tmp_path, + provider_manager=object(), + ) + + +def test_score_penalizes_unvalidated_extra_findings(tmp_path) -> None: + case = GroundTruthManifest.load("evaluations/sourcehunt_ground_truth.yaml").cases[0] + spec = _spec(case) + observation = _observation(spec, positive=True, session_dir=tmp_path).model_copy( + update={"false_positives": 2, "finding_count": 3} + ) + + score, objectives = score_sourcehunt_observation(observation, case) + + assert score < 1.0 + assert objectives["precision"] == pytest.approx(1 / 3) diff --git a/tests/test_sourcehunt_lair.py b/tests/test_sourcehunt_lair.py new file mode 100644 index 00000000..7a11d34a --- /dev/null +++ b/tests/test_sourcehunt_lair.py @@ -0,0 +1,316 @@ +"""Leakage and split invariants for LAIR-to-SourceHunt supervision.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from clearwing.eval.sourcehunt_lair import ( + LairGoldenChain, + LairSplitConfig, + RouterContextCategory, + RouterObligation, + adapt_lair_goldens, + lint_router_rows, + load_lair_goldens, + write_lair_adapter_dataset, +) + + +def _citation(revision: str, path: str, line: int, excerpt: str) -> dict: + return { + "revision": revision, + "path": path, + "line_start": line, + "line_end": line, + "excerpt": excerpt, + "supports": "This exact source line supports the causal step.", + } + + +def _golden_payload( + *, + cve: str = "CVE-2099-1001", + repo: str = "example/parser", + vulnerable: str = "1" * 40, + fix: str = "2" * 40, +) -> dict: + vulnerable_path = "src/secret_parser.c" + fixed_path = "src/secret_parser.c" + return { + "schema_version": "cwpro.cve-golden-chain.v2", + "cve": cve, + "repo": repo, + "vulnerable_commit": vulnerable, + "fix_commit": fix, + "title": "Secret sentinel collision", + "vulnerability_class": "CWE-787 Out-of-bounds Write", + "summary": "An attacker-controlled counter collides with a reserved sentinel value.", + "chain": { + "discovery": { + "id": "discovery", + "candidate": { + "title": "Secret counter collision", + "location": "src/secret_parser.c:parse_secret", + "hypothesis": "A remote counter may collide with a reserved internal value.", + "why_prioritize": "The same representation is used for state and a boundary marker.", + "evidence": [ + _citation( + "vulnerable", + vulnerable_path, + 10, + "state->secret_slot = remote_counter;", + ) + ], + }, + }, + "investigation": { + "id": "investigation", + "preconditions": ["A remote peer can repeatedly advance the secret counter."], + "causal_trace": [ + { + "id": "input", + "kind": "attack_source", + "claim": "A remote packet controls the initial counter update.", + "evidence": [ + _citation( + "vulnerable", vulnerable_path, 4, "parse_secret(packet);" + ) + ], + }, + { + "id": "copy", + "kind": "propagation", + "claim": "The remote value propagates into a local counter.", + "evidence": [ + _citation( + "vulnerable", + vulnerable_path, + 8, + "remote_counter = packet->counter;", + ) + ], + }, + { + "id": "store", + "kind": "state_transition", + "claim": "The counter is stored in the secret slot representation.", + "evidence": [ + _citation( + "vulnerable", + vulnerable_path, + 10, + "state->secret_slot = remote_counter;", + ) + ], + }, + { + "id": "guard", + "kind": "guard_failure", + "claim": "No terminating guard excludes the reserved representation.", + "evidence": [ + _citation( + "vulnerable", + vulnerable_path, + 11, + "if (state->secret_slot == SECRET_SENTINEL) {", + ) + ], + }, + { + "id": "write", + "kind": "vulnerable_operation", + "claim": "The sentinel branch performs a write through a shifted pointer.", + "evidence": [ + _citation( + "vulnerable", vulnerable_path, 12, "shifted[-1] = packet->byte;" + ) + ], + }, + { + "id": "effect", + "kind": "security_effect", + "claim": "The shifted write can modify memory outside the destination.", + "evidence": [ + _citation( + "vulnerable", vulnerable_path, 12, "shifted[-1] = packet->byte;" + ) + ], + }, + ], + "security_impact": "A remote peer can cause an out-of-bounds memory write.", + }, + "challenge": { + "id": "challenge", + "verdict": "confirmed", + "checks": [ + { + "assumption": "A caller may reject the reserved counter before parsing.", + "conclusion": "All callers forward the remote counter without that rejection.", + "evidence": [ + _citation( + "vulnerable", vulnerable_path, 4, "parse_secret(packet);" + ) + ], + } + ], + "fix_validation": { + "strategy": "Reject the reserved value before storing the remote counter.", + "behavior_before": "The reserved value reached the shifted pointer write.", + "behavior_after": "The reserved value returns an error before state mutation.", + "changed_files": [fixed_path], + "evidence": [ + _citation( + "fix", + fixed_path, + 9, + "if (remote_counter == SECRET_SENTINEL) return ERROR;", + ) + ], + "regression_tests": [ + { + "status": "proposed", + "description": "Send the reserved remote counter value.", + "expected_result": "Parsing fails without writing through shifted.", + } + ], + }, + }, + }, + } + + +def _golden(**kwargs: str) -> LairGoldenChain: + return LairGoldenChain.model_validate(_golden_payload(**kwargs)) + + +def test_adapter_emits_only_delexicalized_routing_state() -> None: + golden = _golden() + dataset = adapt_lair_goldens( + [golden], + split_config=LairSplitConfig(train=1.0, development=0.0, test=0.0), + ) + rows = dataset.rows["train"] + + assert len(rows) == len(golden.chain.investigation.causal_trace) + 1 + assert rows[0].target.next_obligation == RouterObligation.ATTACKER_REACHES_ENTRY + assert rows[2].target.context_category == ( + RouterContextCategory.STATE_WRITERS_AND_REPRESENTATION + ) + assert rows[-1].target.next_obligation == RouterObligation.CANDIDATE_SURVIVES_CHALLENGE + assert lint_router_rows(rows, [golden]) == [] + + serialized = "\n".join(row.model_dump_json() for row in rows).casefold() + for forbidden in ( + "cve-2099-1001", + "example/parser", + "secret_parser.c", + "parse_secret", + "secret_sentinel", + "out-of-bounds", + "111111111111", + "222222222222", + ): + assert forbidden not in serialized + + +def test_leakage_linter_rejects_answer_bearing_identifier() -> None: + golden = _golden() + dataset = adapt_lair_goldens( + [golden], + split_config=LairSplitConfig(train=1.0, development=0.0, test=0.0), + ) + clean = dataset.rows["train"][0] + leaked = clean.model_copy( + update={"target": clean.target.model_copy(update={"action": "src/secret_parser.c"})} + ) + + leaks = lint_router_rows([leaked], [golden]) + + assert leaks + assert any(leak.value == "secret_parser.c" for leak in leaks) + + +def test_leakage_linter_allows_source_identifier_collision_with_router_ontology() -> None: + payload = _golden_payload() + payload["chain"]["investigation"]["causal_trace"][0]["evidence"][0]["excerpt"] = ( + "context = packet->counter;" + ) + golden = LairGoldenChain.model_validate(payload) + + dataset = adapt_lair_goldens( + [golden], + split_config=LairSplitConfig(train=1.0, development=0.0, test=0.0), + ) + + assert dataset.manifest.router_row_count == 7 + assert lint_router_rows(dataset.rows["train"], [golden]) == [] + + +def test_repository_groups_never_cross_splits() -> None: + goldens = [ + _golden(cve="CVE-2099-1001", repo="org/shared", vulnerable="1" * 40, fix="2" * 40), + _golden(cve="CVE-2099-1002", repo="org/shared", vulnerable="3" * 40, fix="4" * 40), + _golden(cve="CVE-2099-1003", repo="org/other", vulnerable="5" * 40, fix="6" * 40), + ] + config = LairSplitConfig(seed="stable-test") + dataset = adapt_lair_goldens(goldens, split_config=config) + shared_split = config.assign("org/shared") + + assert dataset.manifest.splits[shared_split].golden_count >= 2 + assert sum(summary.repository_count for summary in dataset.manifest.splits.values()) == 2 + + +def test_ffmpeg_is_excluded_from_optimization_rows() -> None: + dataset = adapt_lair_goldens( + [ + _golden(repo="FFmpeg/FFmpeg"), + _golden( + cve="CVE-2099-1002", + repo="example/other", + vulnerable="3" * 40, + fix="4" * 40, + ), + ], + split_config=LairSplitConfig(train=1.0, development=0.0, test=0.0), + ) + + assert dataset.manifest.golden_count == 1 + assert dataset.manifest.excluded_golden_count == 1 + assert len(dataset.rows["train"]) == 7 + + +def test_loader_writer_and_manifest_digests_are_reproducible(tmp_path: Path) -> None: + golden_root = tmp_path / "source" / "goldens" + golden_root.mkdir(parents=True) + source = golden_root / "CVE-2099-1001.json" + source.write_text(json.dumps(_golden_payload()), encoding="utf-8") + + goldens = load_lair_goldens(tmp_path / "source") + first = adapt_lair_goldens( + goldens, + split_config=LairSplitConfig(train=1.0, development=0.0, test=0.0), + ) + second = adapt_lair_goldens( + goldens, + split_config=LairSplitConfig(train=1.0, development=0.0, test=0.0), + ) + manifest_path = write_lair_adapter_dataset(first, tmp_path / "output") + + assert first == second + assert manifest_path.is_file() + assert len((tmp_path / "output" / "router" / "train.jsonl").read_text().splitlines()) == 7 + assert json.loads(manifest_path.read_text())["corpus_digest"] == ( + first.manifest.corpus_digest + ) + with pytest.raises(FileExistsError, match="Refusing to overwrite"): + write_lair_adapter_dataset(first, tmp_path / "output") + + +def test_duplicate_cves_and_invalid_split_fractions_fail_closed() -> None: + with pytest.raises(ValueError, match="sum to 1.0"): + LairSplitConfig(train=0.8, development=0.2, test=0.2) + with pytest.raises(ValueError, match="Duplicate LAIR goldens"): + adapt_lair_goldens([_golden(), _golden()]) diff --git a/tests/test_sourcehunt_lair_gepa.py b/tests/test_sourcehunt_lair_gepa.py new file mode 100644 index 00000000..ef935fcc --- /dev/null +++ b/tests/test_sourcehunt_lair_gepa.py @@ -0,0 +1,160 @@ +"""Leakage-boundary tests for LAIR validator GEPA.""" + +from __future__ import annotations + +import asyncio +from pathlib import Path + +import pytest +from test_sourcehunt_lair_validator import _replay_golden, _verdict + +from clearwing.eval.sourcehunt_lair_gepa import ( + VALIDATOR_PROMPT_COMPONENT, + VALIDATOR_REFLECTION_TEMPLATE, + LairValidatorGEPAAdapter, + LairValidatorMetricBudgetStopper, + require_generic_validator_prompt, +) +from clearwing.sourcehunt.validator import VALIDATOR_SOURCE_FIRST_COMPACT_PROMPT + + +class _UnusedClient: + pass + + +def test_prompt_linter_rejects_answer_bearing_and_oversized_text(tmp_path: Path) -> None: + golden = _replay_golden(tmp_path / "repo") + + require_generic_validator_prompt(VALIDATOR_SOURCE_FIRST_COMPACT_PROMPT, [golden]) + with pytest.raises(ValueError, match="leaks LAIR"): + require_generic_validator_prompt( + f"Inspect {golden.chain.discovery.candidate.location}", [golden] + ) + with pytest.raises(ValueError, match="exceeds"): + require_generic_validator_prompt("x" * 2001, [golden]) + with pytest.raises(ValueError, match="evaluation-protocol"): + require_generic_validator_prompt("Compare paired snapshots.", [golden]) + with pytest.raises(ValueError, match="evaluation-protocol"): + require_generic_validator_prompt( + "Set rejected_source_contradicted_claim=true.", [golden] + ) + assert "" in VALIDATOR_REFLECTION_TEMPLATE + assert "" in VALIDATOR_REFLECTION_TEMPLATE + assert "case identities" in VALIDATOR_REFLECTION_TEMPLATE + + +def test_adapter_exposes_only_opaque_examples_and_abstract_reflection( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + repo = tmp_path / "workspaces" / "CVE-2099-1001" / "repo" + repo.parent.mkdir(parents=True) + golden = _replay_golden(repo) + adapter = LairValidatorGEPAAdapter( + _UnusedClient(), # type: ignore[arg-type] + [golden], + tmp_path, + model="small-model", + ) + + async def replay(_golden, _repo, _call, **_kwargs): + from clearwing.eval.sourcehunt_lair_validator import ( + LairValidatorCaseResult, + _verdict_payload, + ) + + vulnerable = _verdict_payload(_verdict(True)) + fixed = _verdict_payload(_verdict(False)) + return LairValidatorCaseResult( + cve=golden.cve, + repository=golden.repo, + finding_digest="a" * 64, + source_window_digest="b" * 64, + vulnerable=vulnerable, + fixed=fixed, + vulnerable_correct=True, + fixed_correct=True, + pair_correct=True, + ) + + monkeypatch.setattr( + "clearwing.eval.sourcehunt_lair_gepa.replay_lair_validator_case", replay + ) + result = adapter.evaluate( + list(adapter.examples), + {VALIDATOR_PROMPT_COMPONENT: VALIDATOR_SOURCE_FIRST_COMPACT_PROMPT}, + capture_traces=True, + ) + reflective = adapter.make_reflective_dataset( + {VALIDATOR_PROMPT_COMPONENT: VALIDATOR_SOURCE_FIRST_COMPACT_PROMPT}, + result, + [VALIDATOR_PROMPT_COMPONENT], + ) + + assert len(adapter.examples) == 1 + assert adapter.propose_new_texts is None + assert adapter.examples[0].case_id.startswith("case-") + assert result.scores == [1.0] + rendered = repr(adapter.examples) + repr(reflective) + assert golden.cve not in rendered + assert golden.repo not in rendered + assert golden.chain.discovery.candidate.location not in rendered + assert "source-supported behavior" in rendered + assert "snapshot" not in rendered + assert "_correct" not in rendered + assert "axis_pattern" not in rendered + + +def test_adapter_cannot_run_inside_event_loop(tmp_path: Path) -> None: + repo = tmp_path / "workspaces" / "CVE-2099-1001" / "repo" + repo.parent.mkdir(parents=True) + golden = _replay_golden(repo) + adapter = LairValidatorGEPAAdapter( + _UnusedClient(), # type: ignore[arg-type] + [golden], + tmp_path, + model="small-model", + ) + + async def invoke() -> None: + with pytest.raises(RuntimeError, match="cannot run in an event loop"): + adapter.evaluate( + list(adapter.examples), + {VALIDATOR_PROMPT_COMPONENT: VALIDATOR_SOURCE_FIRST_COMPACT_PROMPT}, + ) + + asyncio.run(invoke()) + + +def test_adapter_enforces_hard_metric_call_budget(tmp_path: Path) -> None: + repo = tmp_path / "workspaces" / "CVE-2099-1001" / "repo" + repo.parent.mkdir(parents=True) + golden = _replay_golden(repo) + adapter = LairValidatorGEPAAdapter( + _UnusedClient(), # type: ignore[arg-type] + [golden], + tmp_path, + model="small-model", + max_metric_calls=1, + ) + adapter.metric_calls = 1 + + with pytest.raises(RuntimeError, match="budget exhausted"): + adapter.evaluate( + list(adapter.examples), + {VALIDATOR_PROMPT_COMPONENT: VALIDATOR_SOURCE_FIRST_COMPACT_PROMPT}, + ) + + +def test_budget_stopper_reserves_worst_case_iteration() -> None: + adapter = object.__new__(LairValidatorGEPAAdapter) + adapter.metric_calls = 58 + stopper = LairValidatorMetricBudgetStopper( + adapter, + max_metric_calls=72, + max_iteration_calls=14, + ) + + assert stopper(object()) is False + adapter.metric_calls = 59 + assert stopper(object()) is True diff --git a/tests/test_sourcehunt_lair_replicates.py b/tests/test_sourcehunt_lair_replicates.py new file mode 100644 index 00000000..a4996c41 --- /dev/null +++ b/tests/test_sourcehunt_lair_replicates.py @@ -0,0 +1,135 @@ +"""Tests for replicated LAIR validator aggregation.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from clearwing.eval.sourcehunt_lair_replicates import ( + aggregate_replicates, + load_replay, + wilson_interval, +) +from clearwing.eval.sourcehunt_lair_validator import ( + LairValidatorCaseResult, + ReplayVerdict, + summarize_lair_validator_replay, +) + + +def _verdict(advance: bool, *, error: bool = False) -> ReplayVerdict: + return ReplayVerdict( + advance=advance, + severity_validated="high" if advance else None, + evidence_level="static_corroboration", + axes={} if error else {"real": {"passed": advance}}, + pro_argument="", + counter_argument="", + tie_breaker="", + model_error=error, + ) + + +def _case(name: str, vulnerable: bool, fixed: bool) -> LairValidatorCaseResult: + return LairValidatorCaseResult( + cve=f"CVE-2099-{name}", + repository=f"example/{name}", + finding_digest=name.zfill(64), + source_window_digest=(name + "f").zfill(64), + vulnerable=_verdict(vulnerable), + fixed=_verdict(not fixed), + vulnerable_correct=vulnerable, + fixed_correct=fixed, + pair_correct=vulnerable and fixed, + ) + + +def _write_run(path: Path, profile: str, cases: list[LairValidatorCaseResult]) -> None: + summary = summarize_lair_validator_replay( + cases, + model="small-model", + prompt_profile=profile, + max_output_tokens=1024, + temperature=0.0, + ) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(summary.model_dump_json(indent=2) + "\n", encoding="utf-8") + + +def test_wilson_interval_bounds_rate() -> None: + interval = wilson_interval(9, 10) + + assert interval["confidence"] == 0.95 + assert interval["lower"] < 0.9 < interval["upper"] + assert wilson_interval(0, 10)["lower"] == 0.0 + + +def test_aggregate_reports_pooled_metrics_and_opaque_stability(tmp_path: Path) -> None: + paths: list[Path] = [] + for replicate, cases in enumerate( + ( + [_case("1", True, True), _case("2", False, True)], + [_case("1", True, False), _case("2", True, True)], + ), + start=1, + ): + path = tmp_path / f"run-{replicate:02d}.json" + _write_run(path, "legacy-v1", cases) + paths.append(path) + + result = aggregate_replicates( + {"legacy-v1": paths}, + model="small-model", + max_output_tokens=1024, + temperature=0.0, + context_radius=18, + max_context_chars=20_000, + max_parallel=2, + ) + + arm = result["arms"]["legacy-v1"] + assert arm["aggregate"]["vulnerable_recall"]["rate"] == 0.75 + assert arm["aggregate"]["fixed_rejection_rate"]["rate"] == 0.75 + assert arm["aggregate"]["pair_accuracy"]["rate"] == 0.5 + assert arm["unanimous_case_count"] == 0 + serialized = json.dumps(result) + assert "CVE-" not in serialized + assert "example/" not in serialized + assert all( + case["case_id"].startswith("case-") + for case in arm["per_case_stability"] + ) + + +def test_aggregate_rejects_coordinate_drift(tmp_path: Path) -> None: + first = tmp_path / "run-01.json" + second = tmp_path / "run-02.json" + _write_run(first, "legacy-v1", [_case("1", True, True)]) + changed = _case("1", True, True).model_copy( + update={"source_window_digest": "e" * 64} + ) + _write_run(second, "legacy-v1", [changed]) + + with pytest.raises(ValueError, match="coordinate drift"): + aggregate_replicates( + {"legacy-v1": [first, second]}, + model="small-model", + max_output_tokens=1024, + temperature=0.0, + context_radius=18, + max_context_chars=20_000, + max_parallel=2, + ) + + +def test_load_replay_rejects_tampered_metrics(tmp_path: Path) -> None: + path = tmp_path / "run.json" + _write_run(path, "legacy-v1", [_case("1", True, True)]) + payload = json.loads(path.read_text()) + payload["pair_accuracy"] = 0.0 + path.write_text(json.dumps(payload), encoding="utf-8") + + with pytest.raises(ValueError, match="inconsistent pair_accuracy"): + load_replay(path) diff --git a/tests/test_sourcehunt_lair_validator.py b/tests/test_sourcehunt_lair_validator.py new file mode 100644 index 00000000..f192859f --- /dev/null +++ b/tests/test_sourcehunt_lair_validator.py @@ -0,0 +1,308 @@ +"""Differential replay tests for the LAIR validator harness.""" + +from __future__ import annotations + +import asyncio +import subprocess +from pathlib import Path + +from clearwing.eval.sourcehunt_lair import LairGoldenChain +from clearwing.eval.sourcehunt_lair_validator import ( + SourceWindow, + build_lair_validator_finding, + render_revision_context, + replay_lair_validator_case, + source_windows_for_golden, + summarize_lair_validator_replay, +) +from clearwing.sourcehunt.state import Axes, AxisResult, ValidatorVerdict + + +def _git(repo: Path, *args: str) -> str: + result = subprocess.run( + ["git", *args], + cwd=repo, + text=True, + capture_output=True, + check=True, + ) + return result.stdout.strip() + + +def _replay_golden(repo: Path) -> LairGoldenChain: + repo.mkdir() + _git(repo, "init", "-q") + _git(repo, "config", "user.name", "Test") + _git(repo, "config", "user.email", "test@example.invalid") + source = repo / "src" / "secret_parser.c" + source.parent.mkdir() + vulnerable_lines = ["int filler = 0;" for _ in range(14)] + vulnerable_lines[3] = "parse_secret(packet);" + vulnerable_lines[7] = "remote_counter = packet->counter;" + vulnerable_lines[9] = "state->secret_slot = remote_counter;" + vulnerable_lines[10] = "if (state->secret_slot == SECRET_SENTINEL) {" + vulnerable_lines[11] = "shifted[-1] = packet->byte;" + source.write_text("\n".join(vulnerable_lines) + "\n", encoding="utf-8") + _git(repo, "add", "src/secret_parser.c") + _git(repo, "commit", "-qm", "vulnerable") + vulnerable = _git(repo, "rev-parse", "HEAD") + vulnerable_lines[8] = "if (remote_counter == SECRET_SENTINEL) return ERROR;" + source.write_text("\n".join(vulnerable_lines) + "\n", encoding="utf-8") + _git(repo, "add", "src/secret_parser.c") + _git(repo, "commit", "-qm", "fixed") + fix = _git(repo, "rev-parse", "HEAD") + def citation(revision: str, line: int, excerpt: str) -> dict: + return { + "revision": revision, + "path": "src/secret_parser.c", + "line_start": line, + "line_end": line, + "excerpt": excerpt, + "supports": "This exact source line supports the causal step.", + } + + payload = { + "schema_version": "cwpro.cve-golden-chain.v2", + "cve": "CVE-2099-1001", + "repo": "example/parser", + "vulnerable_commit": vulnerable, + "fix_commit": fix, + "title": "Secret sentinel collision", + "vulnerability_class": "CWE-787 Out-of-bounds Write", + "summary": "An attacker-controlled counter collides with a reserved sentinel value.", + "chain": { + "discovery": { + "id": "discovery", + "candidate": { + "title": "Secret counter collision", + "location": "src/secret_parser.c:parse_secret", + "hypothesis": "A remote counter may collide with a reserved internal value.", + "why_prioritize": "The representation is shared by state and a boundary marker.", + "evidence": [ + citation("vulnerable", 10, "state->secret_slot = remote_counter;") + ], + }, + }, + "investigation": { + "id": "investigation", + "preconditions": ["A remote peer can supply the secret counter."], + "causal_trace": [ + { + "id": "input", + "kind": "attack_source", + "claim": "A remote packet reaches the secret parser entry point.", + "evidence": [citation("vulnerable", 4, "parse_secret(packet);")], + }, + { + "id": "copy", + "kind": "propagation", + "claim": "The packet value propagates into the local counter.", + "evidence": [ + citation("vulnerable", 8, "remote_counter = packet->counter;") + ], + }, + { + "id": "write", + "kind": "vulnerable_operation", + "claim": "The reserved state causes a shifted pointer write.", + "evidence": [ + citation("vulnerable", 12, "shifted[-1] = packet->byte;") + ], + }, + { + "id": "effect", + "kind": "security_effect", + "claim": "The shifted write can modify memory outside the destination.", + "evidence": [ + citation("vulnerable", 12, "shifted[-1] = packet->byte;") + ], + }, + ], + "security_impact": "A remote peer can cause an out-of-bounds memory write.", + }, + "challenge": { + "id": "challenge", + "verdict": "confirmed", + "checks": [ + { + "assumption": "A caller may reject the reserved counter before parsing.", + "conclusion": "The caller forwards the packet without that rejection.", + "evidence": [citation("vulnerable", 4, "parse_secret(packet);")], + } + ], + "fix_validation": { + "strategy": "Reject the reserved counter before changing the state.", + "behavior_before": "The reserved value reached the shifted pointer write.", + "behavior_after": "The reserved value returns an error before state mutation.", + "changed_files": ["src/secret_parser.c"], + "evidence": [ + citation( + "fix", + 9, + "if (remote_counter == SECRET_SENTINEL) return ERROR;", + ) + ], + "regression_tests": [ + { + "status": "proposed", + "description": "Send the reserved remote counter value.", + "expected_result": "Parsing fails before the shifted write.", + } + ], + }, + }, + }, + } + return LairGoldenChain.model_validate(payload) + + +def _verdict(advance: bool) -> ValidatorVerdict: + result = AxisResult(passed=advance, confidence="high", rationale="source-backed") + return ValidatorVerdict( + finding_id="test", + axes=Axes(real=result, triggerable=result, impactful=result, general=result), + advance=advance, + severity_validated="high" if advance else None, + evidence_level="static_corroboration", + pro_argument="strong source chain", + counter_argument="fixed guard", + tie_breaker="current source snapshot", + duplicate_cve=None, + ) + + +def test_same_finding_and_windows_are_used_for_both_revisions(tmp_path: Path) -> None: + golden = _replay_golden(tmp_path / "repo") + finding = build_lair_validator_finding(golden) + windows = source_windows_for_golden(golden, radius=1) + vulnerable = render_revision_context(tmp_path / "repo", golden.vulnerable_commit, windows) + fixed = render_revision_context(tmp_path / "repo", golden.fix_commit, windows) + + assert finding.code_snippet == "shifted[-1] = packet->byte;" + assert [window.path for window in windows] == ["src/secret_parser.c"] + assert "return ERROR" not in vulnerable + assert "return ERROR" in fixed + + +def test_renderer_balances_paths_and_centers_on_vulnerable_anchors(tmp_path: Path) -> None: + repo = tmp_path / "repo" + repo.mkdir() + _git(repo, "init", "-q") + _git(repo, "config", "user.name", "Test") + _git(repo, "config", "user.email", "test@example.invalid") + for name, marker in (("a.c", "FIRST_ANCHOR"), ("z.c", "LAST_ANCHOR")): + lines = [f"int filler_{index};" for index in range(1, 81)] + lines[39] = marker + (repo / name).write_text("\n".join(lines) + "\n", encoding="utf-8") + _git(repo, "add", "a.c", "z.c") + _git(repo, "commit", "-qm", "source") + revision = _git(repo, "rev-parse", "HEAD") + windows = [ + SourceWindow(path="a.c", start=1, end=80, anchors=(40,)), + SourceWindow(path="z.c", start=1, end=80, anchors=(40,)), + ] + + context = render_revision_context(repo, revision, windows, max_chars=1200) + + assert "FIRST_ANCHOR" in context + assert "LAST_ANCHOR" in context + assert len(context) <= 1200 + + +def test_renderer_uses_identical_line_selection_across_revisions(tmp_path: Path) -> None: + repo = tmp_path / "repo" + repo.mkdir() + _git(repo, "init", "-q") + _git(repo, "config", "user.name", "Test") + _git(repo, "config", "user.email", "test@example.invalid") + source = repo / "source.c" + source.write_text( + "\n".join(f"int old_{index};" for index in range(1, 101)) + "\n", + encoding="utf-8", + ) + _git(repo, "add", "source.c") + _git(repo, "commit", "-qm", "old") + old = _git(repo, "rev-parse", "HEAD") + source.write_text( + "\n".join(f"int new_{index}_with_longer_text;" for index in range(1, 101)) + + "\n", + encoding="utf-8", + ) + _git(repo, "add", "source.c") + _git(repo, "commit", "-qm", "new") + new = _git(repo, "rev-parse", "HEAD") + windows = [SourceWindow(path="source.c", start=1, end=100, anchors=(50,))] + + old_context = render_revision_context(repo, old, windows, max_chars=1200) + new_context = render_revision_context(repo, new, windows, max_chars=1200) + old_lines = [line.split(":", 1)[0] for line in old_context.splitlines()[1:]] + new_lines = [line.split(":", 1)[0] for line in new_context.splitlines()[1:]] + + assert old_lines == new_lines + + +def test_replay_scores_vulnerable_advance_and_fixed_rejection(tmp_path: Path) -> None: + golden = _replay_golden(tmp_path / "repo") + calls: list[tuple[str, str]] = [] + + async def validate(finding, source_context): + calls.append((finding.description, source_context)) + return _verdict("return ERROR" not in source_context) + + result = asyncio.run(replay_lair_validator_case(golden, tmp_path / "repo", validate)) + + assert len(calls) == 2 + assert calls[0][0] == calls[1][0] + assert result.vulnerable_correct is True + assert result.fixed_correct is True + assert result.pair_correct is True + + +def test_summary_reports_false_negatives_and_fixed_false_positives(tmp_path: Path) -> None: + golden = _replay_golden(tmp_path / "repo") + + async def reject_both(_finding, _source_context): + return _verdict(False) + + result = asyncio.run( + replay_lair_validator_case(golden, tmp_path / "repo", reject_both) + ) + summary = summarize_lair_validator_replay([result], model="small-model") + + assert summary.prompt_profile == "legacy-v1" + assert summary.temperature is None + assert summary.vulnerable_recall == 0.0 + assert summary.fixed_rejection_rate == 1.0 + assert summary.pair_accuracy == 0.0 + assert summary.vulnerable_false_negatives == 1 + assert summary.fixed_false_positives == 0 + + +def test_model_error_is_never_scored_as_a_correct_fixed_rejection(tmp_path: Path) -> None: + golden = _replay_golden(tmp_path / "repo") + empty = ValidatorVerdict( + finding_id="test", + axes=Axes(), + advance=False, + severity_validated=None, + evidence_level="suspicion", + pro_argument="", + counter_argument="", + tie_breaker="validator error", + duplicate_cve=None, + ) + + async def fail_both(_finding, _source_context): + return empty + + result = asyncio.run( + replay_lair_validator_case(golden, tmp_path / "repo", fail_both) + ) + summary = summarize_lair_validator_replay([result], model="small-model") + + assert result.vulnerable.model_error is True + assert result.fixed.model_error is True + assert result.fixed_correct is False + assert summary.fixed_rejection_rate == 0.0 + assert summary.fixed_false_positives == 1 + assert summary.model_errors == 2 diff --git a/tests/test_sourcehunt_optimization.py b/tests/test_sourcehunt_optimization.py new file mode 100644 index 00000000..0f332f0e --- /dev/null +++ b/tests/test_sourcehunt_optimization.py @@ -0,0 +1,1067 @@ +"""Prompt/scaffold optimization seams and benchmark leakage guards.""" + +from __future__ import annotations + +import asyncio +import json +from pathlib import Path +from unittest.mock import MagicMock + +import pytest +from genai_pyo3 import ChatMessage, ToolCall, Usage + +from clearwing.agent.tools.hunt.candidates import build_candidate_tools +from clearwing.agent.tools.hunt.reporting import build_reporting_tools +from clearwing.agent.tools.hunt.sandbox import HunterContext +from clearwing.agent.tools.hunt.windows import rank_source_windows +from clearwing.eval.sourcehunt import GroundTruthManifest +from clearwing.sourcehunt.context import SourceHuntContextManager, estimate_request_tokens +from clearwing.sourcehunt.hunter import build_hunter_agent +from clearwing.sourcehunt.optimization import ( + GENERIC_DISCOVERY_V1, + get_context_profile, + get_prompt_bundle, + get_scaffold_profile, + lint_prompt_candidate, + redact_benchmark_terms, + require_generic_prompt, +) +from clearwing.sourcehunt.static_signals import ( + is_production_source_path, + score_source_security_signals, +) + + +def _target(path: str = "src/parser.c") -> dict: + return { + "path": path, + "tier": "B", + "language": "c", + "loc": 100, + "tags": ["parser", "memory_unsafe"], + "imports_by": 0, + } + + +def test_generic_prompt_template_passes_full_manifest_leakage_lint() -> None: + manifest = GroundTruthManifest.load("evaluations/sourcehunt_ground_truth.yaml") + + require_generic_prompt(GENERIC_DISCOVERY_V1, manifest=manifest) + + +def test_leakage_linter_rejects_answer_bearing_prompt() -> None: + manifest = GroundTruthManifest.load("evaluations/sourcehunt_ground_truth.yaml") + case = manifest.cases[0] + candidate = ( + f"Audit {case.repository} and {case.ground_truth.target_files[0]} " + f"at {case.vulnerable_commit}. " + f"Look for {case.ground_truth.expected_mechanisms[0]} and CWE-787." + ) + + leaks = lint_prompt_candidate(candidate, manifest=manifest) + + categories = {leak.category for leak in leaks} + assert {"repository", "commit", "target_file", "mechanism", "expected_cwe"} <= categories + with pytest.raises(ValueError, match="leaks benchmark answers"): + require_generic_prompt(candidate, manifest=manifest) + + redacted = redact_benchmark_terms(candidate, manifest) + assert not lint_prompt_candidate(redacted, manifest=manifest) + + +def test_generic_bundle_excludes_historical_solution_context() -> None: + hunter, ctx = build_hunter_agent( + file_target=_target(), + repo_path=str(Path("tests/fixtures/vuln_samples/c_propagation")), + sandbox=None, + llm=MagicMock(), + session_id="generic", + prompt_mode="specialist", + prompt_bundle="generic-security-v1", + seed_context="CVE-2099-99999 known_solution_symbol", + ) + + assert ctx.specialist == "generic" + assert "ORIENT" in hunter.prompt + assert "CHALLENGE" in hunter.prompt + assert "CVE-2099-99999" not in hunter.prompt + assert "known_solution_symbol" not in hunter.prompt + assert "SENTINEL / COUNTER COLLISIONS" not in hunter.prompt + + +def test_minimal_linear_scaffold_reduces_constrained_tool_surface() -> None: + hunter, _ = build_hunter_agent( + file_target=_target(), + repo_path=str(Path("tests/fixtures/vuln_samples/c_propagation")), + sandbox=None, + llm=MagicMock(), + session_id="minimal", + prompt_bundle="generic-security-v1", + scaffold_profile="minimal-linear-v1", + ) + + assert {tool.name for tool in hunter.tools} == { + "read_source_file", + "grep_source", + "record_trace_step", + "record_finding", + } + assert "READ -> CANDIDATES -> INVESTIGATE -> CHALLENGE -> SUBMIT" in hunter.prompt + + +def test_minimal_linear_scaffold_reduces_deep_tool_surface() -> None: + hunter, _ = build_hunter_agent( + file_target=_target(), + repo_path=str(Path("tests/fixtures/vuln_samples/c_propagation")), + sandbox=None, + llm=MagicMock(), + session_id="minimal-deep", + agent_mode="deep", + prompt_bundle="generic-security-v1", + scaffold_profile="minimal-linear-v1", + ) + + assert {tool.name for tool in hunter.tools} == { + "execute", + "read_file", + "record_trace_step", + "record_finding", + } + + +def test_hunter_step_override_bounds_deep_optimization_runs() -> None: + hunter, _ = build_hunter_agent( + file_target=_target(), + repo_path=str(Path("tests/fixtures/vuln_samples/c_propagation")), + sandbox=None, + llm=MagicMock(), + session_id="bounded-deep", + agent_mode="deep", + prompt_bundle="generic-security-v1", + scaffold_profile="minimal-linear-v1", + max_steps_override=40, + ) + + assert hunter.max_steps == 40 + + +def test_candidate_ledger_scaffold_persists_explicit_hypotheses() -> None: + hunter, ctx = build_hunter_agent( + file_target=_target(), + repo_path=str(Path("tests/fixtures/vuln_samples/c_propagation")), + sandbox=None, + llm=MagicMock(), + session_id="candidate-ledger", + prompt_bundle="generic-security-v1", + scaffold_profile="candidate-ledger-v1", + ) + tools = {tool.name: tool for tool in hunter.tools} + + output = tools["record_candidate"].invoke( + { + "candidate_id": "C1", + "status": "investigating", + "file": "src/parser.c", + "line": 42, + "hypothesis": "unchecked length may exceed an allocation", + "attacker_control": "packet length", + "invariant": "copy length stays within the destination", + "effect": "out-of-bounds write", + "counterargument": "a caller may cap the length", + "next_check": "inspect all callers for a dominating cap", + "evidence": "allocation and copy use different expressions", + } + ) + + assert set(tools) == { + "read_source_file", + "grep_source", + "record_candidate", + "record_trace_step", + "record_finding", + } + assert ctx.candidates["C1"]["next_check"] == "inspect all callers for a dominating cap" + assert "Active queue (1)" in output + assert "Do not sweep the file sequentially" in hunter.prompt + + +def test_candidate_ledger_closure_is_a_versioned_runtime_treatment() -> None: + hunter, _ = build_hunter_agent( + file_target=_target(), + repo_path=str(Path("tests/fixtures/vuln_samples/c_propagation")), + sandbox=None, + llm=MagicMock(), + session_id="candidate-ledger-closure", + prompt_bundle="generic-security-v1", + scaffold_profile="candidate-ledger-closure-v1", + ) + + assert {tool.name for tool in hunter.tools} == { + "read_source_file", + "grep_source", + "record_candidate", + "record_trace_step", + "record_finding", + } + assert hunter.closing_steps == 3 + assert "Budget closure" not in hunter.prompt + + +def test_candidate_ledger_source_retry_is_a_versioned_runtime_treatment() -> None: + hunter, _ = build_hunter_agent( + file_target=_target(), + repo_path=str(Path("tests/fixtures/vuln_samples/c_propagation")), + sandbox=None, + llm=MagicMock(), + session_id="candidate-ledger-source-retry", + prompt_bundle="generic-security-v1", + scaffold_profile="candidate-ledger-source-retry-v1", + ) + + assert hunter.initial_source_action_retries == 1 + assert "No source tool ran" not in hunter.prompt + + +def test_candidate_ledger_active_submission_gate_is_versioned() -> None: + hunter, ctx = build_hunter_agent( + file_target=_target(), + repo_path=str(Path("tests/fixtures/vuln_samples/c_propagation")), + sandbox=None, + llm=MagicMock(), + session_id="candidate-ledger-source-retry-active", + prompt_bundle="generic-security-v1", + scaffold_profile="candidate-ledger-source-retry-active-v1", + ) + + assert hunter.initial_source_action_retries == 1 + assert ctx.require_active_candidate_before_finding is True + assert "No source tool ran" not in hunter.prompt + + +def test_generic_source_window_ranker_prioritizes_diverse_risky_regions() -> None: + lines = ["int harmless = 0;" for _ in range(220)] + lines[19] = "char *buf = malloc(user_len);" + lines[109] = "state->generation = ++global_generation;" + lines[189] = "memcpy(dst, packet, packet_len);" + + windows = rank_source_windows("\n".join(lines), max_windows=3, window_lines=40) + + assert [window["anchor_line"] for window in windows] == [190, 20, 110] + assert {signal for window in windows for signal in window["signals"]} >= { + "memory_operation", + "allocation_lifetime", + "representation_transition", + } + + +def test_file_signal_score_caps_repetition_and_rewards_category_coverage() -> None: + repeated = "\n".join("memcpy(dst, src, len);" for _ in range(100)) + diverse = """ + packet = read_input(); + if (packet_size + offset > buffer_len) return -1; + dst[index] = (uint16_t)packet[value]; + memset(state, -1, sizeof(*state)); + free(state); + """ + + repeated_score = score_source_security_signals(repeated) + diverse_score = score_source_security_signals(diverse) + + assert diverse_score.score > repeated_score.score + assert diverse_score.diversity > repeated_score.diversity + assert repeated_score.counts["memory_operation"] == 100 + + +def test_production_source_filter_is_repository_independent() -> None: + assert is_production_source_path("src/codec/decode.c") is True + assert is_production_source_path("docs/examples/decode.c") is False + assert is_production_source_path("tools/target_decoder_fuzzer.c") is False + + +def test_window_ledger_requires_ranked_windows_before_candidates() -> None: + hunter, ctx = build_hunter_agent( + file_target=_target("src/codec_a.c"), + repo_path=str(Path("tests/fixtures/vuln_samples/c_propagation")), + sandbox=None, + llm=MagicMock(), + session_id="window-ledger", + prompt_bundle="generic-security-v1", + scaffold_profile="window-ledger-v1", + ) + tools = {tool.name: tool for tool in hunter.tools} + + result = tools["rank_source_windows"].invoke( + {"path": "src/codec_a.c", "max_windows": 4, "window_lines": 40} + ) + + assert set(tools) == { + "rank_source_windows", + "read_source_file", + "grep_source", + "record_candidate", + "record_trace_step", + "record_finding", + } + assert ctx.source_windows_ranked is True + assert result["path"] == "src/codec_a.c" + assert hunter.require_source_windows is True + + +def test_guided_window_ledger_reads_ranked_windows_without_line_translation() -> None: + hunter, ctx = build_hunter_agent( + file_target=_target("src/codec_a.c"), + repo_path=str(Path("tests/fixtures/vuln_samples/c_propagation")), + sandbox=None, + llm=MagicMock(), + session_id="guided-window-ledger", + prompt_bundle="generic-security-v1", + scaffold_profile="guided-window-ledger-v1", + context_profile="compact-small-model-v1", + ) + tools = {tool.name: tool for tool in hunter.tools} + + plan = tools["rank_source_windows"].invoke( + {"path": "src/codec_a.c", "max_windows": 4, "window_lines": 40} + ) + first = tools["read_ranked_window"].invoke({"window_id": "W1"}) + repeated_plan = tools["rank_source_windows"].invoke({"path": "src/codec_a.c"}) + + assert plan["windows"][0]["window_id"] == "W1" + assert "src/codec_a.c:" in first + assert ctx.source_windows_read == {"W1"} + assert "already exists" in repeated_plan["instruction"] + assert hunter.ranked_windows_before_candidate == 3 + + +def test_state_interaction_packet_connects_generic_state_roles(tmp_path) -> None: + source = tmp_path / "src" / "target.c" + header = tmp_path / "src" / "target.h" + source.parent.mkdir() + source.write_text( + """ + #include "target.h" + void reset(Context *ctx) { memset(ctx->slot_state, -1, sizeof(ctx->slot_state)); } + void update(Context *ctx, int i) { ctx->slot_state[i] = ctx->generation; } + int check(Context *ctx, int i) { return ctx->slot_state[i] == 0xFFFF; } + void consume(Context *ctx, int i, char *dst, const char *src) { + int use_neighbor = ctx->slot_state[i] == 0xFFFF; + char *selected = ctx->borders[i - 1]; + if (use_neighbor) { + memcpy(selected, src, 8); + } + } + void advance(Context *ctx) { ctx->generation = ++global_generation; } + """, + encoding="utf-8", + ) + header.write_text( + """ + typedef struct Context { + uint16_t slot_state[64]; + int generation; + } Context; + """, + encoding="utf-8", + ) + hunter, ctx = build_hunter_agent( + file_target=_target("src/target.c"), + repo_path=str(tmp_path), + sandbox=None, + llm=MagicMock(), + session_id="state-interactions", + prompt_bundle="generic-security-v1", + scaffold_profile="state-interaction-ledger-v1", + context_profile="compact-small-model-v1", + ) + tools = {tool.name: tool for tool in hunter.tools} + + plan = tools["rank_source_windows"].invoke( + {"path": "src/target.c", "max_windows": 3, "window_lines": 20} + ) + tools["read_ranked_window"].invoke({"window_id": "W1"}) + packet = tools["read_state_interactions"].invoke({"window_id": "W1"}) + + assert set(tools) == { + "rank_source_windows", + "read_ranked_window", + "read_state_interactions", + "read_domain_consequences", + "record_value_domain", + "record_domain_consequence", + "record_domain_proof", + "read_source_file", + "grep_source", + "record_candidate", + "record_trace_step", + "record_finding", + } + assert plan["windows"][0]["window_id"] == "W1" + assert "Primary state: slot_state" in packet + assert "uint16_t slot_state[64]" in packet + assert "slot_state[i] = ctx->generation" in packet + assert "slot_state[i] == 0xFFFF" in packet + assert "generation = ++global_generation" in packet + assert ctx.value_domain_plans["D1"]["producer_tokens"] == ["generation"] + tools["record_value_domain"].invoke( + { + "domain_id": "D1", + "guard": "none observed", + "assessment": "overlap_possible", + "evidence": "the producer reaches storage and storage reserves 0xFFFF", + "next_check": "check whether generation can equal 0xFFFF", + } + ) + consequence_packet = tools["read_domain_consequences"].invoke({"domain_id": "D1"}) + assert "Derived branch-to-effect chains:" in consequence_packet + assert "use_neighbor" in consequence_packet + assert "memcpy(selected, src, 8)" in consequence_packet + assert {"use_neighbor", "memcpy", "selected"} <= set( + ctx.domain_consequence_plans["D1"]["impact_tokens"] + ) + assert ctx.domain_consequence_plans["D1"]["boundary_facts"] == [ + {"line": 8, "token": "i", "expression": "i - 1"} + ] + tools["record_candidate"].invoke( + { + "candidate_id": "C1", + "status": "investigating", + "file": "src/target.c", + "line": 5, + "hypothesis": "slot_state can store a generation that aliases reserved state", + "attacker_control": "input-driven generation advances", + "invariant": "slot_state live generations differ from distinguished values", + "effect": "the changed branch reaches a boundary memory effect", + "counterargument": "a producer guard may exclude the value", + "next_check": "resolve the four domain proof obligations", + "evidence": "slot_state and generation interact across the packet", + } + ) + proof = tools["record_domain_proof"].invoke( + { + "domain_id": "D1", + "candidate_id": "C1", + "attacker_reaches_producer": True, + "producer_reaches_distinguished": True, + "changed_branch_reaches_effect": True, + "boundary_effect_unguarded": True, + "evidence": "src/target.c:5, src/target.c:6, src/target.c:7, src/target.c:8", + "counterevidence": "none observed", + } + ) + + assert "validated C1" in proof + assert ctx.candidates["C1"]["status"] == "validated" + assert [step.note for step in ctx.trace_steps] == [ + "source", + "state sink", + "condition", + "effect sink", + ] + finding = tools["record_finding"].invoke( + { + "candidate_id": "C1", + "file": "src/target.c", + "line_number": 8, + "finding_type": "memory_safety", + "severity": "high", + "description": "A validated state-domain collision reaches a boundary memory effect.", + "evidence_level": "static_corroboration", + } + ) + assert "Finding recorded" in finding + assert [step["note"] for step in ctx.findings[0].vulnerability_trace["steps"]] == [ + "source", + "state sink", + "condition", + "effect sink", + ] + assert ctx.state_packets_read == {"W1"} + assert hunter.state_packets_before_candidate == 1 + + +def test_proof_refinement_is_on_demand_and_obligation_specific(tmp_path) -> None: + source = tmp_path / "src" / "target.c" + source.parent.mkdir() + source.write_text( + """ + int produce(Context *ctx) { return ++ctx->generation; } + int dispatch(Context *ctx) { return produce(ctx); } + int parse(Context *ctx, int count) { + for (int i = 0; i < count; i++) { + dispatch(ctx); + } + return 0; + } + """, + encoding="utf-8", + ) + hunter, ctx = build_hunter_agent( + file_target=_target("src/target.c"), + repo_path=str(tmp_path), + sandbox=None, + llm=MagicMock(), + session_id="refinement", + prompt_bundle="generic-security-v1", + scaffold_profile="proof-refinement-ledger-v1", + context_profile="compact-small-model-v1", + ) + ctx.value_domains["D1"] = { + "domain_id": "D1", + "target_path": "src/target.c", + "trace_facts": [ + { + "role": "source", + "file": "src/target.c", + "line": 2, + "code_snippet": "return ++ctx->generation;", + } + ], + } + ctx.domain_consequence_plans["D1"] = {"trace_facts": [], "boundary_facts": []} + tools = {tool.name: tool for tool in hunter.tools} + + before_proof = tools["read_domain_proof_refinement"].invoke( + {"domain_id": "D1", "obligation": "attacker_reaches_producer"} + ) + assert "call record_domain_proof" in before_proof + + ctx.domain_proof_obligations["D1"] = ["attacker_reaches_producer"] + wrong_obligation = tools["read_domain_proof_refinement"].invoke( + {"domain_id": "D1", "obligation": "changed_branch_reaches_effect"} + ) + packet = tools["read_domain_proof_refinement"].invoke( + {"domain_id": "D1", "obligation": "attacker_reaches_producer"} + ) + repeated = tools["read_domain_proof_refinement"].invoke( + {"domain_id": "D1", "obligation": "attacker_reaches_producer"} + ) + + assert "refine only a recorded unresolved obligation" in wrong_obligation + assert "Proof refinement for attacker_reaches_producer" in packet + assert "for (int i = 0; i < count; i++)" in packet + assert "dispatch(ctx)" in packet + assert len(packet) < 1_200 + assert "already read" in repeated + assert ctx.domain_refinement_pending_proof == {"D1"} + + +def test_refinement_schema_is_hidden_until_proof_records_a_gap() -> None: + hunter, ctx = build_hunter_agent( + file_target=_target(), + repo_path=str(Path("tests/fixtures/vuln_samples/c_propagation")), + sandbox=None, + llm=MagicMock(), + session_id="refinement-schema", + prompt_bundle="generic-security-v1", + scaffold_profile="proof-refinement-ledger-v1", + context_profile="compact-small-model-v1", + ) + + assert "read_domain_proof_refinement" not in { + tool.name for tool in hunter._request_tools() + } + ctx.domain_proof_obligations["D1"] = ["attacker_reaches_producer"] + assert "read_domain_proof_refinement" in { + tool.name for tool in hunter._request_tools() + } + + +def test_value_domain_closure_requires_producer_bound_and_distinguished_value( + tmp_path, +) -> None: + ctx = HunterContext(repo_path=str(tmp_path)) + ctx.value_domain_plans["D1"] = { + "stored_state": "slot_state", + "producer_state": "generation", + "producer_tokens": ["generation", "global_generation"], + "blocking_guard_locations": ["src/target.c:9"], + "distinguished_tokens": ["0xFFFF"], + } + ctx.domain_consequence_plans["D1"] = {"domain_id": "D1"} + tools = {tool.name: tool for tool in build_candidate_tools(ctx)} + + opened = tools["record_value_domain"].invoke( + { + "domain_id": "D1", + "guard": "none observed", + "assessment": "overlap_possible", + "evidence": "producer reaches storage and storage reserves 0xFFFF", + "next_check": "check whether generation can equal 0xFFFF", + } + ) + consequence = tools["record_domain_consequence"].invoke( + { + "domain_id": "D1", + "branch_effect": "reserved-state branch accepts a live value", + "state_effect": "live state is misclassified", + "security_effect": "memory access uses the wrong neighbor state", + "assessment": "security_effect_possible", + "evidence": "consumer branch and downstream memory operation", + "next_check": "trace generation reaching 0xFFFF", + } + ) + + unrelated = tools["record_value_domain"].invoke( + { + "domain_id": "D1", + "guard": "allocation size bounds all table indexes", + "assessment": "overlap_blocked", + "evidence": "the allocation is in bounds and generation would need 0xFFFF", + "next_check": "none", + } + ) + premature_benign = tools["record_domain_consequence"].invoke( + { + "domain_id": "D1", + "branch_effect": "table indexes are in bounds", + "state_effect": "allocation is large enough", + "security_effect": "none", + "assessment": "benign", + "evidence": "allocation calculation and maximum index", + "next_check": "none", + } + ) + practical_limit = tools["record_value_domain"].invoke( + { + "domain_id": "D1", + "guard": "generation needs 65535 increments after each reset, which is impractical", + "assessment": "overlap_blocked", + "evidence": "generation would need to reach distinguished value 0xFFFF", + "next_check": "none", + } + ) + + assert "overlap_possible" in opened + assert "security_effect_possible" in consequence + assert "must constrain the extracted producer chain" in unrelated + assert "requires an extracted source guard" in practical_limit + assert "cannot be marked benign" in premature_benign + assert ctx.value_domains["D1"]["assessment"] == "overlap_possible" + assert ctx.domain_consequences["D1"]["assessment"] == "security_effect_possible" + + closed = tools["record_value_domain"].invoke( + { + "domain_id": "D1", + "guard": "src/target.c:9 rejects when global_generation >= 0xFFFF", + "assessment": "disjoint", + "evidence": ( + "src/target.c:9 returns before assignment and prevents generation from taking " + "distinguished value 0xFFFF" + ), + "next_check": "confirm the guard dominates the transfer", + } + ) + + assert "disjoint" in closed + assert ctx.value_domains["D1"]["assessment"] == "disjoint" + + +def test_candidate_domain_matching_ignores_unresolved_placeholders() -> None: + ctx = HunterContext(repo_path="/repo") + ctx.value_domains["D1"] = { + "stored_state": "interim", + "producer_state": "unknown", + "producer_tokens": [], + "distinguished_tokens": [], + "assessment": "overlap_possible", + } + ctx.domain_consequences["D1"] = {"assessment": "security_effect_possible"} + tools = {tool.name: tool for tool in build_candidate_tools(ctx)} + + result = tools["record_candidate"].invoke( + { + "candidate_id": "C1", + "status": "investigating", + "file": "src/decoder.c", + "hypothesis": "interim samples may cross the allocated block boundary", + "invariant": "interim remains within its allocation", + "evidence": "the copy length and allocation use different bounds", + } + ) + + assert "Candidate C1 saved" in result + assert ctx.domain_candidate_ids == {"D1": "C1"} + assert "unknown" not in ctx.candidates["C1"]["next_check"] + + +def test_domain_proof_requires_guard_reassessment_and_all_obligations(tmp_path) -> None: + ctx = HunterContext(repo_path=str(tmp_path)) + ctx.enable_domain_proof_refinement = True + ctx.value_domains["D1"] = { + "domain_id": "D1", + "stored_state": "slot_state", + "producer_state": "generation", + "assessment": "overlap_possible", + "blocking_guard_locations": ["src/target.c:9"], + "trace_facts": [], + } + ctx.domain_consequence_plans["D1"] = { + "domain_id": "D1", + "boundary_facts": [{"line": 12, "token": "i", "expression": "i - 1"}], + "trace_facts": [], + } + ctx.domain_candidate_ids["D1"] = "C1" + ctx.candidates["C1"] = {"candidate_id": "C1", "status": "investigating"} + tools = {tool.name: tool for tool in build_candidate_tools(ctx)} + proof_args = { + "domain_id": "D1", + "candidate_id": "C1", + "attacker_reaches_producer": True, + "producer_reaches_distinguished": True, + "changed_branch_reaches_effect": True, + "boundary_effect_unguarded": True, + "evidence": "src/target.c:3, src/target.c:7, src/target.c:12", + "counterevidence": "src/target.c:9 may terminate the producer path", + } + + guarded = tools["record_domain_proof"].invoke(proof_args) + assert "record_value_domain first" in guarded + assert ctx.candidates["C1"]["status"] == "investigating" + + ctx.value_domains["D1"]["blocking_guard_locations"] = [] + unresolved = tools["record_domain_proof"].invoke( + { + **proof_args, + "producer_reaches_distinguished": False, + "changed_branch_reaches_effect": False, + } + ) + assert "producer reaches distinguished value" in unresolved + assert "changed branch reaches effect" not in unresolved + assert "read_domain_proof_refinement" in unresolved + assert ctx.candidates["C1"]["status"] == "investigating" + assert ctx.candidates["C1"]["next_check"] == ( + "Resolve: producer reaches distinguished value." + ) + assert ctx.domain_proof_obligations["D1"] == ["producer_reaches_distinguished"] + + ctx.require_validated_candidate_before_finding = True + reporting = {tool.name: tool for tool in build_reporting_tools(ctx)} + rejected = reporting["record_finding"].invoke( + { + "candidate_id": "C1", + "file": "src/target.c", + "line_number": 12, + "finding_type": "memory_safety", + "severity": "high", + "description": "Unresolved candidate must not escape.", + "evidence_level": "static_corroboration", + } + ) + assert "already marked validated" in rejected + + +def test_state_interaction_gate_sequence_can_reach_candidate(tmp_path) -> None: + source = tmp_path / "src" / "target.c" + source.parent.mkdir() + source.write_text( + """ + void reset(State *s) { memset(s->slots, -1, sizeof(s->slots)); } + void update(State *s, int i) { s->slots[i] = s->generation; } + int check(State *s, int i) { return s->slots[i] == 0xFFFF; } + """, + encoding="utf-8", + ) + candidate_arguments = { + "candidate_id": "C1", + "status": "investigating", + "file": "src/target.c", + "line": 2, + "hypothesis": "slots may store a generation that aliases reserved state", + "attacker_control": "number of updates", + "invariant": "live generation differs from the reserved value", + "effect": "state validation bypass", + "counterargument": "generation may be range limited", + "next_check": "inspect whether generation can equal the -1 distinguished value", + "evidence": "different representations share a comparison", + } + calls = [ + ToolCall("rank", "rank_source_windows", '{"path":"src/target.c"}'), + ToolCall("w1", "read_ranked_window", '{"window_id":"W1"}'), + ToolCall("packet", "read_state_interactions", '{"window_id":"W1"}'), + ToolCall( + "domain", + "record_value_domain", + json.dumps( + { + "domain_id": "D1", + "guard": "none observed", + "assessment": "overlap_possible", + "evidence": "src/target.c:2 and src/target.c:3", + "next_check": "inspect whether generation can equal the -1 value", + } + ), + ), + ToolCall( + "consequence_packet", + "read_domain_consequences", + '{"domain_id":"D1"}', + ), + ToolCall( + "consequence", + "record_domain_consequence", + json.dumps( + { + "domain_id": "D1", + "branch_effect": "reserved-state branch accepts a live generation", + "state_effect": "neighbor state is misclassified", + "security_effect": "unsafe state may reach a memory access", + "assessment": "security_effect_possible", + "evidence": "src/target.c:2 and src/target.c:3", + "next_check": "trace the changed branch to its first memory effect", + } + ), + ), + ToolCall("candidate", "record_candidate", json.dumps(candidate_arguments)), + ToolCall( + "candidate-drift", + "record_candidate", + json.dumps( + { + **candidate_arguments, + "hypothesis": "slots allocation may be too small for generation writes", + "invariant": "slots allocation covers generation indexes", + "next_check": "inspect the allocation-size calculation", + } + ), + ), + ] + + class StubLLM: + model_name = "stub" + + def __init__(self): + self.index = 0 + + async def achat(self, **_: object): + class Response: + first_text = "" + texts = [] + reasoning_content = None + provider_model_name = "stub" + + def __init__(self, tool_calls): + self.tool_calls = tool_calls + self.usage = Usage(prompt_tokens=10, completion_tokens=2, total_tokens=12) + + if self.index < len(calls): + call = calls[self.index] + self.index += 1 + return Response([call]) + response = Response([]) + response.first_text = "done" + response.texts = ["done"] + return response + + hunter, ctx = build_hunter_agent( + file_target=_target("src/target.c"), + repo_path=str(tmp_path), + sandbox=None, + llm=StubLLM(), + session_id="state-interaction-sequence", + prompt_bundle="generic-security-v1", + scaffold_profile="state-interaction-ledger-v1", + context_profile="compact-small-model-v1", + max_steps_override=9, + input_price_per_million=0.0, + output_price_per_million=0.0, + ) + ctx.trajectory_dir = tmp_path / "trajectory" + + result = asyncio.run(hunter.arun()) + + assert result.stop_reason == "completed" + assert ctx.source_windows_read == {"W1"} + assert ctx.state_packets_read == {"W1"} + assert ctx.value_domains["D1"]["assessment"] == "overlap_possible" + assert ctx.domain_consequences["D1"]["assessment"] == "security_effect_possible" + assert ctx.candidates["C1"]["status"] == "investigating" + assert ctx.candidates["C1"]["hypothesis"] == candidate_arguments["hypothesis"] + + +def test_guided_window_gate_sequence_can_reach_candidate(tmp_path) -> None: + source = tmp_path / "src" / "target.c" + source.parent.mkdir() + lines = ["int harmless = 0;" for _ in range(240)] + lines[20] = "memcpy(dst, input, input_len);" + lines[110] = "state->generation = ++global_generation;" + lines[200] = "if (index >= count) return -1;" + source.write_text("\n".join(lines), encoding="utf-8") + + candidate_arguments = { + "candidate_id": "C1", + "status": "investigating", + "file": "src/target.c", + "line": 21, + "hypothesis": "copy length may exceed the destination", + "attacker_control": "input_len", + "invariant": "copy fits", + "effect": "out-of-bounds write", + "counterargument": "a caller may cap input_len", + "next_check": "inspect callers", + "evidence": "copy uses external length", + } + calls = [ + ToolCall("rank", "rank_source_windows", '{"path":"src/target.c"}'), + ToolCall("w1", "read_ranked_window", '{"window_id":"W1"}'), + ToolCall("w2", "read_ranked_window", '{"window_id":"W2"}'), + ToolCall("w3", "read_ranked_window", '{"window_id":"W3"}'), + ToolCall("candidate", "record_candidate", json.dumps(candidate_arguments)), + ] + + class StubLLM: + model_name = "stub" + + def __init__(self): + self.index = 0 + + async def achat(self, **_: object): + class Response: + first_text = "" + texts = [] + reasoning_content = None + provider_model_name = "stub" + + def __init__(self, tool_calls): + self.tool_calls = tool_calls + self.usage = Usage( + prompt_tokens=10, + completion_tokens=2, + total_tokens=12, + ) + + if self.index < len(calls): + call = calls[self.index] + self.index += 1 + return Response([call]) + response = Response([]) + response.first_text = "done" + response.texts = ["done"] + return response + + hunter, ctx = build_hunter_agent( + file_target=_target("src/target.c"), + repo_path=str(tmp_path), + sandbox=None, + llm=StubLLM(), + session_id="guided-sequence", + prompt_bundle="generic-security-v1", + scaffold_profile="guided-window-ledger-v1", + context_profile="compact-small-model-v1", + max_steps_override=6, + input_price_per_million=0.0, + output_price_per_million=0.0, + ) + ctx.trajectory_dir = tmp_path / "trajectory" + + result = asyncio.run(hunter.arun()) + + assert result.stop_reason == "completed" + assert ctx.source_windows_read == {"W1", "W2", "W3"} + assert ctx.candidates["C1"]["status"] == "investigating" + + +def test_compact_context_reduces_static_payload_and_clips_tool_results() -> None: + legacy, _ = build_hunter_agent( + file_target=_target("src/codec_a.c"), + repo_path=str(Path("tests/fixtures/vuln_samples/c_propagation")), + sandbox=None, + llm=MagicMock(), + session_id="legacy-context", + prompt_bundle="generic-security-v1", + scaffold_profile="window-ledger-v1", + context_profile="legacy-context-v1", + ) + compact, _ = build_hunter_agent( + file_target=_target("src/codec_a.c"), + repo_path=str(Path("tests/fixtures/vuln_samples/c_propagation")), + sandbox=None, + llm=MagicMock(), + session_id="compact-context", + prompt_bundle="generic-security-v1", + scaffold_profile="window-ledger-v1", + context_profile="compact-small-model-v1", + ) + initial = [ChatMessage("user", "Hunt.")] + manifest = GroundTruthManifest.load("evaluations/sourcehunt_ground_truth.yaml") + + assert compact.context_manager is not None + assert compact.summarizer is None + assert compact.tool_result_chars == 3_500 + assert estimate_request_tokens( + initial, system=compact.prompt, tools=compact.tools + ) < estimate_request_tokens(initial, system=legacy.prompt, tools=legacy.tools) + assert len(compact._tool_output_text("read_source_file", {}, "x" * 10_000)) < 3_600 + require_generic_prompt(compact.prompt, manifest=manifest) + + +def test_compaction_preserves_durable_state_and_complete_protocol_groups() -> None: + ctx = HunterContext(repo_path="/repo", file_path="src/parser.c") + ctx.source_windows_ranked = True + ctx.candidates = { + "C1": { + "candidate_id": "C1", + "status": "investigating", + "file": "src/parser.c", + "line": 42, + "hypothesis": "length may exceed allocation", + "attacker_control": "packet length", + "invariant": "copy fits", + "effect": "out-of-bounds write", + "counterargument": "caller may cap length", + "next_check": "inspect caller cap", + "evidence": "different size expressions", + }, + "C0": { + "candidate_id": "C0", + "status": "rejected", + "file": "src/parser.c", + "line": 12, + "hypothesis": "signed index", + "counterargument": "range check dominates", + "evidence": "all callers reject negatives", + "next_check": "", + }, + } + ctx.trace_steps = [ + { + "file": "src/parser.c", + "line": 42, + "function": "parse", + "code_snippet": "copy(dst, src, len);", + "note": "sink reached from input", + } + ] + profile = get_context_profile("compact-small-model-v1") + manager = SourceHuntContextManager(profile, ctx) + messages = [ChatMessage("user", "Hunt for vulnerabilities.")] + for index in range(8): + call = ToolCall(f"call_{index}", "read_source_file", '{"path":"src/parser.c"}') + messages.extend( + [ + ChatMessage("assistant", "checking", tool_calls=[call]), + ChatMessage("tool", "source\n" + "x" * 5_000, tool_response_call_id=call.call_id), + ] + ) + + result = manager.compact(messages, system="short", tools=[]) + roles = [message.role for message in result.messages] + checkpoint = next(message.content for message in result.messages if message.role == "system") + + assert result.after_tokens < result.before_tokens + assert result.after_tokens <= profile.compact_to_tokens + assert roles[:2] == ["user", "system"] + assert roles[2:] == ["assistant", "tool", "assistant", "tool", "assistant", "tool"] + assert "length may exceed allocation" in checkpoint + assert "caller may cap length" in checkpoint + assert "all callers reject negatives" in checkpoint + assert "sink reached from input" in checkpoint + + +def test_unknown_profile_names_fail_closed() -> None: + with pytest.raises(ValueError, match="Unknown prompt bundle"): + get_prompt_bundle("unknown") + with pytest.raises(ValueError, match="Unknown scaffold profile"): + get_scaffold_profile("unknown") + with pytest.raises(ValueError, match="Unknown context profile"): + get_context_profile("unknown") diff --git a/tests/test_sourcehunt_phase0_eval.py b/tests/test_sourcehunt_phase0_eval.py index 2b4733cc..c1497b38 100644 --- a/tests/test_sourcehunt_phase0_eval.py +++ b/tests/test_sourcehunt_phase0_eval.py @@ -17,6 +17,7 @@ aggregate_baseline, build_ablation_plan, execute_sourcehunt_run, + include_fixed_negative_cases, inspect_ablation_session, run_ablation_campaign, ) @@ -54,6 +55,9 @@ def _observation(run, *, found: bool) -> RunObservation: flow=run.flow, model_tier=run.model_tier, model=run.model, + prompt_bundle=run.prompt_bundle, + scaffold_profile=run.scaffold_profile, + context_profile=run.context_profile, level=run.level, replicate=run.replicate, session_dir=f"sessions/{run.id}", @@ -91,6 +95,20 @@ def test_representative_manifest_has_full_intermediate_ground_truth( assert set(truth.expected_predicates) <= predicates +def test_fixed_commits_expand_into_clean_negative_controls( + manifest: GroundTruthManifest, +) -> None: + expanded = include_fixed_negative_cases(manifest) + original = next(case for case in manifest.cases if case.fixed_commit) + negative = expanded.case(f"{original.id}-fixed-negative") + + assert len(expanded.cases) == len(manifest.cases) + 1 + assert negative.vulnerable_commit == original.fixed_commit + assert negative.fixed_commit is None + assert negative.cves == [] + assert negative.ground_truth.expected_decision == "disproven" + + def test_ablation_levels_never_leak_later_hints( manifest: GroundTruthManifest, ) -> None: @@ -136,6 +154,35 @@ def test_local_and_frontier_arms_receive_identical_contexts( assert all(len(packets) == 1 for packets in packets_by_cell.values()) +def test_prompt_and_scaffold_profiles_are_independent_ablation_cells( + manifest: GroundTruthManifest, +) -> None: + arms = [ + AblationArm( + flow="legacy", + model_tier=tier, + model=f"{tier}-model", + prompt_bundle="generic-security-v1", + scaffold_profile=scaffold, + ) + for scaffold in ("native-v1", "minimal-linear-v1") + for tier in ("local", "frontier") + ] + + plan = build_ablation_plan( + GroundTruthManifest(cases=[manifest.cases[0]]), + arms, + levels=[AblationLevel.REPOSITORY], + ) + + assert len(plan.runs) == 4 + assert {run.scaffold_profile for run in plan.runs} == { + "native-v1", + "minimal-linear-v1", + } + assert len({run.id for run in plan.runs}) == 4 + + def test_baseline_requires_complete_matrix_and_reports_failure_stage( manifest: GroundTruthManifest, ) -> None: @@ -283,6 +330,15 @@ async def arun(self): output_dir=tmp_path / "results", provider_manager=object(), budget_usd=1.0, + input_price_per_million=0.0, + output_price_per_million=0.0, + max_hunt_files=24, + max_hunter_steps=40, + ranker_chunk_size=25, + ranker_max_inflight_chunks=1, + ranker_chunk_max_retries=1, + depth="standard", + no_rank=True, ) assert observation.run_id == spec.id @@ -290,6 +346,20 @@ async def arun(self): assert captured["campaign_hint"] == spec.campaign_hint() assert captured["flow"] == "proof" assert captured["model_override"] == spec.model + assert captured["input_price_per_million"] == 0.0 + assert captured["output_price_per_million"] == 0.0 + assert captured["max_hunt_files"] == 24 + assert captured["max_hunter_steps"] == 40 + assert captured["ranker_chunk_size"] == 25 + assert captured["ranker_max_inflight_chunks"] == 1 + assert captured["ranker_chunk_max_retries"] == 1 + assert captured["max_parallel"] == 4 + assert captured["starting_band"] == "fast" + assert captured["redundancy_override"] == 1 + assert captured["depth"] == "standard" + assert captured["no_rank"] is True + assert captured["prompt_bundle"] == spec.prompt_bundle + assert captured["scaffold_profile"] == spec.scaffold_profile (checkout / "app.py").write_text("print('changed')\n", encoding="utf-8") with pytest.raises(ValueError, match="tracked modifications"): @@ -458,7 +528,23 @@ def test_legacy_session_scorer_uses_instrumented_working_set( "file": case.ground_truth.target_files[0], "cwe": case.ground_truth.expected_cwes[0], "evidence_level": "root_cause_explained", - "vulnerability_trace": {"steps": [{"file": "target.c"}]}, + "description": ( + f"{case.ground_truth.target_functions[0]} propagates " + f"{case.ground_truth.expected_fact_symbols[0]} into " + f"{case.ground_truth.expected_fact_symbols[1]}" + ), + "vulnerability_trace": { + "steps": [ + { + "file": case.ground_truth.target_files[0], + "note": "ENTRY: attacker-controlled state enters", + }, + { + "file": case.ground_truth.target_files[0], + "note": "SINK: the violated state reaches a memory access", + }, + ] + }, "poc": "trigger", } ] @@ -487,3 +573,38 @@ def test_legacy_session_scorer_uses_instrumented_working_set( assert observation.report_failures == 1 assert observation.input_tokens == 40 assert observation.output_tokens == 10 + + +def test_legacy_session_scorer_rejects_file_and_cwe_only_match( + manifest: GroundTruthManifest, + tmp_path, +) -> None: + case = manifest.cases[0] + spec = build_ablation_plan( + GroundTruthManifest(cases=[case]), + _arms(flow="legacy"), + levels=[AblationLevel.REPOSITORY], + ).runs[0] + session = tmp_path / spec.id + (session / "instrumentation").mkdir(parents=True) + (session / "instrumentation" / "summary.json").write_text("{}", encoding="utf-8") + (session / "findings.json").write_text( + json.dumps( + [ + { + "file": case.ground_truth.target_files[0], + "cwe": case.ground_truth.expected_cwes[0], + "evidence_level": "suspicion", + "description": "generic warning with no mechanism", + } + ] + ), + encoding="utf-8", + ) + (session / "manifest.json").write_text('{"status":"completed"}', encoding="utf-8") + + observation = inspect_ablation_session(spec, case, session) + + assert observation.true_positives == 0 + assert observation.false_positives == 1 + assert observation.false_negatives == 1 diff --git a/tests/test_sourcehunt_pool_budget.py b/tests/test_sourcehunt_pool_budget.py index 379c053b..4a74ce24 100644 --- a/tests/test_sourcehunt_pool_budget.py +++ b/tests/test_sourcehunt_pool_budget.py @@ -11,6 +11,7 @@ import pytest +import clearwing.sourcehunt.pool as pool_module from clearwing.sourcehunt.pool import ( HunterPool, HuntPoolConfig, @@ -88,6 +89,30 @@ async def arun(self): return factory +def test_pool_propagates_run_scoped_zero_prices_to_native_hunters(monkeypatch): + captured = {} + + def factory(**kwargs): + captured.update(kwargs) + return MagicMock(), MagicMock() + + monkeypatch.setattr(pool_module, "_DEFAULT_HUNTER_FACTORY", factory) + pool = HunterPool( + HuntPoolConfig( + files=[_ft("src/parser.c", 4, 2)], + repo_path="/repo", + llm=MagicMock(), + input_price_per_million=0.0, + output_price_per_million=0.0, + ) + ) + + pool._build_hunter_for_file(pool.config.files[0], None, budget_usd=5.0) + + assert captured["input_price_per_million"] == 0.0 + assert captured["output_price_per_million"] == 0.0 + + def _make_pool(files, budget=10.0, tier_split=(0.7, 0.25, 0.05), per_call_cost=0.5, max_parallel=4): config = HuntPoolConfig( files=files, @@ -107,6 +132,37 @@ def _make_pool(files, budget=10.0, tier_split=(0.7, 0.25, 0.05), per_call_cost=0 return HunterPool(config) +def _recording_pool(files, dispatch_order, *, prior_spend): + def factory(file_target, sandbox, session_id): + dispatch_order.append(file_target["path"]) + + class _StubHunter: + async def arun(self): + return _StubRunResult( + findings=[], cost_usd=1.0, tokens_used=0, stop_reason="completed" + ) + + return _StubHunter(), MagicMock( + findings=[], session_id=session_id, cleanup_variants=MagicMock() + ) + + return HunterPool( + HuntPoolConfig( + files=files, + repo_path="/tmp/repo", + hunter_factory=factory, + max_parallel=1, + budget_usd=10.0, + tier_budget=TierBudget(0.7, 0.25, 0.05), + prior_spend_per_tier=prior_spend, + session_id_prefix="hunt", + starting_band="deep", + max_band="deep", + redundancy_override=1, + ) + ) + + # --- Tier assignment in __init__ ------------------------------------------- @@ -147,16 +203,19 @@ def order_recording_factory(file_target, sandbox, session_id): class _StubHunter: async def arun(self): return _StubRunResult( - findings=[], cost_usd=0.0, tokens_used=0, + findings=[], + cost_usd=0.0, + tokens_used=0, stop_reason="completed", ) - return _StubHunter(), MagicMock(findings=[], session_id=session_id, - cleanup_variants=MagicMock()) + return _StubHunter(), MagicMock( + findings=[], session_id=session_id, cleanup_variants=MagicMock() + ) # Both are tier A (priority >= 3.0). Low-priority listed first. files = [ - _ft("aaa_low_priority.c", 4, 4), # priority 3.7 → A (lower) + _ft("aaa_low_priority.c", 4, 4), # priority 3.7 → A (lower) _ft("zzz_high_priority.c", 5, 5), # priority 4.4 → A (highest) ] config = HuntPoolConfig( @@ -186,7 +245,6 @@ async def arun(self): ) - # --- Tier A spending -------------------------------------------------------- @@ -292,6 +350,37 @@ def test_unused_b_rolls_into_c(self): assert spent["C"] == pytest.approx(0.8) +class TestResumeTierBudget: + def test_prior_tier_a_spend_reduces_lifetime_allocation(self): + dispatch_order: list[str] = [] + pool = _recording_pool( + [ + *[_ft(f"a{i}.c", 5, 5) for i in range(4)], + _ft("b.c", 2, 2), + ], + dispatch_order, + prior_spend={"A": 6.0, "B": 0.0, "C": 0.0}, + ) + + pool.run() + + assert sum(path.startswith("a") for path in dispatch_order) == 1 + assert "b.c" in dispatch_order + + def test_prior_a_spend_is_not_rolled_over_again_when_a_is_complete(self): + dispatch_order: list[str] = [] + pool = _recording_pool( + [_ft(f"b{i}.c", 2, 2) for i in range(5)], + dispatch_order, + prior_spend={"A": 6.0, "B": 2.0, "C": 0.0}, + ) + + pool.run() + + # Lifetime A+B allowance is $9.50. With $8 already settled, the + # sliding window admits two indivisible $1 calls, but not all five. + assert dispatch_order == ["b0.c", "b1.c"] + # --- Skip-tier-c ----------------------------------------------------------- diff --git a/tests/test_sourcehunt_preprocessor.py b/tests/test_sourcehunt_preprocessor.py index ba589425..a3feea15 100644 --- a/tests/test_sourcehunt_preprocessor.py +++ b/tests/test_sourcehunt_preprocessor.py @@ -169,6 +169,22 @@ def test_respect_gitignore_filters_file_targets_and_static_findings(self, tmp_pa assert [ft["path"] for ft in result.file_targets] == ["src.js"] assert [Path(f.file_path).name for f in result.static_findings] == ["src.js"] + def test_excluded_root_is_removed_from_all_selected_inputs(self, tmp_path): + source = tmp_path / "src.py" + output = tmp_path / "results" / "sh-test" + output.mkdir(parents=True) + source.write_text("value = 1\n") + (output / "generated.py").write_text("from src import value\n") + + result = Preprocessor( + repo_url=str(tmp_path), + local_path=str(tmp_path), + excluded_roots=[output], + ).run() + + assert [target["path"] for target in result.file_targets] == ["src.py"] + assert result.file_targets[0]["imports_by"] == 0 + def test_codec_limits_h_tagged_memory_unsafe(self): pp = Preprocessor( repo_url=str(FIXTURE_C_PROPAGATION), diff --git a/tests/test_sourcehunt_provider_exhaustion.py b/tests/test_sourcehunt_provider_exhaustion.py new file mode 100644 index 00000000..5627135f --- /dev/null +++ b/tests/test_sourcehunt_provider_exhaustion.py @@ -0,0 +1,110 @@ +from __future__ import annotations + +import asyncio + +import pytest + +from clearwing.llm import ( + ProviderExhaustedError, + ProviderExhaustionState, + is_provider_exhausted_error, +) +from clearwing.llm.budget import SpendLedger +from clearwing.llm.native import AsyncLLMClient + + +class _ProviderResponse: + status_code = 403 + + +class _ProviderError(RuntimeError): + def __init__(self, message): + super().__init__(message) + self.response = _ProviderResponse() + + +@pytest.mark.parametrize( + ("message", "expected"), + [ + ( + "HTTP 403 type: access_terminated_error message: " + "You've reached your usage limit for this billing cycle", + True, + ), + ("HTTP 401 unauthorized", False), + ("HTTP 403 forbidden", False), + ("HTTP 429 rate limit", False), + ], +) +def test_provider_exhaustion_classification_is_narrow(message, expected): + assert is_provider_exhausted_error(RuntimeError(message)) is expected + + +def test_provider_exhaustion_uses_structured_http_status(): + error = _ProviderError( + "type: access_terminated_error; usage limit reached for this billing cycle" + ) + + assert is_provider_exhausted_error(error) is True + + +def test_provider_exhaustion_is_not_caught_by_ordinary_fallbacks(): + caught = False + try: + raise ProviderExhaustedError("quota") + except Exception: + caught = True + except ProviderExhaustedError: + pass + + assert caught is False + + +def test_exhaustion_marks_shared_clients_and_stops_before_dispatch(tmp_path): + ledger = SpendLedger( + limit_usd=0, + session_id="sh-provider", + repo_url="repo", + output_dir=tmp_path, + ) + state = ProviderExhaustionState() + client = AsyncLLMClient(model_name="test", provider_name="openai", api_key="test") + first = client.with_spend_ledger( + ledger, + stage="hunt", + provider_exhaustion_state=state, + ) + second = client.with_spend_ledger( + ledger, + stage="verify", + provider_exhaustion_state=state, + ) + state.mark(RuntimeError("quota")) + + with pytest.raises(ProviderExhaustedError, match="quota"): + second._reserve_spend_call(messages=[], system="", tools=None, max_tokens=None) + assert first._provider_exhaustion_state is second._provider_exhaustion_state + + +def test_rate_limit_retry_behavior_remains_bounded(monkeypatch): + client = AsyncLLMClient( + model_name="test", + provider_name="openai", + api_key="test", + rate_limit_max_retries=2, + rate_limit_initial_backoff_seconds=0.1, + ) + attempts = 0 + + async def fail(): + nonlocal attempts + attempts += 1 + raise RuntimeError("HTTP 429 rate limit") + + async def no_sleep(_delay): + return None + + monkeypatch.setattr(asyncio, "sleep", no_sleep) + with pytest.raises(RuntimeError, match="429"): + asyncio.run(client._with_retries(fail)) + assert attempts == 3 diff --git a/tests/test_sourcehunt_ranker.py b/tests/test_sourcehunt_ranker.py index 9a70a140..77b4e9ec 100644 --- a/tests/test_sourcehunt_ranker.py +++ b/tests/test_sourcehunt_ranker.py @@ -245,6 +245,61 @@ def test_rank_makes_one_call_per_chunk(self): Ranker(llm, RankerConfig(chunk_size=100)).rank(files) assert llm.aask_json.call_count == 3 + def test_one_failed_chunk_discards_all_partial_llm_scores(self): + llm = AsyncMock() + llm.aask_json.side_effect = [ + ( + { + "results": [ + { + "path": "a.c", + "surface": 5, + "influence": 5, + "surface_rationale": "LLM score", + "influence_rationale": "LLM score", + } + ] + }, + ChatResponse(), + ), + ({"results": []}, ChatResponse()), + ] + files = [_make_file("a.c"), _make_file("b.c")] + + ranker = Ranker( + llm, + RankerConfig(chunk_size=1, max_inflight_chunks=1), + ) + ranker.rank(files) + + assert llm.aask_json.call_count == 2 + assert ranker.completed_successfully is False + assert all(file["surface_rationale"] == "fallback (LLM did not score)" for file in files) + assert files[0]["surface"] != 5 + assert files[0]["influence"] != 5 + + def test_partial_chunk_response_discards_all_llm_scores(self): + llm = _mock_llm_returning( + [ + { + "path": "a.c", + "surface": 5, + "influence": 5, + "surface_rationale": "LLM score", + "influence_rationale": "LLM score", + } + ] + ) + files = [_make_file("a.c"), _make_file("b.c")] + + ranker = Ranker(llm, RankerConfig(chunk_size=2)) + ranker.rank(files) + + assert ranker.completed_successfully is False + assert all(file["surface_rationale"] == "fallback (LLM did not score)" for file in files) + assert files[0]["surface"] != 5 + assert files[0]["influence"] != 5 + def test_large_repo_reranks_only_top_heuristic_candidates(self): llm = _mock_llm_returning([]) files = [ diff --git a/tests/test_sourcehunt_resume_cli.py b/tests/test_sourcehunt_resume_cli.py new file mode 100644 index 00000000..8c8eaa53 --- /dev/null +++ b/tests/test_sourcehunt_resume_cli.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +import pytest + +from clearwing.ui.cli import CLI +from clearwing.ui.commands.sourcehunt import _parse_resume_options + + +def test_resume_makes_repository_optional(): + args = CLI()._create_parser().parse_args(["sourcehunt", "--resume", "sh-12345678"]) + assert args.repo is None + assert args.resume == "sh-12345678" + + +def test_resume_parser_accepts_only_runtime_provider_options(tmp_path): + options = _parse_resume_options( + [ + "sourcehunt", + "--resume", + "sh-12345678", + "--output-dir", + str(tmp_path), + "--model", + "replacement", + "--api-key", + "fresh-secret", + ], + default_output_dir="unused", + ) + + assert options.session_id == "sh-12345678" + assert options.output_dir == str(tmp_path) + assert options.model_override == "replacement" + + +def test_resume_parser_rejects_behavior_override(tmp_path): + with pytest.raises(SystemExit): + _parse_resume_options( + [ + "sourcehunt", + "--resume", + "sh-12345678", + "--output-dir", + str(tmp_path), + "--no-verify", + ], + default_output_dir="unused", + ) diff --git a/tests/test_sourcehunt_resume_pool.py b/tests/test_sourcehunt_resume_pool.py new file mode 100644 index 00000000..2bfd678a --- /dev/null +++ b/tests/test_sourcehunt_resume_pool.py @@ -0,0 +1,253 @@ +from __future__ import annotations + +import asyncio +from types import SimpleNamespace +from unittest.mock import MagicMock + +from clearwing.findings.types import Finding +from clearwing.sourcehunt.findings_pool import FindingsPool +from clearwing.sourcehunt.pool import HunterPool, HuntPoolConfig, TierBudget, WorkItem +from clearwing.sourcehunt.resume import SourceHuntResumeStore + + +def _target(path): + return { + "path": path, + "absolute_path": f"/repo/{path}", + "surface": 5, + "influence": 5, + "reachability": 3, + "priority": 4.4, + "tier": "A", + "tags": [], + "language": "python", + "loc": 10, + } + + +def _store(tmp_path, session_id="sh-pool"): + store = SourceHuntResumeStore(tmp_path / session_id) + paths = ["done.py", "interrupted.py", "new.py"] + store.create_session( + repository={"url": "repo", "branch": "main"}, + config={"test": True}, + source_identity={ + "algorithm": "sha256-path-content-v1", + "fingerprint": "0" * 64, + "paths": paths, + }, + ) + return store + + +def _save(store, item, *, findings=(), stop_reason="completed", transcript=None): + store.save_work_result( + work_id=item.stable_identifier(store.session_id, "A"), + file=item.file_target["path"], + tier="A", + band=item.band, + attempt=item.attempt, + entry_point=None, + seed_context=item.seed_context, + seed_transcript=item.seed_transcript, + findings=findings, + cost_usd=0, + tokens_used=0, + stop_reason=stop_reason, + promotion_transcript=transcript, + ) + + +def _pool(store, files, calls, *, findings_pool=None, max_band="fast"): + def factory(file_target, sandbox, session_id): + calls.append(file_target["path"]) + + class Hunter: + async def arun(self): + return SimpleNamespace( + findings=[], cost_usd=0, tokens_used=0, stop_reason="completed" + ) + + return Hunter(), MagicMock(cleanup_variants=MagicMock()) + + return HunterPool( + HuntPoolConfig( + files=files, + repo_path="/repo", + hunter_factory=factory, + max_parallel=1, + tier_budget=TierBudget(1, 0, 0), + session_id_prefix=store.session_id, + starting_band="fast", + max_band=max_band, + redundancy_override=1, + findings_pool=findings_pool, + resume_store=store, + ) + ) + + +def test_completed_zero_finding_work_is_skipped_and_missing_work_runs(tmp_path): + store = _store(tmp_path) + files = [_target("done.py"), _target("interrupted.py"), _target("new.py")] + _save(store, WorkItem(files[0], "fast")) + calls = [] + + pool = _pool(store, files, calls) + asyncio.run(pool.arun()) + + assert calls == ["interrupted.py", "new.py"] + assert pool.completed_target_count == 3 + + +def test_restored_findings_and_clusters_are_live_before_dispatch(tmp_path): + store = _store(tmp_path) + done = _target("done.py") + finding = Finding( + id="f-restored", + file="done.py", + description="restored", + primitive_type="memory_corruption", + cluster_id="cluster-restored", + ) + _save(store, WorkItem(done, "fast"), findings=[finding]) + pool_state = FindingsPool() + calls = [] + pool = _pool(store, [done, _target("new.py")], calls, findings_pool=pool_state) + + async def observe_restored_state(file_target, *_args, **_kwargs): + calls.append(file_target["path"]) + assert file_target["path"] == "new.py" + assert [item.id for item in pool_state.all_findings()] == ["f-restored"] + assert pool_state.clusters()[0].finding_ids == ["f-restored"] + return [], 0, 0, "completed" + + pool._run_one_hunter = observe_restored_state + + asyncio.run(pool.arun()) + + assert calls == ["new.py"] + assert [item.id for item in pool_state.all_findings()] == ["f-restored"] + assert pool_state.clusters()[0].finding_ids == ["f-restored"] + + +def test_restored_cluster_descriptor_is_available_before_dispatch(tmp_path): + store = _store(tmp_path) + done = _target("done.py") + finding = Finding( + id="f-clustered", + file="done.py", + description="member", + primitive_type="memory_corruption", + cluster_id="cluster-restored", + ) + item = WorkItem(done, "fast") + store.save_work_result( + work_id=item.stable_identifier(store.session_id, "A"), + file="done.py", + tier="A", + band="fast", + attempt=0, + entry_point=None, + seed_context=None, + seed_transcript=None, + findings=[finding], + clusters=[ + { + "cluster_id": "cluster-restored", + "root_cause_summary": "original root cause", + "primitive_type": "memory_corruption", + "cwe": "CWE-787", + } + ], + cost_usd=0, + tokens_used=0, + stop_reason="completed", + promotion_transcript=None, + ) + pool_state = FindingsPool() + pool = _pool(store, [done], [], findings_pool=pool_state) + + asyncio.run(pool.arun()) + + cluster = pool_state.clusters()[0] + assert cluster.root_cause_summary == "original root cause" + assert cluster.cwe == "CWE-787" + + +def test_completed_result_reconstructs_promotion(tmp_path): + store = _store(tmp_path) + target = _target("done.py") + finding = Finding(id="f-promote", file="done.py", description="promote") + fast = WorkItem(target, "fast") + _save(store, fast, findings=[finding], transcript="seed transcript") + pool = _pool(store, [target], [], max_band="standard") + observed = [] + + async def run_one(file_target, cost_limit, seed_transcript=None, **kwargs): + observed.append(("standard", seed_transcript)) + return [], 0, 0, "completed" + + pool._run_one_hunter = run_one + asyncio.run(pool.arun()) + + assert observed == [("standard", "seed transcript")] + + +def test_completed_promoted_work_is_not_duplicated(tmp_path): + store = _store(tmp_path) + target = _target("done.py") + finding = Finding(id="f-promote", file="done.py", description="promote") + fast = WorkItem(target, "fast") + standard = WorkItem(target, "standard", seed_transcript="seed transcript") + _save(store, fast, findings=[finding], transcript="seed transcript") + _save(store, standard) + calls = [] + + asyncio.run(_pool(store, [target], calls, max_band="standard").arun()) + + assert calls == [] + + +def test_interrupted_and_resumed_pool_matches_uninterrupted_result(tmp_path): + files = [_target("done.py"), _target("new.py")] + full_store = _store(tmp_path / "full", "sh-full") + resumed_store = _store(tmp_path / "resumed", "sh-resumed") + full_calls = [] + resumed_calls = [] + + full_pool = _pool(full_store, files, full_calls) + resumed_pool = _pool(resumed_store, files, resumed_calls) + + async def deterministic_hunt(file_target, *_args, **_kwargs): + finding = Finding( + id=f"finding-{file_target['path']}", + file=file_target["path"], + description="deterministic", + primitive_type="memory_corruption", + ) + return [finding], 0, 0, "completed" + + full_pool._run_one_hunter = deterministic_hunt + resumed_pool._run_one_hunter = deterministic_hunt + asyncio.run(full_pool.arun()) + done = WorkItem(files[0], "fast") + _save( + resumed_store, + done, + findings=[ + Finding( + id="finding-done.py", + file="done.py", + description="deterministic", + primitive_type="memory_corruption", + ) + ], + transcript="seed transcript", + ) + asyncio.run(resumed_pool.arun()) + + assert {(item.file, item.band) for item in full_store.load_completed_work().values()} == { + (item.file, item.band) for item in resumed_store.load_completed_work().values() + } + assert full_store.completed_findings() == resumed_store.completed_findings() diff --git a/tests/test_sourcehunt_resume_runner.py b/tests/test_sourcehunt_resume_runner.py new file mode 100644 index 00000000..26bcb0b1 --- /dev/null +++ b/tests/test_sourcehunt_resume_runner.py @@ -0,0 +1,431 @@ +from __future__ import annotations + +import json +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest + +from clearwing.llm import ProviderExhaustedError +from clearwing.sourcehunt.preprocessor import PreprocessResult +from clearwing.sourcehunt.resume import SourceHuntResumeError, SourceHuntResumeStore +from clearwing.sourcehunt.runner import SourceHuntRunner + + +def _target(repo: Path, name: str) -> dict: + return { + "path": name, + "absolute_path": str(repo / name), + "surface": 0, + "influence": 0, + "reachability": 3, + "priority": 0.0, + "tier": "C", + "tags": [], + "language": "python", + "loc": 1, + "surface_rationale": "", + "influence_rationale": "", + "reachability_rationale": "", + "static_hint": 0, + "semgrep_hint": 0, + "taint_hits": 0, + "imports_by": 0, + "transitive_callers": 0, + "defines_constants": False, + "has_fuzz_entry_point": False, + "fuzz_harness_path": None, + } + + +def _preprocess(repo: Path, names: tuple[str, ...] = ("a.py", "b.py")) -> PreprocessResult: + repo.mkdir(parents=True, exist_ok=True) + for name in names: + path = repo / name + if not path.exists(): + path.write_text(f"value = {name!r}\n", encoding="utf-8") + return PreprocessResult( + repo_path=str(repo), + file_targets=[_target(repo, name) for name in names], + static_findings=[], + ) + + +def _runner(repo: Path, output: Path, **overrides) -> SourceHuntRunner: + options = { + "repo_url": str(repo), + "local_path": str(repo), + "depth": "standard", + "output_dir": str(output), + "max_parallel": 1, + "ranker_llm": SimpleNamespace(provider_name="test"), + "hunter_llm": object(), + "sandbox_factory": lambda: None, + "no_verify": True, + "no_exploit": True, + "enable_calibration": False, + "enable_mechanism_memory": False, + "enable_patch_oracle": False, + "enable_stability_verification": False, + "enable_variant_loop": False, + "enable_knowledge_graph": False, + "enable_findings_pool": False, + "enable_behavior_monitor": False, + } + options.update(overrides) + return SourceHuntRunner(**options) + + +def _prepare(runner: SourceHuntRunner, result: PreprocessResult, monkeypatch) -> None: + monkeypatch.setattr(runner, "_preprocess", lambda: result) + monkeypatch.setattr(runner, "_ensure_sandbox_factory", lambda *_args: None) + monkeypatch.setattr(runner, "_write_report", lambda **_kwargs: {}) + + +class _EmptyPool: + def __init__(self, config): + self.config = config + self.spent_per_tier = {"A": 0.0, "B": 0.0, "C": 0.0} + self.spent_per_band = {"fast": 0.0, "standard": 0.0, "deep": 0.0} + self.runs_per_band = {"fast": 0, "standard": 0, "deep": 0} + self.promotion_counts = {"fast→standard": 0, "standard→deep": 0} + self.total_spent = 0.0 + self.budget_exhausted = False + + @property + def completed_target_count(self): + return self.config.resume_store.completed_target_count() + + async def arun(self): + return self.config.resume_store.completed_findings() + + +def test_partial_ranking_restarts_from_pristine_inputs(tmp_path, monkeypatch): + repo = tmp_path / "repo" + output = tmp_path / "output" + preprocess = _preprocess(repo) + runner = _runner(repo, output) + _prepare(runner, preprocess, monkeypatch) + calls: list[list[float]] = [] + + async def rank(self, files): + calls.append([target["priority"] for target in files]) + files[0]["priority"] = 5.0 + if len(calls) == 1: + raise KeyboardInterrupt + for index, target in enumerate(files): + target["surface"] = 5 - index + target["influence"] = 4 + target["priority"] = 4.3 - index + self.completed_successfully = True + return files + + monkeypatch.setattr("clearwing.sourcehunt.runner.Ranker.arank", rank) + monkeypatch.setattr("clearwing.sourcehunt.runner.HunterPool", _EmptyPool) + + with pytest.raises(KeyboardInterrupt): + runner.run() + + store = SourceHuntResumeStore.load(output / runner.session_id) + assert store.load_rank_plan() is None + + resumed = SourceHuntRunner.resume(runner.session_id, output_dir=str(output)) + resumed.ranker_llm = SimpleNamespace(provider_name="test") + resumed.hunter_llm = object() + _prepare(resumed, _preprocess(repo), monkeypatch) + result = resumed.run() + + assert result.status == "completed" + assert calls == [[0.0, 0.0], [0.0, 0.0]] + assert [target["priority"] for target in store.load_rank_plan()] == [4.3, 3.3] + + +def test_completed_rank_plan_is_restored_without_ranking(tmp_path, monkeypatch): + repo = tmp_path / "repo" + output = tmp_path / "output" + runner = _runner(repo, output) + _prepare(runner, _preprocess(repo), monkeypatch) + captured: list[list[dict]] = [] + + async def rank(self, files): + for index, target in enumerate(files): + target["surface"] = 5 - index + target["influence"] = index + 1 + target["priority"] = 4.75 - index + target["surface_rationale"] = f"rank-{index}" + self.completed_successfully = True + return files + + class CapturingPool(_EmptyPool): + async def arun(self): + captured.append([dict(target) for target in self.config.files]) + return [] + + monkeypatch.setattr("clearwing.sourcehunt.runner.Ranker.arank", rank) + monkeypatch.setattr("clearwing.sourcehunt.runner.HunterPool", CapturingPool) + runner.run() + saved = SourceHuntResumeStore.load(output / runner.session_id).load_rank_plan() + + async def should_not_rank(self, files): + raise AssertionError("completed rank plan must be reused") + + resumed = SourceHuntRunner.resume(runner.session_id, output_dir=str(output)) + resumed.hunter_llm = object() + _prepare(resumed, _preprocess(repo), monkeypatch) + monkeypatch.setattr("clearwing.sourcehunt.runner.Ranker.arank", should_not_rank) + resumed.run() + + restored = [ + {key: value for key, value in target.items() if key != "absolute_path"} + for target in captured[-1] + ] + assert restored == saved + + +def test_degraded_ranking_is_not_committed(tmp_path, monkeypatch): + repo = tmp_path / "repo" + output = tmp_path / "output" + runner = _runner(repo, output) + _prepare(runner, _preprocess(repo), monkeypatch) + + async def degraded_rank(self, files): + for target in files: + target["surface"] = 3 + target["influence"] = 2 + target["priority"] = 2.8 + self.completed_successfully = False + return files + + monkeypatch.setattr("clearwing.sourcehunt.runner.Ranker.arank", degraded_rank) + monkeypatch.setattr("clearwing.sourcehunt.runner.HunterPool", _EmptyPool) + + runner.run() + + assert SourceHuntResumeStore.load(output / runner.session_id).load_rank_plan() is None + + +def test_provider_exhaustion_is_resumable_with_replacement_provider(tmp_path, monkeypatch): + repo = tmp_path / "repo" + output = tmp_path / "output" + preprocess = _preprocess(repo, ("a.py", "b.py", "c.py")) + calls: list[tuple[str, str]] = [] + active_provider = "old" + + class Provider: + def __init__(self, name): + self.name = name + + def get_native_client(self, _task): + return self + + def build_hunter_agent(*, file_target, **_kwargs): + calls.append((active_provider, file_target["path"])) + + class Hunter: + async def arun(self): + if active_provider == "old" and file_target["path"] == "b.py": + raise ProviderExhaustedError("old credentials exhausted") + return SimpleNamespace( + findings=[], cost_usd=0.0, tokens_used=0, stop_reason="completed" + ) + + return Hunter(), MagicMock(cleanup_variants=MagicMock()) + + monkeypatch.setattr("clearwing.sourcehunt.hunter.build_hunter_agent", build_hunter_agent) + monkeypatch.setattr("clearwing.sourcehunt.pool._DEFAULT_HUNTER_FACTORY", build_hunter_agent) + runner = _runner( + repo, + output, + no_rank=True, + ranker_llm=None, + hunter_llm=None, + provider_manager=Provider("old"), + ) + _prepare(runner, preprocess, monkeypatch) + first = runner.run() + + assert first.status == "provider_exhausted" + assert calls == [("old", "a.py"), ("old", "b.py")] + store = SourceHuntResumeStore.load(output / runner.session_id) + assert {item.file for item in store.load_completed_work().values()} == {"a.py"} + + active_provider = "replacement" + resumed = SourceHuntRunner.resume( + runner.session_id, + output_dir=str(output), + provider_manager=Provider("replacement"), + ) + _prepare(resumed, _preprocess(repo, ("a.py", "b.py", "c.py")), monkeypatch) + second = resumed.run() + + assert second.status == "completed" + assert calls[-2:] == [("replacement", "b.py"), ("replacement", "c.py")] + reloaded = SourceHuntResumeStore.load(output / runner.session_id) + assert {item.file for item in reloaded.load_completed_work().values()} == { + "a.py", + "b.py", + "c.py", + } + + +def test_reporting_may_rerun_without_new_work_results(tmp_path, monkeypatch): + repo = tmp_path / "repo" + output = tmp_path / "output" + runner = _runner(repo, output, no_rank=True) + _prepare(runner, _preprocess(repo), monkeypatch) + hunter_runs = 0 + + def build_hunter_agent(**_kwargs): + nonlocal hunter_runs + hunter_runs += 1 + + class Hunter: + async def arun(self): + return SimpleNamespace( + findings=[], cost_usd=0.0, tokens_used=0, stop_reason="completed" + ) + + return Hunter(), MagicMock(cleanup_variants=MagicMock()) + + monkeypatch.setattr("clearwing.sourcehunt.hunter.build_hunter_agent", build_hunter_agent) + monkeypatch.setattr("clearwing.sourcehunt.pool._DEFAULT_HUNTER_FACTORY", build_hunter_agent) + reports = 0 + + def report(**_kwargs): + nonlocal reports + reports += 1 + return {} + + monkeypatch.setattr(runner, "_write_report", report) + runner.run() + store = SourceHuntResumeStore.load(output / runner.session_id) + before = sorted(store.work_results_dir.glob("*.json")) + + resumed = SourceHuntRunner.resume(runner.session_id, output_dir=str(output)) + resumed.hunter_llm = object() + _prepare(resumed, _preprocess(repo), monkeypatch) + monkeypatch.setattr(resumed, "_write_report", report) + resumed.run() + + assert reports == 2 + assert hunter_runs == 2 + assert len(before) == 2 + assert sorted(store.work_results_dir.glob("*.json")) == before + + +def test_completed_findings_are_returned_when_hunting_is_disabled_on_resume( + tmp_path, + monkeypatch, +): + repo = tmp_path / "repo" + output = tmp_path / "output" + runner = _runner(repo, output, no_rank=True) + _prepare(runner, _preprocess(repo, ("a.py",)), monkeypatch) + + from clearwing.findings.types import Finding + + finding = Finding(id="restored", file="a.py", description="saved") + + def build_hunter_agent(**_kwargs): + class Hunter: + async def arun(self): + return SimpleNamespace( + findings=[finding], + cost_usd=0.0, + tokens_used=0, + stop_reason="completed", + ) + + return Hunter(), MagicMock(cleanup_variants=MagicMock()) + + monkeypatch.setattr("clearwing.sourcehunt.hunter.build_hunter_agent", build_hunter_agent) + monkeypatch.setattr("clearwing.sourcehunt.pool._DEFAULT_HUNTER_FACTORY", build_hunter_agent) + runner.run() + + resumed = SourceHuntRunner.resume(runner.session_id, output_dir=str(output)) + resumed._no_per_file_hunt = True + _prepare(resumed, _preprocess(repo, ("a.py",)), monkeypatch) + + result = resumed.run() + + assert [item.id for item in result.findings] == ["restored"] + assert result.files_hunted == 1 + + +def test_runner_rejects_concurrent_resume_and_releases_lock(tmp_path, monkeypatch): + repo = tmp_path / "repo" + output = tmp_path / "output" + runner = _runner(repo, output, no_rank=True) + _prepare(runner, _preprocess(repo), monkeypatch) + monkeypatch.setattr("clearwing.sourcehunt.runner.HunterPool", _EmptyPool) + runner.run() + + session_dir = output / runner.session_id + from clearwing.sourcehunt.resume import SourceHuntSessionLock + + held = SourceHuntSessionLock(session_dir) + held.acquire() + try: + with pytest.raises(SourceHuntResumeError, match="already running"): + SourceHuntRunner.resume(runner.session_id, output_dir=str(output)).run() + finally: + held.release() + + resumed = SourceHuntRunner.resume(runner.session_id, output_dir=str(output)) + resumed.hunter_llm = object() + _prepare(resumed, _preprocess(repo), monkeypatch) + assert resumed.run().status == "completed" + + +def test_session_output_is_excluded_but_source_change_rejects_resume(tmp_path): + repo = tmp_path / "repo" + output = repo / "results" + source = repo / "app.py" + repo.mkdir() + source.write_text("value = 1\n", encoding="utf-8") + runner = SourceHuntRunner( + repo_url=str(repo), + local_path=str(repo), + depth="quick", + no_rank=True, + output_dir=str(output), + enable_knowledge_graph=False, + ) + first = runner.run() + session_dir = output / runner.session_id + (session_dir / "generated.py").write_text("generated = True\n", encoding="utf-8") + + unchanged = SourceHuntRunner.resume(runner.session_id, output_dir=str(output)).run() + assert unchanged.status == "completed" + + source.write_text("value = 2\n", encoding="utf-8") + with pytest.raises(SourceHuntResumeError, match="source inputs changed"): + SourceHuntRunner.resume(runner.session_id, output_dir=str(output)).run() + + session = json.loads(Path(first.output_paths["session"]).read_text(encoding="utf-8")) + assert session["source_identity"]["paths"] == ["app.py"] + + +def test_parent_owned_campaign_run_does_not_create_standalone_resume_state( + tmp_path, + monkeypatch, +): + repo = tmp_path / "repo" + output = tmp_path / "output" + runner = _runner( + repo, + output, + parent_session_id="campaign-parent-project", + depth="quick", + no_rank=True, + ranker_llm=None, + hunter_llm=None, + ) + _prepare(runner, _preprocess(repo, ("a.py",)), monkeypatch) + + result = runner.run() + + assert result.status == "completed" + assert "session" not in result.output_paths + assert not (output / runner.session_id / "session.json").exists() diff --git a/tests/test_sourcehunt_resume_spend.py b/tests/test_sourcehunt_resume_spend.py new file mode 100644 index 00000000..d4e12356 --- /dev/null +++ b/tests/test_sourcehunt_resume_spend.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +import json + +import pytest + +from clearwing.llm.budget import SpendLedger + + +def _ledger(tmp_path, session_id="sh-spend", *, limit=10, resume=False): + return SpendLedger( + limit_usd=limit, + session_id=session_id, + repo_url="repo", + output_dir=tmp_path, + input_price_per_million=0, + output_price_per_million=1_000_000, + resume=resume, + ) + + +def test_settled_spend_is_restored_once(tmp_path): + ledger = _ledger(tmp_path) + reservation = ledger.reserve_call( + model="test", + provider="test", + stage="hunt", + input_token_upper_bound=0, + requested_max_output_tokens=1, + supports_output_limit=True, + ) + ledger.settle_call(reservation, input_tokens=0, output_tokens=1) + + resumed = _ledger(tmp_path, resume=True) + resumed_again = _ledger(tmp_path, resume=True) + + assert resumed.spent_usd == pytest.approx(1) + assert resumed_again.spent_usd == pytest.approx(1) + + +def test_orphan_reservation_uses_cap_active_when_reserved(tmp_path): + uncapped = _ledger(tmp_path, "sh-uncapped", limit=0) + uncapped.reserve_call( + model="test", + provider="test", + stage="hunt", + input_token_upper_bound=0, + requested_max_output_tokens=1, + supports_output_limit=True, + ) + + resumed = _ledger(tmp_path, "sh-uncapped", limit=1, resume=True) + + assert resumed.spent_usd == 0 + + +def test_capped_orphan_is_charged_once_even_when_new_cap_is_higher(tmp_path): + capped = _ledger(tmp_path, "sh-capped", limit=1) + reservation = capped.reserve_call( + model="test", + provider="test", + stage="hunt", + input_token_upper_bound=0, + requested_max_output_tokens=1, + supports_output_limit=True, + ) + + resumed = _ledger(tmp_path, "sh-capped", limit=10, resume=True) + resumed_again = _ledger(tmp_path, "sh-capped", limit=10, resume=True) + events = [ + json.loads(line) for line in resumed.ledger_path.read_text(encoding="utf-8").splitlines() + ] + + assert resumed.spent_usd == pytest.approx(1) + assert resumed_again.spent_usd == pytest.approx(1) + assert sum( + event.get("event") == "call_settled" and event.get("call_id") == reservation.call_id + for event in events + ) == 1 diff --git a/tests/test_sourcehunt_resume_store.py b/tests/test_sourcehunt_resume_store.py new file mode 100644 index 00000000..c15bf513 --- /dev/null +++ b/tests/test_sourcehunt_resume_store.py @@ -0,0 +1,271 @@ +from __future__ import annotations + +import json +import time + +import pytest + +from clearwing.findings.types import Finding +from clearwing.sourcehunt.config import SourceHuntConfig +from clearwing.sourcehunt.resume import ( + SourceHuntResumeError, + SourceHuntResumeStore, + deterministic_work_id, + source_input_identity, +) + + +def _target(path, absolute_path): + return { + "path": path, + "absolute_path": str(absolute_path), + "surface": 5, + "influence": 4, + "reachability": 3, + "priority": 4.1, + "tier": "A", + "tags": [], + "language": "python", + "loc": 1, + "surface_rationale": "ranked", + "influence_rationale": "ranked", + "reachability_rationale": "default", + "static_hint": 0, + "semgrep_hint": 0, + "taint_hits": 0, + "imports_by": 0, + "transitive_callers": 0, + "defines_constants": False, + "has_fuzz_entry_point": False, + "fuzz_harness_path": None, + } + + +def _store(tmp_path, *, session_id="sh-store"): + repo = tmp_path / "repo" + repo.mkdir(exist_ok=True) + source = repo / "app.py" + source.write_text("print('ok')\n", encoding="utf-8") + target = _target("app.py", source) + store = SourceHuntResumeStore(tmp_path / session_id) + store.create_session( + repository={"url": str(repo), "branch": "main", "resolved_commit": None}, + config={ + "target": { + "repo_url": str(repo), + "local_path": str(repo), + "branch": "main", + "depth": "standard", + }, + "budget": {"budget_usd": 10.0}, + "output": {"output_formats": ["json"]}, + "proof": {"flow": "legacy"}, + }, + source_identity=source_input_identity(repo, [target]), + ) + return store, target + + +def _work_id(store, *, band="fast", seed_transcript=None): + return deterministic_work_id( + store.session_id, + file="app.py", + tier="A", + band=band, + attempt=0, + entry_point=None, + seed_context=None, + seed_transcript=seed_transcript, + ) + + +def _work_payload(store, *, findings=(), work_id=None): + work_id = work_id or _work_id(store) + return { + "schema_version": 1, + "work_id": work_id, + "work": { + "file": "app.py", + "tier": "A", + "band": "fast", + "attempt": 0, + "entry_point": None, + "seed_context": None, + "seed_transcript": None, + }, + "result": { + "status": "completed", + "findings": list(findings), + "clusters": [], + }, + } + + +def _save_work(store, *, findings=(), attempt=0, **overrides): + work_id = deterministic_work_id( + store.session_id, + file="app.py", + tier="A", + band="fast", + attempt=attempt, + entry_point=None, + seed_context=None, + seed_transcript=None, + ) + options = { + "work_id": work_id, + "file": "app.py", + "tier": "A", + "band": "fast", + "attempt": attempt, + "entry_point": None, + "seed_context": None, + "seed_transcript": None, + "findings": findings, + "clusters": [], + "cost_usd": 0, + "tokens_used": 0, + "stop_reason": "completed", + "promotion_transcript": None, + } + options.update(overrides) + return store.save_work_result(**options) + + +def test_session_rank_and_work_results_round_trip(tmp_path): + store, target = _store(tmp_path) + ranked = [{key: value for key, value in target.items() if key != "absolute_path"}] + store.save_rank_plan(ranked) + finding = Finding( + id="f-1", + file="app.py", + description="bug", + primitive_type="memory_corruption", + cluster_id="cluster-1", + ) + _save_work( + store, + findings=[finding], + cost_usd=1.25, + tokens_used=30, + promotion_transcript="previous work", + ) + + loaded = SourceHuntResumeStore.load(store.session_dir) + + assert loaded.load_rank_plan() == ranked + result = loaded.load_completed_work()[_work_id(store)] + assert result.findings == [finding] + assert result.promotion_transcript == "previous work" + + +def test_effective_config_round_trips_behavior_options(): + config = SourceHuntConfig.from_options( + { + "repo_url": "repo", + "branch": "feature", + "local_path": "/repo", + "depth": "deep", + "budget_usd": 12.5, + "max_parallel": 3, + "output_dir": "/results", + "output_formats": ["json"], + "no_verify": True, + "mechanism_store_path": "/mechanisms", + "respect_gitignore": True, + "flow": "legacy", + "proof_compile_commands": "/repo/compile_commands.json", + "retain_incomplete_certificates": False, + "emit_rejection_certificates": False, + "falsify": False, + } + ) + + restored = SourceHuntConfig.from_dict(config.to_dict()) + + assert restored == config + assert restored.target.branch == "feature" + assert restored.features.no_verify is True + assert restored.tuning.mechanism_store_path == "/mechanisms" + assert restored.tuning.respect_gitignore is True + assert restored.proof.compile_commands == "/repo/compile_commands.json" + assert restored.proof.retain_incomplete_certificates is False + + +@pytest.mark.parametrize("invalid", ["truncated", "malformed_finding", "wrong_id"]) +def test_invalid_work_results_are_ignored(tmp_path, invalid): + store, _target_value = _store(tmp_path, session_id=f"sh-{invalid}") + store.work_results_dir.mkdir() + work_id = _work_id(store) + if invalid == "truncated": + path = store.work_results_dir / f"{work_id}.json" + path.write_text('{"schema_version":1,"work_id":', encoding="utf-8") + else: + if invalid == "wrong_id": + work_id = "work-0123456789abcdef" + payload = _work_payload(store, work_id=work_id) + if invalid == "malformed_finding": + payload["result"]["findings"] = [{}] + (store.work_results_dir / f"{work_id}.json").write_text( + json.dumps(payload), encoding="utf-8" + ) + + assert store.load_completed_work() == {} + + +def test_existing_valid_work_result_is_immutable(tmp_path): + store, _target_value = _store(tmp_path) + work_id = _work_id(store) + first = Finding(id="first", file="app.py", description="first") + second = Finding(id="second", file="app.py", description="second") + _save_work(store, findings=[first]) + path = store.work_results_dir / f"{work_id}.json" + original = path.read_bytes() + + result = _save_work(store, findings=[second]) + + assert result.findings == [first] + assert path.read_bytes() == original + + +def test_small_result_persistence_benchmark(tmp_path): + store, _target_value = _store(tmp_path) + started = time.perf_counter() + for attempt in range(100): + _save_work(store, attempt=attempt) + loaded = SourceHuntResumeStore.load(store.session_dir).load_completed_work() + + assert len(loaded) == 100 + assert time.perf_counter() - started < 5 + + +def test_partial_or_invalid_rank_plan_is_ignored(tmp_path): + store, _target_value = _store(tmp_path) + store.rank_plan_path.write_text('{"schema_version":1,"targets":', encoding="utf-8") + assert store.load_rank_plan() is None + + store.rank_plan_path.write_text( + json.dumps({"schema_version": 1, "targets": [{"path": "other.py"}]}), + encoding="utf-8", + ) + assert store.load_rank_plan() is None + + store.rank_plan_path.write_text( + json.dumps({"schema_version": 1, "targets": [{"path": "app.py"}]}), + encoding="utf-8", + ) + assert store.load_rank_plan() is None + + +def test_selected_path_changes_reject_resume(tmp_path): + store, target = _store(tmp_path) + extra = tmp_path / "repo" / "extra.py" + extra.write_text("value = 1\n", encoding="utf-8") + + with pytest.raises(SourceHuntResumeError, match="source inputs changed"): + store.validate_source_identity( + source_input_identity( + tmp_path / "repo", + [target, _target("extra.py", extra)], + ) + ) diff --git a/tests/test_sourcehunt_runner.py b/tests/test_sourcehunt_runner.py index eb84685f..3383b7a0 100644 --- a/tests/test_sourcehunt_runner.py +++ b/tests/test_sourcehunt_runner.py @@ -168,6 +168,225 @@ def test_quick_runs_without_hunter_llm(self, tmp_path): class TestStandardDepth: + def test_deterministic_ranker_uses_static_signals_without_llm(self): + files = [ + {"path": "z.c", "tags": ["memory_unsafe"], "static_hint": 0}, + {"path": "a_parser.c", "tags": ["memory_unsafe", "parser"], "static_hint": 0}, + ] + + Ranker(None).rank_heuristically(files) # type: ignore[arg-type] + + assert files[1]["priority"] > files[0]["priority"] + + def test_deterministic_ranker_uses_file_contents_without_solution_hints(self, tmp_path): + dense = tmp_path / "dense.c" + diverse = tmp_path / "diverse.c" + dense.write_text("\n".join("memcpy(dst, src, len);" for _ in range(100))) + diverse.write_text( + "packet = read_input();\n" + "if (packet_size + offset > buffer_len) return -1;\n" + "dst[index] = (uint16_t)packet[value];\n" + "memset(state, -1, sizeof(*state));\n" + "free(state);\n" + ) + files = [ + { + "path": "src/dense.c", + "absolute_path": str(dense), + "language": "c", + "tags": ["memory_unsafe"], + }, + { + "path": "src/diverse.c", + "absolute_path": str(diverse), + "language": "c", + "tags": ["memory_unsafe"], + }, + ] + + Ranker(None).rank_heuristically(files) # type: ignore[arg-type] + + assert files[1]["security_signal_score"] > files[0]["security_signal_score"] + assert files[1]["deterministic_rank_score"] > files[0]["deterministic_rank_score"] + + def test_max_hunt_files_caps_ranked_fanout(self, tmp_path): + file_paths = [ + "include/codec_limits.h", + "src/codec_a.c", + "src/codec_b.c", + "src/codec_c.c", + ] + hunter_llm = AsyncMock() + hunter_llm.achat.return_value = ChatResponse(content=[{"text": "No finding."}]) + runner = SourceHuntRunner( + repo_url=str(FIXTURE_C_PROPAGATION), + local_path=str(FIXTURE_C_PROPAGATION), + depth="standard", + budget_usd=1.0, + max_parallel=1, + max_hunt_files=1, + output_dir=str(tmp_path), + ranker_llm=_make_ranker_llm(file_paths), + hunter_llm=hunter_llm, + verifier_llm=_make_verifier_llm(), + no_exploit=True, + ) + + result = runner.run() + + assert result.files_hunted == 1 + + def test_hunt_file_offset_selects_non_overlapping_rank_wave(self, tmp_path): + runner = SourceHuntRunner( + repo_url=str(FIXTURE_C_PROPAGATION), + local_path=str(FIXTURE_C_PROPAGATION), + depth="standard", + budget_usd=1.0, + max_parallel=1, + max_hunt_files=1, + hunt_file_offset=1, + output_dir=str(tmp_path), + ranker_llm=_make_ranker_llm( + [ + "include/codec_limits.h", + "src/codec_a.c", + "src/codec_b.c", + "src/codec_c.c", + ] + ), + hunter_llm=AsyncMock(), + verifier_llm=_make_verifier_llm(), + no_exploit=True, + ) + runner.hunter_llm.achat.return_value = ChatResponse(content=[{"text": "No finding."}]) + + result = runner.run() + + assert result.files_hunted == 1 + events = [ + json.loads(line) + for line in (tmp_path / runner.session_id / "instrumentation" / "events.jsonl") + .read_text(encoding="utf-8") + .splitlines() + ] + bounded = next( + event + for event in events + if event.get("stage") == "rank" and event.get("status") == "bounded" + ) + assert bounded["files"] == ["src/codec_b.c"] + + def test_hunt_file_offsets_select_exact_sparse_rank_slots(self, tmp_path): + runner = SourceHuntRunner( + repo_url=str(FIXTURE_C_PROPAGATION), + local_path=str(FIXTURE_C_PROPAGATION), + depth="standard", + budget_usd=1.0, + max_parallel=1, + hunt_file_offsets=[0, 2], + output_dir=str(tmp_path), + ranker_llm=_make_ranker_llm( + [ + "include/codec_limits.h", + "src/codec_a.c", + "src/codec_b.c", + "src/codec_c.c", + ] + ), + hunter_llm=AsyncMock(), + verifier_llm=_make_verifier_llm(), + no_exploit=True, + ) + runner.hunter_llm.achat.return_value = ChatResponse(content=[{"text": "No finding."}]) + + result = runner.run() + + assert result.files_hunted == 2 + events = [ + json.loads(line) + for line in (tmp_path / runner.session_id / "instrumentation" / "events.jsonl") + .read_text(encoding="utf-8") + .splitlines() + ] + bounded = next( + event + for event in events + if event.get("stage") == "rank" and event.get("status") == "bounded" + ) + assert bounded["files"] == ["src/codec_a.c", "src/codec_c.c"] + + def test_sparse_rank_slots_reject_ambiguous_window_options(self, tmp_path): + with pytest.raises(ValueError, match="cannot be combined"): + SourceHuntRunner( + repo_url=str(FIXTURE_C_PROPAGATION), + local_path=str(FIXTURE_C_PROPAGATION), + max_hunt_files=1, + hunt_file_offsets=[0], + output_dir=str(tmp_path), + ) + + def test_hunt_file_paths_pin_selection_across_rank_changes(self, tmp_path): + runner = SourceHuntRunner( + repo_url=str(FIXTURE_C_PROPAGATION), + local_path=str(FIXTURE_C_PROPAGATION), + depth="standard", + budget_usd=1.0, + max_parallel=1, + hunt_file_paths=["src/codec_c.c", "include/codec_limits.h"], + output_dir=str(tmp_path), + ranker_llm=_make_ranker_llm( + [ + "include/codec_limits.h", + "src/codec_a.c", + "src/codec_b.c", + "src/codec_c.c", + ] + ), + hunter_llm=AsyncMock(), + verifier_llm=_make_verifier_llm(), + no_exploit=True, + ) + runner.hunter_llm.achat.return_value = ChatResponse(content=[{"text": "No finding."}]) + + result = runner.run() + + assert result.files_hunted == 2 + events = [ + json.loads(line) + for line in (tmp_path / runner.session_id / "instrumentation" / "events.jsonl") + .read_text(encoding="utf-8") + .splitlines() + ] + bounded = next( + event + for event in events + if event.get("stage") == "rank" and event.get("status") == "bounded" + ) + assert set(bounded["files"]) == {"src/codec_c.c", "include/codec_limits.h"} + + def test_hunt_file_paths_fail_closed_when_a_path_is_absent(self, tmp_path): + runner = SourceHuntRunner( + repo_url=str(FIXTURE_C_PROPAGATION), + local_path=str(FIXTURE_C_PROPAGATION), + depth="standard", + hunt_file_paths=["src/missing.c"], + output_dir=str(tmp_path), + ranker_llm=_make_ranker_llm( + [ + "include/codec_limits.h", + "src/codec_a.c", + "src/codec_b.c", + "src/codec_c.c", + ] + ), + hunter_llm=AsyncMock(), + verifier_llm=_make_verifier_llm(), + no_exploit=True, + ) + + with pytest.raises(ValueError, match="absent from ranked source"): + runner.run() + def test_standard_pipeline_completes(self, tmp_path): runner = SourceHuntRunner( repo_url=str(FIXTURE_C_PROPAGATION), diff --git a/tests/test_sourcehunt_subsystem.py b/tests/test_sourcehunt_subsystem.py index 67b4213c..b8d00f34 100644 --- a/tests/test_sourcehunt_subsystem.py +++ b/tests/test_sourcehunt_subsystem.py @@ -2,11 +2,11 @@ from __future__ import annotations -import asyncio -from unittest.mock import AsyncMock, MagicMock +from unittest.mock import MagicMock import pytest +from clearwing.llm import ProviderExhaustedError, ProviderExhaustionState from clearwing.sourcehunt.state import FileTarget, SubsystemTarget from clearwing.sourcehunt.subsystem import ( SubsystemHuntConfig, @@ -430,3 +430,32 @@ async def test_subsystem_hunt_runner_no_subsystems(): )) result = await runner.arun() assert result == [] + + +@pytest.mark.asyncio +async def test_subsystem_hunt_stops_after_provider_exhaustion(): + state = ProviderExhaustionState() + runner = SubsystemHuntRunner( + SubsystemHuntConfig( + subsystems=[ + SubsystemTarget(name="first", root_path="first", files=[]), + SubsystemTarget(name="second", root_path="second", files=[]), + ], + repo_path="/tmp", + llm=MagicMock(), + max_parallel=1, + provider_exhaustion_state=state, + ) + ) + calls: list[str] = [] + + async def fail_first(subsystem, *_args, **_kwargs): + calls.append(subsystem.name) + raise state.mark(RuntimeError("provider quota exhausted")) + + runner._run_one_subsystem = fail_first + + with pytest.raises(ProviderExhaustedError): + await runner.arun() + + assert calls == ["first"] diff --git a/tests/test_sourcehunt_validator.py b/tests/test_sourcehunt_validator.py index 0cb52292..da2f297e 100644 --- a/tests/test_sourcehunt_validator.py +++ b/tests/test_sourcehunt_validator.py @@ -25,6 +25,8 @@ from clearwing.sourcehunt.state import Axes, AxisResult, ValidatorVerdict from clearwing.sourcehunt.validator import ( VALIDATOR_QUICK_PROMPT, + VALIDATOR_SOURCE_FIRST_COMPACT_PROMPT, + VALIDATOR_SOURCE_FIRST_PROMPT, VALIDATOR_SYSTEM_PROMPT, Validator, _VerdictSchema, @@ -131,6 +133,57 @@ def test_quick_pass_disabled(self): f = _make_finding(evidence_level="suspicion") assert val._prompt_for_finding(f) is VALIDATOR_SYSTEM_PROMPT + def test_source_first_profile_selects_compact_full_prompt(self): + val = Validator( + MagicMock(), + enable_quick_pass=False, + prompt_profile="source-first-high-recall-v1", + ) + + prompt = val._prompt_for_finding(_make_finding()) + + assert prompt is VALIDATOR_SOURCE_FIRST_PROMPT + assert "current source is authoritative" in prompt + assert len(prompt) < len(VALIDATOR_SYSTEM_PROMPT) + + def test_unknown_prompt_profile_fails_closed(self): + with pytest.raises(ValueError, match="Unknown validator prompt profile"): + Validator(MagicMock(), prompt_profile="missing") + + def test_source_first_compact_profile_includes_decision_rule(self): + val = Validator( + MagicMock(), + enable_quick_pass=False, + prompt_profile="source-first-compact-v2", + ) + + prompt = val._prompt_for_finding(_make_finding()) + + assert prompt is VALIDATOR_SOURCE_FIRST_COMPACT_PROMPT + assert "Set advance=true only when" in prompt + assert "Return the schema now" in prompt + assert len(prompt) < len(VALIDATOR_SOURCE_FIRST_PROMPT) + + def test_custom_system_prompt_overrides_selected_profile(self): + val = Validator( + MagicMock(), + enable_quick_pass=False, + prompt_profile="source-first-compact-v2", + system_prompt="Custom generic validator instruction.", + ) + + assert val._prompt_for_finding(_make_finding()) == ( + "Custom generic validator instruction." + ) + + def test_invalid_output_budget_fails_closed(self): + with pytest.raises(ValueError, match="max_output_tokens must be positive"): + Validator(MagicMock(), max_output_tokens=0) + + def test_invalid_temperature_fails_closed(self): + with pytest.raises(ValueError, match="temperature must be between 0 and 2"): + Validator(MagicMock(), temperature=2.1) + # --- Independent context tests ----------------------------------------------- @@ -151,6 +204,45 @@ def test_user_message_includes_finding_metadata(self): assert "src/codec_a.c" in msg assert "memcpy overflow" in msg + def test_user_message_includes_source_backed_trace(self): + val = Validator(MagicMock()) + f = _make_finding( + vulnerability_trace={ + "steps": [ + { + "file": "src/entry.c", + "line": 12, + "code_snippet": "parse_one(input);", + "note": "entry dispatch", + } + ] + } + ) + + msg = val._build_user_message(f, "") + + assert "src/entry.c" in msg + assert "parse_one(input);" in msg + assert "entry dispatch" in msg + + def test_user_message_treats_trace_as_an_allegation(self): + val = Validator(MagicMock()) + msg = val._build_user_message(_make_finding(), "") + + assert "reporter's alleged source chain" in msg + assert "independently verify every step" in msg + + def test_user_message_includes_independent_source_context(self): + val = Validator(MagicMock()) + msg = val._build_user_message( + _make_finding(), + "", + "--- src/codec_a.c:1-2 ---\ncurrent source", + ) + + assert "current source snapshot" in msg + assert "current source" in msg + # --- Response parsing tests --------------------------------------------------- @@ -437,6 +529,40 @@ async def test_avalidate_parses_response(self): assert verdict.severity_validated == "high" assert verdict.axes.impactful.boundary_crossed == "privilege" + @pytest.mark.asyncio + async def test_avalidate_applies_output_budget_to_first_call(self): + mock_llm = AsyncMock() + mock_response = MagicMock() + mock_response.first_text = json.dumps({ + "axes": { + "real": {"passed": True, "confidence": "high", "rationale": "yes"}, + "triggerable": { + "passed": True, + "confidence": "high", + "rationale": "yes", + }, + "impactful": { + "passed": True, + "confidence": "high", + "rationale": "yes", + }, + "general": {"passed": True, "confidence": "high", "rationale": "yes"}, + }, + "advance": True, + "severity": "high", + "evidence_level": "static_corroboration", + }) + mock_llm.aask_text = AsyncMock(return_value=mock_response) + + await Validator( + mock_llm, + max_output_tokens=4096, + temperature=0.0, + ).avalidate(_make_finding()) + + assert mock_llm.aask_text.await_args.kwargs["max_tokens"] == 4096 + assert mock_llm.aask_text.await_args.kwargs["temperature"] == 0.0 + @pytest.mark.asyncio async def test_avalidate_llm_error_returns_error_verdict(self): mock_llm = AsyncMock() @@ -446,6 +572,48 @@ async def test_avalidate_llm_error_returns_error_verdict(self): verdict = await val.avalidate(_make_finding()) assert verdict.advance is False assert "validator error" in verdict.tie_breaker + assert mock_llm.aask_text.await_count == 2 + + @pytest.mark.asyncio + async def test_avalidate_retries_empty_response_with_compact_prompt(self): + mock_llm = AsyncMock() + empty_response = MagicMock() + empty_response.first_text = "" + empty_response.texts = [] + valid_response = MagicMock() + valid_response.first_text = json.dumps({ + "axes": { + "real": {"passed": True, "confidence": "high", "rationale": "yes"}, + "triggerable": { + "passed": True, + "confidence": "medium", + "rationale": "reachable", + }, + "impactful": { + "passed": True, + "confidence": "high", + "rationale": "memory corruption", + "boundary_crossed": "user", + }, + "general": { + "passed": True, + "confidence": "medium", + "rationale": "default parser", + }, + }, + "advance": True, + "severity": "high", + "evidence_level": "static_corroboration", + }) + mock_llm.aask_text = AsyncMock(side_effect=[empty_response, valid_response]) + + verdict = await Validator(mock_llm).avalidate(_make_finding()) + + assert verdict.advance is True + assert mock_llm.aask_text.await_count == 2 + retry = mock_llm.aask_text.await_args_list[1].kwargs + assert retry["max_tokens"] == 8192 + assert "Return the structured verdict immediately" in retry["system"] # --- File context tests ------------------------------------------------------- diff --git a/uv.lock b/uv.lock index 9b4c4d47..c58990b9 100644 --- a/uv.lock +++ b/uv.lock @@ -798,6 +798,7 @@ all = [ { name = "black" }, { name = "build" }, { name = "flake8" }, + { name = "gepa" }, { name = "inspect-ai" }, { name = "mkdocs" }, { name = "mkdocs-material" }, @@ -835,6 +836,9 @@ inspect-ai = [ metasploit = [ { name = "pymetasploit3" }, ] +optimization = [ + { name = "gepa" }, +] vector = [ { name = "sentence-transformers" }, ] @@ -854,11 +858,12 @@ requires-dist = [ { name = "black", marker = "extra == 'dev'", specifier = ">=23.0.0" }, { name = "build", marker = "extra == 'dev'", specifier = ">=1.2.0" }, { name = "chromadb", specifier = ">=1.0.0" }, - { name = "clearwing", extras = ["metasploit", "browser", "vector", "dev", "docs", "inspect-ai"], marker = "extra == 'all'" }, + { name = "clearwing", extras = ["metasploit", "browser", "vector", "dev", "docs", "inspect-ai", "optimization"], marker = "extra == 'all'" }, { name = "docker", specifier = ">=7.0.0" }, { name = "fastapi", specifier = ">=0.100.0" }, { name = "flake8", marker = "extra == 'dev'", specifier = ">=6.0.0" }, { name = "genai-pyo3", specifier = ">=0.7.0b14.dev3" }, + { name = "gepa", marker = "extra == 'optimization'", specifier = ">=0.1.4,<0.2" }, { name = "inspect-ai", marker = "extra == 'inspect-ai'", specifier = ">=0.3.0" }, { name = "jinja2", specifier = ">=3.1.0" }, { name = "libpnet-pyo3", specifier = ">=0.1.2" }, @@ -895,7 +900,7 @@ requires-dist = [ { name = "uvicorn", specifier = ">=0.20.0" }, { name = "websockets", specifier = ">=11.0" }, ] -provides-extras = ["metasploit", "browser", "vector", "dev", "docs", "inspect-ai", "all"] +provides-extras = ["metasploit", "browser", "vector", "dev", "docs", "inspect-ai", "optimization", "all"] [package.metadata.requires-dev] dev = [ @@ -990,7 +995,7 @@ name = "cuda-bindings" version = "13.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cuda-pathfinder", marker = "sys_platform == 'linux'" }, + { name = "cuda-pathfinder" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/1a/fe/7351d7e586a8b4c9f89731bfe4cf0148223e8f9903ff09571f78b3fb0682/cuda_bindings-13.2.0-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:08b395f79cb89ce0cd8effff07c4a1e20101b873c256a1aeb286e8fd7bd0f556", size = 5744254, upload-time = "2026-03-11T00:12:29.798Z" }, @@ -1025,37 +1030,37 @@ wheels = [ [package.optional-dependencies] cublas = [ - { name = "nvidia-cublas", marker = "sys_platform == 'linux'" }, + { name = "nvidia-cublas" }, ] cudart = [ - { name = "nvidia-cuda-runtime", marker = "sys_platform == 'linux'" }, + { name = "nvidia-cuda-runtime" }, ] cufft = [ - { name = "nvidia-cufft", marker = "sys_platform == 'linux'" }, + { name = "nvidia-cufft" }, ] cufile = [ - { name = "nvidia-cufile", marker = "sys_platform == 'linux'" }, + { name = "nvidia-cufile" }, ] cupti = [ - { name = "nvidia-cuda-cupti", marker = "sys_platform == 'linux'" }, + { name = "nvidia-cuda-cupti" }, ] curand = [ - { name = "nvidia-curand", marker = "sys_platform == 'linux'" }, + { name = "nvidia-curand" }, ] cusolver = [ - { name = "nvidia-cusolver", marker = "sys_platform == 'linux'" }, + { name = "nvidia-cusolver" }, ] cusparse = [ - { name = "nvidia-cusparse", marker = "sys_platform == 'linux'" }, + { name = "nvidia-cusparse" }, ] nvjitlink = [ - { name = "nvidia-nvjitlink", marker = "sys_platform == 'linux'" }, + { name = "nvidia-nvjitlink" }, ] nvrtc = [ - { name = "nvidia-cuda-nvrtc", marker = "sys_platform == 'linux'" }, + { name = "nvidia-cuda-nvrtc" }, ] nvtx = [ - { name = "nvidia-nvtx", marker = "sys_platform == 'linux'" }, + { name = "nvidia-nvtx" }, ] [[package]] @@ -1151,7 +1156,7 @@ name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ @@ -1380,6 +1385,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7a/ed/61af17587798a61a8a71fabbce0ed2b8665146739797cc758729587f5e59/genai_pyo3-0.7.0b14.dev3-cp314-cp314-win_arm64.whl", hash = "sha256:665982b6c75606b7c3d31e39d50428e7d23328d1c0242bce394212c0fc2b0fa4", size = 4032876, upload-time = "2026-07-15T03:22:18.526Z" }, ] +[[package]] +name = "gepa" +version = "0.1.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/95/56/925779e5690971f1b022f7d107caf015c33ec09560261273ec137e23a8f2/gepa-0.1.4.tar.gz", hash = "sha256:6dd153a676ae5481764860d19286a9c0e8ddb5ef70d7f13044faf24978bdb6b8", size = 351343, upload-time = "2026-07-15T14:53:59.929Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fd/77/5b3a281cfd9caaa9e68349b434cf27f1ca448003ee0067a1ae2184dc52d1/gepa-0.1.4-py3-none-any.whl", hash = "sha256:12b971039599625c156d2231f6d72a29c31a22e9c237689459b5f1a3c353f532", size = 290167, upload-time = "2026-07-15T14:53:58.422Z" }, +] + [[package]] name = "ghp-import" version = "2.1.0" @@ -1784,7 +1798,7 @@ name = "importlib-metadata" version = "8.7.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "zipp", marker = "python_full_version < '3.13'" }, + { name = "zipp" }, ] sdist = { url = "https://files.pythonhosted.org/packages/f3/49/3b30cad09e7771a4982d9975a8cbf64f00d4a1ececb53297f1d9a7be1b10/importlib_metadata-8.7.1.tar.gz", hash = "sha256:49fef1ae6440c182052f407c8d34a68f72efc36db9ca90dc0113398f2fdde8bb", size = 57107, upload-time = "2025-12-21T10:00:19.278Z" } wheels = [ @@ -3198,7 +3212,7 @@ name = "nvidia-cudnn-cu13" version = "9.19.0.56" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas", marker = "sys_platform == 'linux'" }, + { name = "nvidia-cublas" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/f1/84/26025437c1e6b61a707442184fa0c03d083b661adf3a3eecfd6d21677740/nvidia_cudnn_cu13-9.19.0.56-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:6ed29ffaee1176c612daf442e4dd6cfeb6a0caa43ddcbeb59da94953030b1be4", size = 433781201, upload-time = "2026-02-03T20:40:53.805Z" }, @@ -3210,7 +3224,7 @@ name = "nvidia-cufft" version = "12.0.0.61" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink", marker = "sys_platform == 'linux'" }, + { name = "nvidia-nvjitlink" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/8b/ae/f417a75c0259e85c1d2f83ca4e960289a5f814ed0cea74d18c353d3e989d/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2708c852ef8cd89d1d2068bdbece0aa188813a0c934db3779b9b1faa8442e5f5", size = 214053554, upload-time = "2025-09-04T08:31:38.196Z" }, @@ -3240,9 +3254,9 @@ name = "nvidia-cusolver" version = "12.0.4.66" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas", marker = "sys_platform == 'linux'" }, - { name = "nvidia-cusparse", marker = "sys_platform == 'linux'" }, - { name = "nvidia-nvjitlink", marker = "sys_platform == 'linux'" }, + { name = "nvidia-cublas" }, + { name = "nvidia-cusparse" }, + { name = "nvidia-nvjitlink" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/c8/c3/b30c9e935fc01e3da443ec0116ed1b2a009bb867f5324d3f2d7e533e776b/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:02c2457eaa9e39de20f880f4bd8820e6a1cfb9f9a34f820eb12a155aa5bc92d2", size = 223467760, upload-time = "2025-09-04T08:33:04.222Z" }, @@ -3254,7 +3268,7 @@ name = "nvidia-cusparse" version = "12.6.3.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink", marker = "sys_platform == 'linux'" }, + { name = "nvidia-nvjitlink" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/f8/94/5c26f33738ae35276672f12615a64bd008ed5be6d1ebcb23579285d960a9/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:80bcc4662f23f1054ee334a15c72b8940402975e0eab63178fc7e670aa59472c", size = 162155568, upload-time = "2025-09-04T08:33:42.864Z" }, @@ -3324,11 +3338,11 @@ resolution-markers = [ "python_full_version < '3.11' and sys_platform != 'linux'", ] dependencies = [ - { name = "flatbuffers", marker = "python_full_version < '3.11'" }, - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "packaging", marker = "python_full_version < '3.11'" }, - { name = "protobuf", marker = "python_full_version < '3.11'" }, - { name = "sympy", marker = "python_full_version < '3.11'" }, + { name = "flatbuffers" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, + { name = "packaging" }, + { name = "protobuf" }, + { name = "sympy" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/15/41/3253db975a90c3ce1d475e2a230773a21cd7998537f0657947df6fb79861/onnxruntime-1.24.3-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:3e6456801c66b095c5cd68e690ca25db970ea5202bd0c5b84a2c3ef7731c5a3c", size = 17332766, upload-time = "2026-03-05T17:18:59.714Z" }, @@ -3370,11 +3384,11 @@ resolution-markers = [ "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform != 'linux'", ] dependencies = [ - { name = "flatbuffers", marker = "python_full_version >= '3.11'" }, - { name = "numpy", version = "2.4.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "packaging", marker = "python_full_version >= '3.11'" }, - { name = "protobuf", marker = "python_full_version >= '3.11'" }, - { name = "sympy", marker = "python_full_version >= '3.11'" }, + { name = "flatbuffers" }, + { name = "numpy", version = "2.4.4", source = { registry = "https://pypi.org/simple" } }, + { name = "packaging" }, + { name = "protobuf" }, + { name = "sympy" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/60/69/6c40720201012c6af9aa7d4ecdd620e521bd806dc6269d636fdd5c5aeebe/onnxruntime-1.24.4-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:0bdfce8e9a6497cec584aab407b71bf697dac5e1b7b7974adc50bf7533bdb3a2", size = 17332131, upload-time = "2026-03-17T22:05:49.005Z" }, @@ -5062,10 +5076,10 @@ resolution-markers = [ "python_full_version < '3.11' and sys_platform != 'linux'", ] dependencies = [ - { name = "joblib", marker = "python_full_version < '3.11'" }, - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "threadpoolctl", marker = "python_full_version < '3.11'" }, + { name = "joblib" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, + { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" } }, + { name = "threadpoolctl" }, ] sdist = { url = "https://files.pythonhosted.org/packages/98/c2/a7855e41c9d285dfe86dc50b250978105dce513d6e459ea66a6aeb0e1e0c/scikit_learn-1.7.2.tar.gz", hash = "sha256:20e9e49ecd130598f1ca38a1d85090e1a600147b9c02fa6f15d69cb53d968fda", size = 7193136, upload-time = "2025-09-09T08:21:29.075Z" } wheels = [ @@ -5114,10 +5128,10 @@ resolution-markers = [ "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform != 'linux'", ] dependencies = [ - { name = "joblib", marker = "python_full_version >= '3.11'" }, - { name = "numpy", version = "2.4.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "threadpoolctl", marker = "python_full_version >= '3.11'" }, + { name = "joblib" }, + { name = "numpy", version = "2.4.4", source = { registry = "https://pypi.org/simple" } }, + { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" } }, + { name = "threadpoolctl" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0e/d4/40988bf3b8e34feec1d0e6a051446b1f66225f8529b9309becaeef62b6c4/scikit_learn-1.8.0.tar.gz", hash = "sha256:9bccbb3b40e3de10351f8f5068e105d0f4083b1a65fa07b6634fbc401a6287fd", size = 7335585, upload-time = "2025-12-10T07:08:53.618Z" } wheels = [ @@ -5168,7 +5182,7 @@ resolution-markers = [ "python_full_version < '3.11' and sys_platform != 'linux'", ] dependencies = [ - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/0f/37/6964b830433e654ec7485e45a00fc9a27cf868d622838f6b6d9c5ec0d532/scipy-1.15.3.tar.gz", hash = "sha256:eae3cf522bc7df64b42cad3925c876e1b0b6c35c1337c93e12c0f366f55b0eaf", size = 59419214, upload-time = "2025-05-08T16:13:05.955Z" } wheels = [ @@ -5232,7 +5246,7 @@ resolution-markers = [ "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform != 'linux'", ] dependencies = [ - { name = "numpy", version = "2.4.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "numpy", version = "2.4.4", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/7a/97/5a3609c4f8d58b039179648e62dd220f89864f56f7357f5d4f45c29eb2cc/scipy-1.17.1.tar.gz", hash = "sha256:95d8e012d8cb8816c226aef832200b1d45109ed4464303e997c5b13122b297c0", size = 30573822, upload-time = "2026-02-23T00:26:24.851Z" } wheels = [ @@ -5303,8 +5317,8 @@ name = "secretstorage" version = "3.5.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cryptography", marker = "sys_platform == 'linux'" }, - { name = "jeepney", marker = "sys_platform == 'linux'" }, + { name = "cryptography" }, + { name = "jeepney" }, ] sdist = { url = "https://files.pythonhosted.org/packages/1c/03/e834bcd866f2f8a49a85eaff47340affa3bfa391ee9912a952a1faa68c7b/secretstorage-3.5.0.tar.gz", hash = "sha256:f04b8e4689cbce351744d5537bf6b1329c6fc68f91fa666f60a380edddcd11be", size = 19884, upload-time = "2025-11-23T19:02:53.191Z" } wheels = [ @@ -6461,7 +6475,7 @@ name = "zipfile-zstd" version = "0.0.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "zstandard", marker = "python_full_version < '3.14'" }, + { name = "zstandard" }, ] sdist = { url = "https://files.pythonhosted.org/packages/f7/2a/2e0941bc0058d10ab37d8c578b94a19f611f6ae54f124140f2fb451f0932/zipfile-zstd-0.0.4.tar.gz", hash = "sha256:c1498e15b7922a3d1af0ea55df8b11b2af4e8f7e0e80e414e25d66899f7def89", size = 4603, upload-time = "2021-12-08T07:38:16.245Z" } wheels = [