diff --git a/.gitignore b/.gitignore index 79640a7..a4a9800 100644 --- a/.gitignore +++ b/.gitignore @@ -22,12 +22,13 @@ Backtest Outputs/ # DhanHQ instrument master dump (regenerated daily by the runner; auto-swept) Dependencies/all_instrument*.csv # Crash-durable session state (rewritten every session; holds live position and -# P&L detail, so it is operational data and never belongs in the repository) -Dependencies/session_state.json -Dependencies/session_state.json.tmp -# Exact prior-run files are rotated here before a replacement session starts. -# They remain operational P&L/position evidence and must stay out of Git too. -Dependencies/session_state.*.recovery.json +# P&L detail, so it is operational data and never belongs in the repository). +# Two files: the durable trades/P&L document, and the best-effort `.marks.` +# sibling holding open positions -- plus the `.recovery.` copies of each that a +# replacement session rotates aside. One glob covers every variant, so a future +# sidecar cannot be committed by accident. +Dependencies/session_state*.json +Dependencies/session_state*.json.tmp # IDE / OS .vscode/ diff --git a/Dependencies/session_state.py b/Dependencies/session_state.py index d722449..79df474 100644 --- a/Dependencies/session_state.py +++ b/Dependencies/session_state.py @@ -88,6 +88,26 @@ # operational warning that the storage device may be delaying a trading worker. SLOW_WRITE_WARNING_SECONDS = 0.25 +# The marks file is written from the supervisor thread and never fsyncs, so a +# slow one delays shutdown supervision rather than a trading decision. It gets +# a looser threshold on purpose: warning at the durable threshold produced 210 +# lines in one session (2026-08-11) and buried the 13 that actually mattered. +SLOW_MARKS_WRITE_WARNING_SECONDS = 2.0 + +# Per-strategy keys that live in the MARKS file rather than the durable one. +# All of them are refreshed wholesale by every snapshot, so losing the newest +# 30 seconds of them in a crash is the documented trade-off (ADR-0012); none is +# needed to answer "what had this strategy banked when the process died". +_MARKS_ONLY_KEYS = ( + "open_position", + "snapshot_valid", + "snapshot_error_at", + "completed_trades", + "realized_pnl", + "execution_mode", + "live_trading", +) + # Position attributes that are deliberately never written out. ``live_leg`` is # broker exposure state (see the module docstring); the private/dunder filter # below removes bookkeeping attributes that are not part of the position shape. @@ -251,6 +271,14 @@ def __init__( log: logging.Logger | None = None, ) -> None: self.path = Path(path) + # Open positions and their marks live in a SEPARATE file, written on the + # snapshot cadence without fsync. They are deliberately not in the + # durable document: the supervisor rewrites them every 30 seconds, and + # `os.replace` is atomic for the NAME but not for the DATA -- a hard kill + # during an un-fsynced rewrite can publish a present-but-garbage file. If + # that file also held the trades and the P&L rollup, one torn snapshot + # would destroy the very record ADR-0012 exists to protect. + self.marks_path = _marks_path_for(self.path) self.session_date = session_date or _now_ist().date() self.snapshot_interval_seconds = max(1.0, float(snapshot_interval_seconds)) self.max_trade_records = max(1, int(max_trade_records)) @@ -285,6 +313,16 @@ def __init__( "trades": [], } self._carry_forward_same_day_bookkeeping() + # Establish the durable file immediately. Everything else only writes it + # on a trade event or at shutdown, so without this a session that dies + # before its first trade would leave NO durable document -- losing the + # session date, the shutdown flags and any trade book carried forward + # from a same-day restart, all of which a recovery needs. + try: + with self._lock: + self._flush_durable_locked() + except Exception: # noqa: BLE001 - persistence must never stop a session + self._log_write_failure("establish the durable state file") def _archive_previous_file(self) -> tuple[dict[str, Any] | None, Path | None]: """Move an existing state file aside and return its parsed document. @@ -309,6 +347,18 @@ def _archive_previous_file(self) -> tuple[dict[str, Any] | None, Path | None]: stem = self.path.name[: -len(suffix)] if suffix else self.path.name archive = self.path.with_name(f"{stem}.{timestamp}.recovery{suffix}") os.replace(self.path, archive) + # The marks file is archived alongside it under the SAME timestamp, so a + # recovery pair stays identifiable. Its absence is normal (a session that + # died before its first snapshot has none), so this never fails the move. + if self.marks_path.is_file(): + marks_archive = _marks_path_for(archive) + try: + os.replace(self.marks_path, marks_archive) + except OSError: + self.log.warning( + "Could not archive the previous marks file at %s; it will be " + "overwritten by this session.", self.marks_path, + ) self.log.warning( "Archived the previous session state before starting a new file: %s", archive, @@ -372,7 +422,7 @@ def record_trade_event(self, event: Mapping[str, Any]) -> None: # Drop oldest first -- a recovery cares about the newest. del trades[: len(trades) - self.max_trade_records] self._apply_pnl_bearing_event_locked(record) - self._flush_locked() + self._flush_durable_locked() except Exception: # noqa: BLE001 - reporting must never break trading self._log_write_failure("record trade event") @@ -454,7 +504,9 @@ def update_worker_snapshot( # A closed position must be REMOVED, not left behind -- # a stale record here would look resumable. entry.pop("open_position", None) - self._flush_locked() + # Marks only: the snapshot loop must never rewrite the durable + # document, or one torn write could destroy the day's books. + self._flush_marks_locked() return True except Exception: # noqa: BLE001 - reporting must never break trading self._log_write_failure("update worker snapshot") @@ -473,30 +525,82 @@ def mark_clean_shutdown(self, *, results_published: bool = False) -> None: with self._lock: self._state["clean_shutdown"] = True self._state["results_published"] = bool(results_published) - self._flush_locked() + # Both halves: the shutdown flags are crash-critical, and the + # final marks write leaves the pair consistent for a reader. + self._flush_durable_locked() + self._flush_marks_locked() except Exception: # noqa: BLE001 - reporting must never break shutdown self._log_write_failure("mark clean shutdown") - def _flush_locked(self) -> None: - """Publish the in-memory state atomically. Caller must hold the lock. + def _write_document(self, target: Path, document: Mapping[str, Any], *, durable: bool) -> float: + """Atomically publish one document. Returns the elapsed seconds. Writes a sibling ``.tmp`` then :func:`os.replace`s it over the target, - so a reader (or a crash) can only ever see a complete file. ``flush`` - plus ``fsync`` before the replace matter here: without them the rename - can reach the disk before the data does, which on a power cut leaves a - published-but-empty file -- precisely the outcome this module exists to - prevent. + so a reader can only ever see a complete file under normal operation. + + ``durable`` adds ``flush`` + ``fsync`` before the replace, and that is + the whole difference between the two files. Without it the rename can + reach the disk before the data does, so a hard kill can publish a + present-but-garbage file. The durable document therefore always pays the + fsync; the marks document never does, because re-deriving it costs at + most one snapshot interval and it is written 200+ times a session. """ started_at = time.monotonic() - self._state["updated_at"] = _now_ist().isoformat() - self.path.parent.mkdir(parents=True, exist_ok=True) - tmp_path = self.path.with_name(self.path.name + ".tmp") + target.parent.mkdir(parents=True, exist_ok=True) + tmp_path = target.with_name(target.name + ".tmp") with open(tmp_path, "w", encoding="utf-8") as handle: - json.dump(self._state, handle, indent=2, sort_keys=False) - handle.flush() - os.fsync(handle.fileno()) - os.replace(tmp_path, self.path) - elapsed = time.monotonic() - started_at + json.dump(document, handle, indent=2, sort_keys=False) + if durable: + handle.flush() + os.fsync(handle.fileno()) + os.replace(tmp_path, target) + return time.monotonic() - started_at + + def _durable_document_locked(self) -> dict[str, Any]: + """The crash-critical half: everything except the volatile marks.""" + document = { + key: value for key, value in self._state.items() if key != "strategies" + } + strategies: dict[str, Any] = {} + for strategy, entry in self._state.get("strategies", {}).items(): + if isinstance(entry, Mapping): + strategies[str(strategy)] = { + key: value + for key, value in entry.items() + if key not in _MARKS_ONLY_KEYS + } + document["strategies"] = strategies + return document + + def _marks_document_locked(self) -> dict[str, Any]: + """The volatile half: open positions, their marks, and live counters.""" + strategies: dict[str, Any] = {} + for strategy, entry in self._state.get("strategies", {}).items(): + if not isinstance(entry, Mapping): + continue + carried = { + key: value for key, value in entry.items() if key in _MARKS_ONLY_KEYS + } + if carried: + strategies[str(strategy)] = carried + return { + "schema_version": SCHEMA_VERSION, + "session_date": self._state.get("session_date"), + "updated_at": self._state.get("updated_at"), + "strategies": strategies, + } + + def _flush_durable_locked(self) -> None: + """Publish the durable document with fsync. Caller must hold the lock. + + Called only from the trade-event and shutdown paths -- a few dozen times + a session -- so the fsync cost is paid rarely and always for a record + that a crash would otherwise make expensive to rebuild by hand. + """ + self._state["updated_at"] = _now_ist().isoformat() + elapsed = self._write_document( + self.path, self._durable_document_locked(), durable=True + ) if elapsed >= SLOW_WRITE_WARNING_SECONDS: self.log.warning( "Session state durable write took %.3fs (path=%s); the local disk " @@ -505,6 +609,26 @@ def _flush_locked(self) -> None: self.path, ) + def _flush_marks_locked(self) -> None: + """Publish the marks document without fsync. Caller must hold the lock. + + Runs on the supervisor thread, which never trades, and skips fsync + because a torn marks file costs one snapshot interval of mark data and + nothing else -- the trades and the P&L rollup are in the durable file + this method does not touch. + """ + self._state["updated_at"] = _now_ist().isoformat() + elapsed = self._write_document( + self.marks_path, self._marks_document_locked(), durable=False + ) + if elapsed >= SLOW_MARKS_WRITE_WARNING_SECONDS: + self.log.warning( + "Session state marks write took %.3fs (path=%s); this delays " + "supervision, not a trading decision.", + elapsed, + self.marks_path, + ) + def _log_write_failure(self, action: str) -> None: """Log the first persistence failure loudly, then stay quiet.""" if self._write_failure_logged: @@ -526,27 +650,84 @@ def snapshot(self) -> dict[str, Any]: return json.loads(json.dumps(self._state)) +def _marks_path_for(path: str | Path) -> Path: + """Sibling marks path for a durable state path (``x.json`` -> ``x.marks.json``).""" + state_path = Path(path) + suffix = state_path.suffix + stem = state_path.name[: -len(suffix)] if suffix else state_path.name + return state_path.with_name(f"{stem}.marks{suffix}") + + +def _read_json_object(path: Path) -> dict[str, Any] | None: + """Read one JSON object, or ``None`` if it is missing or unusable.""" + if not path.exists(): + return None + with open(path, encoding="utf-8") as handle: + document = json.load(handle) + if not isinstance(document, dict): + logger.warning("Session state at %s is not a JSON object; ignoring it.", path) + return None + return document + + def load_session_state(path: str | Path) -> dict[str, Any] | None: - """Read a state file back, or return ``None`` when it is unusable. + """Read a session back as ONE merged document, or ``None`` if unusable. + + The state is stored as two files -- a durable one holding the trades and the + P&L rollup, and a best-effort ``*.marks.json`` holding open positions and + their last marks (see :class:`SessionStateStore`). This merges them so that + every reader (`resumable_open_positions`, `recorded_realized_pnl`, the + runner's resume path) sees the same single-document shape it always has. Deliberately forgiving: a missing file is the normal first-run case, and a - truncated or hand-edited file must not stop the runner from starting. Both - are reported as ``None`` and the caller simply proceeds without recovery. + truncated or hand-edited one must not stop the runner from starting. + + The asymmetry is the point. A corrupt DURABLE file means no recovery, and + is reported as ``None``. A corrupt or missing MARKS file costs only the + open positions: the P&L is still returned, because that is the record whose + loss motivated ADR-0012 and it was never written by the snapshot loop. """ try: state_path = Path(path) - if not state_path.exists(): + state = _read_json_object(state_path) + if state is None: return None - with open(state_path, encoding="utf-8") as handle: - state = json.load(handle) - if not isinstance(state, dict): - logger.warning("Session state at %s is not a JSON object; ignoring it.", state_path) - return None - return state except Exception: # noqa: BLE001 - a bad state file must not stop startup logger.exception("Could not read session state from %s; continuing without recovery.", path) return None + try: + marks = _read_json_object(_marks_path_for(state_path)) + except Exception: # noqa: BLE001 - marks are best-effort by design + logger.exception( + "Could not read the session marks beside %s; continuing with P&L only " + "(no open positions will be offered for resume).", state_path, + ) + return state + + if marks is None: + return state + if str(marks.get("session_date", "")) != str(state.get("session_date", "")): + logger.warning( + "Session marks at %s are from a different session than the durable " + "state; ignoring them.", _marks_path_for(state_path), + ) + return state + + mark_strategies = marks.get("strategies") + if not isinstance(mark_strategies, Mapping): + return state + strategies = state.setdefault("strategies", {}) + if not isinstance(strategies, dict): + return state + for strategy, entry in mark_strategies.items(): + if not isinstance(entry, Mapping): + continue + target = strategies.setdefault(str(strategy), {}) + if isinstance(target, dict): + target.update(entry) + return state + def resumable_open_positions( state: Mapping[str, Any] | None, diff --git a/Nifty Multi Strategy Front Test - Master File.py b/Nifty Multi Strategy Front Test - Master File.py index 5ee8578..f2795d5 100644 --- a/Nifty Multi Strategy Front Test - Master File.py +++ b/Nifty Multi Strategy Front Test - Master File.py @@ -1272,8 +1272,9 @@ def load_module(module_name: str, file_path: Path): ) # CPR Codex AI modules are loaded from a directory whose name contains spaces, -# so their sibling imports temporarily use bare names. The collision guard below -# refuses to replace an unrelated module that already owns one of those names. +# so their cross-file imports use bare ``cpr_ai_*`` sibling names. The collision +# guard below refuses to replace an unrelated module that already owns one of +# those names. # SDK code remains lazy and runs in a sanitized subprocess; importing the master # therefore does not authenticate Codex. Any failure restores the previous # module table and disables only this optional worker. @@ -1318,6 +1319,21 @@ def load_module(module_name: str, file_path: Path): CPR_AI_DECISION_LOGIC = load_module( "master_cpr_ai_decision_log", _cpr_ai_dir / "cpr_ai_decision_log.py" ) + # ``load_module`` exposes a spaced directory only while one file executes. + # CPR AI intentionally delays importing its prompt, schema, and Codex runner + # until the first completed bar, so removing that directory afterwards made + # those production-only imports fail even though the standalone smoke test + # could see its own script folder. The collision guard above has already + # proved that every ``cpr_ai_*`` bare name is either unused or belongs here, + # so retaining this one narrow source directory is safe and intentional. + if str(_cpr_ai_dir) not in sys.path: + sys.path.insert(0, str(_cpr_ai_dir)) + + # The delayed runner imports ``cpr_ai_agent`` by its sibling name. Point + # that name at the exact module the master already loaded; otherwise Python + # would execute the agent file a second time and create incompatible copies + # of its result dataclasses. + sys.modules["cpr_ai_agent"] = CPR_AI_AGENT_LOGIC CPR_AI_AVAILABLE = True except Exception as _cpr_ai_import_exc: # optional strategy, never fatal for _cpr_ai_name in _cpr_ai_sibling_names: diff --git a/Signal Generators/CPR AI Agent/cpr_ai_agent.py b/Signal Generators/CPR AI Agent/cpr_ai_agent.py index ab167ae..cc50e56 100644 --- a/Signal Generators/CPR AI Agent/cpr_ai_agent.py +++ b/Signal Generators/CPR AI Agent/cpr_ai_agent.py @@ -201,16 +201,38 @@ def _trending_entry(self, context: Mapping[str, Any], proposal: Any, direction: Continuation and reversal use different frozen VWAP sequences, but both require at least 40 percent of the entry body on the trade side, directional RSI, ordered/sloping EMAs, and the completed candle extreme - as stop. Model reasoning cannot waive any of these conditions. + as stop. A continuation also cannot chase a long above R2 or a short + below S2. Model reasoning cannot waive any of these conditions. """ if proposal.regime != "TRENDING": return _hold("trending_regime_rejected", "VWAP entries require the TRENDING regime.", proposal) + long = direction == "LONG" + if proposal.setup == "TRENDING_VWAP_CONTINUATION": + levels = self._mapping(context, "session_levels") + level_map = self._mapping(levels, "levels") + entry = levels.get("current_close") + boundary_name = "r2" if long else "s2" + boundary = level_map.get(boundary_name) + + # The comparison is deliberately strict: the new knowledge says + # "above R2" and "below S2." Existing reward/target geometry still + # decides whether an entry exactly at or near a level is practical. + outside_boundary = ( + isinstance(entry, (int, float)) + and isinstance(boundary, (int, float)) + and ((long and entry > boundary) or (not long and entry < boundary)) + ) + if outside_boundary: + return _hold( + "continuation_outside_r2_s2", + "Trend continuation cannot enter long above R2 or short below S2.", + proposal, + ) momentum = self._mapping(context, "momentum_vwap") vwap = self._mapping(momentum, "vwap") sequence = self._mapping(vwap, "sequence_evidence") body = self._mapping(vwap, "entry_candle") - long = direction == "LONG" sequence_key = ( "all_recent_above" if proposal.setup == "TRENDING_VWAP_CONTINUATION" and long diff --git a/Signal Generators/CPR AI Agent/cpr_ai_prompt.py b/Signal Generators/CPR AI Agent/cpr_ai_prompt.py index 8003219..cead689 100644 --- a/Signal Generators/CPR AI Agent/cpr_ai_prompt.py +++ b/Signal Generators/CPR AI Agent/cpr_ai_prompt.py @@ -8,7 +8,7 @@ from __future__ import annotations -CPR_AI_PROMPT_VERSION = "cpr-srsi-vwap-context-v2" +CPR_AI_PROMPT_VERSION = "cpr-srsi-vwap-context-v3" _ROLE = """ROLE AND BOUNDARY You are an advisory CPR context analyst. Assess only complete five-minute bars. @@ -32,6 +32,13 @@ only when market_structure reports its eligible R1 candidate. Prefer HOLD/NONE whenever evidence conflicts or is incomplete.""" +_CONTINUATION_BOUNDARY = """TREND-CONTINUATION CPR BOUNDARY +Never propose a bullish trend-continuation entry above R2. Never propose a +bearish trend-continuation entry below S2. When either strict boundary is +crossed, return HOLD with setup NONE instead of chasing the continuation. +This restriction applies to TRENDING_VWAP_CONTINUATION; it does not create or +change the evidence rules for sideways or reversal setups.""" + def _output_rules(model_used: str) -> str: """Build output rules that echo the host's configured model exactly. @@ -67,7 +74,7 @@ def build_system_prompt( # Prefer the explicitly approved field. The alias exists only so an older # caller does not need to change at the same time as this prompt API. knowledge = operator_approved_knowledge.strip() or discretionary_context.strip() - sections = [_ROLE, _TOOLS, _JUDGMENT] + sections = [_ROLE, _TOOLS, _JUDGMENT, _CONTINUATION_BOUNDARY] # Keeping discretionary prose in a separate section makes later reviews # show exactly what changed without mixing it into permanent safety text. if knowledge: diff --git a/Signal Generators/SL Hunting AI Agent/premarket_note.json b/Signal Generators/SL Hunting AI Agent/premarket_note.json index be749ff..8e6913e 100644 --- a/Signal Generators/SL Hunting AI Agent/premarket_note.json +++ b/Signal Generators/SL Hunting AI Agent/premarket_note.json @@ -1,32 +1,31 @@ { - "for_date": "2026-08-11", - "source": "Intraday Hunter, 'Prediction For 11 AUG 2026' (cOvPKZFervw, uploaded 2026-08-10)", - "context": "NIFTY sold off but the BREAKDOWN FAILED -- it turned straight back up and held above. Buyers who joined were flushed on the closing rejection. BankNIFTY sold with no follow-through; Sensex sold, recovered, then rejected late.", + "for_date": "2026-08-12", + "source": "Intraday Hunter, 'Prediction For 12 AUG 2026' (CoxS77NfnsI, uploaded 2026-08-11)", + "context": "NIFTY and Sensex sold hard but produced no follow-through and never crossed the round number, so few carried shorts overnight. BankNIFTY sold then recovered, hitting the stops of sellers who joined on the retracement.", "plan": [ - "FLAT to GAP-UP: identify BUY-side setups and go WITH the market. Same read on all three indices -- NIFTY, BankNIFTY and Sensex get the identical conditional.", - "GAP-DOWN: identify SELL-side setups. He says the structure becomes DIFFERENT there and traps form differently, so treat it as its own regime rather than a mirror of the up case.", - "The failed breakdown is the whole story: sellers were TURNED AROUND rather than paid, so they are the trapped inventory carried into this session.", - "He explicitly allows that today may only have spun the sellers, and that the NEXT gap-down is where the market could finally deliver momentum.", - "EXPIRY DAY -- flagged in the first ten seconds. Expect the usual expiry premium decay and pinning distortion around round strikes.", - "Buyers who entered on the hold were flushed on the last rejection, so freshly recruited longs are already thin going in.", - "NOTE: this INVERTS yesterday's plan (10 Aug wanted sells on flat, buys on a gap-up). Do not carry the previous session's conditional forward.", - "No stand-aside branch this time, unlike 10 Aug's large-gap-down veto -- both directions are actionable as stated." + "FLAT to GAP-DOWN: identify SELL-side setups. He states this separately for all three indices -- NIFTY, BankNIFTY and Sensex each get the same conditional.", + "A MODERATE gap-up keeps broadly the same plan; he says 'same plan' for Sensex and names the sell side for BankNIFTY in the same breath as the gap-up case.", + "A LARGE gap-up VOIDS the plan: 'if a big gap-up opens, maybe the market has just made a TRAP -- in a big gap-up we cannot make such a plan for now.' Stand aside; do not guess the missing branch.", + "The seller crowd is SPENT, not available to hunt. On BankNIFTY: sellers who entered on the retracement had their stops hit by the recovery, so the market has already taken them out.", + "Nobody carried size short overnight -- the move never crossed the round number and produced no follow-through, so overnight inventory is thin on BOTH sides.", + "AMBIGUITY, recorded rather than resolved: on NIFTY he also says a mild gap-up can be followed WITH the market, 'if not many sellers are seated the market may not find SLs.' That cuts against the sell-side line; do not force a reading.", + "11 Aug was expiry and gave one move then nothing. Today is an ordinary Wednesday session, so expiry distortion is not a factor." ], "levels": [ { "index": "NIFTY", - "resistance": [24610, 24670], - "support": [24440, 24360] + "resistance": [24560, 24610], + "support": [24430, 24345] }, { "index": "BANKNIFTY", - "resistance": [57800, 58000], - "support": [57340, 57150] + "resistance": [57650, 57800], + "support": [57100, 56960] }, { "index": "SENSEX", - "resistance": [78640, 78920], - "support": [78200, 78000] + "resistance": [78475, 78640], + "support": [78046, 77810] } ] } diff --git a/Signal Generators/SL Hunting AI Agent/sl_hunting_doc.md b/Signal Generators/SL Hunting AI Agent/sl_hunting_doc.md index 3f0f695..4c73d5d 100644 --- a/Signal Generators/SL Hunting AI Agent/sl_hunting_doc.md +++ b/Signal Generators/SL Hunting AI Agent/sl_hunting_doc.md @@ -2912,3 +2912,176 @@ with full precision. Advisory candidate levels only, as always. Test updated: `test_shipped_note_matches_august_11_intraday_hunter_plan` replaces the 10 Aug equivalent and asserts the branch DIRECTIONS explicitly, because an inverted plan is the specific failure mode a copy-forward would cause. + +--- + +## Video addendum - the 11 Aug LIVE SESSION (v4e) + +**Source:** Intraday Hunter live session, 11 Aug 2026 (`_JXirKMmI58`, 9:25). A +**LOSS**, and encoded precisely because it is one. Every prior addendum in this +series distilled a winning session; a losing one shows which part of the method +was load-bearing and which part was rationalisation. + +### He named the disqualifying fact, then traded against it + +The plan came from his own pre-open note (`cOvPKZFervw`, shipped as the 11 Aug +note): flat-to-gap-up wants the buying side. The open was flat across all three +indices and a sharp sell-off followed. He then said, twice, that the setup's +precondition was absent: + +> "Around here neither the BUYER's stop losses are available nor the SELLER's." +> "Here not many traders were seated." + +And traded anyway, on a forecast of who *would* arrive: a sharp drop tempts +intraday sellers in, so the market should rise to take them out. He bought calls +on BankNIFTY (1170 qty, 57400 CE), Sensex (900) and NIFTY (1430, expiry day). +The loss began immediately and widened; the expected recovery never started; he +cut at his pre-declared level. + +That is the whole lesson, and it is the most expensive error this method makes +available: **hunting inventory that exists is the strategy; predicting inventory +that might arrive is a different and much weaker activity wearing the same +vocabulary.** When the honest read is "nobody is seated on either side", the +output is HOLD. + +### The refinement that survives the loss + +His reasoning was not worthless - one part of it is a genuine advance on v4d, +independent of the outcome: + +| Open | Who it recruits | Trap quality | +|---|---|---| +| GAP-DOWN | **positional** sellers - they enter at the close and hold overnight | large, committed, worth hunting next day | +| FLAT | **intraday** sellers only - "positional will not take an entry yet" | small, perishable, gone by the close | + +> "If the market really had to create positional sellers' stop losses, it would +> have given a straight GAP-DOWN." + +v4d established *that* a flat open seats people. This names *who*, and it +explains why a flat-open hunt should be sized and targeted more modestly than +the identical shape after a gap-down. + +### Knowledge changes (v4e, all prose) + +- `OPENING_DRIVE`: WHICH CROWD THE OPEN RECRUITS DECIDES HOW BIG THE TRAP IS; + A FORECAST OF WHO WILL ARRIVE IS NOT EVIDENCE OF WHO IS SEATED; A SHARP FIRST + SLIDE BAITS; A SLOW ONE MEANS IT. +- `RISK`: NAME THE LAST POINT, NOT ONLY THE STOP; DISCIPLINE IS ASYMMETRIC + BETWEEN WINNERS AND LOSERS. +- The sharp-slide rule is deliberately encoded as a **weak prior with its own + counter-example attached** - it is the read that lost him the session, so it + is recorded as a tie-breaker and explicitly barred from being a trade premise. +- Test markers: `test_system_prompt_has_v4e_recruitment_and_losing_session_knowledge` + and `test_v4e_empty_book_is_a_no_trade_not_a_forecasting_licence`. The second + is a drift guard: it asserts the forecasting rule still resolves to HOLD and + still reconciles with v4c's MANUFACTURES MORE, so a later edit cannot quietly + turn it into a licence to predict a crowd. +- Prompt size 96,627 -> 101,087 chars (headroom 18,913). + +### How our agent traded the same session + +**Provisional - the session was still running when this was written** (last log +line 13:07; market closes 15:30). Realized so far, from the runner log: + +| Strategy | Legs | Realized | +|---|---|---| +| SL Hunting AI | 7 | -2,647.75 | +| Parabolic SAR | 5 | -1,478.75 | +| RSI Reversal | 1 | -731.25 | +| Long Strangle | 5 | -542.75 | +| Heikin Ashi | 11 | -484.25 | +| EMA | 1 | +191.75 | +| Mean Reversion Z-Score | 4 | +789.75 | +| CPR Algo 3 | 1 | +874.25 | +| Renko | 2 | +2,190.50 | +| **Total** | **37** | **-1,838.50** | + +SL Hunting's figure is cross-checked against its own `Result summary` line +(-2,647.75, Trades=4) and matches exactly. Note the mirror legs log as +`MIRROR EXIT`, not `EXIT`; a first pass that grepped for `| EXIT ` under-counted +the agent by 948.00. + +**The agent got the direction right and lost anyway.** It was SHORT all morning +- the opposite of IH's call-side trade, and correct: NIFTY fell from a 24576 +open through the 24500 round number toward the 24440 support named in the +pre-open note. IH was wrong about direction and the agent was right about it. + +The agent still lost more than any other strategy, because it took **four short +entries in 47 minutes** and cut each one almost immediately: + +| Entry | Setup | Exit | Held | +|---|---|---|---| +| 09:40 | runaway_trend_continuation_short | 09:52 profit_book_stall_cross_index_veto | 12 min | +| 10:02 | trendline_rejection_shooting_star | 10:04 profit_booking_stall_reversal_bias | **2 min** | +| 10:09 | double_top_rejection_bearish_engulfing (target 24400) | 10:18 premise_stall_theta_exit | 9 min | +| 10:27 | fibo_61_bearish_inside_bar_breakdown | 10:28 index_hierarchy_bnf_exit | **1 min** | + +Three of the four exits are *stall* judgements rather than stops, and the fourth +is the BankNIFTY hierarchy rule firing one minute after entry. The 10:09 trade +had a 24400 target and was released at 24469-24492 having never been stopped. + +So the failure is not the read - it is that **the premise-stall exit is firing +faster than the premise can resolve**, converting one correct directional call +into four round-trips on an expiry day, when spread and theta punish churn +hardest. This is the exact inverse of the failure v3y guards against: v3y stops +the agent holding a loser because it feels right; nothing currently stops it +releasing a winner because the tape paused. + +v4e's two RISK rules speak directly to this - DISCIPLINE IS ASYMMETRIC puts the +patience on the winning side, and NAME THE LAST POINT replaces "has it stalled?" +with a pre-declared level and deadline. Whether that is enough, or whether +`premise_stall` needs a minimum-hold or a bar-count floor before it may fire, is +a **candidate for the lessons loop** rather than something to encode as IH +knowledge - it is a property of our agent, not of the method. + +--- + +### Pre-open note for 2026-08-12 (Wednesday) + +**Source:** Intraday Hunter, "Prediction For 12 AUG 2026" (`CoxS77NfnsI`, +uploaded 2026-08-11, 2:00). Note-only; no knowledge version attached. + +**The seller crowd is described as SPENT rather than seated**, which is a +different starting condition from every note in this series so far. He does not +say sellers are sitting there waiting to be hunted; he says the market has +already taken them out: + +> "When the selling came and a retracement happened, sellers would certainly +> have entered there. But the market would have hit their SLs. So if it has +> already taken the sellers out, we can go WITH the market." + +That pairs directly with **v4e's WHICH CROWD THE OPEN RECRUITS**: he then +explains why nobody is carrying size overnight either — + +> "It did not cross the round number, so not many people would have held their +> selling quantity." +> "One momentum came and after that not many people held short positions." + +No round-number breach, no follow-through, therefore no overnight inventory. The +result is a session that starts with thin positioning on **both** sides. + +**The second explicit escape hatch in the series.** 10 Aug had one for a large +gap-down; this one is for a large gap-up: + +> "If a big gap-up opens, maybe the market has just made a TRAP. In a big gap-up +> we cannot make such a plan for now... there the market can start making a +> DIFFERENT type of trap." + +Recorded as stand-aside, not as a guessed branch — the same treatment the 10 Aug +note gave its missing branch. + +**One genuine ambiguity, left unresolved on purpose.** The sell-side conditional +is stated cleanly for all three indices, but on NIFTY he also says a mild gap-up +can be followed *with* the market "if not many sellers are seated, the market may +not find SLs". Read one way that is a long; read another it is a reason to expect +no upward pull at all. The note records the tension rather than picking a side, +which is what v4e's A FORECAST OF WHO WILL ARRIVE rule demands of an unclear read. + +**Transcription caveat:** one BankNIFTY support arrived as "5710", read as 57100 +alongside 56960 — the same dropped-trailing-zero artefact seen on 4, 7 and +10 Aug. Advisory candidate levels only. + +Test updated: `test_shipped_note_matches_august_12_intraday_hunter_plan` +replaces the 11 Aug equivalent. It asserts the escape hatch and the ambiguity +line survive verbatim, because those are the two things a copy-forward or a +tidy-up would silently remove. diff --git a/Signal Generators/SL Hunting AI Agent/sl_hunting_knowledge.py b/Signal Generators/SL Hunting AI Agent/sl_hunting_knowledge.py index aecd9f2..c3e55bb 100644 --- a/Signal Generators/SL Hunting AI Agent/sl_hunting_knowledge.py +++ b/Signal Generators/SL Hunting AI Agent/sl_hunting_knowledge.py @@ -534,6 +534,45 @@ end, and it is why a flat-open rally into a level is normally a SHORT candidate rather than a breakout candidate. Do not apply that default when every condition of the separately scoped Variant B seller-hunt long is satisfied. +- WHICH CROWD THE OPEN RECRUITS DECIDES HOW BIG THE TRAP IS (v4e). v4d established + THAT a flat open seats people. This names WHO, and it changes the size and the + durability of the inventory: + * GAP-DOWN -> recruits POSITIONAL sellers. They enter at the close and hold + overnight, so the inventory is large, committed, and worth hunting the next + day. IH: "if the market really had to create positional sellers' stop + losses, it would have given a straight GAP-DOWN... in a gap-down everyone + comfortably makes a positional trade and sits." + * FLAT -> recruits INTRADAY sellers only. "In a flat open the positional + trader will not take an entry yet. Here the INTRADAY traders come." They are + fewer, they are already looking to book, and they will be flat by the close. + Consequence: a flat-open hunt is aimed at a SMALLER and more perishable crowd + than a gap-down hunt of the same shape. Size and target accordingly (this is the + participation form of CROWD SIZE IS THE THIRD TARGET INPUT), and do not expect a + flat-open trap to pay like a gap-down one. +- A FORECAST OF WHO WILL ARRIVE IS NOT EVIDENCE OF WHO IS SEATED (v4e). The single + most expensive error available in this method, recorded from a LOSING IH session + (11 Aug 2026) so it is not learned the hard way. He stated the disqualifying fact + himself, twice, before entering: "around here neither the BUYER's stop losses are + available nor the SELLER's" and "here not many traders were seated." He then + built the trade on a PREDICTION instead — that a sharp early sell-off would tempt + intraday sellers in, and the market would rise to take them out. It did not; the + selling simply continued and he cut for a loss. + The rule: this method hunts inventory that ALREADY EXISTS and is OBSERVABLE. A + chain of reasoning about who is likely to arrive, however sound, is a different + and much weaker class of evidence. When the honest read is "nobody is seated on + either side", the correct output is HOLD — an empty book is a no-trade condition, + not an invitation to forecast one into existence. Note this does NOT contradict + v4c's WHEN THE TRAPPED INVENTORY IS SPENT, THE MARKET MANUFACTURES MORE: + manufacturing is what the market does over time, but it may not complete inside + your holding period, and you cannot bank on being early to it. +- A SHARP FIRST SLIDE BAITS; A SLOW ONE MEANS IT (v4e). IH's read of the opening + move, stated as a prior: "if it had fallen SLOWLY I would even have accepted that + the market might produce a big move. But the selling was SHARP — the market + suddenly offered greed", i.e. an abrupt drop looks engineered to recruit sellers, + whereas a grinding one looks like genuine supply. Treat this as a weak prior and + nothing more: on the very session that produced it, the sharp slide was followed + by continuous selling and the bait read was WRONG. Use it to break a tie between + two otherwise-equal reads, never as the premise of a trade on its own. - SEATED-BUYER TEST — run this BEFORE the long branch fires (v3y). The whole gap-up-long premise is "a gap-up leaves nobody trapped, so there is no hunt available". That premise is FALSE when the prior session already seated a buying @@ -939,6 +978,25 @@ than a feeling, and it is what lets A REJECTION BEFORE THE FLUSH IS NOISE be applied without it becoming an excuse: a wobble inside the band is noise, and one well beyond it is the read being wrong even if the stop has not been hit. +- NAME THE LAST POINT, NOT ONLY THE STOP (v4e). One price, declared out loud BEFORE + you need it, at which the question stops being "is the read still alive?" and + becomes "did it recover or not?" IH, deep in a losing trade: "let us pause a + little — THIS IS THE LAST POINT. If the market does not recover from here we + will leave. If it recovers from here our position can survive." He then honoured + it: "no recovery is visible, continuous selling is still there... we will have to + cut our trade." Distinct from PRE-COMMIT THE ADVERSE MOVE, which is a magnitude: + this is a LOCATION plus a deadline, and its purpose is to stop the averaging-in + reflex that a thesis about future participants invites. If price is below the + last point and the expected reaction has not begun, exit — do not re-argue the + premise. +- DISCIPLINE IS ASYMMETRIC BETWEEN WINNERS AND LOSERS (v4e). The same session states + both halves in one breath: "when you get a chance to make profit, THERE you make + the target big, wait in the market — those things work. But when there is a loss, + follow proper discipline and cut the trade and leave." So the patience rules + (EXPECT A SECOND LEG AFTER THE PAUSE, a crowd-scaled target) apply on the winning + side ONLY. Applying them to a loser is not patience, it is the premise being + re-argued after the evidence arrived. Never widen, delay, or suspend an exit rule + because the reasoning still feels right. - CROWD SIZE IS THE THIRD TARGET INPUT (v4c). Alongside how recently the crowd was recruited (v4a) and whether it has averaged down (v4b), HOW MANY are seated scales the move available against them — and for a reason worth knowing: a diff --git a/Tests/Dependencies/test_repository_policy.py b/Tests/Dependencies/test_repository_policy.py index a38b383..2b4289f 100644 --- a/Tests/Dependencies/test_repository_policy.py +++ b/Tests/Dependencies/test_repository_policy.py @@ -10,6 +10,7 @@ import re import sys import tomllib +import urllib.parse from pathlib import Path import yaml @@ -271,6 +272,10 @@ def test_current_architecture_docs_distinguish_core_from_optional_agents(): ROOT / "AGENTS.md", ROOT / "CLAUDE.md", ROOT / "Nifty Multi Strategy Front Test - Master File.py", + # The committed HLD is a whole-system overview, so the same rule applies: + # a reader must be able to see both optional agents and must not mistake + # the core count for the enabled total (docs/adr/0011 follow-up). + ROOT / "docs/hld/system-overview.md", ) failures: list[str] = [] for path in architecture_files: @@ -328,6 +333,97 @@ def test_agent_architecture_docs_stay_in_sync_and_cover_the_optional_cpr_agent() assert "double gate" in lower +def _committed_design_documents() -> list[Path]: + """Every tracked design document under docs/, excluding the scratchpad. + + ``docs/superpowers/`` is gitignored session working material, not product + documentation, so it is deliberately outside every gate here (docs/adr/0011). + """ + + return sorted( + path + for folder in ("adr", "lld") + for path in (ROOT / "docs" / folder).glob("*.md") + if path.is_file() + ) + + +def test_every_committed_design_document_is_linked_from_the_docs_index(): + """A new ADR or LLD must reach the index, and the index must not rot. + + ``docs/README.md`` is the only navigation surface for the committed + architecture set. A document that never gets linked is invisible -- it will + not be read, will not be maintained, and will quietly go stale. The reverse + is just as bad: a link left behind by a renamed or deleted file sends a + reader to a 404 and makes the whole index untrustworthy. + + Checked in BOTH directions for that reason. + """ + + index_path = ROOT / "docs/README.md" + index = index_path.read_text(encoding="utf-8") + + linked = { + match.group(1) + for match in re.finditer(r"\((?:\./)?((?:adr|lld)/[^)#]+\.md)[^)]*\)", index) + } + on_disk = { + path.relative_to(ROOT / "docs").as_posix() for path in _committed_design_documents() + } + + unlinked = sorted(on_disk - linked) + assert not unlinked, ( + "these design documents exist but are not linked from docs/README.md: " + + ", ".join(unlinked) + ) + + dangling = sorted(linked - on_disk) + assert not dangling, ( + "docs/README.md links these documents, which do not exist: " + ", ".join(dangling) + ) + + +def test_relative_links_inside_the_committed_docs_resolve(): + """No broken cross-reference anywhere in the committed docs set. + + The HLD, the LLDs and the ADRs reference each other and the source tree + constantly. Renaming a file is the normal way those break, and a broken link + is invisible until somebody follows it -- so it is checked mechanically + rather than by review. + + Only relative targets are resolved. External URLs are not fetched: this + suite must stay network-free. + """ + + docs_root = ROOT / "docs" + link_pattern = re.compile(r"\[[^\]]*\]\(([^)]+)\)") + broken: list[str] = [] + + for document in sorted(docs_root.rglob("*.md")): + # The Superpowers scratchpad is gitignored and may reference paths that + # only existed during one session. + if "superpowers" in document.parts: + continue + for line_number, line in enumerate( + document.read_text(encoding="utf-8").splitlines(), start=1 + ): + for target in link_pattern.findall(line): + target = target.strip() + if target.startswith(("http://", "https://", "mailto:", "#")): + continue + # Strip any "#section" anchor, then undo the %20 escaping the + # spaced-name folders need in markdown links. + relative = urllib.parse.unquote(target.split("#", maxsplit=1)[0]) + if not relative: + continue + if not (document.parent / relative).resolve().exists(): + broken.append( + f"{document.relative_to(ROOT).as_posix()}:{line_number} -> {target}" + ) + + assert not broken, "broken relative links in the committed docs:\n" + "\n".join(broken) + + def test_every_env_setting_the_code_reads_is_documented_in_env_example(): """A new `.env` knob must ship with its `env.example` entry. diff --git a/Tests/Dependencies/test_session_state.py b/Tests/Dependencies/test_session_state.py index 2fade3d..7d1fffe 100644 --- a/Tests/Dependencies/test_session_state.py +++ b/Tests/Dependencies/test_session_state.py @@ -26,6 +26,7 @@ from session_state import ( SCHEMA_VERSION, SessionStateStore, + _marks_path_for, load_session_state, recorded_realized_pnl, resumable_open_positions, @@ -508,3 +509,181 @@ def test_end_to_end_crash_then_recover(state_path: Path): assert resumable["Renko"]["entry_trade_price"] == 112.35 assert resumable["Renko"]["last_mark_ltp"] == 98.1 assert os.path.exists(state_path) + + +# --------------------------------------------------------------------------- +# Durable / marks split -- the fsync-contention fix +# --------------------------------------------------------------------------- +# The supervisor rewrote the whole document every 30s WITH fsync, which on +# 2026-08-11 produced 210 slow-write warnings (median 0.83s, max 8.73s) against +# 13 on trading threads. Dropping fsync from that path is only safe if the +# snapshot stops touching the durable document at all: `os.replace` is atomic +# for the NAME but not the DATA, so a hard kill during an un-fsynced rewrite can +# publish a present-but-garbage file -- which would destroy the trades and the +# P&L rollup that ADR-0012 exists to protect. + + +def test_trade_events_fsync_but_snapshots_do_not(state_path: Path, monkeypatch): + """The whole point of the split: one path pays fsync, the other never does.""" + calls: list[str] = [] + real_fsync = os.fsync + monkeypatch.setattr( + "session_state.os.fsync", lambda fd: (calls.append("fsync"), real_fsync(fd))[1] + ) + + store = _store(state_path) + calls.clear() # construction establishes the durable file; ignore that write. + + store.update_worker_snapshot([{"strategy": "Renko"}], force=True) + assert calls == [], "the 30s snapshot loop must never fsync" + + store.record_trade_event({"action": "EXIT", "strategy": "Renko", "pnl": -929.5}) + assert calls == ["fsync"], "a realized-P&L event must be durable before returning" + + +def test_a_snapshot_never_rewrites_the_durable_file(state_path: Path): + """The safety property. A torn snapshot must not be able to eat the books.""" + store = _store(state_path) + store.record_trade_event({"action": "EXIT", "strategy": "Renko", "pnl": -929.5}) + durable_before = state_path.read_bytes() + mtime_before = state_path.stat().st_mtime_ns + + for _ in range(5): + store.update_worker_snapshot( + [{"strategy": "Renko", "open_position": serialize_position(_FakePosition())}], + force=True, + ) + + assert state_path.read_bytes() == durable_before + assert state_path.stat().st_mtime_ns == mtime_before + # ...and the marks landed in their own file. + assert _marks_path_for(state_path).exists() + + +def test_durable_file_exists_from_construction(state_path: Path): + """A session that dies before its first trade must still leave a document. + + Without this the session date, the shutdown flags and any trade book carried + forward from a same-day restart would all be missing from recovery. + """ + _store(state_path) + state = load_session_state(state_path) + assert state is not None + assert state["session_date"] == "2026-08-10" + assert state["clean_shutdown"] is False + + +def test_positions_and_pnl_merge_back_into_one_document(state_path: Path): + """Readers keep seeing the single-document shape they always have.""" + store = _store(state_path) + store.record_trade_event({"action": "EXIT", "strategy": "Renko", "pnl": -929.5}) + store.update_worker_snapshot( + [ + { + "strategy": "Renko", + "realized_pnl": -929.5, + "completed_trades": 1, + "live_trading": False, + "execution_mode": "PAPER", + "open_position": serialize_position( + _FakePosition(), leg_marks={"option": 98.1} + ), + } + ], + force=True, + ) + + merged = load_session_state(state_path) + assert merged is not None + assert recorded_realized_pnl(merged) == {"Renko": -929.5} + entry = merged["strategies"]["Renko"] + assert entry["completed_trades"] == 1 # from marks + assert entry["recorded_pnl"] == -929.5 # from durable + assert entry["open_position"]["entry_trade_price"] == 112.35 + assert resumable_open_positions(merged, session_date=TODAY)["Renko"] + + +def test_a_corrupt_marks_file_costs_positions_but_never_the_pnl(state_path: Path): + """The asymmetry that justifies skipping fsync on the marks path.""" + store = _store(state_path) + store.record_trade_event({"action": "EXIT", "strategy": "Renko", "pnl": -929.5}) + store.update_worker_snapshot( + [{"strategy": "Renko", "open_position": serialize_position(_FakePosition())}], + force=True, + ) + # Simulate the torn write that skipping fsync makes possible. + _marks_path_for(state_path).write_text("{ not json", encoding="utf-8") + + recovered = load_session_state(state_path) + assert recovered is not None + assert recorded_realized_pnl(recovered) == {"Renko": -929.5} + assert resumable_open_positions(recovered, session_date=TODAY) == {} + + +def test_a_missing_marks_file_is_not_an_error(state_path: Path): + store = _store(state_path) + store.record_trade_event({"action": "EXIT", "strategy": "Renko", "pnl": 5.0}) + _marks_path_for(state_path).unlink(missing_ok=True) + + recovered = load_session_state(state_path) + assert recovered is not None + assert recorded_realized_pnl(recovered) == {"Renko": 5.0} + + +def test_marks_from_another_session_are_ignored(state_path: Path): + """A stale marks file must never graft yesterday's positions onto today.""" + store = _store(state_path) + store.record_trade_event({"action": "EXIT", "strategy": "Renko", "pnl": 5.0}) + store.update_worker_snapshot( + [{"strategy": "Renko", "open_position": serialize_position(_FakePosition())}], + force=True, + ) + stale = json.loads(_marks_path_for(state_path).read_text(encoding="utf-8")) + stale["session_date"] = "2020-01-01" + _marks_path_for(state_path).write_text(json.dumps(stale), encoding="utf-8") + + recovered = load_session_state(state_path) + assert recovered is not None + assert recorded_realized_pnl(recovered) == {"Renko": 5.0} + assert resumable_open_positions(recovered, session_date=TODAY) == {} + + +def test_a_corrupt_durable_file_is_still_a_total_loss(state_path: Path): + """Unchanged contract: the durable half has no fallback, which is why it fsyncs.""" + store = _store(state_path) + store.record_trade_event({"action": "EXIT", "strategy": "Renko", "pnl": 5.0}) + state_path.write_text("{ not json", encoding="utf-8") + assert load_session_state(state_path) is None + + +def test_restart_archives_both_files_under_one_timestamp(state_path: Path): + store = _store(state_path) + store.record_trade_event({"action": "EXIT", "strategy": "Renko", "pnl": -100.0}) + store.update_worker_snapshot( + [{"strategy": "Renko", "open_position": serialize_position(_FakePosition())}], + force=True, + ) + + replacement = _store(state_path) + archive = replacement.archive_path + assert archive is not None and archive.exists() + assert _marks_path_for(archive).exists(), "the marks file must be archived too" + # The live marks path was moved aside rather than left pointing at old state. + assert not _marks_path_for(state_path).exists() + # Same-day P&L still carried into the replacement session. + assert recorded_realized_pnl(replacement.snapshot()) == {"Renko": -100.0} + + +def test_slow_marks_write_warns_about_supervision_not_trading(state_path: Path, caplog): + """The old message blamed the trading loop for a supervisor-thread write.""" + import session_state as module + + store = _store(state_path) + with caplog.at_level("WARNING"), pytest.MonkeyPatch.context() as mp: + mp.setattr(module, "SLOW_MARKS_WRITE_WARNING_SECONDS", 0.0) + store.update_worker_snapshot([{"strategy": "Renko"}], force=True) + + messages = [record.getMessage() for record in caplog.records] + assert any("marks write took" in m and "not a trading decision" in m for m in messages) + # And it must not blame the trading loop, which was the old message's error. + assert not any("delaying the caller's trading loop" in m for m in messages) diff --git a/Tests/Signal Generators/CPR AI Agent/test_cpr_ai_context.py b/Tests/Signal Generators/CPR AI Agent/test_cpr_ai_context.py index 100e8bd..43b52ca 100644 --- a/Tests/Signal Generators/CPR AI Agent/test_cpr_ai_context.py +++ b/Tests/Signal Generators/CPR AI Agent/test_cpr_ai_context.py @@ -505,6 +505,11 @@ def test_prompt_requires_tools_judgment_risk_boundary_and_future_knowledge_seam( assert "SIDEWAYS" in prompt and "TRENDING" in prompt and "UNDECIDED" in prompt assert "breakout" in prompt.lower() and "breakdown" in prompt.lower() assert "SRSI" in prompt and "VWAP" in prompt and "PREMISE_EXIT" in prompt + # This is model-facing safety knowledge, so both directional boundaries + # must be visible in the fully assembled prompt that Codex actually sees. + assert "bullish trend-continuation" in prompt.lower() and "above R2" in prompt + assert "bearish trend-continuation" in prompt.lower() and "below S2" in prompt + assert "HOLD" in prompt and "NONE" in prompt assert "host-owned" in prompt.lower() assert "confidence" in prompt and "0 through 10" in prompt assert "model_used" in prompt and "configured-test-model" in prompt diff --git a/Tests/Signal Generators/CPR AI Agent/test_cpr_ai_core.py b/Tests/Signal Generators/CPR AI Agent/test_cpr_ai_core.py index d86219a..77c5ff6 100644 --- a/Tests/Signal Generators/CPR AI Agent/test_cpr_ai_core.py +++ b/Tests/Signal Generators/CPR AI Agent/test_cpr_ai_core.py @@ -93,6 +93,53 @@ def test_long_continuation_rejects_each_independent_hard_gate(change, code): assert outcome.validation_code == code +@pytest.mark.parametrize( + ("action", "entry", "boundary_name", "boundary", "expected_code"), + [ + ("ENTER_LONG", 121.0, "r2", 120.0, "continuation_outside_r2_s2"), + ("ENTER_SHORT", 79.0, "s2", 80.0, "continuation_outside_r2_s2"), + ], +) +def test_trend_continuation_rejects_closes_beyond_the_final_cpr_boundary( + action, + entry, + boundary_name, + boundary, + expected_code, +): + """A continuation cannot chase price above R2 or sell below S2. + + The literal entry and boundary pairs independently encode the two strict + inequalities. This test expects a dedicated policy rejection rather than + relying on the later reward/target geometry to reject the trade by accident. + """ + + context = _context() + context["session_levels"]["current_close"] = entry + context["session_levels"]["levels"][boundary_name] = boundary + if action == "ENTER_SHORT": + # Keep every other short continuation fact valid so the new S2 boundary + # is the first and only host-policy reason for rejecting this proposal. + context["momentum_vwap"].update( + { + "rsi14": 60.0, + "ema": {"order": "EMA5_BELOW_EMA20", "ema5_slope": -1.0, "ema20_slope": -0.5}, + "candle": {"low": 75.0, "high": 84.0}, + } + ) + context["momentum_vwap"]["vwap"] = { + "sequence_evidence": {"all_recent_below": True}, + "entry_candle": {"body_fraction_below": 0.5}, + } + + outcome = CPRHostPolicy().validate( + context, + _proposal(action, "TRENDING", "TRENDING_VWAP_CONTINUATION"), + ) + + assert outcome.validation_code == expected_code + + @pytest.mark.parametrize( ("setup", "sequence", "code"), [ diff --git a/Tests/Signal Generators/SL Hunting AI Agent/test_sl_hunting_premarket.py b/Tests/Signal Generators/SL Hunting AI Agent/test_sl_hunting_premarket.py index bca1814..d767a69 100644 --- a/Tests/Signal Generators/SL Hunting AI Agent/test_sl_hunting_premarket.py +++ b/Tests/Signal Generators/SL Hunting AI Agent/test_sl_hunting_premarket.py @@ -201,17 +201,18 @@ def test_shipped_note_targets_the_next_TRADING_day_not_the_next_calendar_day(): ) -def test_shipped_note_matches_august_11_intraday_hunter_plan(): - """The committed advisory must match the hand-checked 10 Aug transcript. +def test_shipped_note_matches_august_12_intraday_hunter_plan(): + """The committed advisory must match the hand-checked 11 Aug transcript. This catches a stale prior-session note, an inverted gap plan, or a mistyped chart level before the dated note is injected into the live prompt. - The distinguishing fact this time is a FAILED BREAKDOWN: NIFTY sold, broke - down, and immediately turned back up and held. That flips the whole - conditional relative to 10 Aug -- a flat open now wants BUYS, where the day - before it wanted SELLS -- so the assertion below is deliberately exact about - which side each gap branch takes. It is also an EXPIRY session. + Two things make this one distinctive. The seller crowd is described as + already SPENT -- BankNIFTY's recovery hit the stops of sellers who joined on + the retracement -- so there is little left to hunt on that side. And it + carries the series' second explicit ESCAPE HATCH: a large gap-up voids the + plan outright, which the note records as stand-aside rather than inventing + the branch he does not state. """ import os @@ -220,31 +221,33 @@ def test_shipped_note_matches_august_11_intraday_hunter_plan(): note = load_premarket_note(shipped) assert note is not None - assert note.for_date == "2026-08-11" - assert "cOvPKZFervw" in note.source - # The distinguishing fact: the breakdown failed and the market held above. - assert "BREAKDOWN FAILED" in note.context - # The plan must be the INVERSE of the previous session's, so assert the - # direction of each branch rather than merely that a plan exists. - assert note.plan[0].startswith("FLAT to GAP-UP: identify BUY-side setups") - assert note.plan[1].startswith("GAP-DOWN: identify SELL-side setups") - assert any("EXPIRY DAY" in line for line in note.plan) - assert any("INVERTS yesterday's plan" in line for line in note.plan) - assert len(note.plan) == 8 + assert note.for_date == "2026-08-12" + assert "CoxS77NfnsI" in note.source + # The distinguishing facts: no follow-through, and no overnight short carry. + assert "never crossed the round number" in note.context + assert note.plan[0].startswith("FLAT to GAP-DOWN: identify SELL-side setups") + # The escape hatch must survive verbatim -- it is the one branch he refuses + # to specify, and inventing it is exactly the failure this test guards. + assert any("LARGE gap-up VOIDS the plan" in line for line in note.plan) + assert any("Stand aside" in line for line in note.plan) + # The ambiguity is recorded, not smoothed away. + assert any("AMBIGUITY, recorded rather than resolved" in line for line in note.plan) + assert any("SPENT, not available to hunt" in line for line in note.plan) + assert len(note.plan) == 7 assert [level.model_dump() for level in note.levels] == [ { "index": "NIFTY", - "resistance": [24610.0, 24670.0], - "support": [24440.0, 24360.0], + "resistance": [24560.0, 24610.0], + "support": [24430.0, 24345.0], }, { "index": "BANKNIFTY", - "resistance": [57800.0, 58000.0], - "support": [57340.0, 57150.0], + "resistance": [57650.0, 57800.0], + "support": [57100.0, 56960.0], }, { "index": "SENSEX", - "resistance": [78640.0, 78920.0], - "support": [78200.0, 78000.0], + "resistance": [78475.0, 78640.0], + "support": [78046.0, 77810.0], }, ] diff --git a/Tests/Signal Generators/SL Hunting AI Agent/test_sl_hunting_schema.py b/Tests/Signal Generators/SL Hunting AI Agent/test_sl_hunting_schema.py index 8fdbd74..1e0b26e 100644 --- a/Tests/Signal Generators/SL Hunting AI Agent/test_sl_hunting_schema.py +++ b/Tests/Signal Generators/SL Hunting AI Agent/test_sl_hunting_schema.py @@ -866,6 +866,48 @@ def test_target_sizing_inputs_are_all_present_and_distinct(): assert "a property of YOU" in prompt +def test_system_prompt_has_v4e_recruitment_and_losing_session_knowledge(): + """v4e (11 Aug live session): IH's LOSS, which is why it is worth encoding. + + He named the disqualifying fact himself -- no stops seated on either side -- + then traded a FORECAST of who would arrive, and the market simply kept + selling. The session also refines v4d: a gap-down recruits POSITIONAL + sellers, a flat open only INTRADAY ones, so the same-shaped trap is smaller + and more perishable after a flat open. + """ + prompt = build_system_prompt() + assert "WHICH CROWD THE OPEN RECRUITS DECIDES HOW BIG THE TRAP IS" in prompt + assert "A FORECAST OF WHO WILL ARRIVE IS NOT EVIDENCE OF WHO IS SEATED" in prompt + assert "A SHARP FIRST SLIDE BAITS; A SLOW ONE MEANS IT" in prompt + assert "NAME THE LAST POINT, NOT ONLY THE STOP" in prompt + assert "DISCIPLINE IS ASYMMETRIC BETWEEN WINNERS AND LOSERS" in prompt + # The recruitment distinction is the point; both halves must be present. + assert "recruits POSITIONAL sellers" in prompt + assert "recruits INTRADAY sellers only" in prompt + + +def test_v4e_empty_book_is_a_no_trade_not_a_forecasting_licence(): + """The v4e lesson must not be readable as "predict the crowd instead". + + The whole method rests on hunting inventory that already exists. If this + rule ever drifted into permitting a trade built on who is LIKELY to arrive, + it would license exactly the loss it was distilled from. + """ + prompt = build_system_prompt() + section = prompt[prompt.index("A FORECAST OF WHO WILL ARRIVE IS NOT EVIDENCE"):] + section = section[: section.index("\n- ")] if "\n- " in section else section + assert "the correct output is HOLD" in section + assert "not an invitation to forecast one into existence" in section + # It must also reconcile with v4c rather than silently contradicting it. + assert "MANUFACTURES MORE" in section + + # And the bait prior must stay a tie-breaker, never a standalone premise. + bait = prompt[prompt.index("A SHARP FIRST SLIDE BAITS"):] + bait = bait[: bait.index("\n- ")] if "\n- " in bait else bait + assert "weak prior" in bait + assert "never as the premise of a trade on its own" in bait + + def test_reentry_gate_does_not_contradict_the_exit_rules(): """The re-entry gate must never be readable as a reason to delay an EXIT. diff --git a/Tests/test_nifty_multi_strategy_master.py b/Tests/test_nifty_multi_strategy_master.py index 661ca42..3d71743 100644 --- a/Tests/test_nifty_multi_strategy_master.py +++ b/Tests/test_nifty_multi_strategy_master.py @@ -1,4 +1,5 @@ import hashlib +import importlib import importlib.util import json import os @@ -9467,6 +9468,50 @@ def test_sheet_failure_keeps_results_marked_unpublished(self): self.assertFalse(finalized.results_published) +class TestCPRAIMasterImportBoundary(unittest.TestCase): + """Exercise lazy CPR imports exactly as the master exposes them at runtime. + + The focused CPR tests permanently add the spaced source directory to + ``sys.path`` through their local ``conftest.py``. Production does not run + that test hook, so these checks belong in the master suite and deliberately + use the module objects created while the master itself was imported. + """ + + def test_master_loaded_agent_can_resolve_lazy_prompt_and_schema(self): + """A completed-bar turn must reach an injected runner after startup.""" + + sentinel = object() + captured: dict[str, object] = {} + + def fake_runner(**kwargs): + """Capture advisory inputs without starting Codex or an MCP server.""" + + captured.update(kwargs) + return sentinel + + agent = master_file.CPR_AI_AGENT_LOGIC.CPRAgent(runner=fake_runner) + + result = agent._run_turn({}, "production-loader-regression") + + self.assertIs(result, sentinel) + self.assertIn("prompt", captured) + self.assertIn("output_schema", captured) + + def test_lazy_codex_runner_reuses_the_master_agent_result_types(self): + """Delayed runner imports must not create a second agent module copy.""" + + runner_module = importlib.import_module("cpr_ai_codex_runner") + + self.assertIs( + runner_module.CPRAgentRunResult, + master_file.CPR_AI_AGENT_LOGIC.CPRAgentRunResult, + ) + self.assertIs( + runner_module.CPRToolCallRecord, + master_file.CPR_AI_AGENT_LOGIC.CPRToolCallRecord, + ) + + class TestCPRAIWorkerFoundation(unittest.TestCase): """Specify CPR AI cadence, mechanics, provenance, and live-ledger safety. diff --git a/docs/adr/0011-committed-docs-untracked-superpowers.md b/docs/adr/0011-committed-docs-untracked-superpowers.md index c0ff5b3..c14b228 100644 --- a/docs/adr/0011-committed-docs-untracked-superpowers.md +++ b/docs/adr/0011-committed-docs-untracked-superpowers.md @@ -106,12 +106,25 @@ from a guess about it. **Harder:** ~24 more files to keep current. The mitigation is convention, not tooling. -**To revisit when:** the docs are observed to be stale. The existing repository -already has precedent for enforcing documentation freshness in CI — -`test_repository_policy.py` fails the build on stale worker-roster claims in -`README.md`, `CLAUDE.md`, `AGENTS.md` and the master file. Extending that gate -to cover `docs/hld/` is a reasonable follow-up; it is deliberately **not** done -in this change, to keep the restructure surgical. +**To revisit when:** the docs are observed to be stale in a way the gate below +does not catch — most likely an LLD whose *prose* drifts from its component +while every link still resolves. No mechanical check can catch that; only +changing the LLD in the same commit as the code can. + +**Update (2026-08-11):** the follow-up recorded here has been done. The +documentation staleness gate now covers this docs set in three ways, all in +`Tests/Dependencies/test_repository_policy.py`: + +1. `docs/hld/system-overview.md` joined the architecture gate, so the HLD must + keep both optional agents visible and must not let the core roster count + masquerade as the enabled total. +2. Every committed ADR and LLD must be linked from `docs/README.md`, and the + index must not link a file that does not exist — checked in both directions, + because an orphaned doc and a dangling link are equally corrosive. +3. Every relative link inside `docs/` must resolve, which is what catches a + rename breaking a cross-reference. + +Each was verified to fail on a deliberate mutation before being committed. ## Action items @@ -121,5 +134,7 @@ in this change, to keep the restructure surgical. - [x] `docs/hld/system-overview.md`. - [x] 12 LLDs under `docs/lld/`. - [x] 11 ADRs under `docs/adr/`. -- [ ] **Follow-up:** consider adding `docs/hld/system-overview.md` to the - architecture-staleness gate in `Tests/Dependencies/test_repository_policy.py`. +- [x] **Follow-up (done 2026-08-11):** `docs/hld/system-overview.md` added to + the architecture-staleness gate, plus index-integrity and link-resolution + gates over the whole committed docs set, in + `Tests/Dependencies/test_repository_policy.py`. diff --git a/docs/adr/0012-crash-durable-session-state.md b/docs/adr/0012-crash-durable-session-state.md index 7b0e249..47eae21 100644 --- a/docs/adr/0012-crash-durable-session-state.md +++ b/docs/adr/0012-crash-durable-session-state.md @@ -84,6 +84,45 @@ is a few dozen events and tens of KB. recovery logic; and the per-strategy rollup would have to be recomputed on every read. Rejected: the write volume never justified it. +#### Amendment (2026-08-11): split into a durable document and a marks sibling + +The first live session measured what the single-document design actually costs. +223 writes crossed the 250 ms warning threshold — **210 of them on the supervisor +thread** (median 0.830 s, max **8.732 s**, 85 over one second) against 13 on +trading threads (median 0.414 s). The file was only 79 KB, so this is disk +contention, not payload size; no amount of shrinking the document would fix it. + +The obvious fix — stop calling `fsync` on the 30-second snapshot — is **not safe +on its own**. `os.replace` is atomic for the *name*, not for the *data*: a hard +kill during an un-fsynced rewrite can publish a present-but-garbage file. Since +that one document also held `trades[]` and the P&L rollup, a single torn +*snapshot* could destroy the record this ADR exists to protect, turning a +performance fix into a reintroduction of the original incident. + +So the state is now **two files**: + +| File | Contents | Written by | fsync | +|---|---|---|---| +| `session_state.json` | schema, session date, shutdown flags, `recorded_pnl` / `recorded_trades`, `trades[]` | trade events, clean shutdown, and once at construction | **yes** | +| `session_state.marks.json` | per-strategy live counters and `open_position` with `last_mark_ltp` | the 30 s supervisor snapshot | no | + +The durable file is now touched *only* by durable writes, so the snapshot loop +cannot corrupt it however it fails. Losing the marks file costs at most one +snapshot interval of mark data, which this ADR already documented as acceptable. + +`load_session_state` merges the pair back into the single-document shape every +existing reader expects, so `resumable_open_positions`, `recorded_realized_pnl` +and the runner's resume path were unchanged. The merge is deliberately +asymmetric: a corrupt **durable** file means no recovery and returns `None`; a +corrupt or missing **marks** file returns the P&L anyway and simply offers no +positions for resume. + +Rejected alongside it: moving the trade-event write to a queue and a writer +thread. It would remove the remaining 13 stalls, but only by weakening +durability from "guaranteed before the call returns" to a sub-second window — +which is the guarantee this ADR was written to provide. At a 0.414 s median, +13 times a session, that trade is not worth making. + ### Whether resume may restore a LIVE position **Rejected.** In live trading the **broker account** is the authority on what is @@ -102,8 +141,13 @@ open paper exposure only. The snapshot cadence is the real trade-off. Marks are refreshed every 30s, so a crash loses at most 30 seconds of *mark* movement on an open position. Trade events are not subject to this — their durable write starts the instant they -happen. Every prior file is archived before replacement, so a restart cannot -erase the realized-P&L journal that was expensive to rebuild by hand. +happen. Every prior file is archived before replacement (both halves, under one +timestamp), so a restart cannot erase the realized-P&L journal that was +expensive to rebuild by hand. + +That cadence trade-off is also what licenses the durable/marks split above: the +marks file may skip `fsync` precisely because its contents were already declared +losable, while the trades and P&L never were. Writing from the supervisor thread rather than from a new thread is deliberate: that loop is already awake once a second, never trades, and holds the canonical diff --git a/docs/hld/system-overview.md b/docs/hld/system-overview.md index b1440ff..fa845ea 100644 --- a/docs/hld/system-overview.md +++ b/docs/hld/system-overview.md @@ -14,6 +14,19 @@ concurrently against one shared market-data feed, decides entries and exits per strategy, and executes those decisions either on paper (default) or through a real broker (explicitly enabled, per strategy). +The two optional agents sit **outside** that core roster and are enabled +independently of each other, so neither the configured nor the running worker +total can be read off the core number: + +| Optional agent | Provider | Default | Detail | +|---|---|---|---| +| **SL Hunting AI Agent** | Claude (`claude-agent-sdk`) | off (`SL_HUNTING_ENABLED`) | [`lld/sl-hunting-ai-agent.md`](../lld/sl-hunting-ai-agent.md) | +| **CPR Codex AI Agent** | Codex (subprocess + MCP) | off (`CPR_AI_ENABLED`) | [`lld/cpr-codex-ai-agent.md`](../lld/cpr-codex-ai-agent.md) | + +With both enabled the configured roster reaches about 29 workers, while the +per-strategy enable and virtual-trading gates keep the *running* roster +configuration-dependent. + It has been running live since May 2026. Every design decision in this document is weighted by that: **the system is allowed to miss a trade; it is not allowed to lose track of a position.** diff --git a/docs/lld/reporting-and-observability.md b/docs/lld/reporting-and-observability.md index 64f0554..d01a9e5 100644 --- a/docs/lld/reporting-and-observability.md +++ b/docs/lld/reporting-and-observability.md @@ -122,10 +122,29 @@ whole morning's books plus thirteen open positions, all rebuilt by hand. | `clean_shutdown: true` | After runner exposure is proven flat and local shutdown completes | Its **absence** is the crash signal for open-position recovery | | `results_published: true` | Only after a real Google Sheet cell batch succeeds | Distinguishes local cleanup from external reporting success | +It is **two files**, split by how much their loss costs: + +| File | Contents | Written by | fsync | +|---|---|---|---| +| `session_state.json` | session date, shutdown flags, `recorded_pnl` / `recorded_trades`, `trades[]` | trade events, clean shutdown, once at construction | **yes** | +| `session_state.marks.json` | live counters and `open_position` with `last_mark_ltp` | the 30 s supervisor snapshot | no | + +The supervisor never touches the durable file. That is a safety property, not an +optimisation: `os.replace` is atomic for the *name* but not the *data*, so a +hard kill during an un-fsynced rewrite can publish a present-but-garbage file — +and if that file also held the trades, one torn snapshot would destroy the books. +The 2026-08-11 session measured 210 supervisor writes over the 250 ms threshold +(max 8.7 s) against 13 on trading threads, which is what forced the split. + +`load_session_state` merges the pair back into one document, so every reader sees +the shape it always has. The merge is asymmetric on purpose: a corrupt **durable** +file means no recovery (`None`); a corrupt or missing **marks** file still +returns the P&L and simply offers no positions for resume. + Properties that make it trustworthy: -- **Atomic** — `.tmp` + `flush` + `fsync` + `os.replace`. A reader never sees a - partial document. +- **Atomic** — `.tmp` + `os.replace` for both files, plus `flush` + `fsync` + before the replace on the durable one. A reader never sees a partial document. - **Restart-safe** — an existing file is moved to a timestamped recovery archive before the replacement run writes anything. Compatible same-day trades and P&L totals seed the new session; old exposure never carries implicitly. @@ -159,7 +178,9 @@ gitignored — it holds live position and P&L detail. | Log file unwritable | Console logging continues; the EOD sheet loses its source for that session. | | Session state file unwritable | None. First failure logs loudly; trading continues without crash recovery for that session. | | Session state durable write exceeds 250ms | Trading still waits for that `fsync` so the event is genuinely durable; a warning identifies local-disk latency for operator action. | -| Session state file corrupt on read | Ignored, logged; the run starts with no recovery rather than refusing to start. | +| Session state marks write exceeds 2s | Warned separately, and the message says so: this delays supervision, not a trading decision. | +| Session state marks file corrupt or missing | P&L is still recovered from the durable file; only open positions are lost, so nothing is offered for resume. | +| Session state durable file corrupt on read | Ignored, logged; the run starts with no recovery rather than refusing to start. | --- diff --git a/docs/lld/testing-and-ci.md b/docs/lld/testing-and-ci.md index 685f966..ad2da48 100644 --- a/docs/lld/testing-and-ci.md +++ b/docs/lld/testing-and-ci.md @@ -185,12 +185,27 @@ runtime. It asserts, without contacting any network: sanity check that the AST walk still works); - `CLAUDE.md` and `AGENTS.md` share one identical runtime section; - architecture docs make both optional agents visible and carry no stale - worker-roster claims. + worker-roster claims; +- every committed ADR and LLD is linked from `docs/README.md`, and the index + links nothing that does not exist; +- every relative link inside `docs/` resolves. -That last group is a **documentation staleness gate**. It currently covers -`README.md`, `Signal Generators/Readme.md`, `AGENTS.md`, `CLAUDE.md` and the -master file. Extending it to `docs/hld/` is an open follow-up recorded in -[ADR-0011](../adr/0011-committed-docs-untracked-superpowers.md). +The last three are the **documentation staleness gate**, and they answer three +different ways docs rot: + +| Failure | Caught by | +|---|---| +| A doc describes a roster or agent set the code no longer has | `test_current_architecture_docs_distinguish_core_from_optional_agents` — covers `README.md`, `Signal Generators/Readme.md`, `AGENTS.md`, `CLAUDE.md`, the master file, **and `docs/hld/system-overview.md`** | +| A new ADR/LLD lands unlinked, or the index points at a deleted file | `test_every_committed_design_document_is_linked_from_the_docs_index` (checked in both directions) | +| A rename breaks a cross-reference between documents | `test_relative_links_inside_the_committed_docs_resolve` | + +`docs/superpowers/` is excluded from all three: it is gitignored session working +material, not product documentation ([ADR-0011](../adr/0011-committed-docs-untracked-superpowers.md)). + +Each of these was verified to **fail** on a deliberate mutation before being +committed — an unlinked ADR, a dangling index link, a renamed cross-reference, +and an agent dropped from the HLD. A policy test that cannot fail is worse than +no test, because it reads like coverage. ---