From 001c9226012c9f4089a5f588dce6886097ee637e Mon Sep 17 00:00:00 2001 From: Hun-ger <48196026+Hun-ger@users.noreply.github.com> Date: Tue, 4 Aug 2026 21:00:39 +0800 Subject: [PATCH 1/5] fix(plugin): prevent foreground starvation and honor Retry-After (#2208) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(plugin): synchronize Hermes release version metadata * fix(plugin): keep Hermes compression turn read-only * fix(plugin): reload Hermes config on restart * fix(plugin): share Hermes bridge across providers * fix(plugin): isolate Hermes bridges by data home * fix(plugin): harden Hermes recovery lifecycle * release: @memtensor/memos-local-plugin v2.0.12-beta.1 * fix(plugin): satisfy Python lint checks * fix(plugin): improve retrieval relevance ranking * fix(plugin): use clean OpenClaw user input * fix(plugin): ignore OpenClaw internal wakeups * fix(plugin): recall compacted same-session history * style(plugin): format Hermes provider pipeline test * fix(plugin): bound foreground retries and shutdown * fix(plugin): unblock Hermes bridge reader * fix(plugin): honor long Retry-After cooldowns * fix(plugin): show memory add roles correctly * style(plugin): format Hermes bridge changes --------- Co-authored-by: 谁在吵着吃糖 Co-authored-by: github-actions[bot] --- .../hermes/memos_provider/__init__.py | 147 +++++++++- .../hermes/memos_provider/bridge_client.py | 137 ++++++--- apps/memos-local-plugin/agent-contract/dto.ts | 5 + apps/memos-local-plugin/bridge/methods.ts | 9 + .../core/embedding/embedder.ts | 18 +- .../core/embedding/fetcher.ts | 129 ++++++++- .../core/embedding/index.ts | 1 + .../core/embedding/providers/cohere.ts | 4 +- .../core/embedding/providers/gemini.ts | 4 +- .../core/embedding/providers/mistral.ts | 4 +- .../core/embedding/providers/openai.ts | 4 +- .../core/embedding/providers/voyage.ts | 4 +- .../core/embedding/retry-worker.ts | 31 +- .../core/embedding/types.ts | 20 +- apps/memos-local-plugin/core/index.ts | 1 + apps/memos-local-plugin/core/llm/client.ts | 11 + apps/memos-local-plugin/core/llm/fetcher.ts | 137 ++++++++- .../core/llm/providers/anthropic.ts | 4 +- .../core/llm/providers/bedrock.ts | 4 +- .../core/llm/providers/gemini.ts | 4 +- .../core/llm/providers/openai.ts | 4 +- apps/memos-local-plugin/core/llm/types.ts | 9 +- apps/memos-local-plugin/core/pipeline/deps.ts | 29 +- .../core/pipeline/memory-core.ts | 88 +++--- .../core/pipeline/orchestrator.ts | 153 ++++++++-- .../core/retrieval/llm-filter.ts | 8 + .../core/retrieval/retrieve.ts | 53 +++- .../core/retrieval/types.ts | 6 +- .../core/session/intent-classifier.ts | 8 +- .../core/session/manager.ts | 3 + .../core/session/relation-classifier.ts | 24 +- apps/memos-local-plugin/core/session/types.ts | 2 + .../core/util/foreground-resources.ts | 274 ++++++++++++++++++ .../core/util/rate-limited-llm.ts | 31 +- .../core/util/request-deadline.ts | 37 +++ .../core/util/retry-after.ts | 200 +++++++++++++ .../memos-local-plugin/core/util/semaphore.ts | 43 ++- .../tests/python/test_bridge_client.py | 165 ++++++++++- .../python/test_hermes_provider_pipeline.py | 69 +++++ .../tests/unit/bridge/methods.test.ts | 11 + .../tests/unit/embedding/embedder.test.ts | 59 ++++ .../tests/unit/embedding/fetcher.test.ts | 82 ++++++ .../tests/unit/embedding/retry-worker.test.ts | 60 ++++ .../tests/unit/llm/client.test.ts | 39 +++ .../tests/unit/llm/fetcher.test.ts | 201 ++++++++++++- .../tests/unit/pipeline/memory-core.test.ts | 47 +++ .../tests/unit/pipeline/orchestrator.test.ts | 33 +++ .../unit/session/intent-classifier.test.ts | 19 ++ .../unit/session/relation-classifier.test.ts | 27 ++ .../unit/util/foreground-resources.test.ts | 138 +++++++++ .../tests/unit/util/request-deadline.test.ts | 35 +++ .../tests/unit/util/retry-after.test.ts | 90 ++++++ .../tests/unit/util/semaphore.test.ts | 19 ++ 53 files changed, 2547 insertions(+), 197 deletions(-) create mode 100644 apps/memos-local-plugin/core/util/foreground-resources.ts create mode 100644 apps/memos-local-plugin/core/util/request-deadline.ts create mode 100644 apps/memos-local-plugin/core/util/retry-after.ts create mode 100644 apps/memos-local-plugin/tests/unit/util/foreground-resources.test.ts create mode 100644 apps/memos-local-plugin/tests/unit/util/request-deadline.test.ts create mode 100644 apps/memos-local-plugin/tests/unit/util/retry-after.test.ts create mode 100644 apps/memos-local-plugin/tests/unit/util/semaphore.test.ts diff --git a/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py b/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py index 6eeaa000e..79b88cbf4 100644 --- a/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py +++ b/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py @@ -201,6 +201,47 @@ def _long_rpc_timeout_default() -> float: _LONG_RPC_TIMEOUT = _long_rpc_timeout_default() + +def _prefetch_rpc_timeout_default() -> float: + """Resolve the latency budget for Hermes' foreground memory lookup. + + Hermes places its own short deadline around ``prefetch``. Reusing the + long capture timeout here lets the bridge continue work after the host + has already moved on. Keep a separate, configurable ceiling below the + host's default and forward the corresponding absolute deadline to core. + """ + raw = os.environ.get("MEMOS_HERMES_PREFETCH_RPC_TIMEOUT", "") + try: + value = float(raw) + except (TypeError, ValueError): + return 6.0 + if not value > 0: + return 6.0 + # Hermes currently abandons external providers after 8 seconds. Keep at + # least one second for Python thread scheduling and response assembly even + # when an operator overrides the default. + return min(value, 7.0) + + +_PREFETCH_RPC_TIMEOUT = _prefetch_rpc_timeout_default() +_PREFETCH_RESPONSE_RESERVE_SECONDS = 0.25 + + +def _remaining_rpc_timeout( + deadline_monotonic: float | None, + requested_timeout: float | None, +) -> float | None: + """Bound one blocking bridge step by a shared end-to-end deadline.""" + if deadline_monotonic is None: + return requested_timeout + remaining = deadline_monotonic - time.monotonic() + if remaining <= 0: + raise BridgeError("timeout", "foreground prefetch deadline exceeded") + if requested_timeout is None: + return remaining + return min(requested_timeout, remaining) + + _HERMES_INTERNAL_REVIEW_PREFIXES = ( "review the conversation above and consider saving to memory if appropriate.", "review the conversation above and update the skill library.", @@ -1022,8 +1063,19 @@ def prefetch(self, query: str, *, session_id: str = "") -> str: # type: ignore[ cached result immediately. Otherwise synchronously run ``turn.start`` against the bridge (small overhead). """ + deadline_monotonic = time.monotonic() + _PREFETCH_RPC_TIMEOUT + started_at_ms = int(time.time() * 1000) + core_budget_seconds = max( + 0.05, + _PREFETCH_RPC_TIMEOUT - _PREFETCH_RESPONSE_RESERVE_SECONDS, + ) + deadline_at_ms = started_at_ms + int(core_budget_seconds * 1000) if self._prefetch_thread and self._prefetch_thread.is_alive(): - self._prefetch_thread.join(timeout=5.0) + try: + join_timeout = _remaining_rpc_timeout(deadline_monotonic, 5.0) + except BridgeError: + return "" + self._prefetch_thread.join(timeout=join_timeout) with self._prefetch_lock: cached = self._prefetch_result self._prefetch_result = "" @@ -1033,10 +1085,25 @@ def prefetch(self, query: str, *, session_id: str = "") -> str: # type: ignore[ suppress_injection = _is_explicit_delegation_request(query) if cached: return "" if suppress_injection else cached - if not self._ensure_bridge(session_id or self._session_id, timeout=10.0): + try: + ensure_timeout = _remaining_rpc_timeout( + deadline_monotonic, + _PREFETCH_RPC_TIMEOUT, + ) + except BridgeError: + return "" + if not self._ensure_bridge( + session_id or self._session_id, + timeout=min(10.0, ensure_timeout or _PREFETCH_RPC_TIMEOUT), + ): return "" try: - context = self._turn_start(query, session_id=session_id) + context = self._turn_start( + query, + session_id=session_id, + deadline_monotonic=deadline_monotonic, + deadline_at_ms=deadline_at_ms, + ) if suppress_injection: # Do not let remembered "do it directly" skills override an # explicit user request to dispatch work to a subagent. @@ -1941,6 +2008,7 @@ def _bridge_request( *, timeout: float | None = None, ensure_session: bool = True, + deadline_monotonic: float | None = None, ) -> dict[str, Any]: bridge = self._bridge if bridge is None: @@ -1958,10 +2026,23 @@ def _bridge_request( bridge.generation, self._session_id, ) - self._open_session(self._session_id, timeout=30.0) - if timeout is None: + session_ceiling = ( + timeout + if deadline_monotonic is not None and timeout is not None + else 30.0 + ) + session_timeout = _remaining_rpc_timeout( + deadline_monotonic, + session_ceiling, + ) + self._open_session( + self._session_id, + timeout=session_timeout or 30.0, + ) + request_timeout = _remaining_rpc_timeout(deadline_monotonic, timeout) + if request_timeout is None: return bridge.request(method, params) - return bridge.request(method, params, timeout=timeout) + return bridge.request(method, params, timeout=request_timeout) def _open_session(self, session_id: str = "", *, timeout: float = 30.0) -> None: bridge = self._bridge @@ -1997,6 +2078,7 @@ def _bridge_request_with_retry( params: Any, *, timeout: float | None = None, + deadline_monotonic: float | None = None, ) -> dict[str, Any]: """Read-path helper: reconnect + retry once on ``transport_closed``. @@ -2012,7 +2094,12 @@ def _bridge_request_with_retry( """ assert self._bridge is not None try: - return self._bridge_request(method, params, timeout=timeout) + return self._bridge_request( + method, + params, + timeout=timeout, + deadline_monotonic=deadline_monotonic, + ) except BridgeError as err: if not self._is_transport_closed(err): raise @@ -2021,9 +2108,24 @@ def _bridge_request_with_retry( method, err, ) - self._reconnect_bridge(self._session_id, timeout=30.0) + reconnect_ceiling = ( + timeout if deadline_monotonic is not None and timeout is not None else 30.0 + ) + reconnect_timeout = _remaining_rpc_timeout( + deadline_monotonic, + reconnect_ceiling, + ) + self._reconnect_bridge( + self._session_id, + timeout=reconnect_timeout or 30.0, + ) assert self._bridge is not None - return self._bridge_request(method, params, timeout=timeout) + return self._bridge_request( + method, + params, + timeout=timeout, + deadline_monotonic=deadline_monotonic, + ) def _is_transport_closed(self, err: Exception) -> bool: if isinstance(err, BridgeError) and err.code == "transport_closed": @@ -2233,7 +2335,14 @@ def _run() -> None: ) self._bridge_keepalive_thread.start() - def _turn_start(self, query: str, *, session_id: str = "") -> str: + def _turn_start( + self, + query: str, + *, + session_id: str = "", + deadline_monotonic: float | None = None, + deadline_at_ms: int | None = None, + ) -> str: assert self._bridge is not None host_runtime = self._host_runtime_context() with self._state_lock: @@ -2251,20 +2360,34 @@ def _turn_start(self, query: str, *, session_id: str = "") -> str: "visibleContextStartTs": visible_context_start_ts, } ) + now_ms = int(time.time() * 1000) + if deadline_monotonic is None: + deadline_monotonic = time.monotonic() + _PREFETCH_RPC_TIMEOUT + if deadline_at_ms is None: + core_budget_seconds = max( + 0.05, + _PREFETCH_RPC_TIMEOUT - _PREFETCH_RESPONSE_RESERVE_SECONDS, + ) + deadline_at_ms = now_ms + int(core_budget_seconds * 1000) payload: dict[str, Any] = { "agent": "hermes", "namespace": self._runtime_namespace(), "sessionId": session_id or self._session_id, "userText": query, "contextHints": context_hints, - "ts": int(time.time() * 1000), + "ts": now_ms, + "deadlineAt": deadline_at_ms, } if turn_key: payload["turnKey"] = turn_key resp = self._bridge_request_with_retry( "turn.start", payload, - timeout=_LONG_RPC_TIMEOUT, + timeout=_remaining_rpc_timeout( + deadline_monotonic, + _PREFETCH_RPC_TIMEOUT, + ), + deadline_monotonic=deadline_monotonic, ) response_query = (resp or {}).get("query") or {} response_session = str(response_query.get("sessionId") or "") diff --git a/apps/memos-local-plugin/adapters/hermes/memos_provider/bridge_client.py b/apps/memos-local-plugin/adapters/hermes/memos_provider/bridge_client.py index 6863ba1ef..cbb2fa84c 100644 --- a/apps/memos-local-plugin/adapters/hermes/memos_provider/bridge_client.py +++ b/apps/memos-local-plugin/adapters/hermes/memos_provider/bridge_client.py @@ -17,6 +17,7 @@ import json import logging import os +import queue import shutil import subprocess import threading @@ -32,6 +33,7 @@ logger = logging.getLogger(__name__) HOST_HANDLER_WAIT_SECONDS = 5.0 +HOST_HANDLER_QUEUE_CAPACITY = 16 # ─── Module-level singleton tracker ───────────────────────────────────── # Each entry maps an ``(agent, no_viewer, runtime_home)`` key to the @@ -152,12 +154,16 @@ def __init__( # Reverse-direction handlers: the bridge can send us a # JSON-RPC request via `serverRequest(...)` (e.g. # `host.llm.complete` for fallback LLM calls). Registered - # methods run on the dedicated reader thread; long-running - # work should spawn its own worker if it needs to. Each - # handler returns a JSON-serialisable value or raises to - # surface a JSON-RPC error back to the bridge. + # methods run on one bounded, daemon worker. Keeping execution + # serial preserves the adapter's previous concurrency contract while + # preventing a slow host LLM call from blocking stdout response + # demultiplexing for every shared provider lease. self._host_handlers: dict[str, Callable[[dict[str, Any]], Any]] = {} self._host_handlers_cv = threading.Condition() + self._host_handler_queue: queue.Queue[tuple[Any, str, dict[str, Any]] | None] = queue.Queue( + maxsize=HOST_HANDLER_QUEUE_CAPACITY + ) + self._host_handler_stop = threading.Event() self._closed = False plugin_root = Path(__file__).resolve().parent.parent.parent.parent @@ -226,6 +232,12 @@ def __init__( env=env, cwd=str(plugin_root), ) + self._host_handler_worker = threading.Thread( + target=self._host_handler_loop, + daemon=True, + name="memos-bridge-host-handler", + ) + self._host_handler_worker.start() self._reader = threading.Thread( target=self._read_loop, daemon=True, @@ -348,7 +360,7 @@ def notify(self, method: str, params: Any = None) -> None: try: self._proc.stdin.write(payload + "\n") self._proc.stdin.flush() - except (BrokenPipeError, OSError): + except (BrokenPipeError, OSError, ValueError): pass def on_event(self, cb: Callable[[dict[str, Any]], None]) -> None: @@ -365,11 +377,9 @@ def register_host_handler( """Register a handler for bridge → adapter (reverse) requests. The Node-side bridge calls these via ``stdio.serverRequest``. - Most-recent registration wins. The handler runs on the reader - thread; if it blocks for a long time it stalls every other - bridge → adapter notification, so handlers that need to do - heavy work (e.g. an LLM call) are still expected to return - within the bridge-side timeout (default 60 s). + Most-recent registration wins. Handlers run serially on a bounded + daemon worker so a long-running host LLM call cannot stall the reader + thread that resolves unrelated foreground JSON-RPC responses. """ with self._host_handlers_cv: self._host_handlers[method] = handler @@ -381,6 +391,7 @@ def close(self) -> None: with self._host_handlers_cv: self._closed = True self._host_handlers_cv.notify_all() + self._stop_host_handler_worker() # Drop self from the module-level singleton tracker (issue #1910) # BEFORE the potentially-slow stdin/SIGTERM/SIGKILL dance. We @@ -435,6 +446,7 @@ def _abort_pending(self, reason: str) -> None: with self._host_handlers_cv: self._closed = True self._host_handlers_cv.notify_all() + self._stop_host_handler_worker() with self._lock: for entry in list(self._pending.values()): entry["error"] = { @@ -477,7 +489,9 @@ def _read_loop(self) -> None: # Reverse-direction request: the bridge is asking the # adapter to do something (e.g. run a fallback LLM call # via `host.llm.complete`). Dispatch to the registered - # handler and write the response back synchronously. + # handler on the bounded worker. The reader must return to + # stdout immediately so a slow host LLM callback cannot + # head-of-line block normal JSON-RPC responses. method = msg.get("method") rpc_id = msg.get("id") if ( @@ -486,33 +500,10 @@ def _read_loop(self) -> None: and "result" not in msg and "error" not in msg ): - handler = self._host_handler_for(method) - if handler is None: - self._send_response( - rpc_id, - error={ - "code": -32601, - "message": f"method not found: {method}", - "data": {"code": "unknown_method"}, - }, - ) - continue params = msg.get("params") or {} if not isinstance(params, dict): params = {} - try: - result = handler(params) - self._send_response(rpc_id, result=result) - except Exception as err: - logger.warning("host handler %s failed: %s", method, err) - self._send_response( - rpc_id, - error={ - "code": -32000, - "message": str(err) or err.__class__.__name__, - "data": {"code": "host_handler_failed"}, - }, - ) + self._dispatch_host_request(rpc_id, method, params) continue except Exception: # Any unexpected exception in the reader loop still needs @@ -527,6 +518,80 @@ def _read_loop(self) -> None: # instead of waiting for each 30 s per-request timeout. self._abort_pending("bridge subprocess exited") + def _dispatch_host_request( + self, + rpc_id: Any, + method: str, + params: dict[str, Any], + ) -> None: + """Queue reverse RPC work without ever blocking the reader thread.""" + if self._closed: + return + try: + self._host_handler_queue.put_nowait((rpc_id, method, params)) + except queue.Full: + logger.warning("host handler queue full; rejecting %s", method) + self._send_response( + rpc_id, + error={ + "code": -32000, + "message": "host handler queue is full", + "data": {"code": "host_handler_busy"}, + }, + ) + + def _host_handler_loop(self) -> None: + """Run reverse RPC handlers serially away from stdout demultiplexing.""" + while True: + request = self._host_handler_queue.get() + try: + if request is None: + return + rpc_id, method, params = request + if self._closed: + continue + handler = self._host_handler_for(method) + if handler is None: + self._send_response( + rpc_id, + error={ + "code": -32601, + "message": f"method not found: {method}", + "data": {"code": "unknown_method"}, + }, + ) + continue + try: + result = handler(params) + self._send_response(rpc_id, result=result) + except Exception as err: + logger.warning("host handler %s failed: %s", method, err) + self._send_response( + rpc_id, + error={ + "code": -32000, + "message": str(err) or err.__class__.__name__, + "data": {"code": "host_handler_failed"}, + }, + ) + finally: + self._host_handler_queue.task_done() + + def _stop_host_handler_worker(self) -> None: + """Discard queued callbacks and ask the daemon worker to exit.""" + if self._host_handler_stop.is_set(): + return + self._host_handler_stop.set() + while True: + try: + self._host_handler_queue.get_nowait() + except queue.Empty: + break + else: + self._host_handler_queue.task_done() + with contextlib.suppress(queue.Full): + self._host_handler_queue.put_nowait(None) + def _host_handler_for( self, method: str, @@ -568,7 +633,7 @@ def _send_response( try: self._proc.stdin.write(json.dumps(payload, ensure_ascii=False) + "\n") self._proc.stdin.flush() - except (BrokenPipeError, OSError): + except (BrokenPipeError, OSError, ValueError): pass def _stderr_loop(self) -> None: diff --git a/apps/memos-local-plugin/agent-contract/dto.ts b/apps/memos-local-plugin/agent-contract/dto.ts index e9ae71101..b76232bee 100644 --- a/apps/memos-local-plugin/agent-contract/dto.ts +++ b/apps/memos-local-plugin/agent-contract/dto.ts @@ -103,6 +103,11 @@ export interface TurnInputDTO { contextHints?: Record; /** Wall-clock when the turn began. */ ts: EpochMs; + /** + * Absolute adapter deadline for foreground work. Every pipeline stage + * shares this budget; it is not reset after relation or intent handling. + */ + deadlineAt?: EpochMs; } export interface TurnResultDTO { diff --git a/apps/memos-local-plugin/bridge/methods.ts b/apps/memos-local-plugin/bridge/methods.ts index 2bc1084a6..1e6cc27a0 100644 --- a/apps/memos-local-plugin/bridge/methods.ts +++ b/apps/memos-local-plugin/bridge/methods.ts @@ -389,6 +389,15 @@ function validateTurnInput(p: Record): void { "turn.start: optional 'turnKey' must be a string", ); } + if ( + p.deadlineAt !== undefined && + (typeof p.deadlineAt !== "number" || !Number.isFinite(p.deadlineAt)) + ) { + throw new MemosError( + "invalid_argument", + "turn.start: optional 'deadlineAt' must be a finite number", + ); + } } function validateTurnResult(p: Record): void { diff --git a/apps/memos-local-plugin/core/embedding/embedder.ts b/apps/memos-local-plugin/core/embedding/embedder.ts index bd4a5fe92..5d035b6ce 100644 --- a/apps/memos-local-plugin/core/embedding/embedder.ts +++ b/apps/memos-local-plugin/core/embedding/embedder.ts @@ -18,6 +18,7 @@ import { ERROR_CODES, MemosError } from "../../agent-contract/errors.js"; import { rootLogger } from "../logger/index.js"; import type { Logger } from "../logger/types.js"; import type { EmbeddingVector } from "../types.js"; +import { extractRetryDiagnostics } from "../util/retry-after.js"; import { LruEmbedCache, NullEmbedCache, @@ -32,6 +33,7 @@ import { MistralEmbeddingProvider } from "./providers/mistral.js"; import { OpenAiEmbeddingProvider } from "./providers/openai.js"; import { VoyageEmbeddingProvider } from "./providers/voyage.js"; import type { + EmbedCallOptions, EmbedInput, EmbedRole, EmbedStats, @@ -87,6 +89,10 @@ export function createEmbedderWithProvider( code?: string; at?: number; durationMs?: number; + retryAfterMs?: number; + retryAt?: number; + retryDecision?: "wait" | "defer" | "stop"; + retryReason?: string; }): void { if (!config.onStatus) return; try { @@ -96,13 +102,17 @@ export function createEmbedderWithProvider( } } - async function embedOne(input: string | EmbedInput): Promise { - const vecs = await embedMany([input]); + async function embedOne( + input: string | EmbedInput, + options?: EmbedCallOptions, + ): Promise { + const vecs = await embedMany([input], options); return vecs[0]!; } async function embedMany( inputs: Array, + options?: EmbedCallOptions, ): Promise { requests += inputs.length; if (inputs.length === 0) return []; @@ -179,6 +189,8 @@ export function createEmbedderWithProvider( const ctx: ProviderCallCtx = { config, log: providerCtxLog, + signal: options?.signal, + deadlineAt: options?.deadlineAt, }; raw = await provider.embed(texts, role, ctx); // Record success but DO NOT clear `lastError` — the viewer @@ -223,6 +235,7 @@ export function createEmbedderWithProvider( message: errMessage, code: err instanceof MemosError ? err.code : undefined, at: errAt, + ...extractRetryDiagnostics(err instanceof MemosError ? err.details : undefined), }); } catch { /* sink errors are non-fatal */ @@ -236,6 +249,7 @@ export function createEmbedderWithProvider( code: err instanceof MemosError ? err.code : undefined, at: errAt, durationMs: errAt - startedAt, + ...extractRetryDiagnostics(err instanceof MemosError ? err.details : undefined), }); throw err instanceof MemosError ? err diff --git a/apps/memos-local-plugin/core/embedding/fetcher.ts b/apps/memos-local-plugin/core/embedding/fetcher.ts index 40524aac7..303dae28e 100644 --- a/apps/memos-local-plugin/core/embedding/fetcher.ts +++ b/apps/memos-local-plugin/core/embedding/fetcher.ts @@ -8,6 +8,15 @@ */ import { ERROR_CODES, MemosError } from "../../agent-contract/errors.js"; +import { + getRetryCooldown, + parseRetryAfterMs, + planRetry, + recordRetryCooldown, + retryCooldownKey, + type RetryPlan, + waitForRetry, +} from "../util/retry-after.js"; import type { EmbeddingProviderName, ProviderLogger } from "./types.js"; export interface HttpPostOpts { @@ -17,6 +26,10 @@ export interface HttpPostOpts { timeoutMs?: number; maxRetries?: number; signal?: AbortSignal; + /** Absolute end-to-end deadline. Unlike timeoutMs, this is not renewed per attempt. */ + deadlineAt?: number; + /** Model/deployment scope; prevents one model cooldown from blocking another. */ + cooldownScope?: string; provider: EmbeddingProviderName; log: ProviderLogger; } @@ -26,11 +39,33 @@ export async function httpPostJson(opts: HttpPostOpts): Promise< const maxRetries = opts.maxRetries ?? 2; let attempt = 0; let lastErr: unknown = null; + const cooldownKey = retryCooldownKey("embedding", opts.provider, opts.url, opts.cooldownScope); while (attempt <= maxRetries) { attempt++; const start = Date.now(); try { + const cooldown = getRetryCooldown(cooldownKey, start); + if (cooldown) { + const details = { + provider: opts.provider, + url: opts.url, + status: cooldown.status, + attempt, + maxRetries, + retryAfterMs: cooldown.retryAfterMs, + retryAt: cooldown.retryAt, + retryDecision: "defer", + retryReason: "cooldown_active", + remainingDeadlineMs: remainingDeadlineMs(opts.deadlineAt, start), + }; + opts.log.warn("http.retry_cooldown", details); + throw new MemosError( + ERROR_CODES.EMBEDDING_UNAVAILABLE, + `${opts.provider} is cooling down until ${new Date(cooldown.retryAt).toISOString()}`, + details, + ); + } const signal = mergeSignals(opts.signal, AbortSignal.timeout(timeoutMs)); const resp = await fetch(opts.url, { method: "POST", @@ -46,21 +81,62 @@ export async function httpPostJson(opts: HttpPostOpts): Promise< if (!resp.ok) { const text = await safeText(resp); const transient = resp.status >= 500 || resp.status === 429; + const retryAfterMs = resp.status === 429 || resp.status === 503 + ? parseRetryAfterMs(resp.headers.get("Retry-After")) + : null; + if (retryAfterMs !== null) { + recordRetryCooldown(cooldownKey, { + retryAfterMs, + retryAt: Date.now() + retryAfterMs, + status: resp.status, + }); + } opts.log.warn("http.non_ok", { url: opts.url, status: resp.status, attempt, transient, + retryAfterMs, durationMs: Date.now() - start, }); if (transient && attempt <= maxRetries) { - await backoff(attempt); + const plan = planRetry({ + attempt, + baseMs: 200, + jitterMaxMs: 100, + retryAfterMs, + deadlineAt: opts.deadlineAt, + }); + const retryDetails = retryPlanDetails(plan, opts, maxRetries, resp.status, attempt); + if (plan.action === "defer") { + opts.log.warn("http.retry_deferred", retryDetails); + throw new MemosError( + ERROR_CODES.EMBEDDING_UNAVAILABLE, + `HTTP ${resp.status} from ${opts.provider}; retry deferred until ${new Date(plan.retryAt).toISOString()}`, + retryDetails, + ); + } + opts.log.warn("http.retry_scheduled", retryDetails); + await waitForRetry(plan.delayMs, opts.signal); continue; } throw new MemosError( ERROR_CODES.EMBEDDING_UNAVAILABLE, `HTTP ${resp.status} from ${opts.provider}`, - { provider: opts.provider, url: opts.url, status: resp.status, body: text }, + { + provider: opts.provider, + url: opts.url, + status: resp.status, + body: text, + ...(retryAfterMs === null + ? {} + : { + retryAfterMs, + retryAt: Date.now() + retryAfterMs, + retryDecision: "stop", + retryReason: "retries_exhausted", + }), + }, ); } @@ -83,7 +159,23 @@ export async function httpPostJson(opts: HttpPostOpts): Promise< durationMs: Date.now() - start, }); if (transient && attempt <= maxRetries) { - await backoff(attempt); + const plan = planRetry({ + attempt, + baseMs: 200, + jitterMaxMs: 100, + deadlineAt: opts.deadlineAt, + }); + const retryDetails = retryPlanDetails(plan, opts, maxRetries, null, attempt); + if (plan.action === "defer") { + opts.log.warn("http.retry_deferred", retryDetails); + throw new MemosError( + ERROR_CODES.EMBEDDING_UNAVAILABLE, + `${opts.provider} retry cannot fit the request deadline`, + retryDetails, + ); + } + opts.log.warn("http.retry_scheduled", retryDetails); + await waitForRetry(plan.delayMs, opts.signal); continue; } throw new MemosError( @@ -123,11 +215,32 @@ function isTransientError(err: unknown): boolean { return false; } -async function backoff(attempt: number): Promise { - const base = 200; - const jitter = Math.floor(Math.random() * 100); - const ms = base * 2 ** (attempt - 1) + jitter; - await new Promise((r) => setTimeout(r, ms)); +function retryPlanDetails( + plan: RetryPlan, + opts: HttpPostOpts, + maxRetries: number, + status: number | null, + attempt: number, +): Record { + return { + provider: opts.provider, + url: opts.url, + status, + attempt, + maxRetries, + backoffMs: plan.backoffMs, + plannedDelayMs: plan.delayMs, + retryAfterMs: plan.retryAfterMs, + retryAt: plan.retryAt, + retrySource: plan.source, + retryDecision: plan.action, + ...(plan.action === "defer" ? { retryReason: plan.reason } : {}), + remainingDeadlineMs: remainingDeadlineMs(opts.deadlineAt), + }; +} + +function remainingDeadlineMs(deadlineAt?: number, nowMs: number = Date.now()): number | null { + return deadlineAt === undefined ? null : Math.max(0, deadlineAt - nowMs); } function mergeSignals(a: AbortSignal | undefined, b: AbortSignal): AbortSignal { diff --git a/apps/memos-local-plugin/core/embedding/index.ts b/apps/memos-local-plugin/core/embedding/index.ts index 99faa0048..f6f4b1ed9 100644 --- a/apps/memos-local-plugin/core/embedding/index.ts +++ b/apps/memos-local-plugin/core/embedding/index.ts @@ -19,6 +19,7 @@ export { l2Normalize, enforceDim, postProcess, toFloat32 } from "./normalize.js" export { createEmbeddingRetryWorker, systemErrorEvent } from "./retry-worker.js"; export type { EmbeddingRetryWorker } from "./retry-worker.js"; export type { + EmbedCallOptions, EmbedInput, EmbedRole, EmbedStats, diff --git a/apps/memos-local-plugin/core/embedding/providers/cohere.ts b/apps/memos-local-plugin/core/embedding/providers/cohere.ts index 891cd4506..58214d341 100644 --- a/apps/memos-local-plugin/core/embedding/providers/cohere.ts +++ b/apps/memos-local-plugin/core/embedding/providers/cohere.ts @@ -21,7 +21,7 @@ export class CohereEmbeddingProvider implements EmbeddingProvider { readonly name: EmbeddingProviderName = "cohere"; async embed(texts: string[], role: EmbedRole, ctx: ProviderCallCtx): Promise { - const { config, log, signal } = ctx; + const { config, log, signal, deadlineAt } = ctx; if (!config.apiKey) { throw new MemosError( ERROR_CODES.EMBEDDING_UNAVAILABLE, @@ -49,6 +49,8 @@ export class CohereEmbeddingProvider implements EmbeddingProvider { timeoutMs: config.timeoutMs, maxRetries: config.maxRetries, signal, + deadlineAt, + cooldownScope: config.model, provider: this.name, log, }); diff --git a/apps/memos-local-plugin/core/embedding/providers/gemini.ts b/apps/memos-local-plugin/core/embedding/providers/gemini.ts index 91d97acba..68ba22708 100644 --- a/apps/memos-local-plugin/core/embedding/providers/gemini.ts +++ b/apps/memos-local-plugin/core/embedding/providers/gemini.ts @@ -22,7 +22,7 @@ export class GeminiEmbeddingProvider implements EmbeddingProvider { readonly name: EmbeddingProviderName = "gemini"; async embed(texts: string[], role: EmbedRole, ctx: ProviderCallCtx): Promise { - const { config, log, signal } = ctx; + const { config, log, signal, deadlineAt } = ctx; if (!config.apiKey) { throw new MemosError( ERROR_CODES.EMBEDDING_UNAVAILABLE, @@ -50,6 +50,8 @@ export class GeminiEmbeddingProvider implements EmbeddingProvider { timeoutMs: config.timeoutMs, maxRetries: config.maxRetries, signal, + deadlineAt, + cooldownScope: config.model, provider: this.name, log, }); diff --git a/apps/memos-local-plugin/core/embedding/providers/mistral.ts b/apps/memos-local-plugin/core/embedding/providers/mistral.ts index 21f50769f..7eace8ab7 100644 --- a/apps/memos-local-plugin/core/embedding/providers/mistral.ts +++ b/apps/memos-local-plugin/core/embedding/providers/mistral.ts @@ -23,7 +23,7 @@ export class MistralEmbeddingProvider implements EmbeddingProvider { readonly name: EmbeddingProviderName = "mistral"; async embed(texts: string[], _role: EmbedRole, ctx: ProviderCallCtx): Promise { - const { config, log, signal } = ctx; + const { config, log, signal, deadlineAt } = ctx; if (!config.apiKey) { throw new MemosError( ERROR_CODES.EMBEDDING_UNAVAILABLE, @@ -46,6 +46,8 @@ export class MistralEmbeddingProvider implements EmbeddingProvider { timeoutMs: config.timeoutMs, maxRetries: config.maxRetries, signal, + deadlineAt, + cooldownScope: config.model, provider: this.name, log, }); diff --git a/apps/memos-local-plugin/core/embedding/providers/openai.ts b/apps/memos-local-plugin/core/embedding/providers/openai.ts index c0df47831..57d852639 100644 --- a/apps/memos-local-plugin/core/embedding/providers/openai.ts +++ b/apps/memos-local-plugin/core/embedding/providers/openai.ts @@ -27,7 +27,7 @@ export class OpenAiEmbeddingProvider implements EmbeddingProvider { readonly name: EmbeddingProviderName = "openai_compatible"; async embed(texts: string[], _role: EmbedRole, ctx: ProviderCallCtx): Promise { - const { config, log, signal } = ctx; + const { config, log, signal, deadlineAt } = ctx; if (!config.apiKey) { throw new MemosError( ERROR_CODES.EMBEDDING_UNAVAILABLE, @@ -53,6 +53,8 @@ export class OpenAiEmbeddingProvider implements EmbeddingProvider { timeoutMs: config.timeoutMs, maxRetries: config.maxRetries, signal, + deadlineAt, + cooldownScope: config.model, provider: this.name, log, }); diff --git a/apps/memos-local-plugin/core/embedding/providers/voyage.ts b/apps/memos-local-plugin/core/embedding/providers/voyage.ts index f89eca832..6f36fe6d8 100644 --- a/apps/memos-local-plugin/core/embedding/providers/voyage.ts +++ b/apps/memos-local-plugin/core/embedding/providers/voyage.ts @@ -23,7 +23,7 @@ export class VoyageEmbeddingProvider implements EmbeddingProvider { readonly name: EmbeddingProviderName = "voyage"; async embed(texts: string[], role: EmbedRole, ctx: ProviderCallCtx): Promise { - const { config, log, signal } = ctx; + const { config, log, signal, deadlineAt } = ctx; if (!config.apiKey) { throw new MemosError( ERROR_CODES.EMBEDDING_UNAVAILABLE, @@ -50,6 +50,8 @@ export class VoyageEmbeddingProvider implements EmbeddingProvider { timeoutMs: config.timeoutMs, maxRetries: config.maxRetries, signal, + deadlineAt, + cooldownScope: config.model, provider: this.name, log, }); diff --git a/apps/memos-local-plugin/core/embedding/retry-worker.ts b/apps/memos-local-plugin/core/embedding/retry-worker.ts index 3620fd9e3..34db37bff 100644 --- a/apps/memos-local-plugin/core/embedding/retry-worker.ts +++ b/apps/memos-local-plugin/core/embedding/retry-worker.ts @@ -35,6 +35,7 @@ export function createEmbeddingRetryWorker( const workerId = `embedding-retry-${ids.span()}`; let timer: ReturnType | null = null; let running: Promise | null = null; + let stopped = false; async function runOnce(): Promise { if (!deps.embedder) return; @@ -103,6 +104,11 @@ export function createEmbeddingRetryWorker( const message = err instanceof Error ? err.message : String(err); const at = now(); const terminal = attemptNo >= job.maxAttempts; + const providerRetryAt = retryAtFromError(err, at); + const nextAttemptAt = Math.max( + at + backoffMs(attemptNo), + providerRetryAt ?? 0, + ); const recorded = terminal ? deps.repos.embeddingRetryQueue.markFailedClaimed(job.id, { ...claim, @@ -113,7 +119,7 @@ export function createEmbeddingRetryWorker( : deps.repos.embeddingRetryQueue.markRetryClaimed(job.id, { ...claim, attempts: attemptNo, - nextAttemptAt: at + backoffMs(attemptNo), + nextAttemptAt, error: message, now: at, }); @@ -121,7 +127,10 @@ export function createEmbeddingRetryWorker( deps.log.debug("embedding_retry.stale_failure_ignored", { jobId: job.id, terminal }); return; } - emitFailure(job, attemptNo, message, terminal, at); + emitFailure(job, attemptNo, message, terminal, at, { + providerRetryAt, + nextAttemptAt: terminal ? null : nextAttemptAt, + }); } } @@ -153,6 +162,7 @@ export function createEmbeddingRetryWorker( message: string, terminal: boolean, at: number, + retry: { providerRetryAt: number | null; nextAttemptAt: number | null }, ): void { const payload = { kind: "embedding.retry_failed", @@ -164,6 +174,8 @@ export function createEmbeddingRetryWorker( maxAttempts: job.maxAttempts, terminal, message, + providerRetryAt: retry.providerRetryAt, + nextAttemptAt: retry.nextAttemptAt, }; deps.log.warn("embedding_retry.failed", payload); try { @@ -182,7 +194,7 @@ export function createEmbeddingRetryWorker( } function tick(): void { - if (running) return; + if (stopped || running) return; running = runOnce().finally(() => { running = null; }); @@ -190,21 +202,30 @@ export function createEmbeddingRetryWorker( return { start(): void { - if (timer || !deps.embedder) return; + if (stopped || timer || !deps.embedder) return; tick(); timer = setInterval(tick, deps.intervalMs ?? DEFAULT_INTERVAL_MS); }, stop(): void { + stopped = true; if (timer) clearInterval(timer); timer = null; }, async flush(): Promise { - tick(); + if (!stopped) tick(); if (running) await running; }, }; } +function retryAtFromError(err: unknown, nowMs: number): number | null { + if (!err || typeof err !== "object") return null; + const details = (err as { details?: unknown }).details; + if (!details || typeof details !== "object") return null; + const retryAt = Number((details as { retryAt?: unknown }).retryAt); + return Number.isSafeInteger(retryAt) && retryAt > nowMs ? retryAt : null; +} + function backoffMs(attemptNo: number): number { return Math.min(MAX_BACKOFF_MS, BASE_BACKOFF_MS * 2 ** Math.max(0, attemptNo - 1)); } diff --git a/apps/memos-local-plugin/core/embedding/types.ts b/apps/memos-local-plugin/core/embedding/types.ts index 4f3f5eb99..95726703c 100644 --- a/apps/memos-local-plugin/core/embedding/types.ts +++ b/apps/memos-local-plugin/core/embedding/types.ts @@ -6,6 +6,7 @@ */ import type { EmbeddingVector } from "../types.js"; +import type { RetryDiagnosticDetails } from "../util/retry-after.js"; // ─── Config ────────────────────────────────────────────────────────────────── @@ -62,7 +63,7 @@ export interface EmbeddingConfig { onStatus?: (detail: EmbeddingStatusDetail) => void; } -export interface EmbeddingErrorDetail { +export interface EmbeddingErrorDetail extends RetryDiagnosticDetails { kind: "embedding"; provider: EmbeddingProviderName | string; model: string; @@ -73,7 +74,7 @@ export interface EmbeddingErrorDetail { at?: number; } -export interface EmbeddingStatusDetail { +export interface EmbeddingStatusDetail extends RetryDiagnosticDetails { kind: "embedding"; status: "ok" | "error"; provider: EmbeddingProviderName | string; @@ -128,6 +129,8 @@ export interface ProviderCallCtx { log: ProviderLogger; /** AbortSignal honored across HTTP + native calls. */ signal?: AbortSignal; + /** Absolute end-to-end deadline shared across provider retry attempts. */ + deadlineAt?: number; } export interface ProviderLogger { @@ -166,13 +169,16 @@ export interface Embedder { /** Model identifier as configured by the operator (e.g. "bge-m3"). */ readonly model: string; - embedOne(input: string | EmbedInput): Promise; + embedOne(input: string | EmbedInput, options?: EmbedCallOptions): Promise; /** * Batch-embed many texts. Results keep input order. Duplicates are deduped * internally so a text repeated N times causes 1 cache miss max. */ - embedMany(inputs: Array): Promise; + embedMany( + inputs: Array, + options?: EmbedCallOptions, + ): Promise; stats(): EmbedStats; @@ -181,6 +187,12 @@ export interface Embedder { close(): Promise; } +export interface EmbedCallOptions { + signal?: AbortSignal; + /** Absolute end-to-end deadline shared across provider retry attempts. */ + deadlineAt?: number; +} + // ─── Errors ────────────────────────────────────────────────────────────────── export interface ProviderHttpFailure { diff --git a/apps/memos-local-plugin/core/index.ts b/apps/memos-local-plugin/core/index.ts index d2b979bce..c5e4fab9d 100644 --- a/apps/memos-local-plugin/core/index.ts +++ b/apps/memos-local-plugin/core/index.ts @@ -110,6 +110,7 @@ export { MistralEmbeddingProvider, type EmbedCache, type EmbedCacheStats, + type EmbedCallOptions, type EmbedInput, type EmbedRole, type EmbedStats, diff --git a/apps/memos-local-plugin/core/llm/client.ts b/apps/memos-local-plugin/core/llm/client.ts index 6bedafa70..ee456ac12 100644 --- a/apps/memos-local-plugin/core/llm/client.ts +++ b/apps/memos-local-plugin/core/llm/client.ts @@ -18,6 +18,7 @@ import { ERROR_CODES, MemosError } from "../../agent-contract/errors.js"; import { rootLogger } from "../logger/index.js"; import type { Logger } from "../logger/types.js"; +import { extractRetryDiagnostics } from "../util/retry-after.js"; import { getHostLlmBridge } from "./host-bridge.js"; import { buildJsonSystemHint, parseLlmJson } from "./json-mode.js"; import { AnthropicLlmProvider } from "./providers/anthropic.js"; @@ -292,6 +293,7 @@ export function createLlmClientWithProvider( }, log: pLog, signal: opts?.signal, + deadlineAt: opts?.deadlineAt, }; } @@ -370,6 +372,7 @@ export function createLlmClientWithProvider( model: config.model, message: summarizeErrMessage(hostErr), code: hostErr instanceof MemosError ? hostErr.code : undefined, + ...extractRetryDiagnostics(hostErr instanceof MemosError ? hostErr.details : undefined), at: failAt, durationMs: Date.now() - startedAt, fallbackProvider: "host", @@ -395,6 +398,7 @@ export function createLlmClientWithProvider( model: config.model, message: summarizeErrMessage(err), code: err instanceof MemosError ? err.code : undefined, + ...extractRetryDiagnostics(err instanceof MemosError ? err.details : undefined), at: failAt, durationMs: Date.now() - startedAt, op, @@ -425,6 +429,7 @@ export function createLlmClientWithProvider( message: summarizeErrMessage(err), code: err instanceof MemosError ? err.code : undefined, at: Date.now(), + ...extractRetryDiagnostics(err instanceof MemosError ? err.details : undefined), }); } catch { /* sink errors are non-fatal */ @@ -444,6 +449,10 @@ export function createLlmClientWithProvider( op?: string; episodeId?: string; phase?: string; + retryAfterMs?: number; + retryAt?: number; + retryDecision?: "wait" | "defer" | "stop"; + retryReason?: string; }): void { if (!config.onStatus) return; try { @@ -616,6 +625,7 @@ export function createLlmClientWithProvider( model: config.model, message: summarizeErrMessage(err), code: err instanceof MemosError ? err.code : undefined, + ...extractRetryDiagnostics(err instanceof MemosError ? err.details : undefined), at: failAt, durationMs: Date.now() - start, op: opts?.op ?? "stream", @@ -669,6 +679,7 @@ export function createLlmClientWithProvider( model: config.model, message: summarizeErrMessage(primaryErr), code: primaryErr instanceof MemosError ? primaryErr.code : undefined, + ...extractRetryDiagnostics(primaryErr instanceof MemosError ? primaryErr.details : undefined), at: fallbackAt, durationMs: completion.durationMs, fallbackProvider: "host", diff --git a/apps/memos-local-plugin/core/llm/fetcher.ts b/apps/memos-local-plugin/core/llm/fetcher.ts index 53eb55ec3..358c73275 100644 --- a/apps/memos-local-plugin/core/llm/fetcher.ts +++ b/apps/memos-local-plugin/core/llm/fetcher.ts @@ -11,6 +11,15 @@ */ import { ERROR_CODES, MemosError } from "../../agent-contract/errors.js"; +import { + getRetryCooldown, + parseRetryAfterMs, + planRetry, + recordRetryCooldown, + retryCooldownKey, + type RetryPlan, + waitForRetry, +} from "../util/retry-after.js"; import type { LlmProviderLogger, LlmProviderName } from "./types.js"; export interface HttpPostOpts { @@ -20,6 +29,10 @@ export interface HttpPostOpts { timeoutMs: number; maxRetries: number; signal?: AbortSignal; + /** Absolute end-to-end deadline. Unlike timeoutMs, this is not renewed per attempt. */ + deadlineAt?: number; + /** Model/deployment scope; prevents one model cooldown from blocking another. */ + cooldownScope?: string; provider: LlmProviderName; log: LlmProviderLogger; onRetry?: (attempt: number) => void; @@ -35,11 +48,33 @@ export async function httpPostJson(opts: HttpPostOpts): Promise< }> { let attempt = 0; let lastErr: unknown = null; + const cooldownKey = retryCooldownKey("llm", opts.provider, opts.url, opts.cooldownScope); while (attempt <= opts.maxRetries) { attempt++; const start = Date.now(); try { + const cooldown = getRetryCooldown(cooldownKey, start); + if (cooldown) { + const details = { + provider: opts.provider, + url: opts.url, + status: cooldown.status, + attempt, + maxRetries: opts.maxRetries, + retryAfterMs: cooldown.retryAfterMs, + retryAt: cooldown.retryAt, + retryDecision: "defer", + retryReason: "cooldown_active", + remainingDeadlineMs: remainingDeadlineMs(opts.deadlineAt, start), + }; + opts.log.warn("http.retry_cooldown", details); + throw new MemosError( + errCodeForStatus(cooldown.status), + `${opts.provider} is cooling down until ${new Date(cooldown.retryAt).toISOString()}`, + details, + ); + } const signal = mergeSignals(opts.signal, AbortSignal.timeout(opts.timeoutMs)); const resp = await fetch(opts.url, { method: "POST", @@ -56,22 +91,63 @@ export async function httpPostJson(opts: HttpPostOpts): Promise< if (!resp.ok) { const text = await safeText(resp); const transient = resp.status >= 500 || resp.status === 429; + const retryAfterMs = resp.status === 429 || resp.status === 503 + ? parseRetryAfterMs(resp.headers.get("Retry-After")) + : null; + if (retryAfterMs !== null) { + recordRetryCooldown(cooldownKey, { + retryAfterMs, + retryAt: Date.now() + retryAfterMs, + status: resp.status, + }); + } opts.log.warn("http.non_ok", { status: resp.status, attempt, transient, durationMs: ms, + retryAfterMs, body: truncateLogBody(text), }); if (transient && attempt <= opts.maxRetries) { + const plan = planRetry({ + attempt, + baseMs: 250, + jitterMaxMs: 120, + retryAfterMs, + deadlineAt: opts.deadlineAt, + }); + const retryDetails = retryPlanDetails(plan, opts, resp.status, attempt); + if (plan.action === "defer") { + opts.log.warn("http.retry_deferred", retryDetails); + throw new MemosError( + errCodeForStatus(resp.status), + `HTTP ${resp.status} from ${opts.provider}; retry deferred until ${new Date(plan.retryAt).toISOString()}`, + retryDetails, + ); + } + opts.log.warn("http.retry_scheduled", retryDetails); opts.onRetry?.(attempt); - await backoff(attempt); + await waitForRetry(plan.delayMs, opts.signal); continue; } throw new MemosError( errCodeForStatus(resp.status), `HTTP ${resp.status} from ${opts.provider}`, - { provider: opts.provider, url: opts.url, status: resp.status, body: text }, + { + provider: opts.provider, + url: opts.url, + status: resp.status, + body: text, + ...(retryAfterMs === null + ? {} + : { + retryAfterMs, + retryAt: Date.now() + retryAfterMs, + retryDecision: "stop", + retryReason: "retries_exhausted", + }), + }, ); } @@ -85,8 +161,15 @@ export async function httpPostJson(opts: HttpPostOpts): Promise< } catch (err) { lastErr = err; if (err instanceof MemosError) throw err; + if (opts.signal?.aborted) { + throw new MemosError( + ERROR_CODES.LLM_TIMEOUT, + `${opts.provider} request was cancelled`, + { provider: opts.provider, url: opts.url, cancelled: true }, + ); + } const transient = isTransientError(err); - const timedOut = isTimeout(err); + const timedOut = isTimeout(err) || opts.signal?.aborted === true; opts.log.warn("http.exception", { attempt, transient, @@ -94,8 +177,24 @@ export async function httpPostJson(opts: HttpPostOpts): Promise< err: toErrDetail(err), }); if ((transient || timedOut) && attempt <= opts.maxRetries) { + const plan = planRetry({ + attempt, + baseMs: 250, + jitterMaxMs: 120, + deadlineAt: opts.deadlineAt, + }); + const retryDetails = retryPlanDetails(plan, opts, null, attempt); + if (plan.action === "defer") { + opts.log.warn("http.retry_deferred", retryDetails); + throw new MemosError( + timedOut ? ERROR_CODES.LLM_TIMEOUT : ERROR_CODES.LLM_UNAVAILABLE, + `${opts.provider} retry cannot fit the request deadline`, + retryDetails, + ); + } + opts.log.warn("http.retry_scheduled", retryDetails); opts.onRetry?.(attempt); - await backoff(attempt); + await waitForRetry(plan.delayMs, opts.signal); continue; } if (timedOut) { @@ -245,11 +344,31 @@ function isTimeout(err: unknown): boolean { return false; } -async function backoff(attempt: number): Promise { - const base = 250; - const jitter = Math.floor(Math.random() * 120); - const ms = base * 2 ** (attempt - 1) + jitter; - await new Promise((r) => setTimeout(r, ms)); +function retryPlanDetails( + plan: RetryPlan, + opts: HttpPostOpts, + status: number | null, + attempt: number, +): Record { + return { + provider: opts.provider, + url: opts.url, + status, + attempt, + maxRetries: opts.maxRetries, + backoffMs: plan.backoffMs, + plannedDelayMs: plan.delayMs, + retryAfterMs: plan.retryAfterMs, + retryAt: plan.retryAt, + retrySource: plan.source, + retryDecision: plan.action, + ...(plan.action === "defer" ? { retryReason: plan.reason } : {}), + remainingDeadlineMs: remainingDeadlineMs(opts.deadlineAt), + }; +} + +function remainingDeadlineMs(deadlineAt?: number, nowMs: number = Date.now()): number | null { + return deadlineAt === undefined ? null : Math.max(0, deadlineAt - nowMs); } function mergeSignals(a: AbortSignal | undefined, b: AbortSignal): AbortSignal { diff --git a/apps/memos-local-plugin/core/llm/providers/anthropic.ts b/apps/memos-local-plugin/core/llm/providers/anthropic.ts index 66a9ee446..6c510ef75 100644 --- a/apps/memos-local-plugin/core/llm/providers/anthropic.ts +++ b/apps/memos-local-plugin/core/llm/providers/anthropic.ts @@ -31,7 +31,7 @@ export class AnthropicLlmProvider implements LlmProvider { opts: ProviderCallInput, ctx: LlmProviderCtx, ): Promise { - const { config, log, signal } = ctx; + const { config, log, signal, deadlineAt } = ctx; if (!config.apiKey) { throw new MemosError( ERROR_CODES.LLM_UNAVAILABLE, @@ -68,6 +68,8 @@ export class AnthropicLlmProvider implements LlmProvider { timeoutMs: config.timeoutMs, maxRetries: config.maxRetries, signal, + deadlineAt, + cooldownScope: config.model, provider: this.name, log, }); diff --git a/apps/memos-local-plugin/core/llm/providers/bedrock.ts b/apps/memos-local-plugin/core/llm/providers/bedrock.ts index 7c00470e7..d956b7111 100644 --- a/apps/memos-local-plugin/core/llm/providers/bedrock.ts +++ b/apps/memos-local-plugin/core/llm/providers/bedrock.ts @@ -36,7 +36,7 @@ export class BedrockLlmProvider implements LlmProvider { opts: ProviderCallInput, ctx: LlmProviderCtx, ): Promise { - const { config, log, signal } = ctx; + const { config, log, signal, deadlineAt } = ctx; if (!config.endpoint || config.endpoint.length === 0) { throw new MemosError( ERROR_CODES.LLM_UNAVAILABLE, @@ -85,6 +85,8 @@ export class BedrockLlmProvider implements LlmProvider { timeoutMs: config.timeoutMs, maxRetries: config.maxRetries, signal, + deadlineAt, + cooldownScope: config.model, provider: this.name, log, }); diff --git a/apps/memos-local-plugin/core/llm/providers/gemini.ts b/apps/memos-local-plugin/core/llm/providers/gemini.ts index 4e6273b55..6bfa323eb 100644 --- a/apps/memos-local-plugin/core/llm/providers/gemini.ts +++ b/apps/memos-local-plugin/core/llm/providers/gemini.ts @@ -39,7 +39,7 @@ export class GeminiLlmProvider implements LlmProvider { opts: ProviderCallInput, ctx: LlmProviderCtx, ): Promise { - const { config, log, signal } = ctx; + const { config, log, signal, deadlineAt } = ctx; if (!config.apiKey) { throw new MemosError( ERROR_CODES.LLM_UNAVAILABLE, @@ -59,6 +59,8 @@ export class GeminiLlmProvider implements LlmProvider { timeoutMs: config.timeoutMs, maxRetries: config.maxRetries, signal, + deadlineAt, + cooldownScope: config.model, provider: this.name, log, }); diff --git a/apps/memos-local-plugin/core/llm/providers/openai.ts b/apps/memos-local-plugin/core/llm/providers/openai.ts index d8a5a20af..562521756 100644 --- a/apps/memos-local-plugin/core/llm/providers/openai.ts +++ b/apps/memos-local-plugin/core/llm/providers/openai.ts @@ -51,7 +51,7 @@ export class OpenAiLlmProvider implements LlmProvider { opts: ProviderCallInput, ctx: LlmProviderCtx, ): Promise { - const { config, log, signal } = ctx; + const { config, log, signal, deadlineAt } = ctx; const url = normalizeEndpoint( config.endpoint && config.endpoint.length > 0 ? config.endpoint @@ -93,6 +93,8 @@ export class OpenAiLlmProvider implements LlmProvider { timeoutMs: config.timeoutMs, maxRetries: config.maxRetries, signal, + deadlineAt, + cooldownScope: config.model, provider: this.name, log, }); diff --git a/apps/memos-local-plugin/core/llm/types.ts b/apps/memos-local-plugin/core/llm/types.ts index a37a2677d..2dd761f32 100644 --- a/apps/memos-local-plugin/core/llm/types.ts +++ b/apps/memos-local-plugin/core/llm/types.ts @@ -6,6 +6,7 @@ */ import type { ReasoningConfig as ConfigReasoningConfig } from "../config/schema.js"; +import type { RetryDiagnosticDetails } from "../util/retry-after.js"; // ─── Providers & config ────────────────────────────────────────────────────── @@ -88,7 +89,7 @@ export interface LlmCircuitBreakerConfig { now?: () => number; } -export interface LlmErrorDetail { +export interface LlmErrorDetail extends RetryDiagnosticDetails { provider: LlmProviderName | string; model: string; message: string; @@ -105,7 +106,7 @@ export interface LlmErrorDetail { role?: "llm" | "skillEvolver"; } -export interface LlmStatusDetail { +export interface LlmStatusDetail extends RetryDiagnosticDetails { status: "ok" | "fallback" | "error" | "circuit_open"; provider: LlmProviderName | string; model: string; @@ -149,6 +150,8 @@ export interface LlmCallOptions { maxTokens?: number; /** Per-call timeout. */ timeoutMs?: number; + /** Absolute end-to-end deadline shared across provider retry attempts. */ + deadlineAt?: number; /** AbortSignal honored across HTTP + host-bridge calls. */ signal?: AbortSignal; /** @@ -213,6 +216,8 @@ export interface LlmProviderCtx { log: LlmProviderLogger; /** Call abort signal; providers must honor it. */ signal?: AbortSignal; + /** Absolute end-to-end deadline; providers must not renew it per retry. */ + deadlineAt?: number; } export interface LlmProviderLogger { diff --git a/apps/memos-local-plugin/core/pipeline/deps.ts b/apps/memos-local-plugin/core/pipeline/deps.ts index 8a0772119..79714b35e 100644 --- a/apps/memos-local-plugin/core/pipeline/deps.ts +++ b/apps/memos-local-plugin/core/pipeline/deps.ts @@ -101,6 +101,10 @@ import type { import { wrapRetrievalRepos } from "./retrieval-repos.js"; import { createSemaphore } from "../util/semaphore.js"; import { rateLimitLlmClient } from "../util/rate-limited-llm.js"; +import { + prioritizeEmbedder, + type ForegroundResources, +} from "../util/foreground-resources.js"; // ─── Algorithm config slice helper ──────────────────────────────────────── @@ -208,19 +212,23 @@ export function buildPipelineSubscribers( buses: PipelineBuses, algorithm: PipelineAlgorithmConfig, session?: PipelineSessionSet, + resources?: ForegroundResources, ): PipelineSubscriberSet { const log = deps.log ?? rootLogger.child({ channel: "core.pipeline" }); const bgLlmSemaphore = createSemaphore(algorithm.session.bgLlmConcurrency); - const bgLlm = rateLimitLlmClient(deps.llm, bgLlmSemaphore); - const bgReflectLlm = rateLimitLlmClient(deps.reflectLlm, bgLlmSemaphore); - const bgL3Llm = rateLimitLlmClient(deps.l3Llm ?? deps.llm, bgLlmSemaphore); + const bgLlm = rateLimitLlmClient(deps.llm, bgLlmSemaphore, resources); + const bgReflectLlm = rateLimitLlmClient(deps.reflectLlm, bgLlmSemaphore, resources); + const bgL3Llm = rateLimitLlmClient(deps.l3Llm ?? deps.llm, bgLlmSemaphore, resources); + const bgEmbedder = resources + ? prioritizeEmbedder(deps.embedder, resources, "background") + : deps.embedder; const lightweightMode = algorithm.lightweightMemory.enabled; const captureRunner = createCaptureRunner({ tracesRepo: deps.repos.traces, embeddingRetryQueue: deps.repos.embeddingRetryQueue, episodesRepo: adaptEpisodesRepo(deps.repos.episodes), - embedder: deps.embedder, + embedder: bgEmbedder, llm: bgLlm, // Issue #2148: capture batch reflection emits JSON, so it must use // the main model rather than the potentially thinking-enabled @@ -327,7 +335,7 @@ export function buildPipelineSubscribers( const skillHandle = attachSkillSubscriber({ repos: deps.repos, - embedder: deps.embedder, + embedder: bgEmbedder, llm: bgLlm, bus: buses.skill, l2Bus: buses.l2, @@ -339,7 +347,7 @@ export function buildPipelineSubscribers( const feedbackHandle = attachFeedbackSubscriber({ repos: deps.repos, llm: bgLlm, - embedder: deps.embedder, + embedder: bgEmbedder, bus: buses.feedback, log: log.child({ channel: "core.feedback" }), config: algorithm.feedback, @@ -404,14 +412,17 @@ export function buildPipelineSession( export function buildRetrievalDeps( deps: PipelineDeps, algorithm: PipelineAlgorithmConfig, + resources?: ForegroundResources, ): RetrievalDeps { - const embedder = deps.embedder; + const embedder = resources + ? prioritizeEmbedder(deps.embedder, resources, "foreground") + : deps.embedder; return { repos: wrapRetrievalRepos(deps.repos, deps.namespace), embedder: embedder ? { - embed: (text, role) => - embedder.embedOne({ text, role: role ?? "query" }), + embed: (text, role, options) => + embedder.embedOne({ text, role: role ?? "query" }, options), } : { // Degraded mode: empty vector so vector-scoring falls back to diff --git a/apps/memos-local-plugin/core/pipeline/memory-core.ts b/apps/memos-local-plugin/core/pipeline/memory-core.ts index c9254e092..a0354bf64 100644 --- a/apps/memos-local-plugin/core/pipeline/memory-core.ts +++ b/apps/memos-local-plugin/core/pipeline/memory-core.ts @@ -1232,18 +1232,59 @@ export function createMemoryCore( const statsLine = `phase=${phase}, stored=${storedCount}` + (r.warnings.length > 0 ? `, warnings=${r.warnings.length}` : ""); - const details = r.traces.map((tc) => ({ - role: inferTurnRole(tc), - action: phase === "lite" ? ("stored" as const) : ("reflected" as const), - summary: tc.reflection?.text ?? null, - content: ( - tc.userText || - tc.agentText || - summarizeToolCalls(tc.toolCalls) || - "" - ).slice(0, 400), - traceId: tc.traceId, - })); + const action = phase === "lite" + ? ("stored" as const) + : ("reflected" as const); + const details = r.traces.flatMap((tc) => { + const items: Array<{ + role: "user" | "assistant" | "tool" | "reflection" | "other"; + action: typeof action; + summary: string | null; + content: string; + traceId: string; + }> = []; + + if (tc.userText) { + items.push({ + role: "user", + action, + summary: null, + content: tc.userText.slice(0, 400), + traceId: tc.traceId, + }); + } + if (tc.agentText) { + items.push({ + role: "assistant", + action, + summary: null, + content: tc.agentText.slice(0, 400), + traceId: tc.traceId, + }); + } + + const toolSummary = summarizeToolCalls(tc.toolCalls); + if (items.length === 0) { + items.push({ + role: toolSummary ? "tool" : "other", + action, + summary: tc.reflection?.text ?? null, + content: toolSummary.slice(0, 400), + traceId: tc.traceId, + }); + } else if (tc.reflection?.text) { + // Keep the existing reflect-phase summary visible without + // presenting it as either side's original chat content. + items.push({ + role: "reflection", + action, + summary: tc.reflection.text, + content: "", + traceId: tc.traceId, + }); + } + return items; + }); handle.repos.apiLogs.insert({ toolName: "memory_add", input: { @@ -6195,26 +6236,3 @@ function summarizeToolCalls( }) .join("\n"); } - -/** - * Heuristic role inference for api_logs "memory_add" rows — mirrors - * the legacy plugin's behaviour where each captured turn showed up - * labelled `user` / `assistant` / `tool` on the Logs page. - * - * Priority: if the step carries userText (the user's query), label it - * "user" even when toolCalls are present — this is the first sub-step - * of a multi-tool turn and semantically represents the user request. - */ -function inferTurnRole(step: { - userText?: string; - agentText?: string; - toolCalls?: readonly unknown[]; -}): "user" | "assistant" | "tool" | "other" { - const u = (step.userText ?? "").length; - const a = (step.agentText ?? "").length; - if (u > 0 && (step.toolCalls?.length ?? 0) > 0) return "user"; - if ((step.toolCalls?.length ?? 0) > 0) return "tool"; - if (u >= a && u > 0) return "user"; - if (a > 0) return "assistant"; - return "other"; -} diff --git a/apps/memos-local-plugin/core/pipeline/orchestrator.ts b/apps/memos-local-plugin/core/pipeline/orchestrator.ts index ac48133ad..4ed0427d1 100644 --- a/apps/memos-local-plugin/core/pipeline/orchestrator.ts +++ b/apps/memos-local-plugin/core/pipeline/orchestrator.ts @@ -83,29 +83,39 @@ import { onBroadcastLog } from "../logger/transports/sse-broadcast.js"; import { createEmbeddingRetryWorker, systemErrorEvent } from "../embedding/index.js"; import type { EpisodeSnapshot } from "../session/index.js"; import type { IntentDecision, RelationDecision, TurnRelation } from "../session/types.js"; +import { + createForegroundResources, + prioritizeEmbedder, +} from "../util/foreground-resources.js"; +import { createRequestDeadline } from "../util/request-deadline.js"; function classifyWithTimeout( classifyFn: () => Promise, timeoutMs: number, log: Logger, ): Promise { + let timer: ReturnType | null = null; return Promise.race([ classifyFn(), - new Promise((_, reject) => - setTimeout(() => reject(new Error("classify_timeout")), timeoutMs), - ), - ]).catch((err) => { - log.warn("relation.classify_timeout", { - timeoutMs, - err: err instanceof Error ? err.message : String(err), + new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error("classify_timeout")), timeoutMs); + }), + ]) + .catch((err) => { + log.warn("relation.classify_timeout", { + timeoutMs, + err: err instanceof Error ? err.message : String(err), + }); + return { + relation: "follow_up" as const, + confidence: 0, + reason: "classify_timeout", + signals: ["classify_timeout"], + }; + }) + .finally(() => { + if (timer) clearTimeout(timer); }); - return { - relation: "new_task" as const, - confidence: 0, - reason: "classify_timeout", - signals: ["classify_timeout"], - }; - }); } // ─── Factory ────────────────────────────────────────────────────────────── @@ -115,6 +125,12 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { const algorithm = extractAlgorithmConfig(deps); const lightweightMode = algorithm.lightweightMemory.enabled; const buses = buildPipelineBuses(); + const foregroundResources = createForegroundResources(); + const backgroundEmbedder = prioritizeEmbedder( + deps.embedder, + foregroundResources, + "background", + ); // Session + intent. const session = buildPipelineSession(deps, buses.session); @@ -123,7 +139,13 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { // Pass `session` so the reward runner's `getEpisodeSnapshot` hook // can resolve the live, in-memory episode (with turns populated) // rather than falling back to the empty row from SQLite. - const subs = buildPipelineSubscribers(deps, buses, algorithm, session); + const subs = buildPipelineSubscribers( + deps, + buses, + algorithm, + session, + foregroundResources, + ); // Core-event aggregator. Every internal bus funnels into one stream. const eventListeners = new Set<(e: CoreEvent) => void>(); @@ -160,7 +182,7 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { let retryEventSeq = 1_000_000; const embeddingRetryWorker = createEmbeddingRetryWorker({ repos: deps.repos, - embedder: deps.embedder, + embedder: backgroundEmbedder, log: log.child({ channel: "core.embedding.retry" }), now: deps.now, onSystemError: (payload, correlationId) => { @@ -366,6 +388,7 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { userText: string, meta: Record, turnTs?: number, + signal?: AbortSignal, ): Promise { const currentEpId = openEpisodeBySession.get(sessionId); if (currentEpId) { @@ -387,6 +410,7 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { userMessage: userText, ts: turnTs, meta: lightweightEpisodeMeta(meta), + signal, }); openEpisodeBySession.set(sessionId, snap.id as EpisodeId); return snap; @@ -433,13 +457,20 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { userText: string, meta: Record, agent: AgentKind, + signal?: AbortSignal, ): Promise<{ episode: EpisodeSnapshot; sessionId: SessionId; relation?: string }> { const mergeMode = algorithm.session.followUpMode === "merge_follow_ups"; const mergeCapMs = algorithm.session.mergeMaxGapMs; const turnTs = timestampFromMeta(meta, "startedAtTurnTs"); if (lightweightMode) { - const snap = await startLightweightEpisode(sessionId, userText, meta, turnTs); + const snap = await startLightweightEpisode( + sessionId, + userText, + meta, + turnTs, + signal, + ); return { episode: snap, sessionId, relation: "lightweight_memory" }; } @@ -465,6 +496,7 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { newUserText: userText, gapMs, prevEpisodeId: currentEpId, + signal, }), algorithm.session.classifyTimeoutMs, log, @@ -582,6 +614,7 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { userMessage: userText, ts: turnTs, meta: { ...meta, relation: "new_task" }, + signal, }); openEpisodeBySession.set(sessionId, snap.id as EpisodeId); return { episode: snap, sessionId, relation: decision.relation }; @@ -601,6 +634,7 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { userMessage: userText, ts: turnTs, meta: { ...meta, relation: decision.relation, gapMs }, + signal, }); openEpisodeBySession.set(sessionId, fresh.id as EpisodeId); return { episode: fresh, sessionId, relation: decision.relation }; @@ -645,6 +679,7 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { newUserText: userText, gapMs, prevEpisodeId: snapshot.id as EpisodeId, + signal, }), algorithm.session.classifyTimeoutMs, log, @@ -748,6 +783,7 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { userMessage: userText, ts: turnTs, meta, + signal, }); openEpisodeBySession.set(sessionId, snap.id as EpisodeId); return { episode: snap, sessionId, relation: "bootstrap" }; @@ -762,6 +798,7 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { newUserText: userText, gapMs, prevEpisodeId: prev.episodeId, + signal, }), algorithm.session.classifyTimeoutMs, log, @@ -861,6 +898,7 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { userMessage: userText, ts: turnTs, meta: { ...meta, relation: "new_task" }, + signal, }); openEpisodeBySession.set(sessionId, snap.id as EpisodeId); return { episode: snap, sessionId, relation: decision.relation }; @@ -871,6 +909,7 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { userMessage: userText, ts: turnTs, meta: { ...meta, relation: decision.relation }, + signal, }); openEpisodeBySession.set(sessionId, snap.id as EpisodeId); return { episode: snap, sessionId, relation: decision.relation }; @@ -1043,7 +1082,7 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { // ─── Retrieval entry points ───────────────────────────────────────────── - const retrievalDeps = buildRetrievalDeps(deps, algorithm); + const retrievalDeps = buildRetrievalDeps(deps, algorithm, foregroundResources); const turnStartRetrievalStats = new Map(); function retrievalDepsFor(namespace = deps.namespace): typeof retrievalDeps { @@ -1057,6 +1096,7 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { async function retrieveTurnStart( input: TurnInputDTO, plan?: RetrievePlan, + signal?: AbortSignal, ): Promise { const ctx = { reason: "turn_start" as const, @@ -1073,6 +1113,8 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { { events: buses.retrieval, skipLlmFilter: input.contextHints?.__memosDeferLlmFilterToCaller === true, + signal, + deadlineAt: input.deadlineAt, plan: plan ? { scenarioId: plan.scenarioId, @@ -1175,13 +1217,45 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { } async function onTurnStartOnce(input: TurnInputDTO): Promise { + const leaveForeground = foregroundResources.enterForeground(); + const deadline = + input.deadlineAt === undefined + ? null + : createRequestDeadline(input.deadlineAt); + const startedAt = Date.now(); + let stage = "ensure_session"; + try { + return await onTurnStartForeground(input, deadline?.signal, (next) => { + stage = next; + }); + } finally { + if (deadline?.signal.aborted) { + log.warn("turn.start.deadline_exceeded", { + sessionId: input.sessionId, + deadlineAt: input.deadlineAt, + elapsedMs: Date.now() - startedAt, + stage, + }); + } + deadline?.dispose(); + leaveForeground(); + } + } + + async function onTurnStartForeground( + input: TurnInputDTO, + signal?: AbortSignal, + setStage: (stage: string) => void = () => {}, + ): Promise { const t0 = now(); + setStage("ensure_session"); const initialSessionId = await ensureSession( input.agent, input.sessionId, input.contextHints, ); + setStage("relation_and_episode"); const routing = await openEpisodeIfNeeded( initialSessionId, input.userText, @@ -1192,6 +1266,7 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { startedAtTurnTs: input.ts, }, input.agent, + signal, ); const sessionId = routing.sessionId; @@ -1203,10 +1278,12 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { sessionId, episodeId: episode.id as EpisodeId, }; + setStage("intent"); const schedulerIntent = await intentForCurrentTurn({ episode, userText: input.userText, ts: input.ts, + signal, }); const retrievePlan = scheduleInjection({ userText: input.userText, @@ -1240,10 +1317,13 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { retrievalTotalMs: 0, elapsedMs: now() - t0, }); + setStage("complete"); return packet; } - const packet = await retrieveTurnStart(normalized, retrievePlan); + setStage("retrieval"); + const packet = await retrieveTurnStart(normalized, retrievePlan, signal); + setStage("complete"); // Always stamp the routed sessionId + episodeId on the packet so // adapters can correlate the subsequent `agent_end` / `turn.end` // call without needing a separate round-trip to the session @@ -1502,12 +1582,28 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { async function shutdown(reason: string = "shutdown"): Promise { log.info("pipeline.shutdown.begin", { reason }); + // Stop admitting retry jobs, but preserve a bounded grace period for raw + // capture and downstream enrichment. Hermes' bridge owns a 20s outer + // shutdown ceiling, so abort before that rather than either hanging or + // discarding every single-shot session's enrichment immediately. + embeddingRetryWorker.stop(); + const flushPromise = flush(); try { - await flush(); + const completed = await settlesWithin(flushPromise, 15_000); + if (!completed) { + log.warn("pipeline.flush_timeout", { reason, timeoutMs: 15_000 }); + foregroundResources.shutdown(reason); + const aborted = await settlesWithin(flushPromise, 4_000); + if (!aborted) { + log.warn("pipeline.flush_abandoned", { reason, abortWaitMs: 4_000 }); + } + } } catch (err) { log.warn("pipeline.flush_failed", { err: err instanceof Error ? err.message : String(err), }); + } finally { + foregroundResources.shutdown(reason); } // Detach subscribers — prevents late events from re-queuing work. subs.subscriptions.capture.stop(); @@ -1516,13 +1612,26 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { subs.l3.detach(); subs.skills.dispose(); subs.feedback.dispose(); - embeddingRetryWorker.stop(); bridge.dispose(); logSubscription(); session.sessionManager.shutdown(reason); log.info("pipeline.shutdown.done", { reason }); } + async function settlesWithin(promise: Promise, timeoutMs: number): Promise { + let timer: ReturnType | null = null; + try { + return await Promise.race([ + promise.then(() => true), + new Promise((resolve) => { + timer = setTimeout(() => resolve(false), timeoutMs); + }), + ]); + } finally { + if (timer) clearTimeout(timer); + } + } + function now(): number { return (deps.now ?? Date.now)(); } @@ -1589,6 +1698,7 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { episode: EpisodeSnapshot; userText: string; ts?: number; + signal?: AbortSignal; }): Promise { const firstTurn = input.episode.turns[0]; const isFreshEpisodeForThisTurn = @@ -1603,6 +1713,7 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { return session.intent.classify(input.userText, { episodeId: input.episode.id as EpisodeId, + signal: input.signal, }); } diff --git a/apps/memos-local-plugin/core/retrieval/llm-filter.ts b/apps/memos-local-plugin/core/retrieval/llm-filter.ts index 142bd626d..f665cb954 100644 --- a/apps/memos-local-plugin/core/retrieval/llm-filter.ts +++ b/apps/memos-local-plugin/core/retrieval/llm-filter.ts @@ -50,6 +50,8 @@ export interface FilterDeps { llm: LlmClient | null; log: Logger; timeoutMs?: number; + deadlineAt?: number; + signal?: AbortSignal; config: Pick< RetrievalConfig, | "llmFilterEnabled" @@ -117,6 +119,10 @@ export async function llmFilterCandidates( if (!deps.llm) { return passthrough(ranked, "no_llm"); } + if (deps.signal?.aborted) { + deps.log.debug("llm_filter.deadline_exceeded", { candidateCount: ranked.length }); + return safeCutoff(ranked, deps); + } const bodyChars = deps.config.llmFilterCandidateBodyChars ?? DEFAULT_CANDIDATE_BODY_CHARS; @@ -148,6 +154,8 @@ ${list}`, episodeId: input.episodeId, temperature: 0, timeoutMs: deps.timeoutMs, + deadlineAt: deps.deadlineAt, + signal: deps.signal, // Output is only ordered indices + one bool, but the list can // legitimately be as long as the ranked candidates. maxTokens: filterOutputTokenBudget(ranked.length), diff --git a/apps/memos-local-plugin/core/retrieval/retrieve.ts b/apps/memos-local-plugin/core/retrieval/retrieve.ts index cec076f62..ff95debf4 100644 --- a/apps/memos-local-plugin/core/retrieval/retrieve.ts +++ b/apps/memos-local-plugin/core/retrieval/retrieve.ts @@ -75,6 +75,10 @@ export interface RetrieveOptions { * one unified final LLM filter across all routes. */ skipLlmFilter?: boolean; + /** Shared foreground cancellation signal. */ + signal?: AbortSignal; + /** Absolute request deadline used to cap optional LLM filtering. */ + deadlineAt?: number; } export interface RetrievePlanOverride { @@ -258,22 +262,28 @@ async function runAll( degraded: false, }; const queryVec = compiled.text - ? await deps.embedder.embed(compiled.text, "query").then((vec) => { - embeddingStats.ok = true; - return vec; - }).catch((err) => { - const code = (err as { code?: string })?.code; - const message = err instanceof Error ? err.message : String(err); - embeddingStats.degraded = true; - embeddingStats.errorCode = code; - embeddingStats.errorMessage = message; - log.warn("embed_failed", { - reason: ctx.reason, - code, - err: message, - }); - return null; - }) + ? await deps.embedder + .embed(compiled.text, "query", { + signal: opts.signal, + deadlineAt: opts.deadlineAt, + }) + .then((vec) => { + embeddingStats.ok = true; + return vec; + }) + .catch((err) => { + const code = (err as { code?: string })?.code; + const message = err instanceof Error ? err.message : String(err); + embeddingStats.degraded = true; + embeddingStats.errorCode = code; + embeddingStats.errorMessage = message; + log.warn("embed_failed", { + reason: ctx.reason, + code, + err: message, + }); + return null; + }) : null; // The keyword channels (FTS + pattern) work even without an embedder, @@ -415,6 +425,9 @@ async function runAll( llm: deps.llm ?? null, log, config: deps.config, + signal: opts.signal, + deadlineAt: opts.deadlineAt, + timeoutMs: filterTimeoutMs(opts.deadlineAt), }, ); @@ -472,6 +485,9 @@ async function runAll( llm: deps.llm ?? null, log, config: deps.config, + signal: opts.signal, + deadlineAt: opts.deadlineAt, + timeoutMs: filterTimeoutMs(opts.deadlineAt), }, ); @@ -668,6 +684,11 @@ async function runAll( } } +function filterTimeoutMs(deadlineAt?: number): number | undefined { + if (deadlineAt === undefined) return undefined; + return Math.max(1, Math.min(2_000, deadlineAt - Date.now())); +} + function emptyResult( reason: RetrievalReason, agent: AgentKind, diff --git a/apps/memos-local-plugin/core/retrieval/types.ts b/apps/memos-local-plugin/core/retrieval/types.ts index 8ed24935e..fe700c611 100644 --- a/apps/memos-local-plugin/core/retrieval/types.ts +++ b/apps/memos-local-plugin/core/retrieval/types.ts @@ -679,7 +679,11 @@ export interface RetrievalRepos { /** Abstract embedder surface consumed by retrieval. Mirrors `Embedder`. */ export interface RetrievalEmbedder { - embed: (text: string, role?: "query" | "document") => Promise; + embed: ( + text: string, + role?: "query" | "document", + options?: { signal?: AbortSignal; deadlineAt?: number }, + ) => Promise; } export interface RetrievalDeps { diff --git a/apps/memos-local-plugin/core/session/intent-classifier.ts b/apps/memos-local-plugin/core/session/intent-classifier.ts index 41206184d..a4896e07b 100644 --- a/apps/memos-local-plugin/core/session/intent-classifier.ts +++ b/apps/memos-local-plugin/core/session/intent-classifier.ts @@ -46,6 +46,8 @@ export interface IntentClassifierOptions { export interface IntentClassifyOptions { /** Episode id this classification is being run for, when known. */ episodeId?: EpisodeId; + /** Foreground request cancellation propagated to the provider call. */ + signal?: AbortSignal; } export interface IntentClassifier { @@ -91,7 +93,7 @@ export function createIntentClassifier(opts: IntentClassifierOptions = {}): Inte if (!llmDisabled && llm) { try { const result = await withTimeout( - callLlm(llm, text, options?.episodeId), + callLlm(llm, text, options?.episodeId, timeoutMs, options?.signal), timeoutMs, "intent.llm.timeout", ); @@ -207,6 +209,8 @@ async function callLlm( llm: LlmClient, text: string, episodeId?: EpisodeId, + timeoutMs?: number, + signal?: AbortSignal, ): Promise { const rsp = await llm.completeJson<{ kind: unknown; confidence: unknown; reason: unknown }>( [ @@ -217,6 +221,8 @@ async function callLlm( op: "session.intent.classify", phase: "session", episodeId, + timeoutMs, + signal, schemaHint: `{"kind":"task"|"memory_probe"|"chitchat"|"meta"|"unknown","confidence":0..1,"reason":"..."}`, validate: (v) => { const o = v as Record; diff --git a/apps/memos-local-plugin/core/session/manager.ts b/apps/memos-local-plugin/core/session/manager.ts index 44da570b2..8f0f2e79f 100644 --- a/apps/memos-local-plugin/core/session/manager.ts +++ b/apps/memos-local-plugin/core/session/manager.ts @@ -60,6 +60,8 @@ export interface StartEpisodeInput { /** Adapter-provided event time for the first user turn. */ ts?: EpochMs; meta?: Record; + /** Foreground cancellation propagated to intent classification. */ + signal?: AbortSignal; } export interface SessionManager { @@ -267,6 +269,7 @@ export function createSessionManager(deps: SessionManagerDeps): SessionManager { const episodeId = (input.id ?? ids.episode()) as EpisodeId; const intent = await deps.intentClassifier.classify(input.userMessage, { episodeId, + signal: input.signal, }); // Wrap the write+emit in a log context so downstream listeners inherit diff --git a/apps/memos-local-plugin/core/session/relation-classifier.ts b/apps/memos-local-plugin/core/session/relation-classifier.ts index bac4679e0..f0e1e2a24 100644 --- a/apps/memos-local-plugin/core/session/relation-classifier.ts +++ b/apps/memos-local-plugin/core/session/relation-classifier.ts @@ -306,7 +306,11 @@ export function createRelationClassifier( // Step 2: LLM classification. if (!llmDisabled && opts.llm) { try { - const result = await withTimeout(callLlm(opts.llm, input), timeoutMs, "relation.llm.timeout"); + const result = await withTimeout( + callLlm(opts.llm, input, timeoutMs), + timeoutMs, + "relation.llm.timeout", + ); log.debug("llm.ok", { relation: result.relation, confidence: result.confidence, @@ -326,7 +330,7 @@ export function createRelationClassifier( }); try { const arb = await withTimeout( - callArbitration(opts.llm, input), + callArbitration(opts.llm, input, timeoutMs), timeoutMs, "relation.arbitration.timeout", ); @@ -516,7 +520,11 @@ function buildLlmUserContent(input: RelationInput): string { return parts.join("\n\n"); } -async function callLlm(llm: LlmClient, input: RelationInput): Promise { +async function callLlm( + llm: LlmClient, + input: RelationInput, + timeoutMs?: number, +): Promise { const userContent = buildLlmUserContent(input); const rsp = await llm.completeJson<{ relation: unknown; confidence: unknown; reason: unknown }>( @@ -528,6 +536,8 @@ async function callLlm(llm: LlmClient, input: RelationInput): Promise { const o = v as Record; @@ -581,7 +591,11 @@ When in doubt, choose follow_up. Reply JSON ONLY: {"relation":"follow_up"|"new_task","reason":"..."}`; -async function callArbitration(llm: LlmClient, input: RelationInput): Promise { +async function callArbitration( + llm: LlmClient, + input: RelationInput, + timeoutMs?: number, +): Promise { const userContent = [ `CURRENT TASK CONTEXT:\n${(input.prevUserText ?? "").slice(0, 600)}`, `ASSISTANT REPLY:\n${(input.prevAssistantText ?? "").slice(0, 800)}`, @@ -597,6 +611,8 @@ async function callArbitration(llm: LlmClient, input: RelationInput): Promise { const o = v as Record; diff --git a/apps/memos-local-plugin/core/session/types.ts b/apps/memos-local-plugin/core/session/types.ts index 3d34438ea..62ea53f97 100644 --- a/apps/memos-local-plugin/core/session/types.ts +++ b/apps/memos-local-plugin/core/session/types.ts @@ -193,6 +193,8 @@ export interface RelationInput { * is "scoring whether to terminate prevEpisodeId". */ prevEpisodeId?: EpisodeId; + /** Foreground request cancellation propagated to LLM classification. */ + signal?: AbortSignal; } // ─── Event bus ────────────────────────────────────────────────────────────── diff --git a/apps/memos-local-plugin/core/util/foreground-resources.ts b/apps/memos-local-plugin/core/util/foreground-resources.ts new file mode 100644 index 000000000..d818d189f --- /dev/null +++ b/apps/memos-local-plugin/core/util/foreground-resources.ts @@ -0,0 +1,274 @@ +import type { + EmbedCallOptions, + Embedder, + EmbedInput, +} from "../embedding/types.js"; +import type { EmbeddingVector } from "../types.js"; + +export type ResourcePriority = "foreground" | "background"; + +export interface ForegroundResources { + readonly shutdownSignal: AbortSignal; + /** Combine a request signal with the pipeline lifecycle signal. */ + signalFor(signal?: AbortSignal): AbortSignal; + /** Mark the complete turn.start path as foreground work. Idempotent release. */ + enterForeground(): () => void; + /** Background LLM work waits here before acquiring its existing semaphore. */ + waitForBackground(signal?: AbortSignal): Promise; + /** Priority-aware, non-preemptive embedding admission. */ + acquireEmbedding( + priority: ResourcePriority, + signal?: AbortSignal, + ): Promise<() => void>; + /** Reject queued work and cancel provider calls before pipeline drain. */ + shutdown(reason?: string): void; +} + +export interface ForegroundResourceOptions { + embeddingConcurrency?: number; + /** Prevent background starvation during a sustained foreground stream. */ + maxForegroundBurst?: number; +} + +interface Waiter { + resolve: (release: () => void) => void; + reject: (error: Error) => void; + signal?: AbortSignal; + onAbort?: () => void; +} + +interface BackgroundWaiter { + resolve: () => void; + reject: (error: Error) => void; + signal?: AbortSignal; + onAbort?: () => void; +} + +export function createForegroundResources( + options: ForegroundResourceOptions = {}, +): ForegroundResources { + const capacity = Math.max(1, Math.floor(options.embeddingConcurrency ?? 1)); + const maxForegroundBurst = Math.max( + 1, + Math.floor(options.maxForegroundBurst ?? 8), + ); + const embeddingWaiters: Record = { + foreground: [], + background: [], + }; + const backgroundWaiters: BackgroundWaiter[] = []; + let embeddingInUse = 0; + let foregroundActive = 0; + let foregroundBurst = 0; + const shutdownController = new AbortController(); + + function signalFor(signal?: AbortSignal): AbortSignal { + return signal + ? AbortSignal.any([signal, shutdownController.signal]) + : shutdownController.signal; + } + + function abortError(signal?: AbortSignal): Error { + return signal?.reason instanceof Error + ? signal.reason + : new DOMException("resource wait aborted", "AbortError"); + } + + function removeAbortListener(waiter: Waiter | BackgroundWaiter): void { + if (waiter.signal && waiter.onAbort) { + waiter.signal.removeEventListener("abort", waiter.onAbort); + } + } + + function nextEmbeddingWaiter(): { + priority: ResourcePriority; + waiter: Waiter; + } | null { + const foreground = embeddingWaiters.foreground; + const background = embeddingWaiters.background; + if ( + background.length > 0 && + (foreground.length === 0 || foregroundBurst >= maxForegroundBurst) + ) { + return { priority: "background", waiter: background.shift()! }; + } + if (foreground.length > 0) { + return { priority: "foreground", waiter: foreground.shift()! }; + } + if (background.length > 0) { + return { priority: "background", waiter: background.shift()! }; + } + return null; + } + + function drainEmbedding(): void { + while (embeddingInUse < capacity) { + const next = nextEmbeddingWaiter(); + if (!next) return; + removeAbortListener(next.waiter); + embeddingInUse++; + foregroundBurst = next.priority === "foreground" ? foregroundBurst + 1 : 0; + next.waiter.resolve(makeEmbeddingRelease()); + } + } + + function makeEmbeddingRelease(): () => void { + let released = false; + return (): void => { + if (released) return; + released = true; + embeddingInUse--; + drainEmbedding(); + }; + } + + function acquireEmbedding( + priority: ResourcePriority, + signal?: AbortSignal, + ): Promise<() => void> { + signal = signalFor(signal); + if (signal.aborted) return Promise.reject(abortError(signal)); + return new Promise((resolve, reject) => { + const waiter: Waiter = { resolve, reject, signal }; + if (signal) { + waiter.onAbort = () => { + const queue = embeddingWaiters[priority]; + const index = queue.indexOf(waiter); + if (index >= 0) queue.splice(index, 1); + reject(abortError(signal)); + }; + signal.addEventListener("abort", waiter.onAbort, { once: true }); + } + embeddingWaiters[priority].push(waiter); + drainEmbedding(); + }); + } + + function drainBackgroundGate(): void { + if (foregroundActive > 0) return; + for (const waiter of backgroundWaiters.splice(0)) { + removeAbortListener(waiter); + waiter.resolve(); + } + } + + function enterForeground(): () => void { + foregroundActive++; + let left = false; + return (): void => { + if (left) return; + left = true; + foregroundActive--; + drainBackgroundGate(); + }; + } + + function waitForBackground(signal?: AbortSignal): Promise { + signal = signalFor(signal); + if (signal.aborted) return Promise.reject(abortError(signal)); + if (foregroundActive === 0) return Promise.resolve(); + return new Promise((resolve, reject) => { + const waiter: BackgroundWaiter = { resolve, reject, signal }; + if (signal) { + waiter.onAbort = () => { + const index = backgroundWaiters.indexOf(waiter); + if (index >= 0) backgroundWaiters.splice(index, 1); + reject(abortError(signal)); + }; + signal.addEventListener("abort", waiter.onAbort, { once: true }); + } + backgroundWaiters.push(waiter); + }); + } + + function shutdown(reason = "pipeline shutdown"): void { + if (shutdownController.signal.aborted) return; + shutdownController.abort(new DOMException(reason, "AbortError")); + } + + return { + shutdownSignal: shutdownController.signal, + signalFor, + enterForeground, + waitForBackground, + acquireEmbedding, + shutdown, + }; +} + +/** + * Keep the Embedder contract intact while moving provider round-trips behind + * the shared priority arbiter. Background batches are deliberately chunked + * so one enrichment pass cannot monopolize the provider for an entire queue. + */ +export function prioritizeEmbedder( + inner: Embedder | null, + resources: ForegroundResources, + priority: ResourcePriority, + backgroundChunkSize = 8, +): Embedder | null { + if (!inner) return null; + + async function embedOne( + input: string | EmbedInput, + options?: EmbedCallOptions, + ): Promise { + const signal = resources.signalFor(options?.signal); + const callOptions = { ...options, signal }; + if (priority === "background") await resources.waitForBackground(signal); + const release = await resources.acquireEmbedding(priority, signal); + try { + return await inner!.embedOne(input, callOptions); + } finally { + release(); + } + } + + async function embedMany( + inputs: Array, + options?: EmbedCallOptions, + ): Promise { + const signal = resources.signalFor(options?.signal); + const callOptions = { ...options, signal }; + if (priority === "foreground" || inputs.length <= backgroundChunkSize) { + if (priority === "background") await resources.waitForBackground(signal); + const release = await resources.acquireEmbedding(priority, signal); + try { + return await inner!.embedMany(inputs, callOptions); + } finally { + release(); + } + } + + const results: EmbeddingVector[] = []; + for (let start = 0; start < inputs.length; start += backgroundChunkSize) { + await resources.waitForBackground(signal); + const release = await resources.acquireEmbedding(priority, signal); + try { + results.push( + ...await inner!.embedMany(inputs.slice(start, start + backgroundChunkSize), callOptions), + ); + } finally { + release(); + } + } + return results; + } + + return { + get dimensions() { + return inner.dimensions; + }, + get provider() { + return inner.provider; + }, + get model() { + return inner.model; + }, + embedOne, + embedMany, + stats: () => inner.stats(), + resetCache: () => inner.resetCache(), + close: () => inner.close(), + }; +} diff --git a/apps/memos-local-plugin/core/util/rate-limited-llm.ts b/apps/memos-local-plugin/core/util/rate-limited-llm.ts index 4d229b4c9..5bcb863bb 100644 --- a/apps/memos-local-plugin/core/util/rate-limited-llm.ts +++ b/apps/memos-local-plugin/core/util/rate-limited-llm.ts @@ -10,20 +10,26 @@ import type { LlmStreamChunk, } from "../llm/types.js"; import type { Semaphore } from "./semaphore.js"; +import type { ForegroundResources } from "./foreground-resources.js"; /** * Wrap an LLM client so expensive background subscribers share one * process-wide concurrency budget without changing call-site semantics. */ -export function rateLimitLlmClient(client: LlmClient | null, semaphore: Semaphore): LlmClient | null { +export function rateLimitLlmClient( + client: LlmClient | null, + semaphore: Semaphore, + resources?: ForegroundResources, +): LlmClient | null { if (!client) return null; - return new RateLimitedLlmClient(client, semaphore); + return new RateLimitedLlmClient(client, semaphore, resources); } class RateLimitedLlmClient implements LlmClient { constructor( private readonly inner: LlmClient, private readonly semaphore: Semaphore, + private readonly resources?: ForegroundResources, ) {} get provider(): LlmProviderName { @@ -42,9 +48,12 @@ class RateLimitedLlmClient implements LlmClient { messages: LlmMessage[] | string, opts?: LlmCallOptions, ): Promise { - const release = await this.semaphore.acquire(); + const signal = this.resources?.signalFor(opts?.signal) ?? opts?.signal; + const callOpts = signal ? { ...opts, signal } : opts; + await this.resources?.waitForBackground(signal); + const release = await this.semaphore.acquire(signal); try { - return await this.inner.complete(messages, opts); + return await this.inner.complete(messages, callOpts); } finally { release(); } @@ -54,9 +63,12 @@ class RateLimitedLlmClient implements LlmClient { messages: LlmMessage[] | string, opts?: LlmCompleteJsonOptions, ): Promise> { - const release = await this.semaphore.acquire(); + const signal = this.resources?.signalFor(opts?.signal) ?? opts?.signal; + const callOpts = signal ? { ...opts, signal } : opts; + await this.resources?.waitForBackground(signal); + const release = await this.semaphore.acquire(signal); try { - return await this.inner.completeJson(messages, opts); + return await this.inner.completeJson(messages, callOpts); } finally { release(); } @@ -66,9 +78,12 @@ class RateLimitedLlmClient implements LlmClient { messages: LlmMessage[] | string, opts?: LlmCallOptions, ): AsyncIterable { - const release = await this.semaphore.acquire(); + const signal = this.resources?.signalFor(opts?.signal) ?? opts?.signal; + const callOpts = signal ? { ...opts, signal } : opts; + await this.resources?.waitForBackground(signal); + const release = await this.semaphore.acquire(signal); try { - yield* this.inner.stream(messages, opts); + yield* this.inner.stream(messages, callOpts); } finally { release(); } diff --git a/apps/memos-local-plugin/core/util/request-deadline.ts b/apps/memos-local-plugin/core/util/request-deadline.ts new file mode 100644 index 000000000..a7fb816a0 --- /dev/null +++ b/apps/memos-local-plugin/core/util/request-deadline.ts @@ -0,0 +1,37 @@ +export interface RequestDeadline { + readonly signal: AbortSignal; + remainingMs(): number; + dispose(): void; +} + +/** + * Convert an adapter-provided absolute epoch deadline into one abort signal. + * The absolute form survives JSON-RPC transport time and prevents every stage + * from accidentally receiving a fresh timeout budget. + */ +export function createRequestDeadline( + deadlineAt: number, + now: () => number = Date.now, +): RequestDeadline { + const controller = new AbortController(); + const remainingMs = (): number => Math.max(0, deadlineAt - now()); + const initialRemaining = remainingMs(); + let timer: ReturnType | null = null; + + if (!Number.isFinite(deadlineAt) || initialRemaining <= 0) { + controller.abort(new DOMException("request deadline exceeded", "TimeoutError")); + } else { + timer = setTimeout(() => { + controller.abort(new DOMException("request deadline exceeded", "TimeoutError")); + }, initialRemaining); + } + + return { + signal: controller.signal, + remainingMs, + dispose(): void { + if (timer) clearTimeout(timer); + timer = null; + }, + }; +} diff --git a/apps/memos-local-plugin/core/util/retry-after.ts b/apps/memos-local-plugin/core/util/retry-after.ts new file mode 100644 index 000000000..e580e5f7b --- /dev/null +++ b/apps/memos-local-plugin/core/util/retry-after.ts @@ -0,0 +1,200 @@ +/** Parse RFC 9110 Retry-After delay-seconds or HTTP-date into milliseconds. */ +export const MAX_INLINE_RETRY_DELAY_MS = 30_000; +/** @deprecated Use MAX_INLINE_RETRY_DELAY_MS. */ +export const MAX_RETRY_DELAY_MS = MAX_INLINE_RETRY_DELAY_MS; + +export type RetryDeferReason = + | "deadline_insufficient" + | "retry_after_too_long"; + +export interface RetryPlanBase { + backoffMs: number; + delayMs: number; + retryAfterMs: number | null; + retryAt: number; + source: "backoff" | "retry_after"; +} + +export type RetryPlan = + | (RetryPlanBase & { action: "wait" }) + | (RetryPlanBase & { action: "defer"; reason: RetryDeferReason }); + +export interface RetryCooldown { + retryAfterMs: number; + retryAt: number; + status: number; +} + +export interface RetryDiagnosticDetails { + retryAfterMs?: number; + retryAt?: number; + retryDecision?: "wait" | "defer" | "stop"; + retryReason?: string; +} + +const retryCooldowns = new Map(); + +export function parseRetryAfterMs( + value: string | null | undefined, + nowMs: number = Date.now(), +): number | null { + const raw = value?.trim(); + if (!raw) return null; + if (/^\d+$/.test(raw)) { + const seconds = Number(raw); + const delayMs = seconds * 1_000; + return Number.isSafeInteger(seconds) && Number.isSafeInteger(delayMs) + ? delayMs + : null; + } + // Retry-After only permits IMF-fixdate here. Keeping the shape strict avoids + // JavaScript accepting ambiguous strings such as "1.5" as a legacy date. + if (!/^[A-Za-z]{3}, \d{2} [A-Za-z]{3} \d{4} \d{2}:\d{2}:\d{2} GMT$/.test(raw)) return null; + const at = Date.parse(raw); + if (!Number.isFinite(at)) return null; + return Math.max(0, at - nowMs); +} + +export function retryDelayMs(input: { + attempt: number; + baseMs: number; + jitterMaxMs: number; + retryAfterMs?: number | null; + maxDelayMs?: number; + random?: () => number; +}): number { + const plan = planRetry({ + ...input, + maxInlineDelayMs: input.maxDelayMs, + }); + return plan.delayMs; +} + +/** + * Decide whether a retry can happen inline without violating Retry-After. + * + * Provider Retry-After values are never clamped downward. When the earliest + * legal retry cannot fit the inline wait or request deadline, callers must + * defer/fallback and carry retryAt into their recovery path. + */ +export function planRetry(input: { + attempt: number; + baseMs: number; + jitterMaxMs: number; + retryAfterMs?: number | null; + maxInlineDelayMs?: number; + deadlineAt?: number; + nowMs?: number; + random?: () => number; +}): RetryPlan { + const nowMs = input.nowMs ?? Date.now(); + const random = input.random ?? Math.random; + const jitter = Math.floor(random() * input.jitterMaxMs); + const rawBackoff = input.baseMs * 2 ** Math.max(0, input.attempt - 1) + jitter; + const maxInlineDelayMs = input.maxInlineDelayMs ?? MAX_INLINE_RETRY_DELAY_MS; + const backoffMs = Math.min(rawBackoff, maxInlineDelayMs); + const retryAfterMs = input.retryAfterMs ?? null; + const delayMs = Math.max(backoffMs, retryAfterMs ?? 0); + const retryAt = nowMs + delayMs; + const source = retryAfterMs !== null && retryAfterMs >= backoffMs + ? "retry_after" as const + : "backoff" as const; + const base: RetryPlanBase = { + backoffMs, + delayMs, + retryAfterMs, + retryAt, + source, + }; + + if (retryAfterMs !== null && retryAfterMs > maxInlineDelayMs) { + return { ...base, action: "defer", reason: "retry_after_too_long" }; + } + if (input.deadlineAt !== undefined && retryAt > input.deadlineAt) { + return { ...base, action: "defer", reason: "deadline_insufficient" }; + } + return { ...base, action: "wait" }; +} + +export function retryCooldownKey( + kind: "llm" | "embedding", + provider: string, + url: string, + scope: string = "", +): string { + return `${kind}\u0000${provider}\u0000${url}\u0000${scope}`; +} + +/** Extend a provider cooldown monotonically; a shorter later response cannot weaken it. */ +export function recordRetryCooldown(key: string, cooldown: RetryCooldown): void { + const current = retryCooldowns.get(key); + if (!current || cooldown.retryAt > current.retryAt) { + retryCooldowns.set(key, { ...cooldown }); + } +} + +export function getRetryCooldown( + key: string, + nowMs: number = Date.now(), +): RetryCooldown | null { + const cooldown = retryCooldowns.get(key); + if (!cooldown) return null; + if (cooldown.retryAt <= nowMs) { + retryCooldowns.delete(key); + return null; + } + return { ...cooldown }; +} + +/** Test/runtime-reset hook; plugin shutdown does not need to await cooldown state. */ +export function clearRetryCooldowns(): void { + retryCooldowns.clear(); +} + +/** Copy only bounded, machine-readable retry fields from an error detail bag. */ +export function extractRetryDiagnostics( + details: Record | undefined, +): RetryDiagnosticDetails { + if (!details) return {}; + const diagnostic: RetryDiagnosticDetails = {}; + if (typeof details.retryAfterMs === "number" && Number.isFinite(details.retryAfterMs)) { + diagnostic.retryAfterMs = details.retryAfterMs; + } + if (typeof details.retryAt === "number" && Number.isFinite(details.retryAt)) { + diagnostic.retryAt = details.retryAt; + } + if ( + details.retryDecision === "wait" + || details.retryDecision === "defer" + || details.retryDecision === "stop" + ) { + diagnostic.retryDecision = details.retryDecision; + } + if (typeof details.retryReason === "string") { + diagnostic.retryReason = details.retryReason; + } + return diagnostic; +} + +/** Abortable retry wait so request cancellation and shutdown do not leave sleepers behind. */ +export function waitForRetry(delayMs: number, signal?: AbortSignal): Promise { + if (signal?.aborted) return Promise.reject(abortReason(signal)); + if (delayMs <= 0) return Promise.resolve(); + + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + signal?.removeEventListener("abort", onAbort); + resolve(); + }, delayMs); + const onAbort = () => { + clearTimeout(timer); + signal?.removeEventListener("abort", onAbort); + reject(signal ? abortReason(signal) : new DOMException("Aborted", "AbortError")); + }; + signal?.addEventListener("abort", onAbort, { once: true }); + }); +} + +function abortReason(signal: AbortSignal): unknown { + return signal.reason ?? new DOMException("Aborted", "AbortError"); +} diff --git a/apps/memos-local-plugin/core/util/semaphore.ts b/apps/memos-local-plugin/core/util/semaphore.ts index 8dd9b75fc..037db2607 100644 --- a/apps/memos-local-plugin/core/util/semaphore.ts +++ b/apps/memos-local-plugin/core/util/semaphore.ts @@ -1,23 +1,37 @@ export interface Semaphore { - acquire(): Promise<() => void>; + acquire(signal?: AbortSignal): Promise<() => void>; +} + +interface Waiter { + resolve: (release: () => void) => void; + reject: (error: Error) => void; + signal?: AbortSignal; + onAbort?: () => void; } export function createSemaphore(max: number): Semaphore { const limit = Math.max(1, Math.floor(max)); let current = 0; - const waiters: Array<() => void> = []; + const waiters: Waiter[] = []; return { - async acquire() { + async acquire(signal?: AbortSignal) { + if (signal?.aborted) throw abortError(signal); if (current < limit) { current++; return release; } - return new Promise<() => void>((resolve) => { - waiters.push(() => { - current++; - resolve(release); - }); + return new Promise<() => void>((resolve, reject) => { + const waiter: Waiter = { resolve, reject, signal }; + if (signal) { + waiter.onAbort = () => { + const index = waiters.indexOf(waiter); + if (index >= 0) waiters.splice(index, 1); + reject(abortError(signal)); + }; + signal.addEventListener("abort", waiter.onAbort, { once: true }); + } + waiters.push(waiter); }); }, }; @@ -25,6 +39,17 @@ export function createSemaphore(max: number): Semaphore { function release() { current = Math.max(0, current - 1); const next = waiters.shift(); - if (next) next(); + if (!next) return; + if (next.signal && next.onAbort) { + next.signal.removeEventListener("abort", next.onAbort); + } + current++; + next.resolve(release); } } + +function abortError(signal: AbortSignal): Error { + return signal.reason instanceof Error + ? signal.reason + : new DOMException("semaphore wait aborted", "AbortError"); +} diff --git a/apps/memos-local-plugin/tests/python/test_bridge_client.py b/apps/memos-local-plugin/tests/python/test_bridge_client.py index 82763fcea..b5c9eb917 100644 --- a/apps/memos-local-plugin/tests/python/test_bridge_client.py +++ b/apps/memos-local-plugin/tests/python/test_bridge_client.py @@ -448,6 +448,128 @@ def test_reverse_request_waits_for_late_host_handler_registration(self) -> None: self.assertNotIn("error", response) client.close() + def test_slow_reverse_handler_does_not_block_regular_rpc_responses(self) -> None: + """A host LLM callback must not stall the stdout response demux. + + ``host.llm.complete`` can legitimately spend several seconds in the + Hermes model client. The bridge reader still has to resolve an + unrelated foreground ``turn.start`` response during that + window; otherwise one background callback head-of-line blocks every + provider lease sharing the process. + """ + client = MemosBridgeClient(bridge_path="/tmp/bridge.cts") + assert self._fake is not None + handler_started = threading.Event() + release_handler = threading.Event() + + def _slow_handler(_params: dict) -> dict: + handler_started.set() + release_handler.wait(timeout=2.0) + return {"text": "host:done", "model": "host-test"} + + client.register_host_handler("host.llm.complete", _slow_handler) + self._fake.stdout._enqueue( + { + "jsonrpc": "2.0", + "id": "srv-slow", + "method": "host.llm.complete", + "params": {"messages": [{"role": "user", "content": "slow"}]}, + } + ) + self.assertTrue(handler_started.wait(timeout=0.5)) + + try: + response = client.request( + "turn.start", + { + "sessionId": "hermes:session:1", + "userText": "foreground recall", + }, + timeout=0.5, + ) + self.assertIn("foreground recall", response["injectedContext"]) + finally: + release_handler.set() + + reverse_response = self._wait_for_client_write(lambda msg: msg.get("id") == "srv-slow") + self.assertEqual(reverse_response["result"]["text"], "host:done") + client.close() + + def test_reverse_handler_queue_rejects_overload_without_blocking_reader(self) -> None: + client = MemosBridgeClient(bridge_path="/tmp/bridge.cts") + assert self._fake is not None + handler_started = threading.Event() + release_handler = threading.Event() + + def _slow_handler(_params: dict) -> dict: + handler_started.set() + release_handler.wait(timeout=2.0) + return {"text": "done"} + + client.register_host_handler("host.llm.complete", _slow_handler) + self._fake.stdout._enqueue( + { + "jsonrpc": "2.0", + "id": "srv-running", + "method": "host.llm.complete", + "params": {}, + } + ) + self.assertTrue(handler_started.wait(timeout=0.5)) + + overflow_id = "srv-overflow" + for index in range(bridge_client_mod.HOST_HANDLER_QUEUE_CAPACITY + 1): + rpc_id = ( + overflow_id + if index == bridge_client_mod.HOST_HANDLER_QUEUE_CAPACITY + else f"srv-{index}" + ) + self._fake.stdout._enqueue( + { + "jsonrpc": "2.0", + "id": rpc_id, + "method": "host.llm.complete", + "params": {}, + } + ) + + try: + response = self._wait_for_client_write(lambda msg: msg.get("id") == overflow_id) + self.assertEqual(response["error"]["data"]["code"], "host_handler_busy") + finally: + client.close() + release_handler.set() + + def test_close_does_not_wait_for_a_running_reverse_handler(self) -> None: + """An uncooperative host callback must not extend bridge shutdown.""" + client = MemosBridgeClient(bridge_path="/tmp/bridge.cts") + assert self._fake is not None + handler_started = threading.Event() + release_handler = threading.Event() + + def _slow_handler(_params: dict) -> dict: + handler_started.set() + release_handler.wait(timeout=2.0) + return {"text": "late", "model": "host-test"} + + client.register_host_handler("host.llm.complete", _slow_handler) + self._fake.stdout._enqueue( + { + "jsonrpc": "2.0", + "id": "srv-close", + "method": "host.llm.complete", + "params": {}, + } + ) + self.assertTrue(handler_started.wait(timeout=0.5)) + + started = time.monotonic() + try: + client.close() + finally: + release_handler.set() + self.assertLess(time.monotonic() - started, 0.5) + def test_reader_exit_marks_pending_as_transport_closed(self) -> None: """R1 (#2028): reader thread EOF must wake pending waiters with transport_closed instead of leaving them parked on their @@ -1111,7 +1233,7 @@ def test_sync_turn_uses_long_rpc_timeout_for_turn_end(self) -> None: "sessions (issue #2028).", ) - def test_prefetch_uses_long_rpc_timeout_for_turn_start(self) -> None: + def test_prefetch_uses_dedicated_foreground_timeout_for_turn_start(self) -> None: p = self._provider_mod.MemTensorProvider() bridge = RecordingBridge() p._bridge = bridge @@ -1122,12 +1244,45 @@ def test_prefetch_uses_long_rpc_timeout_for_turn_start(self) -> None: self.assertIn("turn.start", methods) start_index = methods.index("turn.start") start_kwargs = bridge.call_kwargs[start_index] - self.assertGreaterEqual( + self.assertGreater(start_kwargs.get("timeout", 0.0), 0.0) + self.assertLessEqual( start_kwargs.get("timeout", 0.0), - self._EXPECTED_LONG_TIMEOUT, - "turn.start suffers the same long-tail latency as turn.end and " - "must share the long RPC timeout (issue #2028).", + self._provider_mod._PREFETCH_RPC_TIMEOUT, + "foreground turn.start must finish before the Hermes host deadline; " + "long capture work keeps the separate issue #2028 timeout.", ) + start_payload = bridge.calls[start_index][1] + self.assertIn("deadlineAt", start_payload) + self.assertGreater(start_payload["deadlineAt"], start_payload["ts"]) + + def test_foreground_reconnect_and_retry_share_one_deadline(self) -> None: + class ClosedBridge: + def request(self, *_args, **_kwargs) -> dict: + raise BridgeError("transport_closed", "bridge closed") + + p = self._provider_mod.MemTensorProvider() + p._bridge = ClosedBridge() + recovered = RecordingBridge() + monotonic_now = [100.0] + + def reconnect(_session_id: str, *, timeout: float) -> None: + self.assertLessEqual(timeout, 6.0) + monotonic_now[0] += 4.0 + p._bridge = recovered + + with ( + patch("memos_provider.time.monotonic", side_effect=lambda: monotonic_now[0]), + patch.object(p, "_reconnect_bridge", side_effect=reconnect), + ): + p._bridge_request_with_retry( + "turn.start", + {"sessionId": "s-1"}, + timeout=6.0, + deadline_monotonic=106.0, + ) + + self.assertEqual(recovered.calls[0][0], "turn.start") + self.assertLessEqual(recovered.call_kwargs[0]["timeout"], 2.0) class ViewerDaemonTests(unittest.TestCase): diff --git a/apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py b/apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py index 6b95ae61b..ecd0e1ae1 100644 --- a/apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py +++ b/apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py @@ -480,6 +480,75 @@ def test_prefetch_passes_stable_turn_key_to_bridge(self) -> None: turn_start = next(params for method, params in bridge.calls if method == "turn.start") self.assertEqual(turn_start["turnKey"], "turn-key-session:7") + def test_prefetch_uses_a_dedicated_budget_and_forwards_absolute_deadline(self) -> None: + bridge = FakeBridge() + with ( + patch("memos_provider.ensure_bridge_running", return_value=True), + patch("memos_provider.ensure_viewer_daemon", return_value=True), + patch("memos_provider.MemosBridgeClient", return_value=bridge), + patch("memos_provider._PREFETCH_RPC_TIMEOUT", 6.0), + patch("memos_provider.time.time", return_value=1_700_000_000.0), + ): + provider = memos_provider.MemTensorProvider() + provider.initialize("budget-session") + provider.on_turn_start(1, "recall the build decision") + with patch.object( + provider, + "_bridge_request_with_retry", + wraps=provider._bridge_request_with_retry, + ) as request: + provider.prefetch("recall the build decision") + + turn_start = next(params for method, params in bridge.calls if method == "turn.start") + self.assertEqual(turn_start["deadlineAt"], 1_700_000_005_750) + request.assert_called_once() + self.assertLessEqual(request.call_args.kwargs["timeout"], 6.0) + self.assertIn("deadline_monotonic", request.call_args.kwargs) + + def test_prefetch_budget_includes_bridge_ensure_time(self) -> None: + bridge = FakeBridge() + monotonic_now = [100.0] + + def ensure_bridge(_session_id: str, *, timeout: float) -> bool: + self.assertAlmostEqual(timeout, 6.0, places=3) + monotonic_now[0] += 2.5 + return True + + with ( + patch("memos_provider.ensure_bridge_running", return_value=True), + patch("memos_provider.ensure_viewer_daemon", return_value=True), + patch("memos_provider.MemosBridgeClient", return_value=bridge), + patch("memos_provider._PREFETCH_RPC_TIMEOUT", 6.0), + patch("memos_provider.time.time", return_value=1_700_000_000.0), + patch("memos_provider.time.monotonic", side_effect=lambda: monotonic_now[0]), + ): + provider = memos_provider.MemTensorProvider() + provider.initialize("budget-session") + provider.on_turn_start(1, "recall the build decision") + with ( + patch.object(provider, "_ensure_bridge", side_effect=ensure_bridge), + patch.object( + provider, + "_bridge_request_with_retry", + wraps=provider._bridge_request_with_retry, + ) as request, + ): + provider.prefetch("recall the build decision") + + self.assertLessEqual(request.call_args.kwargs["timeout"], 3.5) + turn_start = next(params for method, params in bridge.calls if method == "turn.start") + self.assertEqual(turn_start["deadlineAt"], 1_700_000_005_750) + + def test_prefetch_timeout_config_rejects_non_positive_values(self) -> None: + with patch.dict("os.environ", {"MEMOS_HERMES_PREFETCH_RPC_TIMEOUT": "0"}): + self.assertEqual(memos_provider._prefetch_rpc_timeout_default(), 6.0) + with patch.dict("os.environ", {"MEMOS_HERMES_PREFETCH_RPC_TIMEOUT": "nan"}): + self.assertEqual(memos_provider._prefetch_rpc_timeout_default(), 6.0) + with patch.dict("os.environ", {"MEMOS_HERMES_PREFETCH_RPC_TIMEOUT": "4.5"}): + self.assertEqual(memos_provider._prefetch_rpc_timeout_default(), 4.5) + with patch.dict("os.environ", {"MEMOS_HERMES_PREFETCH_RPC_TIMEOUT": "30"}): + self.assertEqual(memos_provider._prefetch_rpc_timeout_default(), 7.0) + def test_prefetch_suppresses_memory_injection_for_explicit_delegation(self) -> None: bridge = FakeBridge() with ( diff --git a/apps/memos-local-plugin/tests/unit/bridge/methods.test.ts b/apps/memos-local-plugin/tests/unit/bridge/methods.test.ts index 8ee5fa921..fc9f921e1 100644 --- a/apps/memos-local-plugin/tests/unit/bridge/methods.test.ts +++ b/apps/memos-local-plugin/tests/unit/bridge/methods.test.ts @@ -344,6 +344,17 @@ describe("makeDispatcher", () => { ).rejects.toSatisfy( (err) => err instanceof MemosError && err.code === "invalid_argument", ); + await expect( + dispatch("turn.start", { + agent: "openclaw", + sessionId: "s-1", + userText: "hi", + ts: 123, + deadlineAt: "soon", + }), + ).rejects.toSatisfy( + (err) => err instanceof MemosError && err.code === "invalid_argument", + ); }); it("feedback.submit forwards the DTO shape intact", async () => { diff --git a/apps/memos-local-plugin/tests/unit/embedding/embedder.test.ts b/apps/memos-local-plugin/tests/unit/embedding/embedder.test.ts index 3c608a53a..617bc4a72 100644 --- a/apps/memos-local-plugin/tests/unit/embedding/embedder.test.ts +++ b/apps/memos-local-plugin/tests/unit/embedding/embedder.test.ts @@ -6,8 +6,10 @@ import { initTestLogger } from "../../../core/logger/index.js"; import type { EmbedRole, EmbeddingConfig, + EmbeddingErrorDetail, EmbeddingProvider, EmbeddingProviderName, + EmbeddingStatusDetail, ProviderCallCtx, } from "../../../core/embedding/types.js"; @@ -75,6 +77,24 @@ describe("embedder facade", () => { expect(Array.from(v)).toEqual([3, 97, 0]); // a=97 }); + it("forwards the caller abort signal and deadline to the provider", async () => { + const seen: Array> = []; + const p: EmbeddingProvider = { + name: "openai_compatible", + async embed(texts, _role, ctx) { + seen.push({ signal: ctx.signal, deadlineAt: ctx.deadlineAt }); + return texts.map(() => [1, 2, 3]); + }, + }; + const e = createEmbedderWithProvider(cfg(), p); + const controller = new AbortController(); + + const deadlineAt = Date.now() + 1_000; + await e.embedOne("signal", { signal: controller.signal, deadlineAt }); + + expect(seen).toEqual([{ signal: controller.signal, deadlineAt }]); + }); + it("dedups identical inputs into one provider call", async () => { const p = new FakeProvider(); const e = createEmbedderWithProvider(cfg(), p); @@ -175,6 +195,45 @@ describe("embedder facade", () => { } }); + it("preserves deferred retry diagnostics in error and status sinks", async () => { + const errors: EmbeddingErrorDetail[] = []; + const statuses: EmbeddingStatusDetail[] = []; + const retryAt = Date.now() + 120_000; + const provider: EmbeddingProvider = { + name: "openai_compatible", + async embed() { + throw new MemosError("embedding_unavailable", "provider cooldown", { + retryAfterMs: 120_000, + retryAt, + retryDecision: "defer", + retryReason: "retry_after_too_long", + }); + }, + }; + const e = createEmbedderWithProvider( + cfg({ onError: (detail) => errors.push(detail), onStatus: (detail) => statuses.push(detail) }), + provider, + ); + + await expect(e.embedOne("x")).rejects.toBeInstanceOf(MemosError); + + expect(errors).toContainEqual( + expect.objectContaining({ + retryAfterMs: 120_000, + retryAt, + retryDecision: "defer", + retryReason: "retry_after_too_long", + }), + ); + expect(statuses).toContainEqual( + expect.objectContaining({ + status: "error", + retryAt, + retryDecision: "defer", + }), + ); + }); + it("rejects when provider returns too few rows", async () => { const e = createEmbedderWithProvider(cfg({ provider: "gemini" }), new WrongCountProvider()); await expect(e.embedMany(["x", "y", "z"])).rejects.toBeInstanceOf(MemosError); diff --git a/apps/memos-local-plugin/tests/unit/embedding/fetcher.test.ts b/apps/memos-local-plugin/tests/unit/embedding/fetcher.test.ts index 3b9ee5bf2..2811e94ef 100644 --- a/apps/memos-local-plugin/tests/unit/embedding/fetcher.test.ts +++ b/apps/memos-local-plugin/tests/unit/embedding/fetcher.test.ts @@ -4,6 +4,7 @@ import { MemosError } from "../../../agent-contract/errors.js"; import { initTestLogger } from "../../../core/logger/index.js"; import { httpPostJson } from "../../../core/embedding/fetcher.js"; import type { ProviderLogger } from "../../../core/embedding/types.js"; +import { clearRetryCooldowns } from "../../../core/util/retry-after.js"; function nullLogger(): ProviderLogger { return { @@ -21,6 +22,8 @@ describe("embedding/fetcher", () => { vi.useRealTimers(); // retry backoff uses real setTimeout; keep it real but short }); afterEach(() => { + clearRetryCooldowns(); + vi.useRealTimers(); vi.unstubAllGlobals(); }); @@ -81,6 +84,85 @@ describe("embedding/fetcher", () => { expect(f).toHaveBeenCalledTimes(2); }); + it("honors Retry-After HTTP-date before retrying a 429", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-08-04T00:00:00.000Z")); + const f = mockFetch([ + new Response("rate limited", { + status: 429, + headers: { "Retry-After": "Tue, 04 Aug 2026 00:00:03 GMT" }, + }), + new Response(JSON.stringify({ ok: 1 }), { status: 200 }), + ]); + + const pending = httpPostJson<{ ok: number }>({ + url: "https://x", + body: {}, + provider: "cohere", + log: nullLogger(), + maxRetries: 1, + }); + await vi.advanceTimersByTimeAsync(2_999); + expect(f).toHaveBeenCalledTimes(1); + await vi.advanceTimersByTimeAsync(1); + await expect(pending).resolves.toEqual({ ok: 1 }); + expect(f).toHaveBeenCalledTimes(2); + vi.useRealTimers(); + }); + + it("defers a long Retry-After and short-circuits the provider cooldown", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-08-04T00:00:00.000Z")); + const f = mockFetch([ + new Response("maintenance", { status: 503, headers: { "Retry-After": "120" } }), + ]); + const opts = { + url: "https://embedding-x", + body: {}, + provider: "cohere" as const, + log: nullLogger(), + maxRetries: 1, + }; + + await expect(httpPostJson(opts)).rejects.toMatchObject({ + code: "embedding_unavailable", + details: { + retryAfterMs: 120_000, + retryDecision: "defer", + retryReason: "retry_after_too_long", + }, + }); + await expect(httpPostJson(opts)).rejects.toMatchObject({ + code: "embedding_unavailable", + details: { retryReason: "cooldown_active" }, + }); + expect(f).toHaveBeenCalledTimes(1); + }); + + it("returns structured diagnostics when network backoff cannot fit the deadline", async () => { + vi.useFakeTimers(); + const now = Date.parse("2026-08-04T00:00:00.000Z"); + vi.setSystemTime(now); + const f = mockFetch([new Error("ECONNRESET")]); + + await expect(httpPostJson({ + url: "https://embedding-deadline", + body: {}, + provider: "mistral", + log: nullLogger(), + maxRetries: 1, + deadlineAt: now + 100, + })).rejects.toMatchObject({ + name: "MemosError", + code: "embedding_unavailable", + details: { + retryDecision: "defer", + retryReason: "deadline_insufficient", + }, + }); + expect(f).toHaveBeenCalledTimes(1); + }); + it("does not retry on 400", async () => { mockFetch([new Response("bad", { status: 400 })]); await expect( diff --git a/apps/memos-local-plugin/tests/unit/embedding/retry-worker.test.ts b/apps/memos-local-plugin/tests/unit/embedding/retry-worker.test.ts index f08a37f32..ae72cc60e 100644 --- a/apps/memos-local-plugin/tests/unit/embedding/retry-worker.test.ts +++ b/apps/memos-local-plugin/tests/unit/embedding/retry-worker.test.ts @@ -1,6 +1,7 @@ import { afterEach, beforeEach, describe, expect, it } from "vitest"; import { createEmbeddingRetryWorker } from "../../../core/embedding/retry-worker.js"; +import { ERROR_CODES, MemosError } from "../../../agent-contract/errors.js"; import { rootLogger } from "../../../core/logger/index.js"; import type { EpisodeId, SessionId, TraceId } from "../../../core/types.js"; import { makeTmpDb, type TmpDbHandle } from "../../helpers/tmp-db.js"; @@ -174,6 +175,39 @@ describe("embedding retry worker", () => { expect(handle.repos.apiLogs.list({ toolName: "system_error", limit: 5, offset: 0 })).toHaveLength(1); }); + it("never schedules a durable retry before the provider retryAt", async () => { + const retryAt = NOW + 120_000; + handle.repos.embeddingRetryQueue.enqueue({ + id: "er_retry_after", + targetKind: "trace", + targetId: "tr_retry", + vectorField: "vec_summary", + sourceText: "retry me later", + maxAttempts: 3, + now: NOW, + }); + const worker = createEmbeddingRetryWorker({ + repos: handle.repos, + embedder: fakeEmbedder({ + throwWith: new MemosError( + ERROR_CODES.EMBEDDING_UNAVAILABLE, + "provider cooling down", + { retryAt, retryAfterMs: 120_000, retryDecision: "defer" }, + ), + }), + log: rootLogger.child({ channel: "test.embedding.retry" }), + now: () => NOW, + }); + + await worker.flush(); + + expect(queueRow(handle, "er_retry_after")).toMatchObject({ + status: "pending", + attempts: 1, + next_attempt_at: retryAt, + }); + }); + it("treats missing target rows as retry failures", async () => { handle.repos.embeddingRetryQueue.enqueue({ id: "er_missing", @@ -201,4 +235,30 @@ describe("embedding retry worker", () => { last_error: "embedding retry target not found: trace:tr_missing", }); }); + + it("does not claim new retry jobs after stop during shutdown", async () => { + handle.repos.embeddingRetryQueue.enqueue({ + id: "er_shutdown", + targetKind: "trace", + targetId: "tr_retry", + vectorField: "vec_summary", + sourceText: "do not start during shutdown", + now: NOW, + }); + const worker = createEmbeddingRetryWorker({ + repos: handle.repos, + embedder: fakeEmbedder({ dimensions: 8 }), + log: rootLogger.child({ channel: "test.embedding.retry" }), + now: () => NOW, + }); + + worker.stop(); + await worker.flush(); + + expect(queueRow(handle, "er_shutdown")).toMatchObject({ + status: "pending", + attempts: 0, + claimed_by: null, + }); + }); }); diff --git a/apps/memos-local-plugin/tests/unit/llm/client.test.ts b/apps/memos-local-plugin/tests/unit/llm/client.test.ts index dee0de228..cd125ecd8 100644 --- a/apps/memos-local-plugin/tests/unit/llm/client.test.ts +++ b/apps/memos-local-plugin/tests/unit/llm/client.test.ts @@ -283,6 +283,45 @@ describe("llm/client", () => { await expect(client.complete([] as LlmMessage[])).rejects.toBeInstanceOf(MemosError); }); + it("preserves deferred retry diagnostics in error and status sinks", async () => { + const errors: Array> = []; + const statuses: LlmStatusDetail[] = []; + const retryAt = Date.now() + 120_000; + const provider = new ThrowingProvider( + new MemosError(ERROR_CODES.LLM_RATE_LIMITED, "provider cooldown", { + retryAfterMs: 120_000, + retryAt, + retryDecision: "defer", + retryReason: "retry_after_too_long", + }), + ); + const client = createLlmClientWithProvider( + cfg({ + onError: (detail) => errors.push(detail as unknown as Record), + onStatus: (detail) => statuses.push(detail), + }), + provider, + ); + + await expect(client.complete("x")).rejects.toBeInstanceOf(MemosError); + + expect(errors).toContainEqual( + expect.objectContaining({ + retryAfterMs: 120_000, + retryAt, + retryDecision: "defer", + retryReason: "retry_after_too_long", + }), + ); + expect(statuses).toContainEqual( + expect.objectContaining({ + status: "error", + retryAt, + retryDecision: "defer", + }), + ); + }); + // ─── Circuit breaker (issue #1897) ────────────────────────────────────── describe("circuit breaker", () => { function statusSink(): { rows: LlmStatusDetail[]; push: (d: LlmStatusDetail) => void } { diff --git a/apps/memos-local-plugin/tests/unit/llm/fetcher.test.ts b/apps/memos-local-plugin/tests/unit/llm/fetcher.test.ts index a3941b7aa..9b17e1b16 100644 --- a/apps/memos-local-plugin/tests/unit/llm/fetcher.test.ts +++ b/apps/memos-local-plugin/tests/unit/llm/fetcher.test.ts @@ -4,6 +4,7 @@ import { MemosError } from "../../../agent-contract/errors.js"; import { decodeSse, httpPostJson, httpPostStream } from "../../../core/llm/fetcher.js"; import { initTestLogger } from "../../../core/logger/index.js"; import type { LlmProviderLogger } from "../../../core/llm/types.js"; +import { clearRetryCooldowns } from "../../../core/util/retry-after.js"; function nullLog(): LlmProviderLogger { return { @@ -29,7 +30,205 @@ function mockFetch(replies: Array) { describe("llm/fetcher", () => { beforeAll(() => initTestLogger()); - afterEach(() => vi.unstubAllGlobals()); + afterEach(() => { + clearRetryCooldowns(); + vi.useRealTimers(); + vi.unstubAllGlobals(); + }); + + it("honors Retry-After delay-seconds before retrying a 429", async () => { + vi.useFakeTimers(); + const f = mockFetch([ + new Response("slow down", { status: 429, headers: { "Retry-After": "2" } }), + new Response(JSON.stringify({ ok: 1 }), { status: 200 }), + ]); + + const pending = httpPostJson({ + url: "https://x", + body: {}, + timeoutMs: 5_000, + maxRetries: 1, + provider: "openai_compatible", + log: nullLog(), + }); + await vi.advanceTimersByTimeAsync(1_999); + expect(f).toHaveBeenCalledTimes(1); + await vi.advanceTimersByTimeAsync(1); + await expect(pending).resolves.toMatchObject({ json: { ok: 1 } }); + expect(f).toHaveBeenCalledTimes(2); + vi.useRealTimers(); + }); + + it("aborts while waiting for Retry-After", async () => { + vi.useFakeTimers(); + const ctrl = new AbortController(); + const f = mockFetch([ + new Response("slow down", { status: 429, headers: { "Retry-After": "2" } }), + ]); + + const pending = httpPostJson({ + url: "https://x", + body: {}, + timeoutMs: 5_000, + maxRetries: 1, + signal: ctrl.signal, + provider: "openai_compatible", + log: nullLog(), + }); + await vi.advanceTimersByTimeAsync(0); + ctrl.abort(); + await expect(pending).rejects.toBeInstanceOf(MemosError); + expect(f).toHaveBeenCalledTimes(1); + vi.useRealTimers(); + }); + + it("defers a long Retry-After from a 503 without retrying early", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-08-04T00:00:00.000Z")); + const warn = vi.fn(); + const f = mockFetch([ + new Response("maintenance", { status: 503, headers: { "Retry-After": "120" } }), + ]); + + const pending = httpPostJson({ + url: "https://x", + body: {}, + timeoutMs: 120_000, + maxRetries: 1, + provider: "openai_compatible", + log: { ...nullLog(), warn }, + }); + await expect(pending).rejects.toMatchObject({ + code: "llm_unavailable", + details: { + retryAfterMs: 120_000, + retryDecision: "defer", + retryReason: "retry_after_too_long", + }, + }); + expect(f).toHaveBeenCalledTimes(1); + expect(warn).toHaveBeenCalledWith( + "http.retry_deferred", + expect.objectContaining({ + retryAfterMs: 120_000, + retryDecision: "defer", + retryReason: "retry_after_too_long", + }), + ); + }); + + it("short-circuits calls while the provider Retry-After cooldown is active", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-08-04T00:00:00.000Z")); + const warn = vi.fn(); + const f = mockFetch([ + new Response("slow down", { status: 429, headers: { "Retry-After": "120" } }), + ]); + const opts = { + url: "https://x", + body: {}, + timeoutMs: 5_000, + maxRetries: 1, + provider: "openai_compatible" as const, + log: { ...nullLog(), warn }, + }; + + await expect(httpPostJson(opts)).rejects.toMatchObject({ code: "llm_rate_limited" }); + await expect(httpPostJson(opts)).rejects.toMatchObject({ + code: "llm_rate_limited", + details: { retryDecision: "defer", retryReason: "cooldown_active" }, + }); + expect(f).toHaveBeenCalledTimes(1); + expect(warn).toHaveBeenCalledWith( + "http.retry_cooldown", + expect.objectContaining({ retryReason: "cooldown_active" }), + ); + }); + + it("does not enter a Retry-After wait that cannot fit the absolute deadline", async () => { + vi.useFakeTimers(); + const now = Date.parse("2026-08-04T00:00:00.000Z"); + vi.setSystemTime(now); + const f = mockFetch([ + new Response("slow down", { status: 429, headers: { "Retry-After": "5" } }), + ]); + + await expect(httpPostJson({ + url: "https://deadline", + body: {}, + timeoutMs: 5_000, + maxRetries: 1, + deadlineAt: now + 1_000, + provider: "openai_compatible", + log: nullLog(), + })).rejects.toMatchObject({ + code: "llm_rate_limited", + details: { + retryDecision: "defer", + retryReason: "deadline_insufficient", + }, + }); + expect(f).toHaveBeenCalledTimes(1); + }); + + it("returns structured diagnostics when network backoff cannot fit the deadline", async () => { + vi.useFakeTimers(); + const now = Date.parse("2026-08-04T00:00:00.000Z"); + vi.setSystemTime(now); + const f = mockFetch([new Error("ECONNRESET")]); + + await expect(httpPostJson({ + url: "https://network-deadline", + body: {}, + timeoutMs: 5_000, + maxRetries: 1, + deadlineAt: now + 100, + provider: "openai_compatible", + log: nullLog(), + })).rejects.toMatchObject({ + name: "MemosError", + code: "llm_unavailable", + details: { + retryDecision: "defer", + retryReason: "deadline_insufficient", + }, + }); + expect(f).toHaveBeenCalledTimes(1); + }); + + it("does not let an older in-flight success clear a newer provider cooldown", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-08-04T00:00:00.000Z")); + let resolveSuccess!: (response: Response) => void; + const success = new Promise((resolve) => { resolveSuccess = resolve; }); + const f = vi.fn() + .mockImplementationOnce(() => success) + .mockResolvedValueOnce( + new Response("slow down", { status: 429, headers: { "Retry-After": "120" } }), + ); + vi.stubGlobal("fetch", f); + const opts = { + url: "https://shared-endpoint", + body: {}, + timeoutMs: 5_000, + maxRetries: 1, + provider: "openai_compatible" as const, + log: nullLog(), + }; + + const older = httpPostJson<{ ok: boolean }>(opts); + await vi.waitFor(() => expect(f).toHaveBeenCalledTimes(1)); + await expect(httpPostJson(opts)).rejects.toMatchObject({ + details: { retryReason: "retry_after_too_long" }, + }); + resolveSuccess(new Response(JSON.stringify({ ok: true }), { status: 200 })); + await expect(older).resolves.toMatchObject({ json: { ok: true } }); + + await expect(httpPostJson(opts)).rejects.toMatchObject({ + details: { retryReason: "cooldown_active" }, + }); + expect(f).toHaveBeenCalledTimes(2); + }); it("returns parsed JSON on 200", async () => { mockFetch([new Response(JSON.stringify({ a: 1 }), { status: 200 })]); diff --git a/apps/memos-local-plugin/tests/unit/pipeline/memory-core.test.ts b/apps/memos-local-plugin/tests/unit/pipeline/memory-core.test.ts index cb5140d09..52853ff4a 100644 --- a/apps/memos-local-plugin/tests/unit/pipeline/memory-core.test.ts +++ b/apps/memos-local-plugin/tests/unit/pipeline/memory-core.test.ts @@ -898,6 +898,53 @@ describe("MemoryCore façade", () => { expect(scored.priority).toBe(1); }); + it("logs both user and assistant content with the matching role", async () => { + pipeline = createPipeline(buildDeps(db!)); + core = createMemoryCore( + pipeline, + resolveHome("openclaw", "/tmp/memos-mc-test"), + "test", + ); + await core.init(); + + const userText = "今晚吃什么，推荐一下"; + const agentText = "可以考虑清淡的汤面、盖饭或者附近评价不错的家常菜。"; + const start = await core.onTurnStart({ + agent: "openclaw", + sessionId: "s-memory-add-roles", + userText, + ts: 1_700_000_000_000, + }); + await core.onTurnEnd({ + agent: "openclaw", + sessionId: start.query.sessionId!, + episodeId: start.query.episodeId!, + agentText, + toolCalls: [], + ts: 1_700_000_000_500, + }); + + const { logs } = await core.listApiLogs({ + toolName: "memory_add", + limit: 10, + }); + const liteLog = logs.find((log) => { + const input = JSON.parse(log.inputJson) as { phase?: string }; + return input.phase === "lite"; + }); + expect(liteLog).toBeDefined(); + + const output = JSON.parse(liteLog!.outputJson) as { + details?: Array<{ role?: string; content?: string }>; + }; + expect( + output.details?.map(({ role, content }) => ({ role, content })), + ).toEqual([ + { role: "user", content: userText }, + { role: "assistant", content: agentText }, + ]); + }); + it("onTurnEnd preserves adapter-provided historical timestamps", async () => { pipeline = createPipeline(buildDeps(db!)); core = createMemoryCore( diff --git a/apps/memos-local-plugin/tests/unit/pipeline/orchestrator.test.ts b/apps/memos-local-plugin/tests/unit/pipeline/orchestrator.test.ts index a70fdcf6b..1e79c861a 100644 --- a/apps/memos-local-plugin/tests/unit/pipeline/orchestrator.test.ts +++ b/apps/memos-local-plugin/tests/unit/pipeline/orchestrator.test.ts @@ -77,6 +77,39 @@ afterEach(async () => { }); describe("pipeline/orchestrator", () => { + it("degrades retrieval at the adapter deadline and aborts the provider call", async () => { + const base = fakeEmbedder({ dimensions: 384 }); + let sawAbort = false; + const embedder = { + ...base, + async embedOne(input: Parameters[0], options?: { signal?: AbortSignal }) { + return await new Promise>>((resolve, reject) => { + const onAbort = () => { + sawAbort = true; + reject(new DOMException("deadline", "AbortError")); + }; + if (options?.signal?.aborted) return onAbort(); + options?.signal?.addEventListener("abort", onAbort, { once: true }); + void resolve; + }); + }, + }; + pipeline = createPipeline(buildDeps(dbHandle!, embedder)); + const startedAt = Date.now(); + + const packet = await pipeline.onTurnStart({ + agent: "hermes", + sessionId: "s-deadline", + userText: "find the previous build decision", + ts: Date.now(), + deadlineAt: Date.now() + 25, + }); + + expect(sawAbort).toBe(true); + expect(Date.now() - startedAt).toBeLessThan(500); + expect(packet.reason).toBe("turn_start"); + }); + it("threads a dedicated l3Llm through to the handle", () => { const l3Llm = fakeLlm({ completeJson: {} }); pipeline = createPipeline({ ...buildDeps(dbHandle!), l3Llm }); diff --git a/apps/memos-local-plugin/tests/unit/session/intent-classifier.test.ts b/apps/memos-local-plugin/tests/unit/session/intent-classifier.test.ts index 6b37ae55f..c3321635a 100644 --- a/apps/memos-local-plugin/tests/unit/session/intent-classifier.test.ts +++ b/apps/memos-local-plugin/tests/unit/session/intent-classifier.test.ts @@ -109,6 +109,25 @@ describe("session/intent-classifier", () => { expect(d.signals).toEqual(["llm"]); }); + it("forwards the foreground abort signal and classifier timeout", async () => { + let seen: { signal?: AbortSignal; timeoutMs?: number } | undefined; + const llm = fakeLlm(() => ({ kind: "task", confidence: 0.8, reason: "task" })); + const original = llm.completeJson.bind(llm); + llm.completeJson = async (messages, opts) => { + seen = opts; + return original(messages, opts); + }; + const controller = new AbortController(); + const c = createIntentClassifier({ llm, timeoutMs: 321 }); + + await c.classify("investigate an ambiguous pipeline issue", { + signal: controller.signal, + }); + + expect(seen?.signal).toBe(controller.signal); + expect(seen?.timeoutMs).toBe(321); + }); + it("LLM failure falls back to heuristic", async () => { const c = createIntentClassifier({ llm: fakeLlm(() => { diff --git a/apps/memos-local-plugin/tests/unit/session/relation-classifier.test.ts b/apps/memos-local-plugin/tests/unit/session/relation-classifier.test.ts index 575a4b44a..a17d3d9ce 100644 --- a/apps/memos-local-plugin/tests/unit/session/relation-classifier.test.ts +++ b/apps/memos-local-plugin/tests/unit/session/relation-classifier.test.ts @@ -119,6 +119,33 @@ describe("relation-classifier — V7 §0.1", () => { expect(d.llmModel).toBe("fake/test-model"); }); + it("forwards the foreground abort signal and classifier timeout", async () => { + let seen: { signal?: AbortSignal; timeoutMs?: number } | undefined; + const controller = new AbortController(); + const c = createRelationClassifier({ + timeoutMs: 456, + llm: { + completeJson: async (_messages, opts) => { + seen = opts; + return { + value: { relation: "follow_up", confidence: 0.9, reason: "same task" }, + servedBy: "fake/llm", + } as never; + }, + } as LlmClient, + }); + + await c.classify({ + prevUserText: "investigate retrieval latency", + prevAssistantText: "I found several possible causes.", + newUserText: "could the queue contribute to this behavior?", + signal: controller.signal, + }); + + expect(seen?.signal).toBe(controller.signal); + expect(seen?.timeoutMs).toBe(456); + }); + it("falls back to heuristic when LLM throws", async () => { const llm: Partial = { completeJson: async () => { diff --git a/apps/memos-local-plugin/tests/unit/util/foreground-resources.test.ts b/apps/memos-local-plugin/tests/unit/util/foreground-resources.test.ts new file mode 100644 index 000000000..1380b9a47 --- /dev/null +++ b/apps/memos-local-plugin/tests/unit/util/foreground-resources.test.ts @@ -0,0 +1,138 @@ +import { describe, expect, it } from "vitest"; + +import { + createForegroundResources, + prioritizeEmbedder, +} from "../../../core/util/foreground-resources.js"; +import { fakeEmbedder } from "../../helpers/fake-embedder.js"; + +describe("foreground resources", () => { + it("admits a queued foreground embedding before queued background work", async () => { + const resources = createForegroundResources({ embeddingConcurrency: 1 }); + const first = await resources.acquireEmbedding("background"); + const order: string[] = []; + + const background = resources.acquireEmbedding("background").then((release) => { + order.push("background"); + release(); + }); + const foreground = resources.acquireEmbedding("foreground").then((release) => { + order.push("foreground"); + release(); + }); + + first(); + await Promise.all([foreground, background]); + + expect(order).toEqual(["foreground", "background"]); + }); + + it("lets background work progress after a bounded foreground burst", async () => { + const resources = createForegroundResources({ + embeddingConcurrency: 1, + maxForegroundBurst: 2, + }); + const first = await resources.acquireEmbedding("foreground"); + const order: string[] = []; + + const background = resources.acquireEmbedding("background").then((release) => { + order.push("background"); + release(); + }); + const foreground1 = resources.acquireEmbedding("foreground").then((release) => { + order.push("foreground-1"); + release(); + }); + const foreground2 = resources.acquireEmbedding("foreground").then((release) => { + order.push("foreground-2"); + release(); + }); + + first(); + await Promise.all([background, foreground1, foreground2]); + + expect(order).toEqual(["foreground-1", "background", "foreground-2"]); + }); + + it("does not start background work while a foreground turn is active", async () => { + const resources = createForegroundResources(); + const leaveForeground = resources.enterForeground(); + let started = false; + + const waiting = resources.waitForBackground().then(() => { + started = true; + }); + await Promise.resolve(); + expect(started).toBe(false); + + leaveForeground(); + await waiting; + expect(started).toBe(true); + }); + + it("removes an aborted embedding waiter without consuming capacity", async () => { + const resources = createForegroundResources({ embeddingConcurrency: 1 }); + const first = await resources.acquireEmbedding("background"); + const controller = new AbortController(); + const waiting = resources.acquireEmbedding("foreground", controller.signal); + + controller.abort(); + await expect(waiting).rejects.toMatchObject({ name: "AbortError" }); + first(); + + const release = await resources.acquireEmbedding("background"); + release(); + }); + + it("chunks background embedding batches and yields between chunks", async () => { + const resources = createForegroundResources({ embeddingConcurrency: 1 }); + const base = fakeEmbedder({ dimensions: 4 }); + const batchSizes: number[] = []; + const inner = { + ...base, + async embedMany(...args: Parameters) { + batchSizes.push(args[0].length); + return base.embedMany(...args); + }, + }; + const background = prioritizeEmbedder(inner, resources, "background", 2)!; + + await background.embedMany(["a", "b", "c", "d", "e"]); + + expect(batchSizes).toEqual([2, 2, 1]); + }); + + it("aborts queued and in-flight provider work during shutdown", async () => { + const resources = createForegroundResources({ embeddingConcurrency: 1 }); + const base = fakeEmbedder({ dimensions: 4 }); + let providerSignal: AbortSignal | undefined; + const inner = { + ...base, + async embedOne( + _input: Parameters[0], + options?: Parameters[1], + ) { + providerSignal = options?.signal; + return await new Promise((_resolve, reject) => { + if (options?.signal?.aborted) { + reject(options.signal.reason); + return; + } + options?.signal?.addEventListener( + "abort", + () => reject(options.signal?.reason), + { once: true }, + ); + }); + }, + }; + const background = prioritizeEmbedder(inner, resources, "background")!; + const pending = background.embedOne("slow background work"); + await Promise.resolve(); + + resources.shutdown("test shutdown"); + + await expect(pending).rejects.toMatchObject({ name: "AbortError" }); + expect(providerSignal?.aborted).toBe(true); + }); +}); diff --git a/apps/memos-local-plugin/tests/unit/util/request-deadline.test.ts b/apps/memos-local-plugin/tests/unit/util/request-deadline.test.ts new file mode 100644 index 000000000..8d6b8d5a2 --- /dev/null +++ b/apps/memos-local-plugin/tests/unit/util/request-deadline.test.ts @@ -0,0 +1,35 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { createRequestDeadline } from "../../../core/util/request-deadline.js"; + +afterEach(() => { + vi.useRealTimers(); +}); + +describe("createRequestDeadline", () => { + it("aborts at the absolute deadline and reports no remaining budget", async () => { + vi.useFakeTimers(); + vi.setSystemTime(1_000); + + const deadline = createRequestDeadline(1_250); + expect(deadline.remainingMs()).toBe(250); + expect(deadline.signal.aborted).toBe(false); + + await vi.advanceTimersByTimeAsync(250); + + expect(deadline.signal.aborted).toBe(true); + expect(deadline.remainingMs()).toBe(0); + deadline.dispose(); + }); + + it("treats an already-expired deadline as immediately aborted", () => { + vi.useFakeTimers(); + vi.setSystemTime(2_000); + + const deadline = createRequestDeadline(1_999); + + expect(deadline.signal.aborted).toBe(true); + expect(deadline.remainingMs()).toBe(0); + deadline.dispose(); + }); +}); diff --git a/apps/memos-local-plugin/tests/unit/util/retry-after.test.ts b/apps/memos-local-plugin/tests/unit/util/retry-after.test.ts new file mode 100644 index 000000000..84a4c8ea4 --- /dev/null +++ b/apps/memos-local-plugin/tests/unit/util/retry-after.test.ts @@ -0,0 +1,90 @@ +import { describe, expect, it } from "vitest"; + +import { + clearRetryCooldowns, + getRetryCooldown, + parseRetryAfterMs, + planRetry, + recordRetryCooldown, + retryCooldownKey, +} from "../../../core/util/retry-after.js"; + +describe("parseRetryAfterMs", () => { + it("parses delay-seconds", () => { + expect(parseRetryAfterMs("3", 1_000)).toBe(3_000); + expect(parseRetryAfterMs("0", 1_000)).toBe(0); + }); + + it("parses an HTTP-date relative to the supplied clock", () => { + const now = Date.parse("2026-08-04T00:00:00.000Z"); + expect(parseRetryAfterMs("Tue, 04 Aug 2026 00:00:05 GMT", now)).toBe(5_000); + }); + + it("clamps past HTTP-dates and rejects malformed values", () => { + const now = Date.parse("2026-08-04T00:00:00.000Z"); + expect(parseRetryAfterMs("Mon, 03 Aug 2026 23:59:59 GMT", now)).toBe(0); + expect(parseRetryAfterMs("1.5", now)).toBeNull(); + expect(parseRetryAfterMs("9007199254740991", now)).toBeNull(); + expect(parseRetryAfterMs("later", now)).toBeNull(); + expect(parseRetryAfterMs(null, now)).toBeNull(); + }); + + it("defers instead of retrying before a long provider Retry-After", () => { + expect(planRetry({ + attempt: 1, + baseMs: 200, + jitterMaxMs: 0, + retryAfterMs: 120_000, + maxInlineDelayMs: 30_000, + nowMs: 1_000, + })).toEqual({ + action: "defer", + backoffMs: 200, + delayMs: 120_000, + reason: "retry_after_too_long", + retryAfterMs: 120_000, + retryAt: 121_000, + source: "retry_after", + }); + }); + + it("defers when an otherwise short retry cannot fit the request deadline", () => { + expect(planRetry({ + attempt: 1, + baseMs: 200, + jitterMaxMs: 0, + retryAfterMs: 2_000, + deadlineAt: 2_500, + nowMs: 1_000, + })).toMatchObject({ + action: "defer", + reason: "deadline_insufficient", + retryAt: 3_000, + }); + }); + + it("keeps provider cooldowns monotonic and expires them at retryAt", () => { + clearRetryCooldowns(); + recordRetryCooldown("llm:test", { + retryAfterMs: 2_000, + retryAt: 3_000, + status: 429, + }); + recordRetryCooldown("llm:test", { + retryAfterMs: 500, + retryAt: 1_500, + status: 503, + }); + expect(getRetryCooldown("llm:test", 2_999)).toMatchObject({ + retryAt: 3_000, + status: 429, + }); + expect(getRetryCooldown("llm:test", 3_000)).toBeNull(); + clearRetryCooldowns(); + }); + + it("scopes provider cooldowns by endpoint and model", () => { + expect(retryCooldownKey("llm", "openai_compatible", "https://x", "model-a")) + .not.toBe(retryCooldownKey("llm", "openai_compatible", "https://x", "model-b")); + }); +}); diff --git a/apps/memos-local-plugin/tests/unit/util/semaphore.test.ts b/apps/memos-local-plugin/tests/unit/util/semaphore.test.ts new file mode 100644 index 000000000..2a5308ad2 --- /dev/null +++ b/apps/memos-local-plugin/tests/unit/util/semaphore.test.ts @@ -0,0 +1,19 @@ +import { describe, expect, it } from "vitest"; + +import { createSemaphore } from "../../../core/util/semaphore.js"; + +describe("semaphore", () => { + it("removes an aborted waiter so shutdown cannot hang behind active work", async () => { + const semaphore = createSemaphore(1); + const release = await semaphore.acquire(); + const controller = new AbortController(); + const waiting = semaphore.acquire(controller.signal); + + controller.abort(); + await expect(waiting).rejects.toMatchObject({ name: "AbortError" }); + release(); + + const next = await semaphore.acquire(); + next(); + }); +}); From 411406dd98370f41c745a8041ea511bf86f4bfad Mon Sep 17 00:00:00 2001 From: CovD <2643822566@qq.com> Date: Tue, 4 Aug 2026 22:44:41 +0800 Subject: [PATCH 2/5] fix(plugin): archive idle low-eta skills --- .../core/config/defaults.ts | 1 + apps/memos-local-plugin/core/config/schema.ts | 6 ++ .../core/skill/ALGORITHMS.md | 15 +++ apps/memos-local-plugin/core/skill/README.md | 3 +- .../core/skill/lifecycle.ts | 3 +- .../core/skill/subscriber.ts | 46 ++++++++- apps/memos-local-plugin/core/skill/types.ts | 2 + .../core/storage/repos/skills.ts | 26 +++++ .../docs/CONFIG-ADVANCED.md | 1 + .../templates/config.demo.yaml | 1 + .../adapters/openclaw-full-chain.test.ts | 48 +++++++++- .../tests/unit/config/load.test.ts | 18 ++++ .../tests/unit/skill/_helpers.ts | 6 +- .../tests/unit/skill/lifecycle.test.ts | 49 +++++++++- .../tests/unit/skill/subscriber.test.ts | 96 +++++++++++++++++++ .../tests/unit/storage/repos.test.ts | 61 ++++++++++++ 16 files changed, 372 insertions(+), 10 deletions(-) diff --git a/apps/memos-local-plugin/core/config/defaults.ts b/apps/memos-local-plugin/core/config/defaults.ts index 5c9dff305..b00f71dde 100644 --- a/apps/memos-local-plugin/core/config/defaults.ts +++ b/apps/memos-local-plugin/core/config/defaults.ts @@ -240,6 +240,7 @@ export const DEFAULT_CONFIG: ResolvedConfig = { etaDelta: 0.1, archiveEta: 0.1, minEtaForRetrieval: 0.1, + idleArchiveMs: 30 * 24 * 60 * 60 * 1000, }, feedback: { failureThreshold: 3, diff --git a/apps/memos-local-plugin/core/config/schema.ts b/apps/memos-local-plugin/core/config/schema.ts index 8566f90f3..af9df703f 100644 --- a/apps/memos-local-plugin/core/config/schema.ts +++ b/apps/memos-local-plugin/core/config/schema.ts @@ -346,6 +346,12 @@ const AlgorithmSchema = Type.Object({ archiveEta: NumberInRange(0.1, 0, 1), /** Hide Tier-1 skills whose η is below this. Mirrors retrieval.minSkillEta. */ minEtaForRetrieval: NumberInRange(0.1, 0, 1), + /** Archive low-η active skills after this much retrieval inactivity. */ + idleArchiveMs: NumberInRange( + 30 * 24 * 60 * 60 * 1000, + 0, + 365 * 24 * 60 * 60 * 1000, + ), }, { default: {} }), feedback: Type.Object({ /** Raise a burst after this many failures of the same tool in-window. */ diff --git a/apps/memos-local-plugin/core/skill/ALGORITHMS.md b/apps/memos-local-plugin/core/skill/ALGORITHMS.md index 45cabf6ae..404f49a6d 100644 --- a/apps/memos-local-plugin/core/skill/ALGORITHMS.md +++ b/apps/memos-local-plugin/core/skill/ALGORITHMS.md @@ -236,6 +236,21 @@ can't take down a well-trialled skill. If the blend drives η under `retireEta` we still retire; the skill can rehab later via positive signals. +### Idle archive scan + +The existing lifecycle tick also archives an active skill when both +conditions hold: + +``` +η < minEtaForRetrieval +now - (lastUsedAt ?? createdAt) >= idleArchiveMs +``` + +`lastUsedAt` is updated by the existing recorded-use path. A never-used +skill falls back to `createdAt`; unrelated metadata updates therefore do +not reset its idle clock. The scan runs through the orchestrator's normal +flush lifecycle and does not introduce a separate timer. + --- ## 7. Retrieval surface diff --git a/apps/memos-local-plugin/core/skill/README.md b/apps/memos-local-plugin/core/skill/README.md index 51ed11e25..c1154ebf5 100644 --- a/apps/memos-local-plugin/core/skill/README.md +++ b/apps/memos-local-plugin/core/skill/README.md @@ -208,6 +208,7 @@ See `algorithm.skill` in | `etaDelta` | `0.1` | η step per `user.positive`/`user.negative`. | | `retireEta` | `0.25` | η floor; crossing retires. | | `minEtaForRetrieval` | `0.5` | η gate for Tier-1 retrieval + auto-promotion. | +| `idleArchiveMs` | `2592000000` | Archive low-η active skills after 30 days without use. | ## Logging @@ -232,7 +233,7 @@ log (`logs/audit.jsonl`, never deleted) via the `skill` channel. * `tests/unit/skill/crystallize.test.ts` — LLM draft normalization + failures. * `tests/unit/skill/verifier.test.ts` — coverage + resonance checks. * `tests/unit/skill/packager.test.ts` — row shape, invocation guide, embedder failure. -* `tests/unit/skill/lifecycle.test.ts` — trial counter, thumbs, retire on drift. +* `tests/unit/skill/lifecycle.test.ts` — trial counter, thumbs, reward drift, and idle archive decisions. * `tests/unit/skill/events.test.ts` — bus contract. * `tests/unit/skill/skill.integration.test.ts` — end-to-end against real SQLite. * `tests/unit/skill/subscriber.test.ts` — event-driven trigger + runOnce + flush. diff --git a/apps/memos-local-plugin/core/skill/lifecycle.ts b/apps/memos-local-plugin/core/skill/lifecycle.ts index a4eaee04a..db2ed0a56 100644 --- a/apps/memos-local-plugin/core/skill/lifecycle.ts +++ b/apps/memos-local-plugin/core/skill/lifecycle.ts @@ -196,7 +196,8 @@ export function shouldArchiveIdle( now: number, ): boolean { if (skill.status !== "active") return false; - const age = now - skill.updatedAt; + const idleSince = skill.lastUsedAt ?? skill.createdAt; + const age = now - idleSince; if (age < idleMs) return false; return skill.eta < cfg.minEtaForRetrieval; } diff --git a/apps/memos-local-plugin/core/skill/subscriber.ts b/apps/memos-local-plugin/core/skill/subscriber.ts index 67754c311..bffcfd5b3 100644 --- a/apps/memos-local-plugin/core/skill/subscriber.ts +++ b/apps/memos-local-plugin/core/skill/subscriber.ts @@ -25,7 +25,7 @@ import { runSkill, type RunSkillDeps, } from "./skill.js"; -import { shouldPromoteCandidate } from "./lifecycle.js"; +import { shouldArchiveIdle, shouldPromoteCandidate } from "./lifecycle.js"; import type { RunSkillInput, RunSkillResult, @@ -210,12 +210,12 @@ export function attachSkillSubscriber( } } - /** Periodic lifecycle pass: promote eligible candidate skills to active. */ + /** Promote eligible candidates and archive stale low-η active skills. */ async function lifecycleTick(): Promise { + const at = nowMs(); const candidates = deps.repos.skills.list({ status: "candidate", limit: 500 }); for (const s of candidates) { if (!shouldPromoteCandidate(s, deps.config)) continue; - const at = nowMs(); deps.repos.skills.setStatus(s.id, "active", at); log.info("skill.auto_promoted", { skillId: s.id, name: s.name, eta: s.eta }); deps.bus.emit({ @@ -227,6 +227,46 @@ export function attachSkillSubscriber( transition: "promoted", }); } + + const archiveBatchSize = 500; + const cutoff = at - deps.config.idleArchiveMs; + while (true) { + const candidates = deps.repos.skills.listIdleArchiveCandidates({ + minEtaForRetrieval: deps.config.minEtaForRetrieval, + cutoff, + limit: archiveBatchSize, + }); + let archivedThisBatch = 0; + for (const s of candidates) { + if (!shouldArchiveIdle(s, deps.config.idleArchiveMs, deps.config, at)) continue; + deps.repos.skills.setStatus(s.id, "archived", at); + archivedThisBatch += 1; + log.info("skill.idle_archived", { + skillId: s.id, + name: s.name, + eta: s.eta, + lastUsedAt: s.lastUsedAt ?? null, + idleArchiveMs: deps.config.idleArchiveMs, + }); + deps.bus.emit({ + kind: "skill.status.changed", + at, + skillId: s.id, + previous: "active", + next: "archived", + transition: "archived", + }); + } + if (candidates.length < archiveBatchSize) break; + if (archivedThisBatch === 0) { + log.warn("skill.idle_archive_stalled", { + candidateCount: candidates.length, + cutoff, + minEtaForRetrieval: deps.config.minEtaForRetrieval, + }); + break; + } + } } return { dispose, runOnce, applyFeedback, flush, lifecycleTick }; diff --git a/apps/memos-local-plugin/core/skill/types.ts b/apps/memos-local-plugin/core/skill/types.ts index a2799b061..b04e3bc35 100644 --- a/apps/memos-local-plugin/core/skill/types.ts +++ b/apps/memos-local-plugin/core/skill/types.ts @@ -118,6 +118,8 @@ export interface SkillConfig { archiveEta: number; /** Below this η, skills never surface in Tier-1 — matches retrieval config. */ minEtaForRetrieval: number; + /** Archive a low-η active skill after it has not been retrieved for this long. */ + idleArchiveMs: number; } /** diff --git a/apps/memos-local-plugin/core/storage/repos/skills.ts b/apps/memos-local-plugin/core/storage/repos/skills.ts index 2fef2d71e..cb978142d 100644 --- a/apps/memos-local-plugin/core/storage/repos/skills.ts +++ b/apps/memos-local-plugin/core/storage/repos/skills.ts @@ -140,6 +140,32 @@ export function makeSkillsRepo(db: StorageDb) { return db.prepare(sql).all(params).map(mapRow); }, + /** + * Return one oldest-first batch of active skills that already satisfy + * the idle-archive predicate. Filtering in SQLite prevents unrelated + * recently-updated skills from starving older candidates. + */ + listIdleArchiveCandidates(input: { + minEtaForRetrieval: number; + cutoff: number; + limit?: number; + }): SkillRow[] { + const params = { + min_eta: input.minEtaForRetrieval, + cutoff: input.cutoff, + limit: Math.max(1, Math.min(500, Math.floor(input.limit ?? 500))), + }; + const sql = ` + SELECT ${COLUMNS.join(", ")} + FROM skills + WHERE status = 'active' + AND eta < @min_eta + AND COALESCE(last_used_at, created_at) <= @cutoff + ORDER BY COALESCE(last_used_at, created_at) ASC + LIMIT @limit`; + return db.prepare(sql).all(params).map(mapRow); + }, + count(filter: Omit = {}): number { const fragments: string[] = []; const params: Record = {}; diff --git a/apps/memos-local-plugin/docs/CONFIG-ADVANCED.md b/apps/memos-local-plugin/docs/CONFIG-ADVANCED.md index 8cfe175e9..11b80a562 100644 --- a/apps/memos-local-plugin/docs/CONFIG-ADVANCED.md +++ b/apps/memos-local-plugin/docs/CONFIG-ADVANCED.md @@ -156,6 +156,7 @@ algorithm: etaDelta: 0.1 # η step per user.positive/user.negative thumbs archiveEta: 0.25 # η floor; crossing archives minEtaForRetrieval: 0.5 # η gate for Tier-1 retrieval + auto-promotion + idleArchiveMs: 2592000000 # archive low-η skills after 30d without retrieval feedback: failureThreshold: 3 # failures in `failureWindow` that trigger a burst (V7 §6.3) failureWindow: 5 # rolling tool-call window per (toolId, context) diff --git a/apps/memos-local-plugin/templates/config.demo.yaml b/apps/memos-local-plugin/templates/config.demo.yaml index ccaca7012..a2578c121 100644 --- a/apps/memos-local-plugin/templates/config.demo.yaml +++ b/apps/memos-local-plugin/templates/config.demo.yaml @@ -62,3 +62,4 @@ algorithm: minGain: 0.0 candidateTrials: 1 cooldownMs: 0 + idleArchiveMs: 2592000000 # 30 days diff --git a/apps/memos-local-plugin/tests/integration/adapters/openclaw-full-chain.test.ts b/apps/memos-local-plugin/tests/integration/adapters/openclaw-full-chain.test.ts index 6ec654173..fa12b9189 100644 --- a/apps/memos-local-plugin/tests/integration/adapters/openclaw-full-chain.test.ts +++ b/apps/memos-local-plugin/tests/integration/adapters/openclaw-full-chain.test.ts @@ -55,7 +55,7 @@ import { makeTmpDb, type TmpDbHandle } from "../../helpers/tmp-db.js"; import { fakeLlm, type FakeLlmScript } from "../../helpers/fake-llm.js"; import type { LlmClient } from "../../../core/llm/types.js"; import type { EmbedInput, EmbedStats, Embedder } from "../../../core/embedding/types.js"; -import type { EmbeddingVector } from "../../../core/types.js"; +import type { EmbeddingVector, SkillId, SkillRow } from "../../../core/types.js"; import type { AgentKind } from "../../../agent-contract/dto.js"; // ─── Helpers ───────────────────────────────────────────────────────────── @@ -641,4 +641,50 @@ describe("OpenClaw adapter integration — multi-session full V7 chain", () => { JSON.stringify(snapshot, null, 2), ); }); + + it("archives a stale low-η skill when OpenClaw closes its session", async () => { + const thirtyOneDaysMs = 31 * 24 * 60 * 60 * 1_000; + const stale: SkillRow = { + id: "sk_openclaw_idle_archive" as SkillId, + name: "openclaw_idle_archive", + status: "active", + invocationGuide: "# OpenClaw idle archive integration fixture", + procedureJson: null, + eta: 0.05, + support: 3, + gain: 0.05, + trialsAttempted: 0, + trialsPassed: 0, + sourcePolicyIds: [], + sourceWorldModelIds: [], + evidenceAnchors: [], + vec: unitFromSeed("skill:openclaw_idle_archive") as unknown as EmbeddingVector, + createdAt: (NOW - thirtyOneDaysMs) as SkillRow["createdAt"], + updatedAt: NOW as SkillRow["updatedAt"], + lastUsedAt: (NOW - thirtyOneDaysMs) as SkillRow["lastUsedAt"], + version: 1, + }; + db!.repos.skills.upsert(stale); + const bridge = createOpenClawBridge({ + agent: AGENT, + core: core!, + log: { + trace: (_m: string, _c?: unknown) => undefined, + info: (_m: string, _c?: unknown) => undefined, + warn: (_m: string, _c?: unknown) => undefined, + error: (_m: string, _c?: unknown) => undefined, + debug: (_m: string, _c?: unknown) => undefined, + }, + now: () => NOW, + }); + const session = new OpenClawSimulator({ bridge, sessionKey: "s-idle-archive" }); + + await session.turn( + "用 Python 返回字符串 hello", + '```python\ndef hello() -> str:\n return "hello"\n```', + ); + await session.close(); + + expect(db!.repos.skills.getById(stale.id)?.status).toBe("archived"); + }); }); diff --git a/apps/memos-local-plugin/tests/unit/config/load.test.ts b/apps/memos-local-plugin/tests/unit/config/load.test.ts index 5d85fd6e0..759f9fbb5 100644 --- a/apps/memos-local-plugin/tests/unit/config/load.test.ts +++ b/apps/memos-local-plugin/tests/unit/config/load.test.ts @@ -28,6 +28,24 @@ describe("config/loadConfig", () => { expect(cfg.logging.timezone).toBe("America/Los_Angeles"); }); + it("defaults skill idle archival to 30 days and accepts an override", () => { + const thirtyDaysMs = 30 * 24 * 60 * 60 * 1000; + expect(resolveConfig({}).algorithm.skill.idleArchiveMs).toBe(thirtyDaysMs); + expect(resolveConfig({ + algorithm: { skill: { idleArchiveMs: 1_000 } }, + }).algorithm.skill.idleArchiveMs).toBe(1_000); + }); + + it("rejects skill idle archival outside the supported 0-to-365-day range", () => { + const overOneYearMs = 365 * 24 * 60 * 60 * 1000 + 1; + expect(() => resolveConfig({ + algorithm: { skill: { idleArchiveMs: -1 } }, + })).toThrow(/schema validation/); + expect(() => resolveConfig({ + algorithm: { skill: { idleArchiveMs: overOneYearMs } }, + })).toThrow(/schema validation/); + }); + it("rejects invalid logging.timezone with config_invalid", () => { expect(() => resolveConfig({ logging: { timezone: "Not/AZone" } })).toThrow(MemosError); try { diff --git a/apps/memos-local-plugin/tests/unit/skill/_helpers.ts b/apps/memos-local-plugin/tests/unit/skill/_helpers.ts index 549a9f477..9cef40506 100644 --- a/apps/memos-local-plugin/tests/unit/skill/_helpers.ts +++ b/apps/memos-local-plugin/tests/unit/skill/_helpers.ts @@ -40,6 +40,7 @@ export function makeSkillConfig(partial: Partial = {}): SkillConfig etaDelta: 0.1, archiveEta: 0.1, minEtaForRetrieval: 0.1, + idleArchiveMs: 30 * 24 * 60 * 60 * 1000, ...partial, }; } @@ -145,7 +146,9 @@ export interface SeedSkillArgs { trialsPassed?: number; sourcePolicyIds?: readonly PolicyId[]; invocationGuide?: string; + createdAt?: EpochMs; updatedAt?: EpochMs; + lastUsedAt?: EpochMs | null; vec?: EmbeddingVector | null; } @@ -165,8 +168,9 @@ export function seedSkill(handle: TmpDbHandle, args: SeedSkillArgs = {}): SkillR sourceWorldModelIds: [], evidenceAnchors: [], vec: args.vec ?? vec([1, 0, 0]), - createdAt: (args.updatedAt ?? NOW) as SkillRow["createdAt"], + createdAt: (args.createdAt ?? args.updatedAt ?? NOW) as SkillRow["createdAt"], updatedAt: (args.updatedAt ?? NOW) as SkillRow["updatedAt"], + lastUsedAt: args.lastUsedAt ?? null, version: 1, }; handle.repos.skills.upsert(row); diff --git a/apps/memos-local-plugin/tests/unit/skill/lifecycle.test.ts b/apps/memos-local-plugin/tests/unit/skill/lifecycle.test.ts index 86101908e..946cfb663 100644 --- a/apps/memos-local-plugin/tests/unit/skill/lifecycle.test.ts +++ b/apps/memos-local-plugin/tests/unit/skill/lifecycle.test.ts @@ -22,6 +22,7 @@ function mkSkill(partial: Partial = {}): SkillRow { vec: null, createdAt: partial.createdAt ?? NOW, updatedAt: partial.updatedAt ?? NOW, + lastUsedAt: partial.lastUsedAt ?? null, version: partial.version ?? 1, }; } @@ -104,9 +105,51 @@ describe("skill/lifecycle", () => { expect(recomputeEta(s, policy, cfg)).toBeCloseTo(0.7, 5); }); - it("shouldArchiveIdle picks up stale active skills with low η", () => { + it("archives a low-η active skill after its last use exceeds idleArchiveMs", () => { + const cfg = makeSkillConfig({ minEtaForRetrieval: 0.6, idleArchiveMs: 1_000 }); + const s = mkSkill({ + status: "active", + eta: 0.4, + lastUsedAt: 1_000 as SkillRow["lastUsedAt"], + }); + expect(shouldArchiveIdle(s, 1_000, cfg, 10_000)).toBe(true); + }); + + it("uses createdAt as the idle baseline for a skill that has never been used", () => { + const cfg = makeSkillConfig({ minEtaForRetrieval: 0.6, idleArchiveMs: 1_000 }); + const s = mkSkill({ + status: "active", + eta: 0.4, + createdAt: 1_000 as SkillRow["createdAt"], + updatedAt: 9_500 as SkillRow["updatedAt"], + lastUsedAt: null, + }); + expect(shouldArchiveIdle(s, 1_000, cfg, 10_000)).toBe(true); + }); + + it("keeps recently used or retrievable active skills", () => { + const cfg = makeSkillConfig({ minEtaForRetrieval: 0.6, idleArchiveMs: 1_000 }); + const recent = mkSkill({ + status: "active", + eta: 0.4, + lastUsedAt: 9_500 as SkillRow["lastUsedAt"], + }); + const retrievable = mkSkill({ + status: "active", + eta: 0.6, + lastUsedAt: 1_000 as SkillRow["lastUsedAt"], + }); + expect(shouldArchiveIdle(recent, 1_000, cfg, 10_000)).toBe(false); + expect(shouldArchiveIdle(retrievable, 1_000, cfg, 10_000)).toBe(false); + }); + + it("archives exactly at the configured idle boundary", () => { const cfg = makeSkillConfig({ minEtaForRetrieval: 0.6 }); - const s = mkSkill({ status: "active", eta: 0.4, updatedAt: 0 as SkillRow["updatedAt"] }); - expect(shouldArchiveIdle(s, 1000, cfg, 10_000)).toBe(true); + const skill = mkSkill({ + status: "active", + eta: 0.4, + lastUsedAt: 9_000 as SkillRow["lastUsedAt"], + }); + expect(shouldArchiveIdle(skill, 1_000, cfg, 10_000)).toBe(true); }); }); diff --git a/apps/memos-local-plugin/tests/unit/skill/subscriber.test.ts b/apps/memos-local-plugin/tests/unit/skill/subscriber.test.ts index dbda7394c..88ec1eafb 100644 --- a/apps/memos-local-plugin/tests/unit/skill/subscriber.test.ts +++ b/apps/memos-local-plugin/tests/unit/skill/subscriber.test.ts @@ -16,6 +16,7 @@ import { makeSkillConfig, seedPolicy, seedSessionOnly, + seedSkill, seedTrace, } from "./_helpers.js"; @@ -154,4 +155,99 @@ describe("skill/subscriber", () => { expect(r.crystallized).toBe(1); sub.dispose(); }); + + it("archives each stale low-η active skill once without regressing candidate promotion", async () => { + handle = makeTmpDb(); + const h = handle; + const l2Bus = createL2EventBus(); + const rewardBus = createRewardEventBus(); + const bus = createSkillEventBus(); + const events: Array<{ + skillId: string; + previous: string; + next: string; + transition: string; + }> = []; + bus.on("skill.status.changed", (event) => { + if (event.kind !== "skill.status.changed") return; + events.push({ + skillId: event.skillId, + previous: event.previous, + next: event.next, + transition: event.transition, + }); + }); + + const stale = seedSkill(h, { + id: "sk_stale" as never, + name: "stale_skill", + status: "active", + eta: 0.05, + createdAt: 1 as never, + updatedAt: 9_000 as never, + lastUsedAt: 1_000 as never, + }); + const candidate = seedSkill(h, { + id: "sk_candidate" as never, + name: "candidate_skill", + status: "candidate", + eta: 0.7, + createdAt: 1 as never, + updatedAt: 1 as never, + }); + + const sub = attachSkillSubscriber({ + l2Bus, + rewardBus, + bus, + repos: h.repos, + embedder: null, + llm: null, + log: rootLogger.child({ channel: "core.skill.subscriber" }), + config: makeSkillConfig({ minEtaForRetrieval: 0.1, idleArchiveMs: 1_000 }), + }); + + await sub.lifecycleTick(); + await sub.lifecycleTick(); + + expect(h.repos.skills.getById(stale.id)?.status).toBe("archived"); + expect(h.repos.skills.getById(candidate.id)?.status).toBe("active"); + expect(events.filter((event) => event.skillId === stale.id)).toEqual([ + { skillId: stale.id, previous: "active", next: "archived", transition: "archived" }, + ]); + expect(events.filter((event) => event.skillId === candidate.id)).toHaveLength(1); + sub.dispose(); + }); + + it("drains more than one 500-skill idle archive batch in one lifecycle tick", async () => { + handle = makeTmpDb(); + const h = handle; + for (let i = 0; i < 501; i++) { + seedSkill(h, { + id: `sk_stale_${i}` as never, + name: `stale_skill_${i}`, + status: "active", + eta: 0.05, + createdAt: 1 as never, + updatedAt: (i + 1) as never, + lastUsedAt: 1 as never, + }); + } + const sub = attachSkillSubscriber({ + l2Bus: createL2EventBus(), + rewardBus: createRewardEventBus(), + bus: createSkillEventBus(), + repos: h.repos, + embedder: null, + llm: null, + log: rootLogger.child({ channel: "core.skill.subscriber" }), + config: makeSkillConfig({ minEtaForRetrieval: 0.1, idleArchiveMs: 1_000 }), + }); + + await sub.lifecycleTick(); + + expect(h.repos.skills.count({ status: "archived" })).toBe(501); + expect(h.repos.skills.count({ status: "active" })).toBe(0); + sub.dispose(); + }); }); diff --git a/apps/memos-local-plugin/tests/unit/storage/repos.test.ts b/apps/memos-local-plugin/tests/unit/storage/repos.test.ts index 3c859fc19..aef629696 100644 --- a/apps/memos-local-plugin/tests/unit/storage/repos.test.ts +++ b/apps/memos-local-plugin/tests/unit/storage/repos.test.ts @@ -314,6 +314,67 @@ describe("storage/repos — happy paths", () => { } }); + it("skills: selects idle archive candidates and excludes a skill after recorded use", () => { + const { repos, cleanup } = makeTmpDb(); + try { + const insertSkill = ( + id: string, + status: "active" | "archived", + eta: number, + createdAt: number, + lastUsedAt: number | null, + ) => { + repos.skills.insert({ + id, + name: id, + status, + invocationGuide: "fixture", + procedureJson: null, + eta, + support: 1, + gain: 0, + trialsAttempted: 0, + trialsPassed: 0, + sourcePolicyIds: [], + sourceWorldModelIds: [], + evidenceAnchors: [], + vec: null, + createdAt, + updatedAt: 10_000, + lastUsedAt, + version: 1, + }); + }; + insertSkill("never_used", "active", 0.05, 50, null); + insertSkill("old_used", "active", 0.05, 1, 100); + insertSkill("recent", "active", 0.05, 1, 9_500); + insertSkill("retrievable", "active", 0.1, 1, 100); + insertSkill("already_archived", "archived", 0.05, 1, 100); + + const candidates = repos.skills.listIdleArchiveCandidates({ + minEtaForRetrieval: 0.1, + cutoff: 9_000, + limit: 500, + }); + expect(candidates.map((skill) => skill.id)).toEqual(["never_used", "old_used"]); + expect(repos.skills.listIdleArchiveCandidates({ + minEtaForRetrieval: 0.1, + cutoff: 9_000, + limit: 1, + }).map((skill) => skill.id)).toEqual(["never_used"]); + + expect(repos.skills.recordUse("old_used", 9_500)).toBe(true); + expect(repos.skills.getById("old_used")?.lastUsedAt).toBe(9_500); + expect(repos.skills.listIdleArchiveCandidates({ + minEtaForRetrieval: 0.1, + cutoff: 9_000, + limit: 500, + }).map((skill) => skill.id)).toEqual(["never_used"]); + } finally { + cleanup(); + } + }); + it("feedback: insert, scoped list, polarity filter", () => { const { repos, cleanup } = makeTmpDb(); try { From c5c0d0cd45ca52c075fa343baed038d51b550bda Mon Sep 17 00:00:00 2001 From: CovD <2643822566@qq.com> Date: Tue, 4 Aug 2026 23:15:13 +0800 Subject: [PATCH 3/5] fix(plugin): address idle archive review --- apps/memos-local-plugin/core/config/schema.ts | 4 ++-- apps/memos-local-plugin/core/skill/ALGORITHMS.md | 4 ++++ apps/memos-local-plugin/core/skill/README.md | 2 +- apps/memos-local-plugin/core/skill/subscriber.ts | 14 +++++++------- .../core/storage/repos/skills.ts | 10 +++++++++- apps/memos-local-plugin/docs/CONFIG-ADVANCED.md | 2 +- .../templates/config.demo.yaml | 2 +- .../tests/unit/config/load.test.ts | 16 ++++++++++++---- 8 files changed, 37 insertions(+), 17 deletions(-) diff --git a/apps/memos-local-plugin/core/config/schema.ts b/apps/memos-local-plugin/core/config/schema.ts index af9df703f..0ac5ade8c 100644 --- a/apps/memos-local-plugin/core/config/schema.ts +++ b/apps/memos-local-plugin/core/config/schema.ts @@ -346,10 +346,10 @@ const AlgorithmSchema = Type.Object({ archiveEta: NumberInRange(0.1, 0, 1), /** Hide Tier-1 skills whose η is below this. Mirrors retrieval.minSkillEta. */ minEtaForRetrieval: NumberInRange(0.1, 0, 1), - /** Archive low-η active skills after this much retrieval inactivity. */ + /** Archive low-η active skills after this much retrieval inactivity (minimum 1 hour). */ idleArchiveMs: NumberInRange( 30 * 24 * 60 * 60 * 1000, - 0, + 60 * 60 * 1000, 365 * 24 * 60 * 60 * 1000, ), }, { default: {} }), diff --git a/apps/memos-local-plugin/core/skill/ALGORITHMS.md b/apps/memos-local-plugin/core/skill/ALGORITHMS.md index 404f49a6d..94daaf7ab 100644 --- a/apps/memos-local-plugin/core/skill/ALGORITHMS.md +++ b/apps/memos-local-plugin/core/skill/ALGORITHMS.md @@ -246,6 +246,10 @@ conditions hold: now - (lastUsedAt ?? createdAt) >= idleArchiveMs ``` +Configuration validation enforces a one-hour minimum for `idleArchiveMs` to +prevent an accidental zero value from archiving every low-η active Skill on +the next lifecycle tick. + `lastUsedAt` is updated by the existing recorded-use path. A never-used skill falls back to `createdAt`; unrelated metadata updates therefore do not reset its idle clock. The scan runs through the orchestrator's normal diff --git a/apps/memos-local-plugin/core/skill/README.md b/apps/memos-local-plugin/core/skill/README.md index c1154ebf5..a0e573b8c 100644 --- a/apps/memos-local-plugin/core/skill/README.md +++ b/apps/memos-local-plugin/core/skill/README.md @@ -208,7 +208,7 @@ See `algorithm.skill` in | `etaDelta` | `0.1` | η step per `user.positive`/`user.negative`. | | `retireEta` | `0.25` | η floor; crossing retires. | | `minEtaForRetrieval` | `0.5` | η gate for Tier-1 retrieval + auto-promotion. | -| `idleArchiveMs` | `2592000000` | Archive low-η active skills after 30 days without use. | +| `idleArchiveMs` | `2592000000` | Archive low-η active skills after 30 days without use (minimum 1 hour). | ## Logging diff --git a/apps/memos-local-plugin/core/skill/subscriber.ts b/apps/memos-local-plugin/core/skill/subscriber.ts index bffcfd5b3..af1b0a57c 100644 --- a/apps/memos-local-plugin/core/skill/subscriber.ts +++ b/apps/memos-local-plugin/core/skill/subscriber.ts @@ -35,6 +35,7 @@ import type { } from "./types.js"; import type { SkillId } from "../types.js"; import { now as nowMs } from "../time.js"; +import { IDLE_ARCHIVE_BATCH_LIMIT } from "../storage/repos/skills.js"; export interface SkillSubscriberDeps extends Omit { @@ -228,16 +229,15 @@ export function attachSkillSubscriber( }); } - const archiveBatchSize = 500; const cutoff = at - deps.config.idleArchiveMs; while (true) { - const candidates = deps.repos.skills.listIdleArchiveCandidates({ + const archiveCandidates = deps.repos.skills.listIdleArchiveCandidates({ minEtaForRetrieval: deps.config.minEtaForRetrieval, cutoff, - limit: archiveBatchSize, + limit: IDLE_ARCHIVE_BATCH_LIMIT, }); let archivedThisBatch = 0; - for (const s of candidates) { + for (const s of archiveCandidates) { if (!shouldArchiveIdle(s, deps.config.idleArchiveMs, deps.config, at)) continue; deps.repos.skills.setStatus(s.id, "archived", at); archivedThisBatch += 1; @@ -257,15 +257,15 @@ export function attachSkillSubscriber( transition: "archived", }); } - if (candidates.length < archiveBatchSize) break; - if (archivedThisBatch === 0) { + if (archiveCandidates.length > 0 && archivedThisBatch === 0) { log.warn("skill.idle_archive_stalled", { - candidateCount: candidates.length, + candidateCount: archiveCandidates.length, cutoff, minEtaForRetrieval: deps.config.minEtaForRetrieval, }); break; } + if (archiveCandidates.length < IDLE_ARCHIVE_BATCH_LIMIT) break; } } diff --git a/apps/memos-local-plugin/core/storage/repos/skills.ts b/apps/memos-local-plugin/core/storage/repos/skills.ts index cb978142d..01b91eba2 100644 --- a/apps/memos-local-plugin/core/storage/repos/skills.ts +++ b/apps/memos-local-plugin/core/storage/repos/skills.ts @@ -14,6 +14,8 @@ import { toJsonText, } from "./_helpers.js"; +export const IDLE_ARCHIVE_BATCH_LIMIT = 500; + const COLUMNS = [ "id", "owner_agent_kind", @@ -153,7 +155,13 @@ export function makeSkillsRepo(db: StorageDb) { const params = { min_eta: input.minEtaForRetrieval, cutoff: input.cutoff, - limit: Math.max(1, Math.min(500, Math.floor(input.limit ?? 500))), + limit: Math.max( + 1, + Math.min( + IDLE_ARCHIVE_BATCH_LIMIT, + Math.floor(input.limit ?? IDLE_ARCHIVE_BATCH_LIMIT), + ), + ), }; const sql = ` SELECT ${COLUMNS.join(", ")} diff --git a/apps/memos-local-plugin/docs/CONFIG-ADVANCED.md b/apps/memos-local-plugin/docs/CONFIG-ADVANCED.md index 11b80a562..82e03230d 100644 --- a/apps/memos-local-plugin/docs/CONFIG-ADVANCED.md +++ b/apps/memos-local-plugin/docs/CONFIG-ADVANCED.md @@ -156,7 +156,7 @@ algorithm: etaDelta: 0.1 # η step per user.positive/user.negative thumbs archiveEta: 0.25 # η floor; crossing archives minEtaForRetrieval: 0.5 # η gate for Tier-1 retrieval + auto-promotion - idleArchiveMs: 2592000000 # archive low-η skills after 30d without retrieval + idleArchiveMs: 2592000000 # archive low-η skills after 30d without retrieval (minimum 1h) feedback: failureThreshold: 3 # failures in `failureWindow` that trigger a burst (V7 §6.3) failureWindow: 5 # rolling tool-call window per (toolId, context) diff --git a/apps/memos-local-plugin/templates/config.demo.yaml b/apps/memos-local-plugin/templates/config.demo.yaml index a2578c121..7359f38bc 100644 --- a/apps/memos-local-plugin/templates/config.demo.yaml +++ b/apps/memos-local-plugin/templates/config.demo.yaml @@ -62,4 +62,4 @@ algorithm: minGain: 0.0 candidateTrials: 1 cooldownMs: 0 - idleArchiveMs: 2592000000 # 30 days + idleArchiveMs: 2592000000 # 30 days; minimum 1 hour diff --git a/apps/memos-local-plugin/tests/unit/config/load.test.ts b/apps/memos-local-plugin/tests/unit/config/load.test.ts index 759f9fbb5..d7f394772 100644 --- a/apps/memos-local-plugin/tests/unit/config/load.test.ts +++ b/apps/memos-local-plugin/tests/unit/config/load.test.ts @@ -30,16 +30,24 @@ describe("config/loadConfig", () => { it("defaults skill idle archival to 30 days and accepts an override", () => { const thirtyDaysMs = 30 * 24 * 60 * 60 * 1000; + const sixHoursMs = 6 * 60 * 60 * 1000; expect(resolveConfig({}).algorithm.skill.idleArchiveMs).toBe(thirtyDaysMs); expect(resolveConfig({ - algorithm: { skill: { idleArchiveMs: 1_000 } }, - }).algorithm.skill.idleArchiveMs).toBe(1_000); + algorithm: { skill: { idleArchiveMs: sixHoursMs } }, + }).algorithm.skill.idleArchiveMs).toBe(sixHoursMs); }); - it("rejects skill idle archival outside the supported 0-to-365-day range", () => { + it("rejects skill idle archival outside the supported one-hour-to-365-day range", () => { + const oneHourMs = 60 * 60 * 1000; const overOneYearMs = 365 * 24 * 60 * 60 * 1000 + 1; + expect(resolveConfig({ + algorithm: { skill: { idleArchiveMs: oneHourMs } }, + }).algorithm.skill.idleArchiveMs).toBe(oneHourMs); + expect(() => resolveConfig({ + algorithm: { skill: { idleArchiveMs: 0 } }, + })).toThrow(/schema validation/); expect(() => resolveConfig({ - algorithm: { skill: { idleArchiveMs: -1 } }, + algorithm: { skill: { idleArchiveMs: oneHourMs - 1 } }, })).toThrow(/schema validation/); expect(() => resolveConfig({ algorithm: { skill: { idleArchiveMs: overOneYearMs } }, From 243b0a63fd79d2f170d3abaf4bf7d33b74a7a97a Mon Sep 17 00:00:00 2001 From: CovD <2643822566@qq.com> Date: Tue, 4 Aug 2026 23:18:42 +0800 Subject: [PATCH 4/5] chore(plugin): align PR with dev-v2.0.29 Remove #2208 changes after maintainers retargeted #2209 from main to dev-v2.0.29. --- .../hermes/memos_provider/__init__.py | 147 +--------- .../hermes/memos_provider/bridge_client.py | 137 +++------ apps/memos-local-plugin/agent-contract/dto.ts | 5 - apps/memos-local-plugin/bridge/methods.ts | 9 - .../core/embedding/embedder.ts | 18 +- .../core/embedding/fetcher.ts | 129 +-------- .../core/embedding/index.ts | 1 - .../core/embedding/providers/cohere.ts | 4 +- .../core/embedding/providers/gemini.ts | 4 +- .../core/embedding/providers/mistral.ts | 4 +- .../core/embedding/providers/openai.ts | 4 +- .../core/embedding/providers/voyage.ts | 4 +- .../core/embedding/retry-worker.ts | 31 +- .../core/embedding/types.ts | 20 +- apps/memos-local-plugin/core/index.ts | 1 - apps/memos-local-plugin/core/llm/client.ts | 11 - apps/memos-local-plugin/core/llm/fetcher.ts | 137 +-------- .../core/llm/providers/anthropic.ts | 4 +- .../core/llm/providers/bedrock.ts | 4 +- .../core/llm/providers/gemini.ts | 4 +- .../core/llm/providers/openai.ts | 4 +- apps/memos-local-plugin/core/llm/types.ts | 9 +- apps/memos-local-plugin/core/pipeline/deps.ts | 29 +- .../core/pipeline/memory-core.ts | 88 +++--- .../core/pipeline/orchestrator.ts | 153 ++-------- .../core/retrieval/llm-filter.ts | 8 - .../core/retrieval/retrieve.ts | 53 +--- .../core/retrieval/types.ts | 6 +- .../core/session/intent-classifier.ts | 8 +- .../core/session/manager.ts | 3 - .../core/session/relation-classifier.ts | 24 +- apps/memos-local-plugin/core/session/types.ts | 2 - .../core/util/foreground-resources.ts | 274 ------------------ .../core/util/rate-limited-llm.ts | 31 +- .../core/util/request-deadline.ts | 37 --- .../core/util/retry-after.ts | 200 ------------- .../memos-local-plugin/core/util/semaphore.ts | 43 +-- .../tests/python/test_bridge_client.py | 165 +---------- .../python/test_hermes_provider_pipeline.py | 69 ----- .../tests/unit/bridge/methods.test.ts | 11 - .../tests/unit/embedding/embedder.test.ts | 59 ---- .../tests/unit/embedding/fetcher.test.ts | 82 ------ .../tests/unit/embedding/retry-worker.test.ts | 60 ---- .../tests/unit/llm/client.test.ts | 39 --- .../tests/unit/llm/fetcher.test.ts | 201 +------------ .../tests/unit/pipeline/memory-core.test.ts | 47 --- .../tests/unit/pipeline/orchestrator.test.ts | 33 --- .../unit/session/intent-classifier.test.ts | 19 -- .../unit/session/relation-classifier.test.ts | 27 -- .../unit/util/foreground-resources.test.ts | 138 --------- .../tests/unit/util/request-deadline.test.ts | 35 --- .../tests/unit/util/retry-after.test.ts | 90 ------ .../tests/unit/util/semaphore.test.ts | 19 -- 53 files changed, 197 insertions(+), 2547 deletions(-) delete mode 100644 apps/memos-local-plugin/core/util/foreground-resources.ts delete mode 100644 apps/memos-local-plugin/core/util/request-deadline.ts delete mode 100644 apps/memos-local-plugin/core/util/retry-after.ts delete mode 100644 apps/memos-local-plugin/tests/unit/util/foreground-resources.test.ts delete mode 100644 apps/memos-local-plugin/tests/unit/util/request-deadline.test.ts delete mode 100644 apps/memos-local-plugin/tests/unit/util/retry-after.test.ts delete mode 100644 apps/memos-local-plugin/tests/unit/util/semaphore.test.ts diff --git a/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py b/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py index 79b88cbf4..6eeaa000e 100644 --- a/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py +++ b/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py @@ -201,47 +201,6 @@ def _long_rpc_timeout_default() -> float: _LONG_RPC_TIMEOUT = _long_rpc_timeout_default() - -def _prefetch_rpc_timeout_default() -> float: - """Resolve the latency budget for Hermes' foreground memory lookup. - - Hermes places its own short deadline around ``prefetch``. Reusing the - long capture timeout here lets the bridge continue work after the host - has already moved on. Keep a separate, configurable ceiling below the - host's default and forward the corresponding absolute deadline to core. - """ - raw = os.environ.get("MEMOS_HERMES_PREFETCH_RPC_TIMEOUT", "") - try: - value = float(raw) - except (TypeError, ValueError): - return 6.0 - if not value > 0: - return 6.0 - # Hermes currently abandons external providers after 8 seconds. Keep at - # least one second for Python thread scheduling and response assembly even - # when an operator overrides the default. - return min(value, 7.0) - - -_PREFETCH_RPC_TIMEOUT = _prefetch_rpc_timeout_default() -_PREFETCH_RESPONSE_RESERVE_SECONDS = 0.25 - - -def _remaining_rpc_timeout( - deadline_monotonic: float | None, - requested_timeout: float | None, -) -> float | None: - """Bound one blocking bridge step by a shared end-to-end deadline.""" - if deadline_monotonic is None: - return requested_timeout - remaining = deadline_monotonic - time.monotonic() - if remaining <= 0: - raise BridgeError("timeout", "foreground prefetch deadline exceeded") - if requested_timeout is None: - return remaining - return min(requested_timeout, remaining) - - _HERMES_INTERNAL_REVIEW_PREFIXES = ( "review the conversation above and consider saving to memory if appropriate.", "review the conversation above and update the skill library.", @@ -1063,19 +1022,8 @@ def prefetch(self, query: str, *, session_id: str = "") -> str: # type: ignore[ cached result immediately. Otherwise synchronously run ``turn.start`` against the bridge (small overhead). """ - deadline_monotonic = time.monotonic() + _PREFETCH_RPC_TIMEOUT - started_at_ms = int(time.time() * 1000) - core_budget_seconds = max( - 0.05, - _PREFETCH_RPC_TIMEOUT - _PREFETCH_RESPONSE_RESERVE_SECONDS, - ) - deadline_at_ms = started_at_ms + int(core_budget_seconds * 1000) if self._prefetch_thread and self._prefetch_thread.is_alive(): - try: - join_timeout = _remaining_rpc_timeout(deadline_monotonic, 5.0) - except BridgeError: - return "" - self._prefetch_thread.join(timeout=join_timeout) + self._prefetch_thread.join(timeout=5.0) with self._prefetch_lock: cached = self._prefetch_result self._prefetch_result = "" @@ -1085,25 +1033,10 @@ def prefetch(self, query: str, *, session_id: str = "") -> str: # type: ignore[ suppress_injection = _is_explicit_delegation_request(query) if cached: return "" if suppress_injection else cached - try: - ensure_timeout = _remaining_rpc_timeout( - deadline_monotonic, - _PREFETCH_RPC_TIMEOUT, - ) - except BridgeError: - return "" - if not self._ensure_bridge( - session_id or self._session_id, - timeout=min(10.0, ensure_timeout or _PREFETCH_RPC_TIMEOUT), - ): + if not self._ensure_bridge(session_id or self._session_id, timeout=10.0): return "" try: - context = self._turn_start( - query, - session_id=session_id, - deadline_monotonic=deadline_monotonic, - deadline_at_ms=deadline_at_ms, - ) + context = self._turn_start(query, session_id=session_id) if suppress_injection: # Do not let remembered "do it directly" skills override an # explicit user request to dispatch work to a subagent. @@ -2008,7 +1941,6 @@ def _bridge_request( *, timeout: float | None = None, ensure_session: bool = True, - deadline_monotonic: float | None = None, ) -> dict[str, Any]: bridge = self._bridge if bridge is None: @@ -2026,23 +1958,10 @@ def _bridge_request( bridge.generation, self._session_id, ) - session_ceiling = ( - timeout - if deadline_monotonic is not None and timeout is not None - else 30.0 - ) - session_timeout = _remaining_rpc_timeout( - deadline_monotonic, - session_ceiling, - ) - self._open_session( - self._session_id, - timeout=session_timeout or 30.0, - ) - request_timeout = _remaining_rpc_timeout(deadline_monotonic, timeout) - if request_timeout is None: + self._open_session(self._session_id, timeout=30.0) + if timeout is None: return bridge.request(method, params) - return bridge.request(method, params, timeout=request_timeout) + return bridge.request(method, params, timeout=timeout) def _open_session(self, session_id: str = "", *, timeout: float = 30.0) -> None: bridge = self._bridge @@ -2078,7 +1997,6 @@ def _bridge_request_with_retry( params: Any, *, timeout: float | None = None, - deadline_monotonic: float | None = None, ) -> dict[str, Any]: """Read-path helper: reconnect + retry once on ``transport_closed``. @@ -2094,12 +2012,7 @@ def _bridge_request_with_retry( """ assert self._bridge is not None try: - return self._bridge_request( - method, - params, - timeout=timeout, - deadline_monotonic=deadline_monotonic, - ) + return self._bridge_request(method, params, timeout=timeout) except BridgeError as err: if not self._is_transport_closed(err): raise @@ -2108,24 +2021,9 @@ def _bridge_request_with_retry( method, err, ) - reconnect_ceiling = ( - timeout if deadline_monotonic is not None and timeout is not None else 30.0 - ) - reconnect_timeout = _remaining_rpc_timeout( - deadline_monotonic, - reconnect_ceiling, - ) - self._reconnect_bridge( - self._session_id, - timeout=reconnect_timeout or 30.0, - ) + self._reconnect_bridge(self._session_id, timeout=30.0) assert self._bridge is not None - return self._bridge_request( - method, - params, - timeout=timeout, - deadline_monotonic=deadline_monotonic, - ) + return self._bridge_request(method, params, timeout=timeout) def _is_transport_closed(self, err: Exception) -> bool: if isinstance(err, BridgeError) and err.code == "transport_closed": @@ -2335,14 +2233,7 @@ def _run() -> None: ) self._bridge_keepalive_thread.start() - def _turn_start( - self, - query: str, - *, - session_id: str = "", - deadline_monotonic: float | None = None, - deadline_at_ms: int | None = None, - ) -> str: + def _turn_start(self, query: str, *, session_id: str = "") -> str: assert self._bridge is not None host_runtime = self._host_runtime_context() with self._state_lock: @@ -2360,34 +2251,20 @@ def _turn_start( "visibleContextStartTs": visible_context_start_ts, } ) - now_ms = int(time.time() * 1000) - if deadline_monotonic is None: - deadline_monotonic = time.monotonic() + _PREFETCH_RPC_TIMEOUT - if deadline_at_ms is None: - core_budget_seconds = max( - 0.05, - _PREFETCH_RPC_TIMEOUT - _PREFETCH_RESPONSE_RESERVE_SECONDS, - ) - deadline_at_ms = now_ms + int(core_budget_seconds * 1000) payload: dict[str, Any] = { "agent": "hermes", "namespace": self._runtime_namespace(), "sessionId": session_id or self._session_id, "userText": query, "contextHints": context_hints, - "ts": now_ms, - "deadlineAt": deadline_at_ms, + "ts": int(time.time() * 1000), } if turn_key: payload["turnKey"] = turn_key resp = self._bridge_request_with_retry( "turn.start", payload, - timeout=_remaining_rpc_timeout( - deadline_monotonic, - _PREFETCH_RPC_TIMEOUT, - ), - deadline_monotonic=deadline_monotonic, + timeout=_LONG_RPC_TIMEOUT, ) response_query = (resp or {}).get("query") or {} response_session = str(response_query.get("sessionId") or "") diff --git a/apps/memos-local-plugin/adapters/hermes/memos_provider/bridge_client.py b/apps/memos-local-plugin/adapters/hermes/memos_provider/bridge_client.py index cbb2fa84c..6863ba1ef 100644 --- a/apps/memos-local-plugin/adapters/hermes/memos_provider/bridge_client.py +++ b/apps/memos-local-plugin/adapters/hermes/memos_provider/bridge_client.py @@ -17,7 +17,6 @@ import json import logging import os -import queue import shutil import subprocess import threading @@ -33,7 +32,6 @@ logger = logging.getLogger(__name__) HOST_HANDLER_WAIT_SECONDS = 5.0 -HOST_HANDLER_QUEUE_CAPACITY = 16 # ─── Module-level singleton tracker ───────────────────────────────────── # Each entry maps an ``(agent, no_viewer, runtime_home)`` key to the @@ -154,16 +152,12 @@ def __init__( # Reverse-direction handlers: the bridge can send us a # JSON-RPC request via `serverRequest(...)` (e.g. # `host.llm.complete` for fallback LLM calls). Registered - # methods run on one bounded, daemon worker. Keeping execution - # serial preserves the adapter's previous concurrency contract while - # preventing a slow host LLM call from blocking stdout response - # demultiplexing for every shared provider lease. + # methods run on the dedicated reader thread; long-running + # work should spawn its own worker if it needs to. Each + # handler returns a JSON-serialisable value or raises to + # surface a JSON-RPC error back to the bridge. self._host_handlers: dict[str, Callable[[dict[str, Any]], Any]] = {} self._host_handlers_cv = threading.Condition() - self._host_handler_queue: queue.Queue[tuple[Any, str, dict[str, Any]] | None] = queue.Queue( - maxsize=HOST_HANDLER_QUEUE_CAPACITY - ) - self._host_handler_stop = threading.Event() self._closed = False plugin_root = Path(__file__).resolve().parent.parent.parent.parent @@ -232,12 +226,6 @@ def __init__( env=env, cwd=str(plugin_root), ) - self._host_handler_worker = threading.Thread( - target=self._host_handler_loop, - daemon=True, - name="memos-bridge-host-handler", - ) - self._host_handler_worker.start() self._reader = threading.Thread( target=self._read_loop, daemon=True, @@ -360,7 +348,7 @@ def notify(self, method: str, params: Any = None) -> None: try: self._proc.stdin.write(payload + "\n") self._proc.stdin.flush() - except (BrokenPipeError, OSError, ValueError): + except (BrokenPipeError, OSError): pass def on_event(self, cb: Callable[[dict[str, Any]], None]) -> None: @@ -377,9 +365,11 @@ def register_host_handler( """Register a handler for bridge → adapter (reverse) requests. The Node-side bridge calls these via ``stdio.serverRequest``. - Most-recent registration wins. Handlers run serially on a bounded - daemon worker so a long-running host LLM call cannot stall the reader - thread that resolves unrelated foreground JSON-RPC responses. + Most-recent registration wins. The handler runs on the reader + thread; if it blocks for a long time it stalls every other + bridge → adapter notification, so handlers that need to do + heavy work (e.g. an LLM call) are still expected to return + within the bridge-side timeout (default 60 s). """ with self._host_handlers_cv: self._host_handlers[method] = handler @@ -391,7 +381,6 @@ def close(self) -> None: with self._host_handlers_cv: self._closed = True self._host_handlers_cv.notify_all() - self._stop_host_handler_worker() # Drop self from the module-level singleton tracker (issue #1910) # BEFORE the potentially-slow stdin/SIGTERM/SIGKILL dance. We @@ -446,7 +435,6 @@ def _abort_pending(self, reason: str) -> None: with self._host_handlers_cv: self._closed = True self._host_handlers_cv.notify_all() - self._stop_host_handler_worker() with self._lock: for entry in list(self._pending.values()): entry["error"] = { @@ -489,9 +477,7 @@ def _read_loop(self) -> None: # Reverse-direction request: the bridge is asking the # adapter to do something (e.g. run a fallback LLM call # via `host.llm.complete`). Dispatch to the registered - # handler on the bounded worker. The reader must return to - # stdout immediately so a slow host LLM callback cannot - # head-of-line block normal JSON-RPC responses. + # handler and write the response back synchronously. method = msg.get("method") rpc_id = msg.get("id") if ( @@ -500,10 +486,33 @@ def _read_loop(self) -> None: and "result" not in msg and "error" not in msg ): + handler = self._host_handler_for(method) + if handler is None: + self._send_response( + rpc_id, + error={ + "code": -32601, + "message": f"method not found: {method}", + "data": {"code": "unknown_method"}, + }, + ) + continue params = msg.get("params") or {} if not isinstance(params, dict): params = {} - self._dispatch_host_request(rpc_id, method, params) + try: + result = handler(params) + self._send_response(rpc_id, result=result) + except Exception as err: + logger.warning("host handler %s failed: %s", method, err) + self._send_response( + rpc_id, + error={ + "code": -32000, + "message": str(err) or err.__class__.__name__, + "data": {"code": "host_handler_failed"}, + }, + ) continue except Exception: # Any unexpected exception in the reader loop still needs @@ -518,80 +527,6 @@ def _read_loop(self) -> None: # instead of waiting for each 30 s per-request timeout. self._abort_pending("bridge subprocess exited") - def _dispatch_host_request( - self, - rpc_id: Any, - method: str, - params: dict[str, Any], - ) -> None: - """Queue reverse RPC work without ever blocking the reader thread.""" - if self._closed: - return - try: - self._host_handler_queue.put_nowait((rpc_id, method, params)) - except queue.Full: - logger.warning("host handler queue full; rejecting %s", method) - self._send_response( - rpc_id, - error={ - "code": -32000, - "message": "host handler queue is full", - "data": {"code": "host_handler_busy"}, - }, - ) - - def _host_handler_loop(self) -> None: - """Run reverse RPC handlers serially away from stdout demultiplexing.""" - while True: - request = self._host_handler_queue.get() - try: - if request is None: - return - rpc_id, method, params = request - if self._closed: - continue - handler = self._host_handler_for(method) - if handler is None: - self._send_response( - rpc_id, - error={ - "code": -32601, - "message": f"method not found: {method}", - "data": {"code": "unknown_method"}, - }, - ) - continue - try: - result = handler(params) - self._send_response(rpc_id, result=result) - except Exception as err: - logger.warning("host handler %s failed: %s", method, err) - self._send_response( - rpc_id, - error={ - "code": -32000, - "message": str(err) or err.__class__.__name__, - "data": {"code": "host_handler_failed"}, - }, - ) - finally: - self._host_handler_queue.task_done() - - def _stop_host_handler_worker(self) -> None: - """Discard queued callbacks and ask the daemon worker to exit.""" - if self._host_handler_stop.is_set(): - return - self._host_handler_stop.set() - while True: - try: - self._host_handler_queue.get_nowait() - except queue.Empty: - break - else: - self._host_handler_queue.task_done() - with contextlib.suppress(queue.Full): - self._host_handler_queue.put_nowait(None) - def _host_handler_for( self, method: str, @@ -633,7 +568,7 @@ def _send_response( try: self._proc.stdin.write(json.dumps(payload, ensure_ascii=False) + "\n") self._proc.stdin.flush() - except (BrokenPipeError, OSError, ValueError): + except (BrokenPipeError, OSError): pass def _stderr_loop(self) -> None: diff --git a/apps/memos-local-plugin/agent-contract/dto.ts b/apps/memos-local-plugin/agent-contract/dto.ts index b76232bee..e9ae71101 100644 --- a/apps/memos-local-plugin/agent-contract/dto.ts +++ b/apps/memos-local-plugin/agent-contract/dto.ts @@ -103,11 +103,6 @@ export interface TurnInputDTO { contextHints?: Record; /** Wall-clock when the turn began. */ ts: EpochMs; - /** - * Absolute adapter deadline for foreground work. Every pipeline stage - * shares this budget; it is not reset after relation or intent handling. - */ - deadlineAt?: EpochMs; } export interface TurnResultDTO { diff --git a/apps/memos-local-plugin/bridge/methods.ts b/apps/memos-local-plugin/bridge/methods.ts index 1e6cc27a0..2bc1084a6 100644 --- a/apps/memos-local-plugin/bridge/methods.ts +++ b/apps/memos-local-plugin/bridge/methods.ts @@ -389,15 +389,6 @@ function validateTurnInput(p: Record): void { "turn.start: optional 'turnKey' must be a string", ); } - if ( - p.deadlineAt !== undefined && - (typeof p.deadlineAt !== "number" || !Number.isFinite(p.deadlineAt)) - ) { - throw new MemosError( - "invalid_argument", - "turn.start: optional 'deadlineAt' must be a finite number", - ); - } } function validateTurnResult(p: Record): void { diff --git a/apps/memos-local-plugin/core/embedding/embedder.ts b/apps/memos-local-plugin/core/embedding/embedder.ts index 5d035b6ce..bd4a5fe92 100644 --- a/apps/memos-local-plugin/core/embedding/embedder.ts +++ b/apps/memos-local-plugin/core/embedding/embedder.ts @@ -18,7 +18,6 @@ import { ERROR_CODES, MemosError } from "../../agent-contract/errors.js"; import { rootLogger } from "../logger/index.js"; import type { Logger } from "../logger/types.js"; import type { EmbeddingVector } from "../types.js"; -import { extractRetryDiagnostics } from "../util/retry-after.js"; import { LruEmbedCache, NullEmbedCache, @@ -33,7 +32,6 @@ import { MistralEmbeddingProvider } from "./providers/mistral.js"; import { OpenAiEmbeddingProvider } from "./providers/openai.js"; import { VoyageEmbeddingProvider } from "./providers/voyage.js"; import type { - EmbedCallOptions, EmbedInput, EmbedRole, EmbedStats, @@ -89,10 +87,6 @@ export function createEmbedderWithProvider( code?: string; at?: number; durationMs?: number; - retryAfterMs?: number; - retryAt?: number; - retryDecision?: "wait" | "defer" | "stop"; - retryReason?: string; }): void { if (!config.onStatus) return; try { @@ -102,17 +96,13 @@ export function createEmbedderWithProvider( } } - async function embedOne( - input: string | EmbedInput, - options?: EmbedCallOptions, - ): Promise { - const vecs = await embedMany([input], options); + async function embedOne(input: string | EmbedInput): Promise { + const vecs = await embedMany([input]); return vecs[0]!; } async function embedMany( inputs: Array, - options?: EmbedCallOptions, ): Promise { requests += inputs.length; if (inputs.length === 0) return []; @@ -189,8 +179,6 @@ export function createEmbedderWithProvider( const ctx: ProviderCallCtx = { config, log: providerCtxLog, - signal: options?.signal, - deadlineAt: options?.deadlineAt, }; raw = await provider.embed(texts, role, ctx); // Record success but DO NOT clear `lastError` — the viewer @@ -235,7 +223,6 @@ export function createEmbedderWithProvider( message: errMessage, code: err instanceof MemosError ? err.code : undefined, at: errAt, - ...extractRetryDiagnostics(err instanceof MemosError ? err.details : undefined), }); } catch { /* sink errors are non-fatal */ @@ -249,7 +236,6 @@ export function createEmbedderWithProvider( code: err instanceof MemosError ? err.code : undefined, at: errAt, durationMs: errAt - startedAt, - ...extractRetryDiagnostics(err instanceof MemosError ? err.details : undefined), }); throw err instanceof MemosError ? err diff --git a/apps/memos-local-plugin/core/embedding/fetcher.ts b/apps/memos-local-plugin/core/embedding/fetcher.ts index 303dae28e..40524aac7 100644 --- a/apps/memos-local-plugin/core/embedding/fetcher.ts +++ b/apps/memos-local-plugin/core/embedding/fetcher.ts @@ -8,15 +8,6 @@ */ import { ERROR_CODES, MemosError } from "../../agent-contract/errors.js"; -import { - getRetryCooldown, - parseRetryAfterMs, - planRetry, - recordRetryCooldown, - retryCooldownKey, - type RetryPlan, - waitForRetry, -} from "../util/retry-after.js"; import type { EmbeddingProviderName, ProviderLogger } from "./types.js"; export interface HttpPostOpts { @@ -26,10 +17,6 @@ export interface HttpPostOpts { timeoutMs?: number; maxRetries?: number; signal?: AbortSignal; - /** Absolute end-to-end deadline. Unlike timeoutMs, this is not renewed per attempt. */ - deadlineAt?: number; - /** Model/deployment scope; prevents one model cooldown from blocking another. */ - cooldownScope?: string; provider: EmbeddingProviderName; log: ProviderLogger; } @@ -39,33 +26,11 @@ export async function httpPostJson(opts: HttpPostOpts): Promise< const maxRetries = opts.maxRetries ?? 2; let attempt = 0; let lastErr: unknown = null; - const cooldownKey = retryCooldownKey("embedding", opts.provider, opts.url, opts.cooldownScope); while (attempt <= maxRetries) { attempt++; const start = Date.now(); try { - const cooldown = getRetryCooldown(cooldownKey, start); - if (cooldown) { - const details = { - provider: opts.provider, - url: opts.url, - status: cooldown.status, - attempt, - maxRetries, - retryAfterMs: cooldown.retryAfterMs, - retryAt: cooldown.retryAt, - retryDecision: "defer", - retryReason: "cooldown_active", - remainingDeadlineMs: remainingDeadlineMs(opts.deadlineAt, start), - }; - opts.log.warn("http.retry_cooldown", details); - throw new MemosError( - ERROR_CODES.EMBEDDING_UNAVAILABLE, - `${opts.provider} is cooling down until ${new Date(cooldown.retryAt).toISOString()}`, - details, - ); - } const signal = mergeSignals(opts.signal, AbortSignal.timeout(timeoutMs)); const resp = await fetch(opts.url, { method: "POST", @@ -81,62 +46,21 @@ export async function httpPostJson(opts: HttpPostOpts): Promise< if (!resp.ok) { const text = await safeText(resp); const transient = resp.status >= 500 || resp.status === 429; - const retryAfterMs = resp.status === 429 || resp.status === 503 - ? parseRetryAfterMs(resp.headers.get("Retry-After")) - : null; - if (retryAfterMs !== null) { - recordRetryCooldown(cooldownKey, { - retryAfterMs, - retryAt: Date.now() + retryAfterMs, - status: resp.status, - }); - } opts.log.warn("http.non_ok", { url: opts.url, status: resp.status, attempt, transient, - retryAfterMs, durationMs: Date.now() - start, }); if (transient && attempt <= maxRetries) { - const plan = planRetry({ - attempt, - baseMs: 200, - jitterMaxMs: 100, - retryAfterMs, - deadlineAt: opts.deadlineAt, - }); - const retryDetails = retryPlanDetails(plan, opts, maxRetries, resp.status, attempt); - if (plan.action === "defer") { - opts.log.warn("http.retry_deferred", retryDetails); - throw new MemosError( - ERROR_CODES.EMBEDDING_UNAVAILABLE, - `HTTP ${resp.status} from ${opts.provider}; retry deferred until ${new Date(plan.retryAt).toISOString()}`, - retryDetails, - ); - } - opts.log.warn("http.retry_scheduled", retryDetails); - await waitForRetry(plan.delayMs, opts.signal); + await backoff(attempt); continue; } throw new MemosError( ERROR_CODES.EMBEDDING_UNAVAILABLE, `HTTP ${resp.status} from ${opts.provider}`, - { - provider: opts.provider, - url: opts.url, - status: resp.status, - body: text, - ...(retryAfterMs === null - ? {} - : { - retryAfterMs, - retryAt: Date.now() + retryAfterMs, - retryDecision: "stop", - retryReason: "retries_exhausted", - }), - }, + { provider: opts.provider, url: opts.url, status: resp.status, body: text }, ); } @@ -159,23 +83,7 @@ export async function httpPostJson(opts: HttpPostOpts): Promise< durationMs: Date.now() - start, }); if (transient && attempt <= maxRetries) { - const plan = planRetry({ - attempt, - baseMs: 200, - jitterMaxMs: 100, - deadlineAt: opts.deadlineAt, - }); - const retryDetails = retryPlanDetails(plan, opts, maxRetries, null, attempt); - if (plan.action === "defer") { - opts.log.warn("http.retry_deferred", retryDetails); - throw new MemosError( - ERROR_CODES.EMBEDDING_UNAVAILABLE, - `${opts.provider} retry cannot fit the request deadline`, - retryDetails, - ); - } - opts.log.warn("http.retry_scheduled", retryDetails); - await waitForRetry(plan.delayMs, opts.signal); + await backoff(attempt); continue; } throw new MemosError( @@ -215,32 +123,11 @@ function isTransientError(err: unknown): boolean { return false; } -function retryPlanDetails( - plan: RetryPlan, - opts: HttpPostOpts, - maxRetries: number, - status: number | null, - attempt: number, -): Record { - return { - provider: opts.provider, - url: opts.url, - status, - attempt, - maxRetries, - backoffMs: plan.backoffMs, - plannedDelayMs: plan.delayMs, - retryAfterMs: plan.retryAfterMs, - retryAt: plan.retryAt, - retrySource: plan.source, - retryDecision: plan.action, - ...(plan.action === "defer" ? { retryReason: plan.reason } : {}), - remainingDeadlineMs: remainingDeadlineMs(opts.deadlineAt), - }; -} - -function remainingDeadlineMs(deadlineAt?: number, nowMs: number = Date.now()): number | null { - return deadlineAt === undefined ? null : Math.max(0, deadlineAt - nowMs); +async function backoff(attempt: number): Promise { + const base = 200; + const jitter = Math.floor(Math.random() * 100); + const ms = base * 2 ** (attempt - 1) + jitter; + await new Promise((r) => setTimeout(r, ms)); } function mergeSignals(a: AbortSignal | undefined, b: AbortSignal): AbortSignal { diff --git a/apps/memos-local-plugin/core/embedding/index.ts b/apps/memos-local-plugin/core/embedding/index.ts index f6f4b1ed9..99faa0048 100644 --- a/apps/memos-local-plugin/core/embedding/index.ts +++ b/apps/memos-local-plugin/core/embedding/index.ts @@ -19,7 +19,6 @@ export { l2Normalize, enforceDim, postProcess, toFloat32 } from "./normalize.js" export { createEmbeddingRetryWorker, systemErrorEvent } from "./retry-worker.js"; export type { EmbeddingRetryWorker } from "./retry-worker.js"; export type { - EmbedCallOptions, EmbedInput, EmbedRole, EmbedStats, diff --git a/apps/memos-local-plugin/core/embedding/providers/cohere.ts b/apps/memos-local-plugin/core/embedding/providers/cohere.ts index 58214d341..891cd4506 100644 --- a/apps/memos-local-plugin/core/embedding/providers/cohere.ts +++ b/apps/memos-local-plugin/core/embedding/providers/cohere.ts @@ -21,7 +21,7 @@ export class CohereEmbeddingProvider implements EmbeddingProvider { readonly name: EmbeddingProviderName = "cohere"; async embed(texts: string[], role: EmbedRole, ctx: ProviderCallCtx): Promise { - const { config, log, signal, deadlineAt } = ctx; + const { config, log, signal } = ctx; if (!config.apiKey) { throw new MemosError( ERROR_CODES.EMBEDDING_UNAVAILABLE, @@ -49,8 +49,6 @@ export class CohereEmbeddingProvider implements EmbeddingProvider { timeoutMs: config.timeoutMs, maxRetries: config.maxRetries, signal, - deadlineAt, - cooldownScope: config.model, provider: this.name, log, }); diff --git a/apps/memos-local-plugin/core/embedding/providers/gemini.ts b/apps/memos-local-plugin/core/embedding/providers/gemini.ts index 68ba22708..91d97acba 100644 --- a/apps/memos-local-plugin/core/embedding/providers/gemini.ts +++ b/apps/memos-local-plugin/core/embedding/providers/gemini.ts @@ -22,7 +22,7 @@ export class GeminiEmbeddingProvider implements EmbeddingProvider { readonly name: EmbeddingProviderName = "gemini"; async embed(texts: string[], role: EmbedRole, ctx: ProviderCallCtx): Promise { - const { config, log, signal, deadlineAt } = ctx; + const { config, log, signal } = ctx; if (!config.apiKey) { throw new MemosError( ERROR_CODES.EMBEDDING_UNAVAILABLE, @@ -50,8 +50,6 @@ export class GeminiEmbeddingProvider implements EmbeddingProvider { timeoutMs: config.timeoutMs, maxRetries: config.maxRetries, signal, - deadlineAt, - cooldownScope: config.model, provider: this.name, log, }); diff --git a/apps/memos-local-plugin/core/embedding/providers/mistral.ts b/apps/memos-local-plugin/core/embedding/providers/mistral.ts index 7eace8ab7..21f50769f 100644 --- a/apps/memos-local-plugin/core/embedding/providers/mistral.ts +++ b/apps/memos-local-plugin/core/embedding/providers/mistral.ts @@ -23,7 +23,7 @@ export class MistralEmbeddingProvider implements EmbeddingProvider { readonly name: EmbeddingProviderName = "mistral"; async embed(texts: string[], _role: EmbedRole, ctx: ProviderCallCtx): Promise { - const { config, log, signal, deadlineAt } = ctx; + const { config, log, signal } = ctx; if (!config.apiKey) { throw new MemosError( ERROR_CODES.EMBEDDING_UNAVAILABLE, @@ -46,8 +46,6 @@ export class MistralEmbeddingProvider implements EmbeddingProvider { timeoutMs: config.timeoutMs, maxRetries: config.maxRetries, signal, - deadlineAt, - cooldownScope: config.model, provider: this.name, log, }); diff --git a/apps/memos-local-plugin/core/embedding/providers/openai.ts b/apps/memos-local-plugin/core/embedding/providers/openai.ts index 57d852639..c0df47831 100644 --- a/apps/memos-local-plugin/core/embedding/providers/openai.ts +++ b/apps/memos-local-plugin/core/embedding/providers/openai.ts @@ -27,7 +27,7 @@ export class OpenAiEmbeddingProvider implements EmbeddingProvider { readonly name: EmbeddingProviderName = "openai_compatible"; async embed(texts: string[], _role: EmbedRole, ctx: ProviderCallCtx): Promise { - const { config, log, signal, deadlineAt } = ctx; + const { config, log, signal } = ctx; if (!config.apiKey) { throw new MemosError( ERROR_CODES.EMBEDDING_UNAVAILABLE, @@ -53,8 +53,6 @@ export class OpenAiEmbeddingProvider implements EmbeddingProvider { timeoutMs: config.timeoutMs, maxRetries: config.maxRetries, signal, - deadlineAt, - cooldownScope: config.model, provider: this.name, log, }); diff --git a/apps/memos-local-plugin/core/embedding/providers/voyage.ts b/apps/memos-local-plugin/core/embedding/providers/voyage.ts index 6f36fe6d8..f89eca832 100644 --- a/apps/memos-local-plugin/core/embedding/providers/voyage.ts +++ b/apps/memos-local-plugin/core/embedding/providers/voyage.ts @@ -23,7 +23,7 @@ export class VoyageEmbeddingProvider implements EmbeddingProvider { readonly name: EmbeddingProviderName = "voyage"; async embed(texts: string[], role: EmbedRole, ctx: ProviderCallCtx): Promise { - const { config, log, signal, deadlineAt } = ctx; + const { config, log, signal } = ctx; if (!config.apiKey) { throw new MemosError( ERROR_CODES.EMBEDDING_UNAVAILABLE, @@ -50,8 +50,6 @@ export class VoyageEmbeddingProvider implements EmbeddingProvider { timeoutMs: config.timeoutMs, maxRetries: config.maxRetries, signal, - deadlineAt, - cooldownScope: config.model, provider: this.name, log, }); diff --git a/apps/memos-local-plugin/core/embedding/retry-worker.ts b/apps/memos-local-plugin/core/embedding/retry-worker.ts index 34db37bff..3620fd9e3 100644 --- a/apps/memos-local-plugin/core/embedding/retry-worker.ts +++ b/apps/memos-local-plugin/core/embedding/retry-worker.ts @@ -35,7 +35,6 @@ export function createEmbeddingRetryWorker( const workerId = `embedding-retry-${ids.span()}`; let timer: ReturnType | null = null; let running: Promise | null = null; - let stopped = false; async function runOnce(): Promise { if (!deps.embedder) return; @@ -104,11 +103,6 @@ export function createEmbeddingRetryWorker( const message = err instanceof Error ? err.message : String(err); const at = now(); const terminal = attemptNo >= job.maxAttempts; - const providerRetryAt = retryAtFromError(err, at); - const nextAttemptAt = Math.max( - at + backoffMs(attemptNo), - providerRetryAt ?? 0, - ); const recorded = terminal ? deps.repos.embeddingRetryQueue.markFailedClaimed(job.id, { ...claim, @@ -119,7 +113,7 @@ export function createEmbeddingRetryWorker( : deps.repos.embeddingRetryQueue.markRetryClaimed(job.id, { ...claim, attempts: attemptNo, - nextAttemptAt, + nextAttemptAt: at + backoffMs(attemptNo), error: message, now: at, }); @@ -127,10 +121,7 @@ export function createEmbeddingRetryWorker( deps.log.debug("embedding_retry.stale_failure_ignored", { jobId: job.id, terminal }); return; } - emitFailure(job, attemptNo, message, terminal, at, { - providerRetryAt, - nextAttemptAt: terminal ? null : nextAttemptAt, - }); + emitFailure(job, attemptNo, message, terminal, at); } } @@ -162,7 +153,6 @@ export function createEmbeddingRetryWorker( message: string, terminal: boolean, at: number, - retry: { providerRetryAt: number | null; nextAttemptAt: number | null }, ): void { const payload = { kind: "embedding.retry_failed", @@ -174,8 +164,6 @@ export function createEmbeddingRetryWorker( maxAttempts: job.maxAttempts, terminal, message, - providerRetryAt: retry.providerRetryAt, - nextAttemptAt: retry.nextAttemptAt, }; deps.log.warn("embedding_retry.failed", payload); try { @@ -194,7 +182,7 @@ export function createEmbeddingRetryWorker( } function tick(): void { - if (stopped || running) return; + if (running) return; running = runOnce().finally(() => { running = null; }); @@ -202,30 +190,21 @@ export function createEmbeddingRetryWorker( return { start(): void { - if (stopped || timer || !deps.embedder) return; + if (timer || !deps.embedder) return; tick(); timer = setInterval(tick, deps.intervalMs ?? DEFAULT_INTERVAL_MS); }, stop(): void { - stopped = true; if (timer) clearInterval(timer); timer = null; }, async flush(): Promise { - if (!stopped) tick(); + tick(); if (running) await running; }, }; } -function retryAtFromError(err: unknown, nowMs: number): number | null { - if (!err || typeof err !== "object") return null; - const details = (err as { details?: unknown }).details; - if (!details || typeof details !== "object") return null; - const retryAt = Number((details as { retryAt?: unknown }).retryAt); - return Number.isSafeInteger(retryAt) && retryAt > nowMs ? retryAt : null; -} - function backoffMs(attemptNo: number): number { return Math.min(MAX_BACKOFF_MS, BASE_BACKOFF_MS * 2 ** Math.max(0, attemptNo - 1)); } diff --git a/apps/memos-local-plugin/core/embedding/types.ts b/apps/memos-local-plugin/core/embedding/types.ts index 95726703c..4f3f5eb99 100644 --- a/apps/memos-local-plugin/core/embedding/types.ts +++ b/apps/memos-local-plugin/core/embedding/types.ts @@ -6,7 +6,6 @@ */ import type { EmbeddingVector } from "../types.js"; -import type { RetryDiagnosticDetails } from "../util/retry-after.js"; // ─── Config ────────────────────────────────────────────────────────────────── @@ -63,7 +62,7 @@ export interface EmbeddingConfig { onStatus?: (detail: EmbeddingStatusDetail) => void; } -export interface EmbeddingErrorDetail extends RetryDiagnosticDetails { +export interface EmbeddingErrorDetail { kind: "embedding"; provider: EmbeddingProviderName | string; model: string; @@ -74,7 +73,7 @@ export interface EmbeddingErrorDetail extends RetryDiagnosticDetails { at?: number; } -export interface EmbeddingStatusDetail extends RetryDiagnosticDetails { +export interface EmbeddingStatusDetail { kind: "embedding"; status: "ok" | "error"; provider: EmbeddingProviderName | string; @@ -129,8 +128,6 @@ export interface ProviderCallCtx { log: ProviderLogger; /** AbortSignal honored across HTTP + native calls. */ signal?: AbortSignal; - /** Absolute end-to-end deadline shared across provider retry attempts. */ - deadlineAt?: number; } export interface ProviderLogger { @@ -169,16 +166,13 @@ export interface Embedder { /** Model identifier as configured by the operator (e.g. "bge-m3"). */ readonly model: string; - embedOne(input: string | EmbedInput, options?: EmbedCallOptions): Promise; + embedOne(input: string | EmbedInput): Promise; /** * Batch-embed many texts. Results keep input order. Duplicates are deduped * internally so a text repeated N times causes 1 cache miss max. */ - embedMany( - inputs: Array, - options?: EmbedCallOptions, - ): Promise; + embedMany(inputs: Array): Promise; stats(): EmbedStats; @@ -187,12 +181,6 @@ export interface Embedder { close(): Promise; } -export interface EmbedCallOptions { - signal?: AbortSignal; - /** Absolute end-to-end deadline shared across provider retry attempts. */ - deadlineAt?: number; -} - // ─── Errors ────────────────────────────────────────────────────────────────── export interface ProviderHttpFailure { diff --git a/apps/memos-local-plugin/core/index.ts b/apps/memos-local-plugin/core/index.ts index c5e4fab9d..d2b979bce 100644 --- a/apps/memos-local-plugin/core/index.ts +++ b/apps/memos-local-plugin/core/index.ts @@ -110,7 +110,6 @@ export { MistralEmbeddingProvider, type EmbedCache, type EmbedCacheStats, - type EmbedCallOptions, type EmbedInput, type EmbedRole, type EmbedStats, diff --git a/apps/memos-local-plugin/core/llm/client.ts b/apps/memos-local-plugin/core/llm/client.ts index ee456ac12..6bedafa70 100644 --- a/apps/memos-local-plugin/core/llm/client.ts +++ b/apps/memos-local-plugin/core/llm/client.ts @@ -18,7 +18,6 @@ import { ERROR_CODES, MemosError } from "../../agent-contract/errors.js"; import { rootLogger } from "../logger/index.js"; import type { Logger } from "../logger/types.js"; -import { extractRetryDiagnostics } from "../util/retry-after.js"; import { getHostLlmBridge } from "./host-bridge.js"; import { buildJsonSystemHint, parseLlmJson } from "./json-mode.js"; import { AnthropicLlmProvider } from "./providers/anthropic.js"; @@ -293,7 +292,6 @@ export function createLlmClientWithProvider( }, log: pLog, signal: opts?.signal, - deadlineAt: opts?.deadlineAt, }; } @@ -372,7 +370,6 @@ export function createLlmClientWithProvider( model: config.model, message: summarizeErrMessage(hostErr), code: hostErr instanceof MemosError ? hostErr.code : undefined, - ...extractRetryDiagnostics(hostErr instanceof MemosError ? hostErr.details : undefined), at: failAt, durationMs: Date.now() - startedAt, fallbackProvider: "host", @@ -398,7 +395,6 @@ export function createLlmClientWithProvider( model: config.model, message: summarizeErrMessage(err), code: err instanceof MemosError ? err.code : undefined, - ...extractRetryDiagnostics(err instanceof MemosError ? err.details : undefined), at: failAt, durationMs: Date.now() - startedAt, op, @@ -429,7 +425,6 @@ export function createLlmClientWithProvider( message: summarizeErrMessage(err), code: err instanceof MemosError ? err.code : undefined, at: Date.now(), - ...extractRetryDiagnostics(err instanceof MemosError ? err.details : undefined), }); } catch { /* sink errors are non-fatal */ @@ -449,10 +444,6 @@ export function createLlmClientWithProvider( op?: string; episodeId?: string; phase?: string; - retryAfterMs?: number; - retryAt?: number; - retryDecision?: "wait" | "defer" | "stop"; - retryReason?: string; }): void { if (!config.onStatus) return; try { @@ -625,7 +616,6 @@ export function createLlmClientWithProvider( model: config.model, message: summarizeErrMessage(err), code: err instanceof MemosError ? err.code : undefined, - ...extractRetryDiagnostics(err instanceof MemosError ? err.details : undefined), at: failAt, durationMs: Date.now() - start, op: opts?.op ?? "stream", @@ -679,7 +669,6 @@ export function createLlmClientWithProvider( model: config.model, message: summarizeErrMessage(primaryErr), code: primaryErr instanceof MemosError ? primaryErr.code : undefined, - ...extractRetryDiagnostics(primaryErr instanceof MemosError ? primaryErr.details : undefined), at: fallbackAt, durationMs: completion.durationMs, fallbackProvider: "host", diff --git a/apps/memos-local-plugin/core/llm/fetcher.ts b/apps/memos-local-plugin/core/llm/fetcher.ts index 358c73275..53eb55ec3 100644 --- a/apps/memos-local-plugin/core/llm/fetcher.ts +++ b/apps/memos-local-plugin/core/llm/fetcher.ts @@ -11,15 +11,6 @@ */ import { ERROR_CODES, MemosError } from "../../agent-contract/errors.js"; -import { - getRetryCooldown, - parseRetryAfterMs, - planRetry, - recordRetryCooldown, - retryCooldownKey, - type RetryPlan, - waitForRetry, -} from "../util/retry-after.js"; import type { LlmProviderLogger, LlmProviderName } from "./types.js"; export interface HttpPostOpts { @@ -29,10 +20,6 @@ export interface HttpPostOpts { timeoutMs: number; maxRetries: number; signal?: AbortSignal; - /** Absolute end-to-end deadline. Unlike timeoutMs, this is not renewed per attempt. */ - deadlineAt?: number; - /** Model/deployment scope; prevents one model cooldown from blocking another. */ - cooldownScope?: string; provider: LlmProviderName; log: LlmProviderLogger; onRetry?: (attempt: number) => void; @@ -48,33 +35,11 @@ export async function httpPostJson(opts: HttpPostOpts): Promise< }> { let attempt = 0; let lastErr: unknown = null; - const cooldownKey = retryCooldownKey("llm", opts.provider, opts.url, opts.cooldownScope); while (attempt <= opts.maxRetries) { attempt++; const start = Date.now(); try { - const cooldown = getRetryCooldown(cooldownKey, start); - if (cooldown) { - const details = { - provider: opts.provider, - url: opts.url, - status: cooldown.status, - attempt, - maxRetries: opts.maxRetries, - retryAfterMs: cooldown.retryAfterMs, - retryAt: cooldown.retryAt, - retryDecision: "defer", - retryReason: "cooldown_active", - remainingDeadlineMs: remainingDeadlineMs(opts.deadlineAt, start), - }; - opts.log.warn("http.retry_cooldown", details); - throw new MemosError( - errCodeForStatus(cooldown.status), - `${opts.provider} is cooling down until ${new Date(cooldown.retryAt).toISOString()}`, - details, - ); - } const signal = mergeSignals(opts.signal, AbortSignal.timeout(opts.timeoutMs)); const resp = await fetch(opts.url, { method: "POST", @@ -91,63 +56,22 @@ export async function httpPostJson(opts: HttpPostOpts): Promise< if (!resp.ok) { const text = await safeText(resp); const transient = resp.status >= 500 || resp.status === 429; - const retryAfterMs = resp.status === 429 || resp.status === 503 - ? parseRetryAfterMs(resp.headers.get("Retry-After")) - : null; - if (retryAfterMs !== null) { - recordRetryCooldown(cooldownKey, { - retryAfterMs, - retryAt: Date.now() + retryAfterMs, - status: resp.status, - }); - } opts.log.warn("http.non_ok", { status: resp.status, attempt, transient, durationMs: ms, - retryAfterMs, body: truncateLogBody(text), }); if (transient && attempt <= opts.maxRetries) { - const plan = planRetry({ - attempt, - baseMs: 250, - jitterMaxMs: 120, - retryAfterMs, - deadlineAt: opts.deadlineAt, - }); - const retryDetails = retryPlanDetails(plan, opts, resp.status, attempt); - if (plan.action === "defer") { - opts.log.warn("http.retry_deferred", retryDetails); - throw new MemosError( - errCodeForStatus(resp.status), - `HTTP ${resp.status} from ${opts.provider}; retry deferred until ${new Date(plan.retryAt).toISOString()}`, - retryDetails, - ); - } - opts.log.warn("http.retry_scheduled", retryDetails); opts.onRetry?.(attempt); - await waitForRetry(plan.delayMs, opts.signal); + await backoff(attempt); continue; } throw new MemosError( errCodeForStatus(resp.status), `HTTP ${resp.status} from ${opts.provider}`, - { - provider: opts.provider, - url: opts.url, - status: resp.status, - body: text, - ...(retryAfterMs === null - ? {} - : { - retryAfterMs, - retryAt: Date.now() + retryAfterMs, - retryDecision: "stop", - retryReason: "retries_exhausted", - }), - }, + { provider: opts.provider, url: opts.url, status: resp.status, body: text }, ); } @@ -161,15 +85,8 @@ export async function httpPostJson(opts: HttpPostOpts): Promise< } catch (err) { lastErr = err; if (err instanceof MemosError) throw err; - if (opts.signal?.aborted) { - throw new MemosError( - ERROR_CODES.LLM_TIMEOUT, - `${opts.provider} request was cancelled`, - { provider: opts.provider, url: opts.url, cancelled: true }, - ); - } const transient = isTransientError(err); - const timedOut = isTimeout(err) || opts.signal?.aborted === true; + const timedOut = isTimeout(err); opts.log.warn("http.exception", { attempt, transient, @@ -177,24 +94,8 @@ export async function httpPostJson(opts: HttpPostOpts): Promise< err: toErrDetail(err), }); if ((transient || timedOut) && attempt <= opts.maxRetries) { - const plan = planRetry({ - attempt, - baseMs: 250, - jitterMaxMs: 120, - deadlineAt: opts.deadlineAt, - }); - const retryDetails = retryPlanDetails(plan, opts, null, attempt); - if (plan.action === "defer") { - opts.log.warn("http.retry_deferred", retryDetails); - throw new MemosError( - timedOut ? ERROR_CODES.LLM_TIMEOUT : ERROR_CODES.LLM_UNAVAILABLE, - `${opts.provider} retry cannot fit the request deadline`, - retryDetails, - ); - } - opts.log.warn("http.retry_scheduled", retryDetails); opts.onRetry?.(attempt); - await waitForRetry(plan.delayMs, opts.signal); + await backoff(attempt); continue; } if (timedOut) { @@ -344,31 +245,11 @@ function isTimeout(err: unknown): boolean { return false; } -function retryPlanDetails( - plan: RetryPlan, - opts: HttpPostOpts, - status: number | null, - attempt: number, -): Record { - return { - provider: opts.provider, - url: opts.url, - status, - attempt, - maxRetries: opts.maxRetries, - backoffMs: plan.backoffMs, - plannedDelayMs: plan.delayMs, - retryAfterMs: plan.retryAfterMs, - retryAt: plan.retryAt, - retrySource: plan.source, - retryDecision: plan.action, - ...(plan.action === "defer" ? { retryReason: plan.reason } : {}), - remainingDeadlineMs: remainingDeadlineMs(opts.deadlineAt), - }; -} - -function remainingDeadlineMs(deadlineAt?: number, nowMs: number = Date.now()): number | null { - return deadlineAt === undefined ? null : Math.max(0, deadlineAt - nowMs); +async function backoff(attempt: number): Promise { + const base = 250; + const jitter = Math.floor(Math.random() * 120); + const ms = base * 2 ** (attempt - 1) + jitter; + await new Promise((r) => setTimeout(r, ms)); } function mergeSignals(a: AbortSignal | undefined, b: AbortSignal): AbortSignal { diff --git a/apps/memos-local-plugin/core/llm/providers/anthropic.ts b/apps/memos-local-plugin/core/llm/providers/anthropic.ts index 6c510ef75..66a9ee446 100644 --- a/apps/memos-local-plugin/core/llm/providers/anthropic.ts +++ b/apps/memos-local-plugin/core/llm/providers/anthropic.ts @@ -31,7 +31,7 @@ export class AnthropicLlmProvider implements LlmProvider { opts: ProviderCallInput, ctx: LlmProviderCtx, ): Promise { - const { config, log, signal, deadlineAt } = ctx; + const { config, log, signal } = ctx; if (!config.apiKey) { throw new MemosError( ERROR_CODES.LLM_UNAVAILABLE, @@ -68,8 +68,6 @@ export class AnthropicLlmProvider implements LlmProvider { timeoutMs: config.timeoutMs, maxRetries: config.maxRetries, signal, - deadlineAt, - cooldownScope: config.model, provider: this.name, log, }); diff --git a/apps/memos-local-plugin/core/llm/providers/bedrock.ts b/apps/memos-local-plugin/core/llm/providers/bedrock.ts index d956b7111..7c00470e7 100644 --- a/apps/memos-local-plugin/core/llm/providers/bedrock.ts +++ b/apps/memos-local-plugin/core/llm/providers/bedrock.ts @@ -36,7 +36,7 @@ export class BedrockLlmProvider implements LlmProvider { opts: ProviderCallInput, ctx: LlmProviderCtx, ): Promise { - const { config, log, signal, deadlineAt } = ctx; + const { config, log, signal } = ctx; if (!config.endpoint || config.endpoint.length === 0) { throw new MemosError( ERROR_CODES.LLM_UNAVAILABLE, @@ -85,8 +85,6 @@ export class BedrockLlmProvider implements LlmProvider { timeoutMs: config.timeoutMs, maxRetries: config.maxRetries, signal, - deadlineAt, - cooldownScope: config.model, provider: this.name, log, }); diff --git a/apps/memos-local-plugin/core/llm/providers/gemini.ts b/apps/memos-local-plugin/core/llm/providers/gemini.ts index 6bfa323eb..4e6273b55 100644 --- a/apps/memos-local-plugin/core/llm/providers/gemini.ts +++ b/apps/memos-local-plugin/core/llm/providers/gemini.ts @@ -39,7 +39,7 @@ export class GeminiLlmProvider implements LlmProvider { opts: ProviderCallInput, ctx: LlmProviderCtx, ): Promise { - const { config, log, signal, deadlineAt } = ctx; + const { config, log, signal } = ctx; if (!config.apiKey) { throw new MemosError( ERROR_CODES.LLM_UNAVAILABLE, @@ -59,8 +59,6 @@ export class GeminiLlmProvider implements LlmProvider { timeoutMs: config.timeoutMs, maxRetries: config.maxRetries, signal, - deadlineAt, - cooldownScope: config.model, provider: this.name, log, }); diff --git a/apps/memos-local-plugin/core/llm/providers/openai.ts b/apps/memos-local-plugin/core/llm/providers/openai.ts index 562521756..d8a5a20af 100644 --- a/apps/memos-local-plugin/core/llm/providers/openai.ts +++ b/apps/memos-local-plugin/core/llm/providers/openai.ts @@ -51,7 +51,7 @@ export class OpenAiLlmProvider implements LlmProvider { opts: ProviderCallInput, ctx: LlmProviderCtx, ): Promise { - const { config, log, signal, deadlineAt } = ctx; + const { config, log, signal } = ctx; const url = normalizeEndpoint( config.endpoint && config.endpoint.length > 0 ? config.endpoint @@ -93,8 +93,6 @@ export class OpenAiLlmProvider implements LlmProvider { timeoutMs: config.timeoutMs, maxRetries: config.maxRetries, signal, - deadlineAt, - cooldownScope: config.model, provider: this.name, log, }); diff --git a/apps/memos-local-plugin/core/llm/types.ts b/apps/memos-local-plugin/core/llm/types.ts index 2dd761f32..a37a2677d 100644 --- a/apps/memos-local-plugin/core/llm/types.ts +++ b/apps/memos-local-plugin/core/llm/types.ts @@ -6,7 +6,6 @@ */ import type { ReasoningConfig as ConfigReasoningConfig } from "../config/schema.js"; -import type { RetryDiagnosticDetails } from "../util/retry-after.js"; // ─── Providers & config ────────────────────────────────────────────────────── @@ -89,7 +88,7 @@ export interface LlmCircuitBreakerConfig { now?: () => number; } -export interface LlmErrorDetail extends RetryDiagnosticDetails { +export interface LlmErrorDetail { provider: LlmProviderName | string; model: string; message: string; @@ -106,7 +105,7 @@ export interface LlmErrorDetail extends RetryDiagnosticDetails { role?: "llm" | "skillEvolver"; } -export interface LlmStatusDetail extends RetryDiagnosticDetails { +export interface LlmStatusDetail { status: "ok" | "fallback" | "error" | "circuit_open"; provider: LlmProviderName | string; model: string; @@ -150,8 +149,6 @@ export interface LlmCallOptions { maxTokens?: number; /** Per-call timeout. */ timeoutMs?: number; - /** Absolute end-to-end deadline shared across provider retry attempts. */ - deadlineAt?: number; /** AbortSignal honored across HTTP + host-bridge calls. */ signal?: AbortSignal; /** @@ -216,8 +213,6 @@ export interface LlmProviderCtx { log: LlmProviderLogger; /** Call abort signal; providers must honor it. */ signal?: AbortSignal; - /** Absolute end-to-end deadline; providers must not renew it per retry. */ - deadlineAt?: number; } export interface LlmProviderLogger { diff --git a/apps/memos-local-plugin/core/pipeline/deps.ts b/apps/memos-local-plugin/core/pipeline/deps.ts index 79714b35e..8a0772119 100644 --- a/apps/memos-local-plugin/core/pipeline/deps.ts +++ b/apps/memos-local-plugin/core/pipeline/deps.ts @@ -101,10 +101,6 @@ import type { import { wrapRetrievalRepos } from "./retrieval-repos.js"; import { createSemaphore } from "../util/semaphore.js"; import { rateLimitLlmClient } from "../util/rate-limited-llm.js"; -import { - prioritizeEmbedder, - type ForegroundResources, -} from "../util/foreground-resources.js"; // ─── Algorithm config slice helper ──────────────────────────────────────── @@ -212,23 +208,19 @@ export function buildPipelineSubscribers( buses: PipelineBuses, algorithm: PipelineAlgorithmConfig, session?: PipelineSessionSet, - resources?: ForegroundResources, ): PipelineSubscriberSet { const log = deps.log ?? rootLogger.child({ channel: "core.pipeline" }); const bgLlmSemaphore = createSemaphore(algorithm.session.bgLlmConcurrency); - const bgLlm = rateLimitLlmClient(deps.llm, bgLlmSemaphore, resources); - const bgReflectLlm = rateLimitLlmClient(deps.reflectLlm, bgLlmSemaphore, resources); - const bgL3Llm = rateLimitLlmClient(deps.l3Llm ?? deps.llm, bgLlmSemaphore, resources); - const bgEmbedder = resources - ? prioritizeEmbedder(deps.embedder, resources, "background") - : deps.embedder; + const bgLlm = rateLimitLlmClient(deps.llm, bgLlmSemaphore); + const bgReflectLlm = rateLimitLlmClient(deps.reflectLlm, bgLlmSemaphore); + const bgL3Llm = rateLimitLlmClient(deps.l3Llm ?? deps.llm, bgLlmSemaphore); const lightweightMode = algorithm.lightweightMemory.enabled; const captureRunner = createCaptureRunner({ tracesRepo: deps.repos.traces, embeddingRetryQueue: deps.repos.embeddingRetryQueue, episodesRepo: adaptEpisodesRepo(deps.repos.episodes), - embedder: bgEmbedder, + embedder: deps.embedder, llm: bgLlm, // Issue #2148: capture batch reflection emits JSON, so it must use // the main model rather than the potentially thinking-enabled @@ -335,7 +327,7 @@ export function buildPipelineSubscribers( const skillHandle = attachSkillSubscriber({ repos: deps.repos, - embedder: bgEmbedder, + embedder: deps.embedder, llm: bgLlm, bus: buses.skill, l2Bus: buses.l2, @@ -347,7 +339,7 @@ export function buildPipelineSubscribers( const feedbackHandle = attachFeedbackSubscriber({ repos: deps.repos, llm: bgLlm, - embedder: bgEmbedder, + embedder: deps.embedder, bus: buses.feedback, log: log.child({ channel: "core.feedback" }), config: algorithm.feedback, @@ -412,17 +404,14 @@ export function buildPipelineSession( export function buildRetrievalDeps( deps: PipelineDeps, algorithm: PipelineAlgorithmConfig, - resources?: ForegroundResources, ): RetrievalDeps { - const embedder = resources - ? prioritizeEmbedder(deps.embedder, resources, "foreground") - : deps.embedder; + const embedder = deps.embedder; return { repos: wrapRetrievalRepos(deps.repos, deps.namespace), embedder: embedder ? { - embed: (text, role, options) => - embedder.embedOne({ text, role: role ?? "query" }, options), + embed: (text, role) => + embedder.embedOne({ text, role: role ?? "query" }), } : { // Degraded mode: empty vector so vector-scoring falls back to diff --git a/apps/memos-local-plugin/core/pipeline/memory-core.ts b/apps/memos-local-plugin/core/pipeline/memory-core.ts index a0354bf64..c9254e092 100644 --- a/apps/memos-local-plugin/core/pipeline/memory-core.ts +++ b/apps/memos-local-plugin/core/pipeline/memory-core.ts @@ -1232,59 +1232,18 @@ export function createMemoryCore( const statsLine = `phase=${phase}, stored=${storedCount}` + (r.warnings.length > 0 ? `, warnings=${r.warnings.length}` : ""); - const action = phase === "lite" - ? ("stored" as const) - : ("reflected" as const); - const details = r.traces.flatMap((tc) => { - const items: Array<{ - role: "user" | "assistant" | "tool" | "reflection" | "other"; - action: typeof action; - summary: string | null; - content: string; - traceId: string; - }> = []; - - if (tc.userText) { - items.push({ - role: "user", - action, - summary: null, - content: tc.userText.slice(0, 400), - traceId: tc.traceId, - }); - } - if (tc.agentText) { - items.push({ - role: "assistant", - action, - summary: null, - content: tc.agentText.slice(0, 400), - traceId: tc.traceId, - }); - } - - const toolSummary = summarizeToolCalls(tc.toolCalls); - if (items.length === 0) { - items.push({ - role: toolSummary ? "tool" : "other", - action, - summary: tc.reflection?.text ?? null, - content: toolSummary.slice(0, 400), - traceId: tc.traceId, - }); - } else if (tc.reflection?.text) { - // Keep the existing reflect-phase summary visible without - // presenting it as either side's original chat content. - items.push({ - role: "reflection", - action, - summary: tc.reflection.text, - content: "", - traceId: tc.traceId, - }); - } - return items; - }); + const details = r.traces.map((tc) => ({ + role: inferTurnRole(tc), + action: phase === "lite" ? ("stored" as const) : ("reflected" as const), + summary: tc.reflection?.text ?? null, + content: ( + tc.userText || + tc.agentText || + summarizeToolCalls(tc.toolCalls) || + "" + ).slice(0, 400), + traceId: tc.traceId, + })); handle.repos.apiLogs.insert({ toolName: "memory_add", input: { @@ -6236,3 +6195,26 @@ function summarizeToolCalls( }) .join("\n"); } + +/** + * Heuristic role inference for api_logs "memory_add" rows — mirrors + * the legacy plugin's behaviour where each captured turn showed up + * labelled `user` / `assistant` / `tool` on the Logs page. + * + * Priority: if the step carries userText (the user's query), label it + * "user" even when toolCalls are present — this is the first sub-step + * of a multi-tool turn and semantically represents the user request. + */ +function inferTurnRole(step: { + userText?: string; + agentText?: string; + toolCalls?: readonly unknown[]; +}): "user" | "assistant" | "tool" | "other" { + const u = (step.userText ?? "").length; + const a = (step.agentText ?? "").length; + if (u > 0 && (step.toolCalls?.length ?? 0) > 0) return "user"; + if ((step.toolCalls?.length ?? 0) > 0) return "tool"; + if (u >= a && u > 0) return "user"; + if (a > 0) return "assistant"; + return "other"; +} diff --git a/apps/memos-local-plugin/core/pipeline/orchestrator.ts b/apps/memos-local-plugin/core/pipeline/orchestrator.ts index 4ed0427d1..ac48133ad 100644 --- a/apps/memos-local-plugin/core/pipeline/orchestrator.ts +++ b/apps/memos-local-plugin/core/pipeline/orchestrator.ts @@ -83,39 +83,29 @@ import { onBroadcastLog } from "../logger/transports/sse-broadcast.js"; import { createEmbeddingRetryWorker, systemErrorEvent } from "../embedding/index.js"; import type { EpisodeSnapshot } from "../session/index.js"; import type { IntentDecision, RelationDecision, TurnRelation } from "../session/types.js"; -import { - createForegroundResources, - prioritizeEmbedder, -} from "../util/foreground-resources.js"; -import { createRequestDeadline } from "../util/request-deadline.js"; function classifyWithTimeout( classifyFn: () => Promise, timeoutMs: number, log: Logger, ): Promise { - let timer: ReturnType | null = null; return Promise.race([ classifyFn(), - new Promise((_, reject) => { - timer = setTimeout(() => reject(new Error("classify_timeout")), timeoutMs); - }), - ]) - .catch((err) => { - log.warn("relation.classify_timeout", { - timeoutMs, - err: err instanceof Error ? err.message : String(err), - }); - return { - relation: "follow_up" as const, - confidence: 0, - reason: "classify_timeout", - signals: ["classify_timeout"], - }; - }) - .finally(() => { - if (timer) clearTimeout(timer); + new Promise((_, reject) => + setTimeout(() => reject(new Error("classify_timeout")), timeoutMs), + ), + ]).catch((err) => { + log.warn("relation.classify_timeout", { + timeoutMs, + err: err instanceof Error ? err.message : String(err), }); + return { + relation: "new_task" as const, + confidence: 0, + reason: "classify_timeout", + signals: ["classify_timeout"], + }; + }); } // ─── Factory ────────────────────────────────────────────────────────────── @@ -125,12 +115,6 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { const algorithm = extractAlgorithmConfig(deps); const lightweightMode = algorithm.lightweightMemory.enabled; const buses = buildPipelineBuses(); - const foregroundResources = createForegroundResources(); - const backgroundEmbedder = prioritizeEmbedder( - deps.embedder, - foregroundResources, - "background", - ); // Session + intent. const session = buildPipelineSession(deps, buses.session); @@ -139,13 +123,7 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { // Pass `session` so the reward runner's `getEpisodeSnapshot` hook // can resolve the live, in-memory episode (with turns populated) // rather than falling back to the empty row from SQLite. - const subs = buildPipelineSubscribers( - deps, - buses, - algorithm, - session, - foregroundResources, - ); + const subs = buildPipelineSubscribers(deps, buses, algorithm, session); // Core-event aggregator. Every internal bus funnels into one stream. const eventListeners = new Set<(e: CoreEvent) => void>(); @@ -182,7 +160,7 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { let retryEventSeq = 1_000_000; const embeddingRetryWorker = createEmbeddingRetryWorker({ repos: deps.repos, - embedder: backgroundEmbedder, + embedder: deps.embedder, log: log.child({ channel: "core.embedding.retry" }), now: deps.now, onSystemError: (payload, correlationId) => { @@ -388,7 +366,6 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { userText: string, meta: Record, turnTs?: number, - signal?: AbortSignal, ): Promise { const currentEpId = openEpisodeBySession.get(sessionId); if (currentEpId) { @@ -410,7 +387,6 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { userMessage: userText, ts: turnTs, meta: lightweightEpisodeMeta(meta), - signal, }); openEpisodeBySession.set(sessionId, snap.id as EpisodeId); return snap; @@ -457,20 +433,13 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { userText: string, meta: Record, agent: AgentKind, - signal?: AbortSignal, ): Promise<{ episode: EpisodeSnapshot; sessionId: SessionId; relation?: string }> { const mergeMode = algorithm.session.followUpMode === "merge_follow_ups"; const mergeCapMs = algorithm.session.mergeMaxGapMs; const turnTs = timestampFromMeta(meta, "startedAtTurnTs"); if (lightweightMode) { - const snap = await startLightweightEpisode( - sessionId, - userText, - meta, - turnTs, - signal, - ); + const snap = await startLightweightEpisode(sessionId, userText, meta, turnTs); return { episode: snap, sessionId, relation: "lightweight_memory" }; } @@ -496,7 +465,6 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { newUserText: userText, gapMs, prevEpisodeId: currentEpId, - signal, }), algorithm.session.classifyTimeoutMs, log, @@ -614,7 +582,6 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { userMessage: userText, ts: turnTs, meta: { ...meta, relation: "new_task" }, - signal, }); openEpisodeBySession.set(sessionId, snap.id as EpisodeId); return { episode: snap, sessionId, relation: decision.relation }; @@ -634,7 +601,6 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { userMessage: userText, ts: turnTs, meta: { ...meta, relation: decision.relation, gapMs }, - signal, }); openEpisodeBySession.set(sessionId, fresh.id as EpisodeId); return { episode: fresh, sessionId, relation: decision.relation }; @@ -679,7 +645,6 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { newUserText: userText, gapMs, prevEpisodeId: snapshot.id as EpisodeId, - signal, }), algorithm.session.classifyTimeoutMs, log, @@ -783,7 +748,6 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { userMessage: userText, ts: turnTs, meta, - signal, }); openEpisodeBySession.set(sessionId, snap.id as EpisodeId); return { episode: snap, sessionId, relation: "bootstrap" }; @@ -798,7 +762,6 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { newUserText: userText, gapMs, prevEpisodeId: prev.episodeId, - signal, }), algorithm.session.classifyTimeoutMs, log, @@ -898,7 +861,6 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { userMessage: userText, ts: turnTs, meta: { ...meta, relation: "new_task" }, - signal, }); openEpisodeBySession.set(sessionId, snap.id as EpisodeId); return { episode: snap, sessionId, relation: decision.relation }; @@ -909,7 +871,6 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { userMessage: userText, ts: turnTs, meta: { ...meta, relation: decision.relation }, - signal, }); openEpisodeBySession.set(sessionId, snap.id as EpisodeId); return { episode: snap, sessionId, relation: decision.relation }; @@ -1082,7 +1043,7 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { // ─── Retrieval entry points ───────────────────────────────────────────── - const retrievalDeps = buildRetrievalDeps(deps, algorithm, foregroundResources); + const retrievalDeps = buildRetrievalDeps(deps, algorithm); const turnStartRetrievalStats = new Map(); function retrievalDepsFor(namespace = deps.namespace): typeof retrievalDeps { @@ -1096,7 +1057,6 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { async function retrieveTurnStart( input: TurnInputDTO, plan?: RetrievePlan, - signal?: AbortSignal, ): Promise { const ctx = { reason: "turn_start" as const, @@ -1113,8 +1073,6 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { { events: buses.retrieval, skipLlmFilter: input.contextHints?.__memosDeferLlmFilterToCaller === true, - signal, - deadlineAt: input.deadlineAt, plan: plan ? { scenarioId: plan.scenarioId, @@ -1217,45 +1175,13 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { } async function onTurnStartOnce(input: TurnInputDTO): Promise { - const leaveForeground = foregroundResources.enterForeground(); - const deadline = - input.deadlineAt === undefined - ? null - : createRequestDeadline(input.deadlineAt); - const startedAt = Date.now(); - let stage = "ensure_session"; - try { - return await onTurnStartForeground(input, deadline?.signal, (next) => { - stage = next; - }); - } finally { - if (deadline?.signal.aborted) { - log.warn("turn.start.deadline_exceeded", { - sessionId: input.sessionId, - deadlineAt: input.deadlineAt, - elapsedMs: Date.now() - startedAt, - stage, - }); - } - deadline?.dispose(); - leaveForeground(); - } - } - - async function onTurnStartForeground( - input: TurnInputDTO, - signal?: AbortSignal, - setStage: (stage: string) => void = () => {}, - ): Promise { const t0 = now(); - setStage("ensure_session"); const initialSessionId = await ensureSession( input.agent, input.sessionId, input.contextHints, ); - setStage("relation_and_episode"); const routing = await openEpisodeIfNeeded( initialSessionId, input.userText, @@ -1266,7 +1192,6 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { startedAtTurnTs: input.ts, }, input.agent, - signal, ); const sessionId = routing.sessionId; @@ -1278,12 +1203,10 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { sessionId, episodeId: episode.id as EpisodeId, }; - setStage("intent"); const schedulerIntent = await intentForCurrentTurn({ episode, userText: input.userText, ts: input.ts, - signal, }); const retrievePlan = scheduleInjection({ userText: input.userText, @@ -1317,13 +1240,10 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { retrievalTotalMs: 0, elapsedMs: now() - t0, }); - setStage("complete"); return packet; } - setStage("retrieval"); - const packet = await retrieveTurnStart(normalized, retrievePlan, signal); - setStage("complete"); + const packet = await retrieveTurnStart(normalized, retrievePlan); // Always stamp the routed sessionId + episodeId on the packet so // adapters can correlate the subsequent `agent_end` / `turn.end` // call without needing a separate round-trip to the session @@ -1582,28 +1502,12 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { async function shutdown(reason: string = "shutdown"): Promise { log.info("pipeline.shutdown.begin", { reason }); - // Stop admitting retry jobs, but preserve a bounded grace period for raw - // capture and downstream enrichment. Hermes' bridge owns a 20s outer - // shutdown ceiling, so abort before that rather than either hanging or - // discarding every single-shot session's enrichment immediately. - embeddingRetryWorker.stop(); - const flushPromise = flush(); try { - const completed = await settlesWithin(flushPromise, 15_000); - if (!completed) { - log.warn("pipeline.flush_timeout", { reason, timeoutMs: 15_000 }); - foregroundResources.shutdown(reason); - const aborted = await settlesWithin(flushPromise, 4_000); - if (!aborted) { - log.warn("pipeline.flush_abandoned", { reason, abortWaitMs: 4_000 }); - } - } + await flush(); } catch (err) { log.warn("pipeline.flush_failed", { err: err instanceof Error ? err.message : String(err), }); - } finally { - foregroundResources.shutdown(reason); } // Detach subscribers — prevents late events from re-queuing work. subs.subscriptions.capture.stop(); @@ -1612,26 +1516,13 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { subs.l3.detach(); subs.skills.dispose(); subs.feedback.dispose(); + embeddingRetryWorker.stop(); bridge.dispose(); logSubscription(); session.sessionManager.shutdown(reason); log.info("pipeline.shutdown.done", { reason }); } - async function settlesWithin(promise: Promise, timeoutMs: number): Promise { - let timer: ReturnType | null = null; - try { - return await Promise.race([ - promise.then(() => true), - new Promise((resolve) => { - timer = setTimeout(() => resolve(false), timeoutMs); - }), - ]); - } finally { - if (timer) clearTimeout(timer); - } - } - function now(): number { return (deps.now ?? Date.now)(); } @@ -1698,7 +1589,6 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { episode: EpisodeSnapshot; userText: string; ts?: number; - signal?: AbortSignal; }): Promise { const firstTurn = input.episode.turns[0]; const isFreshEpisodeForThisTurn = @@ -1713,7 +1603,6 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { return session.intent.classify(input.userText, { episodeId: input.episode.id as EpisodeId, - signal: input.signal, }); } diff --git a/apps/memos-local-plugin/core/retrieval/llm-filter.ts b/apps/memos-local-plugin/core/retrieval/llm-filter.ts index f665cb954..142bd626d 100644 --- a/apps/memos-local-plugin/core/retrieval/llm-filter.ts +++ b/apps/memos-local-plugin/core/retrieval/llm-filter.ts @@ -50,8 +50,6 @@ export interface FilterDeps { llm: LlmClient | null; log: Logger; timeoutMs?: number; - deadlineAt?: number; - signal?: AbortSignal; config: Pick< RetrievalConfig, | "llmFilterEnabled" @@ -119,10 +117,6 @@ export async function llmFilterCandidates( if (!deps.llm) { return passthrough(ranked, "no_llm"); } - if (deps.signal?.aborted) { - deps.log.debug("llm_filter.deadline_exceeded", { candidateCount: ranked.length }); - return safeCutoff(ranked, deps); - } const bodyChars = deps.config.llmFilterCandidateBodyChars ?? DEFAULT_CANDIDATE_BODY_CHARS; @@ -154,8 +148,6 @@ ${list}`, episodeId: input.episodeId, temperature: 0, timeoutMs: deps.timeoutMs, - deadlineAt: deps.deadlineAt, - signal: deps.signal, // Output is only ordered indices + one bool, but the list can // legitimately be as long as the ranked candidates. maxTokens: filterOutputTokenBudget(ranked.length), diff --git a/apps/memos-local-plugin/core/retrieval/retrieve.ts b/apps/memos-local-plugin/core/retrieval/retrieve.ts index ff95debf4..cec076f62 100644 --- a/apps/memos-local-plugin/core/retrieval/retrieve.ts +++ b/apps/memos-local-plugin/core/retrieval/retrieve.ts @@ -75,10 +75,6 @@ export interface RetrieveOptions { * one unified final LLM filter across all routes. */ skipLlmFilter?: boolean; - /** Shared foreground cancellation signal. */ - signal?: AbortSignal; - /** Absolute request deadline used to cap optional LLM filtering. */ - deadlineAt?: number; } export interface RetrievePlanOverride { @@ -262,28 +258,22 @@ async function runAll( degraded: false, }; const queryVec = compiled.text - ? await deps.embedder - .embed(compiled.text, "query", { - signal: opts.signal, - deadlineAt: opts.deadlineAt, - }) - .then((vec) => { - embeddingStats.ok = true; - return vec; - }) - .catch((err) => { - const code = (err as { code?: string })?.code; - const message = err instanceof Error ? err.message : String(err); - embeddingStats.degraded = true; - embeddingStats.errorCode = code; - embeddingStats.errorMessage = message; - log.warn("embed_failed", { - reason: ctx.reason, - code, - err: message, - }); - return null; - }) + ? await deps.embedder.embed(compiled.text, "query").then((vec) => { + embeddingStats.ok = true; + return vec; + }).catch((err) => { + const code = (err as { code?: string })?.code; + const message = err instanceof Error ? err.message : String(err); + embeddingStats.degraded = true; + embeddingStats.errorCode = code; + embeddingStats.errorMessage = message; + log.warn("embed_failed", { + reason: ctx.reason, + code, + err: message, + }); + return null; + }) : null; // The keyword channels (FTS + pattern) work even without an embedder, @@ -425,9 +415,6 @@ async function runAll( llm: deps.llm ?? null, log, config: deps.config, - signal: opts.signal, - deadlineAt: opts.deadlineAt, - timeoutMs: filterTimeoutMs(opts.deadlineAt), }, ); @@ -485,9 +472,6 @@ async function runAll( llm: deps.llm ?? null, log, config: deps.config, - signal: opts.signal, - deadlineAt: opts.deadlineAt, - timeoutMs: filterTimeoutMs(opts.deadlineAt), }, ); @@ -684,11 +668,6 @@ async function runAll( } } -function filterTimeoutMs(deadlineAt?: number): number | undefined { - if (deadlineAt === undefined) return undefined; - return Math.max(1, Math.min(2_000, deadlineAt - Date.now())); -} - function emptyResult( reason: RetrievalReason, agent: AgentKind, diff --git a/apps/memos-local-plugin/core/retrieval/types.ts b/apps/memos-local-plugin/core/retrieval/types.ts index fe700c611..8ed24935e 100644 --- a/apps/memos-local-plugin/core/retrieval/types.ts +++ b/apps/memos-local-plugin/core/retrieval/types.ts @@ -679,11 +679,7 @@ export interface RetrievalRepos { /** Abstract embedder surface consumed by retrieval. Mirrors `Embedder`. */ export interface RetrievalEmbedder { - embed: ( - text: string, - role?: "query" | "document", - options?: { signal?: AbortSignal; deadlineAt?: number }, - ) => Promise; + embed: (text: string, role?: "query" | "document") => Promise; } export interface RetrievalDeps { diff --git a/apps/memos-local-plugin/core/session/intent-classifier.ts b/apps/memos-local-plugin/core/session/intent-classifier.ts index a4896e07b..41206184d 100644 --- a/apps/memos-local-plugin/core/session/intent-classifier.ts +++ b/apps/memos-local-plugin/core/session/intent-classifier.ts @@ -46,8 +46,6 @@ export interface IntentClassifierOptions { export interface IntentClassifyOptions { /** Episode id this classification is being run for, when known. */ episodeId?: EpisodeId; - /** Foreground request cancellation propagated to the provider call. */ - signal?: AbortSignal; } export interface IntentClassifier { @@ -93,7 +91,7 @@ export function createIntentClassifier(opts: IntentClassifierOptions = {}): Inte if (!llmDisabled && llm) { try { const result = await withTimeout( - callLlm(llm, text, options?.episodeId, timeoutMs, options?.signal), + callLlm(llm, text, options?.episodeId), timeoutMs, "intent.llm.timeout", ); @@ -209,8 +207,6 @@ async function callLlm( llm: LlmClient, text: string, episodeId?: EpisodeId, - timeoutMs?: number, - signal?: AbortSignal, ): Promise { const rsp = await llm.completeJson<{ kind: unknown; confidence: unknown; reason: unknown }>( [ @@ -221,8 +217,6 @@ async function callLlm( op: "session.intent.classify", phase: "session", episodeId, - timeoutMs, - signal, schemaHint: `{"kind":"task"|"memory_probe"|"chitchat"|"meta"|"unknown","confidence":0..1,"reason":"..."}`, validate: (v) => { const o = v as Record; diff --git a/apps/memos-local-plugin/core/session/manager.ts b/apps/memos-local-plugin/core/session/manager.ts index 8f0f2e79f..44da570b2 100644 --- a/apps/memos-local-plugin/core/session/manager.ts +++ b/apps/memos-local-plugin/core/session/manager.ts @@ -60,8 +60,6 @@ export interface StartEpisodeInput { /** Adapter-provided event time for the first user turn. */ ts?: EpochMs; meta?: Record; - /** Foreground cancellation propagated to intent classification. */ - signal?: AbortSignal; } export interface SessionManager { @@ -269,7 +267,6 @@ export function createSessionManager(deps: SessionManagerDeps): SessionManager { const episodeId = (input.id ?? ids.episode()) as EpisodeId; const intent = await deps.intentClassifier.classify(input.userMessage, { episodeId, - signal: input.signal, }); // Wrap the write+emit in a log context so downstream listeners inherit diff --git a/apps/memos-local-plugin/core/session/relation-classifier.ts b/apps/memos-local-plugin/core/session/relation-classifier.ts index f0e1e2a24..bac4679e0 100644 --- a/apps/memos-local-plugin/core/session/relation-classifier.ts +++ b/apps/memos-local-plugin/core/session/relation-classifier.ts @@ -306,11 +306,7 @@ export function createRelationClassifier( // Step 2: LLM classification. if (!llmDisabled && opts.llm) { try { - const result = await withTimeout( - callLlm(opts.llm, input, timeoutMs), - timeoutMs, - "relation.llm.timeout", - ); + const result = await withTimeout(callLlm(opts.llm, input), timeoutMs, "relation.llm.timeout"); log.debug("llm.ok", { relation: result.relation, confidence: result.confidence, @@ -330,7 +326,7 @@ export function createRelationClassifier( }); try { const arb = await withTimeout( - callArbitration(opts.llm, input, timeoutMs), + callArbitration(opts.llm, input), timeoutMs, "relation.arbitration.timeout", ); @@ -520,11 +516,7 @@ function buildLlmUserContent(input: RelationInput): string { return parts.join("\n\n"); } -async function callLlm( - llm: LlmClient, - input: RelationInput, - timeoutMs?: number, -): Promise { +async function callLlm(llm: LlmClient, input: RelationInput): Promise { const userContent = buildLlmUserContent(input); const rsp = await llm.completeJson<{ relation: unknown; confidence: unknown; reason: unknown }>( @@ -536,8 +528,6 @@ async function callLlm( op: "session.relation.classify", phase: "session", episodeId: input.prevEpisodeId, - timeoutMs, - signal: input.signal, schemaHint: `{"relation":"revision"|"follow_up"|"new_task"|"unknown","confidence":0..1,"reason":"..."}`, validate: (v) => { const o = v as Record; @@ -591,11 +581,7 @@ When in doubt, choose follow_up. Reply JSON ONLY: {"relation":"follow_up"|"new_task","reason":"..."}`; -async function callArbitration( - llm: LlmClient, - input: RelationInput, - timeoutMs?: number, -): Promise { +async function callArbitration(llm: LlmClient, input: RelationInput): Promise { const userContent = [ `CURRENT TASK CONTEXT:\n${(input.prevUserText ?? "").slice(0, 600)}`, `ASSISTANT REPLY:\n${(input.prevAssistantText ?? "").slice(0, 800)}`, @@ -611,8 +597,6 @@ async function callArbitration( op: "session.relation.arbitrate", phase: "session", episodeId: input.prevEpisodeId, - timeoutMs, - signal: input.signal, schemaHint: `{"relation":"follow_up"|"new_task","reason":"..."}`, validate: (v) => { const o = v as Record; diff --git a/apps/memos-local-plugin/core/session/types.ts b/apps/memos-local-plugin/core/session/types.ts index 62ea53f97..3d34438ea 100644 --- a/apps/memos-local-plugin/core/session/types.ts +++ b/apps/memos-local-plugin/core/session/types.ts @@ -193,8 +193,6 @@ export interface RelationInput { * is "scoring whether to terminate prevEpisodeId". */ prevEpisodeId?: EpisodeId; - /** Foreground request cancellation propagated to LLM classification. */ - signal?: AbortSignal; } // ─── Event bus ────────────────────────────────────────────────────────────── diff --git a/apps/memos-local-plugin/core/util/foreground-resources.ts b/apps/memos-local-plugin/core/util/foreground-resources.ts deleted file mode 100644 index d818d189f..000000000 --- a/apps/memos-local-plugin/core/util/foreground-resources.ts +++ /dev/null @@ -1,274 +0,0 @@ -import type { - EmbedCallOptions, - Embedder, - EmbedInput, -} from "../embedding/types.js"; -import type { EmbeddingVector } from "../types.js"; - -export type ResourcePriority = "foreground" | "background"; - -export interface ForegroundResources { - readonly shutdownSignal: AbortSignal; - /** Combine a request signal with the pipeline lifecycle signal. */ - signalFor(signal?: AbortSignal): AbortSignal; - /** Mark the complete turn.start path as foreground work. Idempotent release. */ - enterForeground(): () => void; - /** Background LLM work waits here before acquiring its existing semaphore. */ - waitForBackground(signal?: AbortSignal): Promise; - /** Priority-aware, non-preemptive embedding admission. */ - acquireEmbedding( - priority: ResourcePriority, - signal?: AbortSignal, - ): Promise<() => void>; - /** Reject queued work and cancel provider calls before pipeline drain. */ - shutdown(reason?: string): void; -} - -export interface ForegroundResourceOptions { - embeddingConcurrency?: number; - /** Prevent background starvation during a sustained foreground stream. */ - maxForegroundBurst?: number; -} - -interface Waiter { - resolve: (release: () => void) => void; - reject: (error: Error) => void; - signal?: AbortSignal; - onAbort?: () => void; -} - -interface BackgroundWaiter { - resolve: () => void; - reject: (error: Error) => void; - signal?: AbortSignal; - onAbort?: () => void; -} - -export function createForegroundResources( - options: ForegroundResourceOptions = {}, -): ForegroundResources { - const capacity = Math.max(1, Math.floor(options.embeddingConcurrency ?? 1)); - const maxForegroundBurst = Math.max( - 1, - Math.floor(options.maxForegroundBurst ?? 8), - ); - const embeddingWaiters: Record = { - foreground: [], - background: [], - }; - const backgroundWaiters: BackgroundWaiter[] = []; - let embeddingInUse = 0; - let foregroundActive = 0; - let foregroundBurst = 0; - const shutdownController = new AbortController(); - - function signalFor(signal?: AbortSignal): AbortSignal { - return signal - ? AbortSignal.any([signal, shutdownController.signal]) - : shutdownController.signal; - } - - function abortError(signal?: AbortSignal): Error { - return signal?.reason instanceof Error - ? signal.reason - : new DOMException("resource wait aborted", "AbortError"); - } - - function removeAbortListener(waiter: Waiter | BackgroundWaiter): void { - if (waiter.signal && waiter.onAbort) { - waiter.signal.removeEventListener("abort", waiter.onAbort); - } - } - - function nextEmbeddingWaiter(): { - priority: ResourcePriority; - waiter: Waiter; - } | null { - const foreground = embeddingWaiters.foreground; - const background = embeddingWaiters.background; - if ( - background.length > 0 && - (foreground.length === 0 || foregroundBurst >= maxForegroundBurst) - ) { - return { priority: "background", waiter: background.shift()! }; - } - if (foreground.length > 0) { - return { priority: "foreground", waiter: foreground.shift()! }; - } - if (background.length > 0) { - return { priority: "background", waiter: background.shift()! }; - } - return null; - } - - function drainEmbedding(): void { - while (embeddingInUse < capacity) { - const next = nextEmbeddingWaiter(); - if (!next) return; - removeAbortListener(next.waiter); - embeddingInUse++; - foregroundBurst = next.priority === "foreground" ? foregroundBurst + 1 : 0; - next.waiter.resolve(makeEmbeddingRelease()); - } - } - - function makeEmbeddingRelease(): () => void { - let released = false; - return (): void => { - if (released) return; - released = true; - embeddingInUse--; - drainEmbedding(); - }; - } - - function acquireEmbedding( - priority: ResourcePriority, - signal?: AbortSignal, - ): Promise<() => void> { - signal = signalFor(signal); - if (signal.aborted) return Promise.reject(abortError(signal)); - return new Promise((resolve, reject) => { - const waiter: Waiter = { resolve, reject, signal }; - if (signal) { - waiter.onAbort = () => { - const queue = embeddingWaiters[priority]; - const index = queue.indexOf(waiter); - if (index >= 0) queue.splice(index, 1); - reject(abortError(signal)); - }; - signal.addEventListener("abort", waiter.onAbort, { once: true }); - } - embeddingWaiters[priority].push(waiter); - drainEmbedding(); - }); - } - - function drainBackgroundGate(): void { - if (foregroundActive > 0) return; - for (const waiter of backgroundWaiters.splice(0)) { - removeAbortListener(waiter); - waiter.resolve(); - } - } - - function enterForeground(): () => void { - foregroundActive++; - let left = false; - return (): void => { - if (left) return; - left = true; - foregroundActive--; - drainBackgroundGate(); - }; - } - - function waitForBackground(signal?: AbortSignal): Promise { - signal = signalFor(signal); - if (signal.aborted) return Promise.reject(abortError(signal)); - if (foregroundActive === 0) return Promise.resolve(); - return new Promise((resolve, reject) => { - const waiter: BackgroundWaiter = { resolve, reject, signal }; - if (signal) { - waiter.onAbort = () => { - const index = backgroundWaiters.indexOf(waiter); - if (index >= 0) backgroundWaiters.splice(index, 1); - reject(abortError(signal)); - }; - signal.addEventListener("abort", waiter.onAbort, { once: true }); - } - backgroundWaiters.push(waiter); - }); - } - - function shutdown(reason = "pipeline shutdown"): void { - if (shutdownController.signal.aborted) return; - shutdownController.abort(new DOMException(reason, "AbortError")); - } - - return { - shutdownSignal: shutdownController.signal, - signalFor, - enterForeground, - waitForBackground, - acquireEmbedding, - shutdown, - }; -} - -/** - * Keep the Embedder contract intact while moving provider round-trips behind - * the shared priority arbiter. Background batches are deliberately chunked - * so one enrichment pass cannot monopolize the provider for an entire queue. - */ -export function prioritizeEmbedder( - inner: Embedder | null, - resources: ForegroundResources, - priority: ResourcePriority, - backgroundChunkSize = 8, -): Embedder | null { - if (!inner) return null; - - async function embedOne( - input: string | EmbedInput, - options?: EmbedCallOptions, - ): Promise { - const signal = resources.signalFor(options?.signal); - const callOptions = { ...options, signal }; - if (priority === "background") await resources.waitForBackground(signal); - const release = await resources.acquireEmbedding(priority, signal); - try { - return await inner!.embedOne(input, callOptions); - } finally { - release(); - } - } - - async function embedMany( - inputs: Array, - options?: EmbedCallOptions, - ): Promise { - const signal = resources.signalFor(options?.signal); - const callOptions = { ...options, signal }; - if (priority === "foreground" || inputs.length <= backgroundChunkSize) { - if (priority === "background") await resources.waitForBackground(signal); - const release = await resources.acquireEmbedding(priority, signal); - try { - return await inner!.embedMany(inputs, callOptions); - } finally { - release(); - } - } - - const results: EmbeddingVector[] = []; - for (let start = 0; start < inputs.length; start += backgroundChunkSize) { - await resources.waitForBackground(signal); - const release = await resources.acquireEmbedding(priority, signal); - try { - results.push( - ...await inner!.embedMany(inputs.slice(start, start + backgroundChunkSize), callOptions), - ); - } finally { - release(); - } - } - return results; - } - - return { - get dimensions() { - return inner.dimensions; - }, - get provider() { - return inner.provider; - }, - get model() { - return inner.model; - }, - embedOne, - embedMany, - stats: () => inner.stats(), - resetCache: () => inner.resetCache(), - close: () => inner.close(), - }; -} diff --git a/apps/memos-local-plugin/core/util/rate-limited-llm.ts b/apps/memos-local-plugin/core/util/rate-limited-llm.ts index 5bcb863bb..4d229b4c9 100644 --- a/apps/memos-local-plugin/core/util/rate-limited-llm.ts +++ b/apps/memos-local-plugin/core/util/rate-limited-llm.ts @@ -10,26 +10,20 @@ import type { LlmStreamChunk, } from "../llm/types.js"; import type { Semaphore } from "./semaphore.js"; -import type { ForegroundResources } from "./foreground-resources.js"; /** * Wrap an LLM client so expensive background subscribers share one * process-wide concurrency budget without changing call-site semantics. */ -export function rateLimitLlmClient( - client: LlmClient | null, - semaphore: Semaphore, - resources?: ForegroundResources, -): LlmClient | null { +export function rateLimitLlmClient(client: LlmClient | null, semaphore: Semaphore): LlmClient | null { if (!client) return null; - return new RateLimitedLlmClient(client, semaphore, resources); + return new RateLimitedLlmClient(client, semaphore); } class RateLimitedLlmClient implements LlmClient { constructor( private readonly inner: LlmClient, private readonly semaphore: Semaphore, - private readonly resources?: ForegroundResources, ) {} get provider(): LlmProviderName { @@ -48,12 +42,9 @@ class RateLimitedLlmClient implements LlmClient { messages: LlmMessage[] | string, opts?: LlmCallOptions, ): Promise { - const signal = this.resources?.signalFor(opts?.signal) ?? opts?.signal; - const callOpts = signal ? { ...opts, signal } : opts; - await this.resources?.waitForBackground(signal); - const release = await this.semaphore.acquire(signal); + const release = await this.semaphore.acquire(); try { - return await this.inner.complete(messages, callOpts); + return await this.inner.complete(messages, opts); } finally { release(); } @@ -63,12 +54,9 @@ class RateLimitedLlmClient implements LlmClient { messages: LlmMessage[] | string, opts?: LlmCompleteJsonOptions, ): Promise> { - const signal = this.resources?.signalFor(opts?.signal) ?? opts?.signal; - const callOpts = signal ? { ...opts, signal } : opts; - await this.resources?.waitForBackground(signal); - const release = await this.semaphore.acquire(signal); + const release = await this.semaphore.acquire(); try { - return await this.inner.completeJson(messages, callOpts); + return await this.inner.completeJson(messages, opts); } finally { release(); } @@ -78,12 +66,9 @@ class RateLimitedLlmClient implements LlmClient { messages: LlmMessage[] | string, opts?: LlmCallOptions, ): AsyncIterable { - const signal = this.resources?.signalFor(opts?.signal) ?? opts?.signal; - const callOpts = signal ? { ...opts, signal } : opts; - await this.resources?.waitForBackground(signal); - const release = await this.semaphore.acquire(signal); + const release = await this.semaphore.acquire(); try { - yield* this.inner.stream(messages, callOpts); + yield* this.inner.stream(messages, opts); } finally { release(); } diff --git a/apps/memos-local-plugin/core/util/request-deadline.ts b/apps/memos-local-plugin/core/util/request-deadline.ts deleted file mode 100644 index a7fb816a0..000000000 --- a/apps/memos-local-plugin/core/util/request-deadline.ts +++ /dev/null @@ -1,37 +0,0 @@ -export interface RequestDeadline { - readonly signal: AbortSignal; - remainingMs(): number; - dispose(): void; -} - -/** - * Convert an adapter-provided absolute epoch deadline into one abort signal. - * The absolute form survives JSON-RPC transport time and prevents every stage - * from accidentally receiving a fresh timeout budget. - */ -export function createRequestDeadline( - deadlineAt: number, - now: () => number = Date.now, -): RequestDeadline { - const controller = new AbortController(); - const remainingMs = (): number => Math.max(0, deadlineAt - now()); - const initialRemaining = remainingMs(); - let timer: ReturnType | null = null; - - if (!Number.isFinite(deadlineAt) || initialRemaining <= 0) { - controller.abort(new DOMException("request deadline exceeded", "TimeoutError")); - } else { - timer = setTimeout(() => { - controller.abort(new DOMException("request deadline exceeded", "TimeoutError")); - }, initialRemaining); - } - - return { - signal: controller.signal, - remainingMs, - dispose(): void { - if (timer) clearTimeout(timer); - timer = null; - }, - }; -} diff --git a/apps/memos-local-plugin/core/util/retry-after.ts b/apps/memos-local-plugin/core/util/retry-after.ts deleted file mode 100644 index e580e5f7b..000000000 --- a/apps/memos-local-plugin/core/util/retry-after.ts +++ /dev/null @@ -1,200 +0,0 @@ -/** Parse RFC 9110 Retry-After delay-seconds or HTTP-date into milliseconds. */ -export const MAX_INLINE_RETRY_DELAY_MS = 30_000; -/** @deprecated Use MAX_INLINE_RETRY_DELAY_MS. */ -export const MAX_RETRY_DELAY_MS = MAX_INLINE_RETRY_DELAY_MS; - -export type RetryDeferReason = - | "deadline_insufficient" - | "retry_after_too_long"; - -export interface RetryPlanBase { - backoffMs: number; - delayMs: number; - retryAfterMs: number | null; - retryAt: number; - source: "backoff" | "retry_after"; -} - -export type RetryPlan = - | (RetryPlanBase & { action: "wait" }) - | (RetryPlanBase & { action: "defer"; reason: RetryDeferReason }); - -export interface RetryCooldown { - retryAfterMs: number; - retryAt: number; - status: number; -} - -export interface RetryDiagnosticDetails { - retryAfterMs?: number; - retryAt?: number; - retryDecision?: "wait" | "defer" | "stop"; - retryReason?: string; -} - -const retryCooldowns = new Map(); - -export function parseRetryAfterMs( - value: string | null | undefined, - nowMs: number = Date.now(), -): number | null { - const raw = value?.trim(); - if (!raw) return null; - if (/^\d+$/.test(raw)) { - const seconds = Number(raw); - const delayMs = seconds * 1_000; - return Number.isSafeInteger(seconds) && Number.isSafeInteger(delayMs) - ? delayMs - : null; - } - // Retry-After only permits IMF-fixdate here. Keeping the shape strict avoids - // JavaScript accepting ambiguous strings such as "1.5" as a legacy date. - if (!/^[A-Za-z]{3}, \d{2} [A-Za-z]{3} \d{4} \d{2}:\d{2}:\d{2} GMT$/.test(raw)) return null; - const at = Date.parse(raw); - if (!Number.isFinite(at)) return null; - return Math.max(0, at - nowMs); -} - -export function retryDelayMs(input: { - attempt: number; - baseMs: number; - jitterMaxMs: number; - retryAfterMs?: number | null; - maxDelayMs?: number; - random?: () => number; -}): number { - const plan = planRetry({ - ...input, - maxInlineDelayMs: input.maxDelayMs, - }); - return plan.delayMs; -} - -/** - * Decide whether a retry can happen inline without violating Retry-After. - * - * Provider Retry-After values are never clamped downward. When the earliest - * legal retry cannot fit the inline wait or request deadline, callers must - * defer/fallback and carry retryAt into their recovery path. - */ -export function planRetry(input: { - attempt: number; - baseMs: number; - jitterMaxMs: number; - retryAfterMs?: number | null; - maxInlineDelayMs?: number; - deadlineAt?: number; - nowMs?: number; - random?: () => number; -}): RetryPlan { - const nowMs = input.nowMs ?? Date.now(); - const random = input.random ?? Math.random; - const jitter = Math.floor(random() * input.jitterMaxMs); - const rawBackoff = input.baseMs * 2 ** Math.max(0, input.attempt - 1) + jitter; - const maxInlineDelayMs = input.maxInlineDelayMs ?? MAX_INLINE_RETRY_DELAY_MS; - const backoffMs = Math.min(rawBackoff, maxInlineDelayMs); - const retryAfterMs = input.retryAfterMs ?? null; - const delayMs = Math.max(backoffMs, retryAfterMs ?? 0); - const retryAt = nowMs + delayMs; - const source = retryAfterMs !== null && retryAfterMs >= backoffMs - ? "retry_after" as const - : "backoff" as const; - const base: RetryPlanBase = { - backoffMs, - delayMs, - retryAfterMs, - retryAt, - source, - }; - - if (retryAfterMs !== null && retryAfterMs > maxInlineDelayMs) { - return { ...base, action: "defer", reason: "retry_after_too_long" }; - } - if (input.deadlineAt !== undefined && retryAt > input.deadlineAt) { - return { ...base, action: "defer", reason: "deadline_insufficient" }; - } - return { ...base, action: "wait" }; -} - -export function retryCooldownKey( - kind: "llm" | "embedding", - provider: string, - url: string, - scope: string = "", -): string { - return `${kind}\u0000${provider}\u0000${url}\u0000${scope}`; -} - -/** Extend a provider cooldown monotonically; a shorter later response cannot weaken it. */ -export function recordRetryCooldown(key: string, cooldown: RetryCooldown): void { - const current = retryCooldowns.get(key); - if (!current || cooldown.retryAt > current.retryAt) { - retryCooldowns.set(key, { ...cooldown }); - } -} - -export function getRetryCooldown( - key: string, - nowMs: number = Date.now(), -): RetryCooldown | null { - const cooldown = retryCooldowns.get(key); - if (!cooldown) return null; - if (cooldown.retryAt <= nowMs) { - retryCooldowns.delete(key); - return null; - } - return { ...cooldown }; -} - -/** Test/runtime-reset hook; plugin shutdown does not need to await cooldown state. */ -export function clearRetryCooldowns(): void { - retryCooldowns.clear(); -} - -/** Copy only bounded, machine-readable retry fields from an error detail bag. */ -export function extractRetryDiagnostics( - details: Record | undefined, -): RetryDiagnosticDetails { - if (!details) return {}; - const diagnostic: RetryDiagnosticDetails = {}; - if (typeof details.retryAfterMs === "number" && Number.isFinite(details.retryAfterMs)) { - diagnostic.retryAfterMs = details.retryAfterMs; - } - if (typeof details.retryAt === "number" && Number.isFinite(details.retryAt)) { - diagnostic.retryAt = details.retryAt; - } - if ( - details.retryDecision === "wait" - || details.retryDecision === "defer" - || details.retryDecision === "stop" - ) { - diagnostic.retryDecision = details.retryDecision; - } - if (typeof details.retryReason === "string") { - diagnostic.retryReason = details.retryReason; - } - return diagnostic; -} - -/** Abortable retry wait so request cancellation and shutdown do not leave sleepers behind. */ -export function waitForRetry(delayMs: number, signal?: AbortSignal): Promise { - if (signal?.aborted) return Promise.reject(abortReason(signal)); - if (delayMs <= 0) return Promise.resolve(); - - return new Promise((resolve, reject) => { - const timer = setTimeout(() => { - signal?.removeEventListener("abort", onAbort); - resolve(); - }, delayMs); - const onAbort = () => { - clearTimeout(timer); - signal?.removeEventListener("abort", onAbort); - reject(signal ? abortReason(signal) : new DOMException("Aborted", "AbortError")); - }; - signal?.addEventListener("abort", onAbort, { once: true }); - }); -} - -function abortReason(signal: AbortSignal): unknown { - return signal.reason ?? new DOMException("Aborted", "AbortError"); -} diff --git a/apps/memos-local-plugin/core/util/semaphore.ts b/apps/memos-local-plugin/core/util/semaphore.ts index 037db2607..8dd9b75fc 100644 --- a/apps/memos-local-plugin/core/util/semaphore.ts +++ b/apps/memos-local-plugin/core/util/semaphore.ts @@ -1,37 +1,23 @@ export interface Semaphore { - acquire(signal?: AbortSignal): Promise<() => void>; -} - -interface Waiter { - resolve: (release: () => void) => void; - reject: (error: Error) => void; - signal?: AbortSignal; - onAbort?: () => void; + acquire(): Promise<() => void>; } export function createSemaphore(max: number): Semaphore { const limit = Math.max(1, Math.floor(max)); let current = 0; - const waiters: Waiter[] = []; + const waiters: Array<() => void> = []; return { - async acquire(signal?: AbortSignal) { - if (signal?.aborted) throw abortError(signal); + async acquire() { if (current < limit) { current++; return release; } - return new Promise<() => void>((resolve, reject) => { - const waiter: Waiter = { resolve, reject, signal }; - if (signal) { - waiter.onAbort = () => { - const index = waiters.indexOf(waiter); - if (index >= 0) waiters.splice(index, 1); - reject(abortError(signal)); - }; - signal.addEventListener("abort", waiter.onAbort, { once: true }); - } - waiters.push(waiter); + return new Promise<() => void>((resolve) => { + waiters.push(() => { + current++; + resolve(release); + }); }); }, }; @@ -39,17 +25,6 @@ export function createSemaphore(max: number): Semaphore { function release() { current = Math.max(0, current - 1); const next = waiters.shift(); - if (!next) return; - if (next.signal && next.onAbort) { - next.signal.removeEventListener("abort", next.onAbort); - } - current++; - next.resolve(release); + if (next) next(); } } - -function abortError(signal: AbortSignal): Error { - return signal.reason instanceof Error - ? signal.reason - : new DOMException("semaphore wait aborted", "AbortError"); -} diff --git a/apps/memos-local-plugin/tests/python/test_bridge_client.py b/apps/memos-local-plugin/tests/python/test_bridge_client.py index b5c9eb917..82763fcea 100644 --- a/apps/memos-local-plugin/tests/python/test_bridge_client.py +++ b/apps/memos-local-plugin/tests/python/test_bridge_client.py @@ -448,128 +448,6 @@ def test_reverse_request_waits_for_late_host_handler_registration(self) -> None: self.assertNotIn("error", response) client.close() - def test_slow_reverse_handler_does_not_block_regular_rpc_responses(self) -> None: - """A host LLM callback must not stall the stdout response demux. - - ``host.llm.complete`` can legitimately spend several seconds in the - Hermes model client. The bridge reader still has to resolve an - unrelated foreground ``turn.start`` response during that - window; otherwise one background callback head-of-line blocks every - provider lease sharing the process. - """ - client = MemosBridgeClient(bridge_path="/tmp/bridge.cts") - assert self._fake is not None - handler_started = threading.Event() - release_handler = threading.Event() - - def _slow_handler(_params: dict) -> dict: - handler_started.set() - release_handler.wait(timeout=2.0) - return {"text": "host:done", "model": "host-test"} - - client.register_host_handler("host.llm.complete", _slow_handler) - self._fake.stdout._enqueue( - { - "jsonrpc": "2.0", - "id": "srv-slow", - "method": "host.llm.complete", - "params": {"messages": [{"role": "user", "content": "slow"}]}, - } - ) - self.assertTrue(handler_started.wait(timeout=0.5)) - - try: - response = client.request( - "turn.start", - { - "sessionId": "hermes:session:1", - "userText": "foreground recall", - }, - timeout=0.5, - ) - self.assertIn("foreground recall", response["injectedContext"]) - finally: - release_handler.set() - - reverse_response = self._wait_for_client_write(lambda msg: msg.get("id") == "srv-slow") - self.assertEqual(reverse_response["result"]["text"], "host:done") - client.close() - - def test_reverse_handler_queue_rejects_overload_without_blocking_reader(self) -> None: - client = MemosBridgeClient(bridge_path="/tmp/bridge.cts") - assert self._fake is not None - handler_started = threading.Event() - release_handler = threading.Event() - - def _slow_handler(_params: dict) -> dict: - handler_started.set() - release_handler.wait(timeout=2.0) - return {"text": "done"} - - client.register_host_handler("host.llm.complete", _slow_handler) - self._fake.stdout._enqueue( - { - "jsonrpc": "2.0", - "id": "srv-running", - "method": "host.llm.complete", - "params": {}, - } - ) - self.assertTrue(handler_started.wait(timeout=0.5)) - - overflow_id = "srv-overflow" - for index in range(bridge_client_mod.HOST_HANDLER_QUEUE_CAPACITY + 1): - rpc_id = ( - overflow_id - if index == bridge_client_mod.HOST_HANDLER_QUEUE_CAPACITY - else f"srv-{index}" - ) - self._fake.stdout._enqueue( - { - "jsonrpc": "2.0", - "id": rpc_id, - "method": "host.llm.complete", - "params": {}, - } - ) - - try: - response = self._wait_for_client_write(lambda msg: msg.get("id") == overflow_id) - self.assertEqual(response["error"]["data"]["code"], "host_handler_busy") - finally: - client.close() - release_handler.set() - - def test_close_does_not_wait_for_a_running_reverse_handler(self) -> None: - """An uncooperative host callback must not extend bridge shutdown.""" - client = MemosBridgeClient(bridge_path="/tmp/bridge.cts") - assert self._fake is not None - handler_started = threading.Event() - release_handler = threading.Event() - - def _slow_handler(_params: dict) -> dict: - handler_started.set() - release_handler.wait(timeout=2.0) - return {"text": "late", "model": "host-test"} - - client.register_host_handler("host.llm.complete", _slow_handler) - self._fake.stdout._enqueue( - { - "jsonrpc": "2.0", - "id": "srv-close", - "method": "host.llm.complete", - "params": {}, - } - ) - self.assertTrue(handler_started.wait(timeout=0.5)) - - started = time.monotonic() - try: - client.close() - finally: - release_handler.set() - self.assertLess(time.monotonic() - started, 0.5) - def test_reader_exit_marks_pending_as_transport_closed(self) -> None: """R1 (#2028): reader thread EOF must wake pending waiters with transport_closed instead of leaving them parked on their @@ -1233,7 +1111,7 @@ def test_sync_turn_uses_long_rpc_timeout_for_turn_end(self) -> None: "sessions (issue #2028).", ) - def test_prefetch_uses_dedicated_foreground_timeout_for_turn_start(self) -> None: + def test_prefetch_uses_long_rpc_timeout_for_turn_start(self) -> None: p = self._provider_mod.MemTensorProvider() bridge = RecordingBridge() p._bridge = bridge @@ -1244,45 +1122,12 @@ def test_prefetch_uses_dedicated_foreground_timeout_for_turn_start(self) -> None self.assertIn("turn.start", methods) start_index = methods.index("turn.start") start_kwargs = bridge.call_kwargs[start_index] - self.assertGreater(start_kwargs.get("timeout", 0.0), 0.0) - self.assertLessEqual( + self.assertGreaterEqual( start_kwargs.get("timeout", 0.0), - self._provider_mod._PREFETCH_RPC_TIMEOUT, - "foreground turn.start must finish before the Hermes host deadline; " - "long capture work keeps the separate issue #2028 timeout.", + self._EXPECTED_LONG_TIMEOUT, + "turn.start suffers the same long-tail latency as turn.end and " + "must share the long RPC timeout (issue #2028).", ) - start_payload = bridge.calls[start_index][1] - self.assertIn("deadlineAt", start_payload) - self.assertGreater(start_payload["deadlineAt"], start_payload["ts"]) - - def test_foreground_reconnect_and_retry_share_one_deadline(self) -> None: - class ClosedBridge: - def request(self, *_args, **_kwargs) -> dict: - raise BridgeError("transport_closed", "bridge closed") - - p = self._provider_mod.MemTensorProvider() - p._bridge = ClosedBridge() - recovered = RecordingBridge() - monotonic_now = [100.0] - - def reconnect(_session_id: str, *, timeout: float) -> None: - self.assertLessEqual(timeout, 6.0) - monotonic_now[0] += 4.0 - p._bridge = recovered - - with ( - patch("memos_provider.time.monotonic", side_effect=lambda: monotonic_now[0]), - patch.object(p, "_reconnect_bridge", side_effect=reconnect), - ): - p._bridge_request_with_retry( - "turn.start", - {"sessionId": "s-1"}, - timeout=6.0, - deadline_monotonic=106.0, - ) - - self.assertEqual(recovered.calls[0][0], "turn.start") - self.assertLessEqual(recovered.call_kwargs[0]["timeout"], 2.0) class ViewerDaemonTests(unittest.TestCase): diff --git a/apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py b/apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py index ecd0e1ae1..6b95ae61b 100644 --- a/apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py +++ b/apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py @@ -480,75 +480,6 @@ def test_prefetch_passes_stable_turn_key_to_bridge(self) -> None: turn_start = next(params for method, params in bridge.calls if method == "turn.start") self.assertEqual(turn_start["turnKey"], "turn-key-session:7") - def test_prefetch_uses_a_dedicated_budget_and_forwards_absolute_deadline(self) -> None: - bridge = FakeBridge() - with ( - patch("memos_provider.ensure_bridge_running", return_value=True), - patch("memos_provider.ensure_viewer_daemon", return_value=True), - patch("memos_provider.MemosBridgeClient", return_value=bridge), - patch("memos_provider._PREFETCH_RPC_TIMEOUT", 6.0), - patch("memos_provider.time.time", return_value=1_700_000_000.0), - ): - provider = memos_provider.MemTensorProvider() - provider.initialize("budget-session") - provider.on_turn_start(1, "recall the build decision") - with patch.object( - provider, - "_bridge_request_with_retry", - wraps=provider._bridge_request_with_retry, - ) as request: - provider.prefetch("recall the build decision") - - turn_start = next(params for method, params in bridge.calls if method == "turn.start") - self.assertEqual(turn_start["deadlineAt"], 1_700_000_005_750) - request.assert_called_once() - self.assertLessEqual(request.call_args.kwargs["timeout"], 6.0) - self.assertIn("deadline_monotonic", request.call_args.kwargs) - - def test_prefetch_budget_includes_bridge_ensure_time(self) -> None: - bridge = FakeBridge() - monotonic_now = [100.0] - - def ensure_bridge(_session_id: str, *, timeout: float) -> bool: - self.assertAlmostEqual(timeout, 6.0, places=3) - monotonic_now[0] += 2.5 - return True - - with ( - patch("memos_provider.ensure_bridge_running", return_value=True), - patch("memos_provider.ensure_viewer_daemon", return_value=True), - patch("memos_provider.MemosBridgeClient", return_value=bridge), - patch("memos_provider._PREFETCH_RPC_TIMEOUT", 6.0), - patch("memos_provider.time.time", return_value=1_700_000_000.0), - patch("memos_provider.time.monotonic", side_effect=lambda: monotonic_now[0]), - ): - provider = memos_provider.MemTensorProvider() - provider.initialize("budget-session") - provider.on_turn_start(1, "recall the build decision") - with ( - patch.object(provider, "_ensure_bridge", side_effect=ensure_bridge), - patch.object( - provider, - "_bridge_request_with_retry", - wraps=provider._bridge_request_with_retry, - ) as request, - ): - provider.prefetch("recall the build decision") - - self.assertLessEqual(request.call_args.kwargs["timeout"], 3.5) - turn_start = next(params for method, params in bridge.calls if method == "turn.start") - self.assertEqual(turn_start["deadlineAt"], 1_700_000_005_750) - - def test_prefetch_timeout_config_rejects_non_positive_values(self) -> None: - with patch.dict("os.environ", {"MEMOS_HERMES_PREFETCH_RPC_TIMEOUT": "0"}): - self.assertEqual(memos_provider._prefetch_rpc_timeout_default(), 6.0) - with patch.dict("os.environ", {"MEMOS_HERMES_PREFETCH_RPC_TIMEOUT": "nan"}): - self.assertEqual(memos_provider._prefetch_rpc_timeout_default(), 6.0) - with patch.dict("os.environ", {"MEMOS_HERMES_PREFETCH_RPC_TIMEOUT": "4.5"}): - self.assertEqual(memos_provider._prefetch_rpc_timeout_default(), 4.5) - with patch.dict("os.environ", {"MEMOS_HERMES_PREFETCH_RPC_TIMEOUT": "30"}): - self.assertEqual(memos_provider._prefetch_rpc_timeout_default(), 7.0) - def test_prefetch_suppresses_memory_injection_for_explicit_delegation(self) -> None: bridge = FakeBridge() with ( diff --git a/apps/memos-local-plugin/tests/unit/bridge/methods.test.ts b/apps/memos-local-plugin/tests/unit/bridge/methods.test.ts index fc9f921e1..8ee5fa921 100644 --- a/apps/memos-local-plugin/tests/unit/bridge/methods.test.ts +++ b/apps/memos-local-plugin/tests/unit/bridge/methods.test.ts @@ -344,17 +344,6 @@ describe("makeDispatcher", () => { ).rejects.toSatisfy( (err) => err instanceof MemosError && err.code === "invalid_argument", ); - await expect( - dispatch("turn.start", { - agent: "openclaw", - sessionId: "s-1", - userText: "hi", - ts: 123, - deadlineAt: "soon", - }), - ).rejects.toSatisfy( - (err) => err instanceof MemosError && err.code === "invalid_argument", - ); }); it("feedback.submit forwards the DTO shape intact", async () => { diff --git a/apps/memos-local-plugin/tests/unit/embedding/embedder.test.ts b/apps/memos-local-plugin/tests/unit/embedding/embedder.test.ts index 617bc4a72..3c608a53a 100644 --- a/apps/memos-local-plugin/tests/unit/embedding/embedder.test.ts +++ b/apps/memos-local-plugin/tests/unit/embedding/embedder.test.ts @@ -6,10 +6,8 @@ import { initTestLogger } from "../../../core/logger/index.js"; import type { EmbedRole, EmbeddingConfig, - EmbeddingErrorDetail, EmbeddingProvider, EmbeddingProviderName, - EmbeddingStatusDetail, ProviderCallCtx, } from "../../../core/embedding/types.js"; @@ -77,24 +75,6 @@ describe("embedder facade", () => { expect(Array.from(v)).toEqual([3, 97, 0]); // a=97 }); - it("forwards the caller abort signal and deadline to the provider", async () => { - const seen: Array> = []; - const p: EmbeddingProvider = { - name: "openai_compatible", - async embed(texts, _role, ctx) { - seen.push({ signal: ctx.signal, deadlineAt: ctx.deadlineAt }); - return texts.map(() => [1, 2, 3]); - }, - }; - const e = createEmbedderWithProvider(cfg(), p); - const controller = new AbortController(); - - const deadlineAt = Date.now() + 1_000; - await e.embedOne("signal", { signal: controller.signal, deadlineAt }); - - expect(seen).toEqual([{ signal: controller.signal, deadlineAt }]); - }); - it("dedups identical inputs into one provider call", async () => { const p = new FakeProvider(); const e = createEmbedderWithProvider(cfg(), p); @@ -195,45 +175,6 @@ describe("embedder facade", () => { } }); - it("preserves deferred retry diagnostics in error and status sinks", async () => { - const errors: EmbeddingErrorDetail[] = []; - const statuses: EmbeddingStatusDetail[] = []; - const retryAt = Date.now() + 120_000; - const provider: EmbeddingProvider = { - name: "openai_compatible", - async embed() { - throw new MemosError("embedding_unavailable", "provider cooldown", { - retryAfterMs: 120_000, - retryAt, - retryDecision: "defer", - retryReason: "retry_after_too_long", - }); - }, - }; - const e = createEmbedderWithProvider( - cfg({ onError: (detail) => errors.push(detail), onStatus: (detail) => statuses.push(detail) }), - provider, - ); - - await expect(e.embedOne("x")).rejects.toBeInstanceOf(MemosError); - - expect(errors).toContainEqual( - expect.objectContaining({ - retryAfterMs: 120_000, - retryAt, - retryDecision: "defer", - retryReason: "retry_after_too_long", - }), - ); - expect(statuses).toContainEqual( - expect.objectContaining({ - status: "error", - retryAt, - retryDecision: "defer", - }), - ); - }); - it("rejects when provider returns too few rows", async () => { const e = createEmbedderWithProvider(cfg({ provider: "gemini" }), new WrongCountProvider()); await expect(e.embedMany(["x", "y", "z"])).rejects.toBeInstanceOf(MemosError); diff --git a/apps/memos-local-plugin/tests/unit/embedding/fetcher.test.ts b/apps/memos-local-plugin/tests/unit/embedding/fetcher.test.ts index 2811e94ef..3b9ee5bf2 100644 --- a/apps/memos-local-plugin/tests/unit/embedding/fetcher.test.ts +++ b/apps/memos-local-plugin/tests/unit/embedding/fetcher.test.ts @@ -4,7 +4,6 @@ import { MemosError } from "../../../agent-contract/errors.js"; import { initTestLogger } from "../../../core/logger/index.js"; import { httpPostJson } from "../../../core/embedding/fetcher.js"; import type { ProviderLogger } from "../../../core/embedding/types.js"; -import { clearRetryCooldowns } from "../../../core/util/retry-after.js"; function nullLogger(): ProviderLogger { return { @@ -22,8 +21,6 @@ describe("embedding/fetcher", () => { vi.useRealTimers(); // retry backoff uses real setTimeout; keep it real but short }); afterEach(() => { - clearRetryCooldowns(); - vi.useRealTimers(); vi.unstubAllGlobals(); }); @@ -84,85 +81,6 @@ describe("embedding/fetcher", () => { expect(f).toHaveBeenCalledTimes(2); }); - it("honors Retry-After HTTP-date before retrying a 429", async () => { - vi.useFakeTimers(); - vi.setSystemTime(new Date("2026-08-04T00:00:00.000Z")); - const f = mockFetch([ - new Response("rate limited", { - status: 429, - headers: { "Retry-After": "Tue, 04 Aug 2026 00:00:03 GMT" }, - }), - new Response(JSON.stringify({ ok: 1 }), { status: 200 }), - ]); - - const pending = httpPostJson<{ ok: number }>({ - url: "https://x", - body: {}, - provider: "cohere", - log: nullLogger(), - maxRetries: 1, - }); - await vi.advanceTimersByTimeAsync(2_999); - expect(f).toHaveBeenCalledTimes(1); - await vi.advanceTimersByTimeAsync(1); - await expect(pending).resolves.toEqual({ ok: 1 }); - expect(f).toHaveBeenCalledTimes(2); - vi.useRealTimers(); - }); - - it("defers a long Retry-After and short-circuits the provider cooldown", async () => { - vi.useFakeTimers(); - vi.setSystemTime(new Date("2026-08-04T00:00:00.000Z")); - const f = mockFetch([ - new Response("maintenance", { status: 503, headers: { "Retry-After": "120" } }), - ]); - const opts = { - url: "https://embedding-x", - body: {}, - provider: "cohere" as const, - log: nullLogger(), - maxRetries: 1, - }; - - await expect(httpPostJson(opts)).rejects.toMatchObject({ - code: "embedding_unavailable", - details: { - retryAfterMs: 120_000, - retryDecision: "defer", - retryReason: "retry_after_too_long", - }, - }); - await expect(httpPostJson(opts)).rejects.toMatchObject({ - code: "embedding_unavailable", - details: { retryReason: "cooldown_active" }, - }); - expect(f).toHaveBeenCalledTimes(1); - }); - - it("returns structured diagnostics when network backoff cannot fit the deadline", async () => { - vi.useFakeTimers(); - const now = Date.parse("2026-08-04T00:00:00.000Z"); - vi.setSystemTime(now); - const f = mockFetch([new Error("ECONNRESET")]); - - await expect(httpPostJson({ - url: "https://embedding-deadline", - body: {}, - provider: "mistral", - log: nullLogger(), - maxRetries: 1, - deadlineAt: now + 100, - })).rejects.toMatchObject({ - name: "MemosError", - code: "embedding_unavailable", - details: { - retryDecision: "defer", - retryReason: "deadline_insufficient", - }, - }); - expect(f).toHaveBeenCalledTimes(1); - }); - it("does not retry on 400", async () => { mockFetch([new Response("bad", { status: 400 })]); await expect( diff --git a/apps/memos-local-plugin/tests/unit/embedding/retry-worker.test.ts b/apps/memos-local-plugin/tests/unit/embedding/retry-worker.test.ts index ae72cc60e..f08a37f32 100644 --- a/apps/memos-local-plugin/tests/unit/embedding/retry-worker.test.ts +++ b/apps/memos-local-plugin/tests/unit/embedding/retry-worker.test.ts @@ -1,7 +1,6 @@ import { afterEach, beforeEach, describe, expect, it } from "vitest"; import { createEmbeddingRetryWorker } from "../../../core/embedding/retry-worker.js"; -import { ERROR_CODES, MemosError } from "../../../agent-contract/errors.js"; import { rootLogger } from "../../../core/logger/index.js"; import type { EpisodeId, SessionId, TraceId } from "../../../core/types.js"; import { makeTmpDb, type TmpDbHandle } from "../../helpers/tmp-db.js"; @@ -175,39 +174,6 @@ describe("embedding retry worker", () => { expect(handle.repos.apiLogs.list({ toolName: "system_error", limit: 5, offset: 0 })).toHaveLength(1); }); - it("never schedules a durable retry before the provider retryAt", async () => { - const retryAt = NOW + 120_000; - handle.repos.embeddingRetryQueue.enqueue({ - id: "er_retry_after", - targetKind: "trace", - targetId: "tr_retry", - vectorField: "vec_summary", - sourceText: "retry me later", - maxAttempts: 3, - now: NOW, - }); - const worker = createEmbeddingRetryWorker({ - repos: handle.repos, - embedder: fakeEmbedder({ - throwWith: new MemosError( - ERROR_CODES.EMBEDDING_UNAVAILABLE, - "provider cooling down", - { retryAt, retryAfterMs: 120_000, retryDecision: "defer" }, - ), - }), - log: rootLogger.child({ channel: "test.embedding.retry" }), - now: () => NOW, - }); - - await worker.flush(); - - expect(queueRow(handle, "er_retry_after")).toMatchObject({ - status: "pending", - attempts: 1, - next_attempt_at: retryAt, - }); - }); - it("treats missing target rows as retry failures", async () => { handle.repos.embeddingRetryQueue.enqueue({ id: "er_missing", @@ -235,30 +201,4 @@ describe("embedding retry worker", () => { last_error: "embedding retry target not found: trace:tr_missing", }); }); - - it("does not claim new retry jobs after stop during shutdown", async () => { - handle.repos.embeddingRetryQueue.enqueue({ - id: "er_shutdown", - targetKind: "trace", - targetId: "tr_retry", - vectorField: "vec_summary", - sourceText: "do not start during shutdown", - now: NOW, - }); - const worker = createEmbeddingRetryWorker({ - repos: handle.repos, - embedder: fakeEmbedder({ dimensions: 8 }), - log: rootLogger.child({ channel: "test.embedding.retry" }), - now: () => NOW, - }); - - worker.stop(); - await worker.flush(); - - expect(queueRow(handle, "er_shutdown")).toMatchObject({ - status: "pending", - attempts: 0, - claimed_by: null, - }); - }); }); diff --git a/apps/memos-local-plugin/tests/unit/llm/client.test.ts b/apps/memos-local-plugin/tests/unit/llm/client.test.ts index cd125ecd8..dee0de228 100644 --- a/apps/memos-local-plugin/tests/unit/llm/client.test.ts +++ b/apps/memos-local-plugin/tests/unit/llm/client.test.ts @@ -283,45 +283,6 @@ describe("llm/client", () => { await expect(client.complete([] as LlmMessage[])).rejects.toBeInstanceOf(MemosError); }); - it("preserves deferred retry diagnostics in error and status sinks", async () => { - const errors: Array> = []; - const statuses: LlmStatusDetail[] = []; - const retryAt = Date.now() + 120_000; - const provider = new ThrowingProvider( - new MemosError(ERROR_CODES.LLM_RATE_LIMITED, "provider cooldown", { - retryAfterMs: 120_000, - retryAt, - retryDecision: "defer", - retryReason: "retry_after_too_long", - }), - ); - const client = createLlmClientWithProvider( - cfg({ - onError: (detail) => errors.push(detail as unknown as Record), - onStatus: (detail) => statuses.push(detail), - }), - provider, - ); - - await expect(client.complete("x")).rejects.toBeInstanceOf(MemosError); - - expect(errors).toContainEqual( - expect.objectContaining({ - retryAfterMs: 120_000, - retryAt, - retryDecision: "defer", - retryReason: "retry_after_too_long", - }), - ); - expect(statuses).toContainEqual( - expect.objectContaining({ - status: "error", - retryAt, - retryDecision: "defer", - }), - ); - }); - // ─── Circuit breaker (issue #1897) ────────────────────────────────────── describe("circuit breaker", () => { function statusSink(): { rows: LlmStatusDetail[]; push: (d: LlmStatusDetail) => void } { diff --git a/apps/memos-local-plugin/tests/unit/llm/fetcher.test.ts b/apps/memos-local-plugin/tests/unit/llm/fetcher.test.ts index 9b17e1b16..a3941b7aa 100644 --- a/apps/memos-local-plugin/tests/unit/llm/fetcher.test.ts +++ b/apps/memos-local-plugin/tests/unit/llm/fetcher.test.ts @@ -4,7 +4,6 @@ import { MemosError } from "../../../agent-contract/errors.js"; import { decodeSse, httpPostJson, httpPostStream } from "../../../core/llm/fetcher.js"; import { initTestLogger } from "../../../core/logger/index.js"; import type { LlmProviderLogger } from "../../../core/llm/types.js"; -import { clearRetryCooldowns } from "../../../core/util/retry-after.js"; function nullLog(): LlmProviderLogger { return { @@ -30,205 +29,7 @@ function mockFetch(replies: Array) { describe("llm/fetcher", () => { beforeAll(() => initTestLogger()); - afterEach(() => { - clearRetryCooldowns(); - vi.useRealTimers(); - vi.unstubAllGlobals(); - }); - - it("honors Retry-After delay-seconds before retrying a 429", async () => { - vi.useFakeTimers(); - const f = mockFetch([ - new Response("slow down", { status: 429, headers: { "Retry-After": "2" } }), - new Response(JSON.stringify({ ok: 1 }), { status: 200 }), - ]); - - const pending = httpPostJson({ - url: "https://x", - body: {}, - timeoutMs: 5_000, - maxRetries: 1, - provider: "openai_compatible", - log: nullLog(), - }); - await vi.advanceTimersByTimeAsync(1_999); - expect(f).toHaveBeenCalledTimes(1); - await vi.advanceTimersByTimeAsync(1); - await expect(pending).resolves.toMatchObject({ json: { ok: 1 } }); - expect(f).toHaveBeenCalledTimes(2); - vi.useRealTimers(); - }); - - it("aborts while waiting for Retry-After", async () => { - vi.useFakeTimers(); - const ctrl = new AbortController(); - const f = mockFetch([ - new Response("slow down", { status: 429, headers: { "Retry-After": "2" } }), - ]); - - const pending = httpPostJson({ - url: "https://x", - body: {}, - timeoutMs: 5_000, - maxRetries: 1, - signal: ctrl.signal, - provider: "openai_compatible", - log: nullLog(), - }); - await vi.advanceTimersByTimeAsync(0); - ctrl.abort(); - await expect(pending).rejects.toBeInstanceOf(MemosError); - expect(f).toHaveBeenCalledTimes(1); - vi.useRealTimers(); - }); - - it("defers a long Retry-After from a 503 without retrying early", async () => { - vi.useFakeTimers(); - vi.setSystemTime(new Date("2026-08-04T00:00:00.000Z")); - const warn = vi.fn(); - const f = mockFetch([ - new Response("maintenance", { status: 503, headers: { "Retry-After": "120" } }), - ]); - - const pending = httpPostJson({ - url: "https://x", - body: {}, - timeoutMs: 120_000, - maxRetries: 1, - provider: "openai_compatible", - log: { ...nullLog(), warn }, - }); - await expect(pending).rejects.toMatchObject({ - code: "llm_unavailable", - details: { - retryAfterMs: 120_000, - retryDecision: "defer", - retryReason: "retry_after_too_long", - }, - }); - expect(f).toHaveBeenCalledTimes(1); - expect(warn).toHaveBeenCalledWith( - "http.retry_deferred", - expect.objectContaining({ - retryAfterMs: 120_000, - retryDecision: "defer", - retryReason: "retry_after_too_long", - }), - ); - }); - - it("short-circuits calls while the provider Retry-After cooldown is active", async () => { - vi.useFakeTimers(); - vi.setSystemTime(new Date("2026-08-04T00:00:00.000Z")); - const warn = vi.fn(); - const f = mockFetch([ - new Response("slow down", { status: 429, headers: { "Retry-After": "120" } }), - ]); - const opts = { - url: "https://x", - body: {}, - timeoutMs: 5_000, - maxRetries: 1, - provider: "openai_compatible" as const, - log: { ...nullLog(), warn }, - }; - - await expect(httpPostJson(opts)).rejects.toMatchObject({ code: "llm_rate_limited" }); - await expect(httpPostJson(opts)).rejects.toMatchObject({ - code: "llm_rate_limited", - details: { retryDecision: "defer", retryReason: "cooldown_active" }, - }); - expect(f).toHaveBeenCalledTimes(1); - expect(warn).toHaveBeenCalledWith( - "http.retry_cooldown", - expect.objectContaining({ retryReason: "cooldown_active" }), - ); - }); - - it("does not enter a Retry-After wait that cannot fit the absolute deadline", async () => { - vi.useFakeTimers(); - const now = Date.parse("2026-08-04T00:00:00.000Z"); - vi.setSystemTime(now); - const f = mockFetch([ - new Response("slow down", { status: 429, headers: { "Retry-After": "5" } }), - ]); - - await expect(httpPostJson({ - url: "https://deadline", - body: {}, - timeoutMs: 5_000, - maxRetries: 1, - deadlineAt: now + 1_000, - provider: "openai_compatible", - log: nullLog(), - })).rejects.toMatchObject({ - code: "llm_rate_limited", - details: { - retryDecision: "defer", - retryReason: "deadline_insufficient", - }, - }); - expect(f).toHaveBeenCalledTimes(1); - }); - - it("returns structured diagnostics when network backoff cannot fit the deadline", async () => { - vi.useFakeTimers(); - const now = Date.parse("2026-08-04T00:00:00.000Z"); - vi.setSystemTime(now); - const f = mockFetch([new Error("ECONNRESET")]); - - await expect(httpPostJson({ - url: "https://network-deadline", - body: {}, - timeoutMs: 5_000, - maxRetries: 1, - deadlineAt: now + 100, - provider: "openai_compatible", - log: nullLog(), - })).rejects.toMatchObject({ - name: "MemosError", - code: "llm_unavailable", - details: { - retryDecision: "defer", - retryReason: "deadline_insufficient", - }, - }); - expect(f).toHaveBeenCalledTimes(1); - }); - - it("does not let an older in-flight success clear a newer provider cooldown", async () => { - vi.useFakeTimers(); - vi.setSystemTime(new Date("2026-08-04T00:00:00.000Z")); - let resolveSuccess!: (response: Response) => void; - const success = new Promise((resolve) => { resolveSuccess = resolve; }); - const f = vi.fn() - .mockImplementationOnce(() => success) - .mockResolvedValueOnce( - new Response("slow down", { status: 429, headers: { "Retry-After": "120" } }), - ); - vi.stubGlobal("fetch", f); - const opts = { - url: "https://shared-endpoint", - body: {}, - timeoutMs: 5_000, - maxRetries: 1, - provider: "openai_compatible" as const, - log: nullLog(), - }; - - const older = httpPostJson<{ ok: boolean }>(opts); - await vi.waitFor(() => expect(f).toHaveBeenCalledTimes(1)); - await expect(httpPostJson(opts)).rejects.toMatchObject({ - details: { retryReason: "retry_after_too_long" }, - }); - resolveSuccess(new Response(JSON.stringify({ ok: true }), { status: 200 })); - await expect(older).resolves.toMatchObject({ json: { ok: true } }); - - await expect(httpPostJson(opts)).rejects.toMatchObject({ - details: { retryReason: "cooldown_active" }, - }); - expect(f).toHaveBeenCalledTimes(2); - }); + afterEach(() => vi.unstubAllGlobals()); it("returns parsed JSON on 200", async () => { mockFetch([new Response(JSON.stringify({ a: 1 }), { status: 200 })]); diff --git a/apps/memos-local-plugin/tests/unit/pipeline/memory-core.test.ts b/apps/memos-local-plugin/tests/unit/pipeline/memory-core.test.ts index 52853ff4a..cb5140d09 100644 --- a/apps/memos-local-plugin/tests/unit/pipeline/memory-core.test.ts +++ b/apps/memos-local-plugin/tests/unit/pipeline/memory-core.test.ts @@ -898,53 +898,6 @@ describe("MemoryCore façade", () => { expect(scored.priority).toBe(1); }); - it("logs both user and assistant content with the matching role", async () => { - pipeline = createPipeline(buildDeps(db!)); - core = createMemoryCore( - pipeline, - resolveHome("openclaw", "/tmp/memos-mc-test"), - "test", - ); - await core.init(); - - const userText = "今晚吃什么，推荐一下"; - const agentText = "可以考虑清淡的汤面、盖饭或者附近评价不错的家常菜。"; - const start = await core.onTurnStart({ - agent: "openclaw", - sessionId: "s-memory-add-roles", - userText, - ts: 1_700_000_000_000, - }); - await core.onTurnEnd({ - agent: "openclaw", - sessionId: start.query.sessionId!, - episodeId: start.query.episodeId!, - agentText, - toolCalls: [], - ts: 1_700_000_000_500, - }); - - const { logs } = await core.listApiLogs({ - toolName: "memory_add", - limit: 10, - }); - const liteLog = logs.find((log) => { - const input = JSON.parse(log.inputJson) as { phase?: string }; - return input.phase === "lite"; - }); - expect(liteLog).toBeDefined(); - - const output = JSON.parse(liteLog!.outputJson) as { - details?: Array<{ role?: string; content?: string }>; - }; - expect( - output.details?.map(({ role, content }) => ({ role, content })), - ).toEqual([ - { role: "user", content: userText }, - { role: "assistant", content: agentText }, - ]); - }); - it("onTurnEnd preserves adapter-provided historical timestamps", async () => { pipeline = createPipeline(buildDeps(db!)); core = createMemoryCore( diff --git a/apps/memos-local-plugin/tests/unit/pipeline/orchestrator.test.ts b/apps/memos-local-plugin/tests/unit/pipeline/orchestrator.test.ts index 1e79c861a..a70fdcf6b 100644 --- a/apps/memos-local-plugin/tests/unit/pipeline/orchestrator.test.ts +++ b/apps/memos-local-plugin/tests/unit/pipeline/orchestrator.test.ts @@ -77,39 +77,6 @@ afterEach(async () => { }); describe("pipeline/orchestrator", () => { - it("degrades retrieval at the adapter deadline and aborts the provider call", async () => { - const base = fakeEmbedder({ dimensions: 384 }); - let sawAbort = false; - const embedder = { - ...base, - async embedOne(input: Parameters[0], options?: { signal?: AbortSignal }) { - return await new Promise>>((resolve, reject) => { - const onAbort = () => { - sawAbort = true; - reject(new DOMException("deadline", "AbortError")); - }; - if (options?.signal?.aborted) return onAbort(); - options?.signal?.addEventListener("abort", onAbort, { once: true }); - void resolve; - }); - }, - }; - pipeline = createPipeline(buildDeps(dbHandle!, embedder)); - const startedAt = Date.now(); - - const packet = await pipeline.onTurnStart({ - agent: "hermes", - sessionId: "s-deadline", - userText: "find the previous build decision", - ts: Date.now(), - deadlineAt: Date.now() + 25, - }); - - expect(sawAbort).toBe(true); - expect(Date.now() - startedAt).toBeLessThan(500); - expect(packet.reason).toBe("turn_start"); - }); - it("threads a dedicated l3Llm through to the handle", () => { const l3Llm = fakeLlm({ completeJson: {} }); pipeline = createPipeline({ ...buildDeps(dbHandle!), l3Llm }); diff --git a/apps/memos-local-plugin/tests/unit/session/intent-classifier.test.ts b/apps/memos-local-plugin/tests/unit/session/intent-classifier.test.ts index c3321635a..6b37ae55f 100644 --- a/apps/memos-local-plugin/tests/unit/session/intent-classifier.test.ts +++ b/apps/memos-local-plugin/tests/unit/session/intent-classifier.test.ts @@ -109,25 +109,6 @@ describe("session/intent-classifier", () => { expect(d.signals).toEqual(["llm"]); }); - it("forwards the foreground abort signal and classifier timeout", async () => { - let seen: { signal?: AbortSignal; timeoutMs?: number } | undefined; - const llm = fakeLlm(() => ({ kind: "task", confidence: 0.8, reason: "task" })); - const original = llm.completeJson.bind(llm); - llm.completeJson = async (messages, opts) => { - seen = opts; - return original(messages, opts); - }; - const controller = new AbortController(); - const c = createIntentClassifier({ llm, timeoutMs: 321 }); - - await c.classify("investigate an ambiguous pipeline issue", { - signal: controller.signal, - }); - - expect(seen?.signal).toBe(controller.signal); - expect(seen?.timeoutMs).toBe(321); - }); - it("LLM failure falls back to heuristic", async () => { const c = createIntentClassifier({ llm: fakeLlm(() => { diff --git a/apps/memos-local-plugin/tests/unit/session/relation-classifier.test.ts b/apps/memos-local-plugin/tests/unit/session/relation-classifier.test.ts index a17d3d9ce..575a4b44a 100644 --- a/apps/memos-local-plugin/tests/unit/session/relation-classifier.test.ts +++ b/apps/memos-local-plugin/tests/unit/session/relation-classifier.test.ts @@ -119,33 +119,6 @@ describe("relation-classifier — V7 §0.1", () => { expect(d.llmModel).toBe("fake/test-model"); }); - it("forwards the foreground abort signal and classifier timeout", async () => { - let seen: { signal?: AbortSignal; timeoutMs?: number } | undefined; - const controller = new AbortController(); - const c = createRelationClassifier({ - timeoutMs: 456, - llm: { - completeJson: async (_messages, opts) => { - seen = opts; - return { - value: { relation: "follow_up", confidence: 0.9, reason: "same task" }, - servedBy: "fake/llm", - } as never; - }, - } as LlmClient, - }); - - await c.classify({ - prevUserText: "investigate retrieval latency", - prevAssistantText: "I found several possible causes.", - newUserText: "could the queue contribute to this behavior?", - signal: controller.signal, - }); - - expect(seen?.signal).toBe(controller.signal); - expect(seen?.timeoutMs).toBe(456); - }); - it("falls back to heuristic when LLM throws", async () => { const llm: Partial = { completeJson: async () => { diff --git a/apps/memos-local-plugin/tests/unit/util/foreground-resources.test.ts b/apps/memos-local-plugin/tests/unit/util/foreground-resources.test.ts deleted file mode 100644 index 1380b9a47..000000000 --- a/apps/memos-local-plugin/tests/unit/util/foreground-resources.test.ts +++ /dev/null @@ -1,138 +0,0 @@ -import { describe, expect, it } from "vitest"; - -import { - createForegroundResources, - prioritizeEmbedder, -} from "../../../core/util/foreground-resources.js"; -import { fakeEmbedder } from "../../helpers/fake-embedder.js"; - -describe("foreground resources", () => { - it("admits a queued foreground embedding before queued background work", async () => { - const resources = createForegroundResources({ embeddingConcurrency: 1 }); - const first = await resources.acquireEmbedding("background"); - const order: string[] = []; - - const background = resources.acquireEmbedding("background").then((release) => { - order.push("background"); - release(); - }); - const foreground = resources.acquireEmbedding("foreground").then((release) => { - order.push("foreground"); - release(); - }); - - first(); - await Promise.all([foreground, background]); - - expect(order).toEqual(["foreground", "background"]); - }); - - it("lets background work progress after a bounded foreground burst", async () => { - const resources = createForegroundResources({ - embeddingConcurrency: 1, - maxForegroundBurst: 2, - }); - const first = await resources.acquireEmbedding("foreground"); - const order: string[] = []; - - const background = resources.acquireEmbedding("background").then((release) => { - order.push("background"); - release(); - }); - const foreground1 = resources.acquireEmbedding("foreground").then((release) => { - order.push("foreground-1"); - release(); - }); - const foreground2 = resources.acquireEmbedding("foreground").then((release) => { - order.push("foreground-2"); - release(); - }); - - first(); - await Promise.all([background, foreground1, foreground2]); - - expect(order).toEqual(["foreground-1", "background", "foreground-2"]); - }); - - it("does not start background work while a foreground turn is active", async () => { - const resources = createForegroundResources(); - const leaveForeground = resources.enterForeground(); - let started = false; - - const waiting = resources.waitForBackground().then(() => { - started = true; - }); - await Promise.resolve(); - expect(started).toBe(false); - - leaveForeground(); - await waiting; - expect(started).toBe(true); - }); - - it("removes an aborted embedding waiter without consuming capacity", async () => { - const resources = createForegroundResources({ embeddingConcurrency: 1 }); - const first = await resources.acquireEmbedding("background"); - const controller = new AbortController(); - const waiting = resources.acquireEmbedding("foreground", controller.signal); - - controller.abort(); - await expect(waiting).rejects.toMatchObject({ name: "AbortError" }); - first(); - - const release = await resources.acquireEmbedding("background"); - release(); - }); - - it("chunks background embedding batches and yields between chunks", async () => { - const resources = createForegroundResources({ embeddingConcurrency: 1 }); - const base = fakeEmbedder({ dimensions: 4 }); - const batchSizes: number[] = []; - const inner = { - ...base, - async embedMany(...args: Parameters) { - batchSizes.push(args[0].length); - return base.embedMany(...args); - }, - }; - const background = prioritizeEmbedder(inner, resources, "background", 2)!; - - await background.embedMany(["a", "b", "c", "d", "e"]); - - expect(batchSizes).toEqual([2, 2, 1]); - }); - - it("aborts queued and in-flight provider work during shutdown", async () => { - const resources = createForegroundResources({ embeddingConcurrency: 1 }); - const base = fakeEmbedder({ dimensions: 4 }); - let providerSignal: AbortSignal | undefined; - const inner = { - ...base, - async embedOne( - _input: Parameters[0], - options?: Parameters[1], - ) { - providerSignal = options?.signal; - return await new Promise((_resolve, reject) => { - if (options?.signal?.aborted) { - reject(options.signal.reason); - return; - } - options?.signal?.addEventListener( - "abort", - () => reject(options.signal?.reason), - { once: true }, - ); - }); - }, - }; - const background = prioritizeEmbedder(inner, resources, "background")!; - const pending = background.embedOne("slow background work"); - await Promise.resolve(); - - resources.shutdown("test shutdown"); - - await expect(pending).rejects.toMatchObject({ name: "AbortError" }); - expect(providerSignal?.aborted).toBe(true); - }); -}); diff --git a/apps/memos-local-plugin/tests/unit/util/request-deadline.test.ts b/apps/memos-local-plugin/tests/unit/util/request-deadline.test.ts deleted file mode 100644 index 8d6b8d5a2..000000000 --- a/apps/memos-local-plugin/tests/unit/util/request-deadline.test.ts +++ /dev/null @@ -1,35 +0,0 @@ -import { afterEach, describe, expect, it, vi } from "vitest"; - -import { createRequestDeadline } from "../../../core/util/request-deadline.js"; - -afterEach(() => { - vi.useRealTimers(); -}); - -describe("createRequestDeadline", () => { - it("aborts at the absolute deadline and reports no remaining budget", async () => { - vi.useFakeTimers(); - vi.setSystemTime(1_000); - - const deadline = createRequestDeadline(1_250); - expect(deadline.remainingMs()).toBe(250); - expect(deadline.signal.aborted).toBe(false); - - await vi.advanceTimersByTimeAsync(250); - - expect(deadline.signal.aborted).toBe(true); - expect(deadline.remainingMs()).toBe(0); - deadline.dispose(); - }); - - it("treats an already-expired deadline as immediately aborted", () => { - vi.useFakeTimers(); - vi.setSystemTime(2_000); - - const deadline = createRequestDeadline(1_999); - - expect(deadline.signal.aborted).toBe(true); - expect(deadline.remainingMs()).toBe(0); - deadline.dispose(); - }); -}); diff --git a/apps/memos-local-plugin/tests/unit/util/retry-after.test.ts b/apps/memos-local-plugin/tests/unit/util/retry-after.test.ts deleted file mode 100644 index 84a4c8ea4..000000000 --- a/apps/memos-local-plugin/tests/unit/util/retry-after.test.ts +++ /dev/null @@ -1,90 +0,0 @@ -import { describe, expect, it } from "vitest"; - -import { - clearRetryCooldowns, - getRetryCooldown, - parseRetryAfterMs, - planRetry, - recordRetryCooldown, - retryCooldownKey, -} from "../../../core/util/retry-after.js"; - -describe("parseRetryAfterMs", () => { - it("parses delay-seconds", () => { - expect(parseRetryAfterMs("3", 1_000)).toBe(3_000); - expect(parseRetryAfterMs("0", 1_000)).toBe(0); - }); - - it("parses an HTTP-date relative to the supplied clock", () => { - const now = Date.parse("2026-08-04T00:00:00.000Z"); - expect(parseRetryAfterMs("Tue, 04 Aug 2026 00:00:05 GMT", now)).toBe(5_000); - }); - - it("clamps past HTTP-dates and rejects malformed values", () => { - const now = Date.parse("2026-08-04T00:00:00.000Z"); - expect(parseRetryAfterMs("Mon, 03 Aug 2026 23:59:59 GMT", now)).toBe(0); - expect(parseRetryAfterMs("1.5", now)).toBeNull(); - expect(parseRetryAfterMs("9007199254740991", now)).toBeNull(); - expect(parseRetryAfterMs("later", now)).toBeNull(); - expect(parseRetryAfterMs(null, now)).toBeNull(); - }); - - it("defers instead of retrying before a long provider Retry-After", () => { - expect(planRetry({ - attempt: 1, - baseMs: 200, - jitterMaxMs: 0, - retryAfterMs: 120_000, - maxInlineDelayMs: 30_000, - nowMs: 1_000, - })).toEqual({ - action: "defer", - backoffMs: 200, - delayMs: 120_000, - reason: "retry_after_too_long", - retryAfterMs: 120_000, - retryAt: 121_000, - source: "retry_after", - }); - }); - - it("defers when an otherwise short retry cannot fit the request deadline", () => { - expect(planRetry({ - attempt: 1, - baseMs: 200, - jitterMaxMs: 0, - retryAfterMs: 2_000, - deadlineAt: 2_500, - nowMs: 1_000, - })).toMatchObject({ - action: "defer", - reason: "deadline_insufficient", - retryAt: 3_000, - }); - }); - - it("keeps provider cooldowns monotonic and expires them at retryAt", () => { - clearRetryCooldowns(); - recordRetryCooldown("llm:test", { - retryAfterMs: 2_000, - retryAt: 3_000, - status: 429, - }); - recordRetryCooldown("llm:test", { - retryAfterMs: 500, - retryAt: 1_500, - status: 503, - }); - expect(getRetryCooldown("llm:test", 2_999)).toMatchObject({ - retryAt: 3_000, - status: 429, - }); - expect(getRetryCooldown("llm:test", 3_000)).toBeNull(); - clearRetryCooldowns(); - }); - - it("scopes provider cooldowns by endpoint and model", () => { - expect(retryCooldownKey("llm", "openai_compatible", "https://x", "model-a")) - .not.toBe(retryCooldownKey("llm", "openai_compatible", "https://x", "model-b")); - }); -}); diff --git a/apps/memos-local-plugin/tests/unit/util/semaphore.test.ts b/apps/memos-local-plugin/tests/unit/util/semaphore.test.ts deleted file mode 100644 index 2a5308ad2..000000000 --- a/apps/memos-local-plugin/tests/unit/util/semaphore.test.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { describe, expect, it } from "vitest"; - -import { createSemaphore } from "../../../core/util/semaphore.js"; - -describe("semaphore", () => { - it("removes an aborted waiter so shutdown cannot hang behind active work", async () => { - const semaphore = createSemaphore(1); - const release = await semaphore.acquire(); - const controller = new AbortController(); - const waiting = semaphore.acquire(controller.signal); - - controller.abort(); - await expect(waiting).rejects.toMatchObject({ name: "AbortError" }); - release(); - - const next = await semaphore.acquire(); - next(); - }); -}); From 0ff0a72d2e1794301b59e7013c9293883a589813 Mon Sep 17 00:00:00 2001 From: CovD <2643822566@qq.com> Date: Wed, 5 Aug 2026 00:23:08 +0800 Subject: [PATCH 5/5] fix(plugin): bound idle archive batches --- .../core/skill/ALGORITHMS.md | 4 +- .../core/skill/subscriber.ts | 15 +++++- .../tests/unit/skill/subscriber.test.ts | 47 +++++++++++++++++++ 3 files changed, 64 insertions(+), 2 deletions(-) diff --git a/apps/memos-local-plugin/core/skill/ALGORITHMS.md b/apps/memos-local-plugin/core/skill/ALGORITHMS.md index 94daaf7ab..205c3e7b2 100644 --- a/apps/memos-local-plugin/core/skill/ALGORITHMS.md +++ b/apps/memos-local-plugin/core/skill/ALGORITHMS.md @@ -253,7 +253,9 @@ the next lifecycle tick. `lastUsedAt` is updated by the existing recorded-use path. A never-used skill falls back to `createdAt`; unrelated metadata updates therefore do not reset its idle clock. The scan runs through the orchestrator's normal -flush lifecycle and does not introduce a separate timer. +flush lifecycle and does not introduce a separate timer. Each tick processes +at most ten 500-row batches; any remaining backlog is deferred to a later tick +so a large archive queue cannot monopolize the event loop. --- diff --git a/apps/memos-local-plugin/core/skill/subscriber.ts b/apps/memos-local-plugin/core/skill/subscriber.ts index af1b0a57c..9dd861a73 100644 --- a/apps/memos-local-plugin/core/skill/subscriber.ts +++ b/apps/memos-local-plugin/core/skill/subscriber.ts @@ -37,6 +37,8 @@ import type { SkillId } from "../types.js"; import { now as nowMs } from "../time.js"; import { IDLE_ARCHIVE_BATCH_LIMIT } from "../storage/repos/skills.js"; +const IDLE_ARCHIVE_MAX_BATCHES_PER_TICK = 10; + export interface SkillSubscriberDeps extends Omit { log?: Logger; @@ -230,17 +232,21 @@ export function attachSkillSubscriber( } const cutoff = at - deps.config.idleArchiveMs; - while (true) { + let batchesProcessed = 0; + let archivedTotal = 0; + while (batchesProcessed < IDLE_ARCHIVE_MAX_BATCHES_PER_TICK) { const archiveCandidates = deps.repos.skills.listIdleArchiveCandidates({ minEtaForRetrieval: deps.config.minEtaForRetrieval, cutoff, limit: IDLE_ARCHIVE_BATCH_LIMIT, }); + batchesProcessed += 1; let archivedThisBatch = 0; for (const s of archiveCandidates) { if (!shouldArchiveIdle(s, deps.config.idleArchiveMs, deps.config, at)) continue; deps.repos.skills.setStatus(s.id, "archived", at); archivedThisBatch += 1; + archivedTotal += 1; log.info("skill.idle_archived", { skillId: s.id, name: s.name, @@ -266,6 +272,13 @@ export function attachSkillSubscriber( break; } if (archiveCandidates.length < IDLE_ARCHIVE_BATCH_LIMIT) break; + if (batchesProcessed === IDLE_ARCHIVE_MAX_BATCHES_PER_TICK) { + log.warn("skill.idle_archive_batch_limit_reached", { + batchCount: batchesProcessed, + archivedCount: archivedTotal, + batchSize: IDLE_ARCHIVE_BATCH_LIMIT, + }); + } } } diff --git a/apps/memos-local-plugin/tests/unit/skill/subscriber.test.ts b/apps/memos-local-plugin/tests/unit/skill/subscriber.test.ts index 88ec1eafb..a8c35468a 100644 --- a/apps/memos-local-plugin/tests/unit/skill/subscriber.test.ts +++ b/apps/memos-local-plugin/tests/unit/skill/subscriber.test.ts @@ -250,4 +250,51 @@ describe("skill/subscriber", () => { expect(h.repos.skills.count({ status: "active" })).toBe(0); sub.dispose(); }); + + it("caps idle archival at ten batches per lifecycle tick", async () => { + handle = makeTmpDb(); + const h = handle; + for (let i = 0; i < 5_001; i++) { + seedSkill(h, { + id: `sk_backlog_${i}` as never, + name: `backlog_skill_${i}`, + status: "active", + eta: 0.05, + createdAt: 1 as never, + updatedAt: (i + 1) as never, + lastUsedAt: 1 as never, + }); + } + const log = rootLogger.child({ channel: "core.skill.subscriber" }); + const infoSpy = vi.spyOn(log, "info").mockImplementation(() => undefined); + const warnSpy = vi.spyOn(log, "warn").mockImplementation(() => undefined); + const sub = attachSkillSubscriber({ + l2Bus: createL2EventBus(), + rewardBus: createRewardEventBus(), + bus: createSkillEventBus(), + repos: h.repos, + embedder: null, + llm: null, + log, + config: makeSkillConfig({ minEtaForRetrieval: 0.1, idleArchiveMs: 1_000 }), + }); + + await sub.lifecycleTick(); + + expect(h.repos.skills.count({ status: "archived" })).toBe(5_000); + expect(h.repos.skills.count({ status: "active" })).toBe(1); + expect(warnSpy).toHaveBeenCalledWith("skill.idle_archive_batch_limit_reached", { + batchCount: 10, + archivedCount: 5_000, + batchSize: 500, + }); + + await sub.lifecycleTick(); + + expect(h.repos.skills.count({ status: "archived" })).toBe(5_001); + expect(h.repos.skills.count({ status: "active" })).toBe(0); + sub.dispose(); + infoSpy.mockRestore(); + warnSpy.mockRestore(); + }); });