diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 6611ad2..044a787 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -38,7 +38,7 @@ jobs:
run: |
python - <<'PY'
from app import create_app
- app = create_app()
+ app = create_app(testing=True)
client = app.test_client()
assert client.get("/").status_code == 200
PY
diff --git a/api/search.py b/api/search.py
index e16427f..6bfd5a0 100644
--- a/api/search.py
+++ b/api/search.py
@@ -3,6 +3,7 @@
from __future__ import annotations
import logging
+import os
from datetime import datetime, timezone
from typing import Any
@@ -11,17 +12,16 @@
from api._flask_types import FlaskReturn, json_response
from api.error_codes import ErrorCode, error_response
from models.search import SearchHitDict
-from models.session import MessageDict
from utils.exclusion_rules import is_session_excluded
from utils.search_index import (
index_is_usable,
index_search_enabled,
+ message_searchable_text,
query_index_hits,
resolve_search_since_ms,
search_snippet,
timestamp_in_search_window_iso,
timestamp_to_ms,
- tool_result_searchable_text,
)
from utils.session_cache import get_cached_session
from utils.session_path import get_claude_projects_dir, list_projects, list_sessions
@@ -32,6 +32,7 @@
_DEFAULT_LIMIT = 50
_MAX_LIMIT = 500
+_MAX_QUERY_LEN = 500
_MAX_SEARCH_SINCE_DAYS = 36_500
@@ -53,10 +54,10 @@ def _parse_since_days(raw: str | None) -> int | None:
return None
try:
days = int(str(raw).strip())
- except ValueError:
- return None
+ except ValueError as exc:
+ raise ValueError("Invalid since_days: must be a positive integer") from exc
if days <= 0:
- return None
+ raise ValueError("Invalid since_days: must be a positive integer")
return min(days, _MAX_SEARCH_SINCE_DAYS)
@@ -71,16 +72,44 @@ def _rank_search_hits(results: list[SearchHitDict]) -> list[SearchHitDict]:
)
-def _message_searchable_text(msg: MessageDict) -> str:
- text = msg.get("text", "") or msg.get("content", "")
- if not isinstance(text, str):
- text = ""
- tool_result = msg.get("tool_result")
- if isinstance(tool_result, dict):
- tool_text = tool_result_searchable_text(tool_result)
- if tool_text:
- text = f"{text}\n{tool_text}" if text else tool_text
- return text
+def _hit_dedup_key(hit: SearchHitDict) -> tuple[str, str, str | None, str]:
+ ts = hit.get("timestamp")
+ return (
+ hit["project"],
+ hit["session_id"],
+ ts if isinstance(ts, str) else None,
+ hit["role"],
+ )
+
+
+def _merge_search_hits(
+ primary: list[SearchHitDict],
+ extra: list[SearchHitDict],
+ *,
+ max_results: int,
+) -> list[SearchHitDict]:
+ seen = {_hit_dedup_key(hit) for hit in primary}
+ merged = list(primary)
+ for hit in extra:
+ key = _hit_dedup_key(hit)
+ if key in seen:
+ continue
+ merged.append(hit)
+ seen.add(key)
+ if len(merged) >= max_results:
+ break
+ return _rank_search_hits(merged)[:max_results]
+
+
+def _projects_dir_inaccessible(projects_dir: str) -> bool:
+ """True when the projects path exists but cannot be listed (503 case)."""
+ try:
+ if not os.path.isdir(projects_dir):
+ return False
+ os.listdir(projects_dir)
+ return False
+ except OSError:
+ return True
def _index_hit_excluded(
@@ -104,7 +133,7 @@ def _index_hit_excluded(
file_path,
exc_info=True,
)
- return False
+ return True
return is_session_excluded(rules, session, project_name)
@@ -116,13 +145,14 @@ def _search_via_index(
*,
since_ms: int | None,
max_results: int,
-) -> list[SearchHitDict] | None:
+) -> tuple[list[SearchHitDict] | None, bool]:
if not index_search_enabled() or not index_is_usable(projects_dir, rules):
- return None
+ return None, False
rules_fp = rules_fingerprint(rules)
results: list[SearchHitDict] = []
sql_offset = 0
+ fts_exhausted = False
while len(results) < max_results:
need = max_results - len(results)
indexed = query_index_hits(
@@ -131,9 +161,18 @@ def _search_via_index(
max_results=need,
sql_offset=sql_offset,
)
+ if indexed["index_locked"]:
+ _logger.warning(
+ "Search index locked during query; %d hit(s) collected so far",
+ len(results),
+ )
+ if results:
+ return _rank_search_hits(results)[:max_results], False
+ return None, False
if not indexed["query_ok"]:
- return None
+ return None, False
if indexed["sql_rows_fetched"] == 0:
+ fts_exhausted = indexed["sql_exhausted"]
break
for hit in indexed["hits"]:
@@ -160,8 +199,50 @@ def _search_via_index(
sql_offset += indexed["sql_rows_fetched"]
if indexed["sql_exhausted"]:
+ fts_exhausted = True
break
- return _rank_search_hits(results)[:max_results]
+ return _rank_search_hits(results)[:max_results], fts_exhausted
+
+
+def _resolve_search_results(
+ base: str,
+ rules: list[Any],
+ query: str,
+ query_lower: str,
+ *,
+ since_ms: int | None,
+ max_results: int,
+) -> list[SearchHitDict]:
+ indexed, _fts_exhausted = _search_via_index(
+ base,
+ rules,
+ query,
+ query_lower,
+ since_ms=since_ms,
+ max_results=max_results,
+ )
+ if indexed is None:
+ return _search_live_scan(
+ base,
+ rules,
+ query,
+ query_lower,
+ since_ms=since_ms,
+ max_results=max_results,
+ )
+
+ if len(indexed) >= max_results:
+ return indexed
+
+ live = _search_live_scan(
+ base,
+ rules,
+ query,
+ query_lower,
+ since_ms=since_ms,
+ max_results=max_results,
+ )
+ return _merge_search_hits(indexed, live, max_results=max_results)
def _search_live_scan(
@@ -192,7 +273,7 @@ def _search_live_scan(
continue
for msg in session["messages"]:
- text = _message_searchable_text(msg)
+ text = message_searchable_text(msg)
if not text or query_lower not in text.lower():
continue
if not timestamp_in_search_window_iso(
@@ -215,9 +296,20 @@ def _search_live_scan(
@search_bp.route("/api/search")
def search() -> FlaskReturn:
- query = request.args.get("q", "").strip()
+ raw_query = request.args.get("q", "")
+ query = raw_query.strip()
if not query:
- return json_response([])
+ return error_response(
+ ErrorCode.SEARCH_EMPTY_QUERY,
+ "Search query is required",
+ 400,
+ )
+ if len(query) > _MAX_QUERY_LEN:
+ return error_response(
+ ErrorCode.SEARCH_QUERY_TOO_LONG,
+ f"Search query must be at most {_MAX_QUERY_LEN} characters",
+ 400,
+ )
try:
max_results = _parse_limit(request.args.get("limit"))
@@ -228,35 +320,49 @@ def search() -> FlaskReturn:
400,
)
+ since_days_raw = request.args.get("since_days")
+ try:
+ since_days = _parse_since_days(since_days_raw)
+ except ValueError:
+ return error_response(
+ ErrorCode.SEARCH_INVALID_SINCE_DAYS,
+ "Invalid since_days: must be a positive integer",
+ 400,
+ )
+
query_lower = query.lower()
all_history = request.args.get("all_history") in ("1", "true")
since_ms = resolve_search_since_ms(
all_history=all_history,
- since_days=_parse_since_days(request.args.get("since_days")),
+ since_days=since_days,
now=datetime.now(timezone.utc),
)
base = current_app.config.get("CLAUDE_PROJECTS_DIR") or get_claude_projects_dir()
- rules = current_app.config.get("EXCLUSION_RULES") or []
+ if _projects_dir_inaccessible(base):
+ return error_response(
+ ErrorCode.SEARCH_PROJECTS_UNAVAILABLE,
+ "Claude projects directory is not accessible",
+ 503,
+ )
- indexed = _search_via_index(
- base,
- rules,
- query,
- query_lower,
- since_ms=since_ms,
- max_results=max_results,
- )
- if indexed is not None:
- return json_response(indexed)
+ rules = current_app.config.get("EXCLUSION_RULES") or []
- return json_response(
- _search_live_scan(
- base,
- rules,
- query,
- query_lower,
- since_ms=since_ms,
- max_results=max_results,
+ try:
+ return json_response(
+ _resolve_search_results(
+ base,
+ rules,
+ query,
+ query_lower,
+ since_ms=since_ms,
+ max_results=max_results,
+ )
+ )
+ except Exception:
+ _logger.exception("Unexpected error during search")
+ return error_response(
+ ErrorCode.INTERNAL_ERROR,
+ "Search failed",
+ 500,
)
- )
diff --git a/app.py b/app.py
index 67b5b21..0144681 100644
--- a/app.py
+++ b/app.py
@@ -92,8 +92,11 @@ def validate_startup_cli(args: argparse.Namespace) -> None:
def create_app(
base_dir: str | None = None,
exclusion_rules_path: str | None = None,
+ *,
+ testing: bool = False,
) -> Flask:
app = Flask(__name__)
+ app.config["TESTING"] = testing
app.config["CLAUDE_PROJECTS_DIR"] = base_dir
resolved = resolve_exclusion_rules_path(exclusion_rules_path)
@@ -106,7 +109,7 @@ def create_app(
app.register_blueprint(export_bp)
app.register_blueprint(schema_report_bp)
- if not app.config.get("TESTING"):
+ if not testing:
try:
from utils.search_index import start_search_index_background
diff --git a/docs/api-reference.md b/docs/api-reference.md
index d979cf3..47d1e77 100644
--- a/docs/api-reference.md
+++ b/docs/api-reference.md
@@ -56,6 +56,10 @@ Extra fields may appear for specific codes (for example `since` on invalid bulk-
| `code` | HTTP | Routes | Meaning |
|--------|------|--------|---------|
| `SEARCH_INVALID_LIMIT` | 400 | `GET /api/search` | Query param `limit` is not a positive integer |
+| `SEARCH_EMPTY_QUERY` | 400 | `GET /api/search` | Query param `q` is empty or whitespace |
+| `SEARCH_QUERY_TOO_LONG` | 400 | `GET /api/search` | Query param `q` exceeds 500 characters |
+| `SEARCH_INVALID_SINCE_DAYS` | 400 | `GET /api/search` | Query param `since_days` is not a positive integer |
+| `SEARCH_PROJECTS_UNAVAILABLE` | 503 | `GET /api/search` | Claude projects directory exists but is not readable |
| `INVALID_PATH` | 400 | Session, stats, export session | Path traversal or rejected URL segment |
| `SESSION_NOT_FOUND` | 404 | Session, stats, export session | File missing on disk or session excluded by rules |
| `INVALID_REQUEST_BODY` | 400 | `POST /api/export` | Body is not a JSON object |
@@ -280,14 +284,18 @@ curl -s "http://127.0.0.1:5000/api/sessions/F--boost-capy/session_abc123/stats"
**Source:** [`api/search.py`](../api/search.py)
-Case-insensitive substring search across all non-excluded messages in all projects. Linear scan — suitable for local history size, not indexed search.
+Case-insensitive substring search across all non-excluded messages in all projects. Uses a local FTS5 index when available; falls back to a live JSONL scan when the index is missing, stale, or the query cannot be served from SQLite.
#### Query parameters
| Name | Type | Default | Description |
|------|------|---------|-------------|
-| `q` | string | `""` | Search string; whitespace stripped; empty → `[]` |
+| `q` | string | — | Search string (required); whitespace stripped; max 500 characters |
| `limit` | integer | `50` | Max results; must be ≥ 1; **capped at 500** |
+| `all_history` | flag | off | When `1` or `true`, search all history (no time window) |
+| `since_days` | integer | `30` | When `all_history` is off, include messages from the last *N* days (positive integer; capped at 36 500) |
+
+By default, messages older than 30 days are excluded. Sessions without a parseable timestamp may still match when windowed.
#### Response — `200 OK`
@@ -306,10 +314,16 @@ Case-insensitive substring search across all non-excluded messages in all projec
| Status | `code` | When |
|--------|--------|------|
+| 400 | `SEARCH_EMPTY_QUERY` | `q` is empty or whitespace |
+| 400 | `SEARCH_QUERY_TOO_LONG` | `q` exceeds 500 characters |
| 400 | `SEARCH_INVALID_LIMIT` | `limit` not a positive integer (e.g. `abc`, `0`, `1.5`) |
+| 400 | `SEARCH_INVALID_SINCE_DAYS` | `since_days` not a positive integer |
+| 503 | `SEARCH_PROJECTS_UNAVAILABLE` | Projects directory exists but is not readable |
+| 500 | `INTERNAL_ERROR` | Unexpected server failure |
```bash
curl -s "http://127.0.0.1:5000/api/search?q=parser&limit=10" | jq '.[0]'
+curl -s "http://127.0.0.1:5000/api/search?q=" # → 400 SEARCH_EMPTY_QUERY
curl -s "http://127.0.0.1:5000/api/search?q=test&limit=abc" # → 400
```
diff --git a/models/error_codes.py b/models/error_codes.py
index 1d81781..4295b68 100644
--- a/models/error_codes.py
+++ b/models/error_codes.py
@@ -7,6 +7,10 @@
class ErrorCode(StrEnum):
SEARCH_INVALID_LIMIT = "SEARCH_INVALID_LIMIT"
+ SEARCH_EMPTY_QUERY = "SEARCH_EMPTY_QUERY"
+ SEARCH_QUERY_TOO_LONG = "SEARCH_QUERY_TOO_LONG"
+ SEARCH_INVALID_SINCE_DAYS = "SEARCH_INVALID_SINCE_DAYS"
+ SEARCH_PROJECTS_UNAVAILABLE = "SEARCH_PROJECTS_UNAVAILABLE"
INVALID_PATH = "INVALID_PATH"
SESSION_NOT_FOUND = "SESSION_NOT_FOUND"
INVALID_REQUEST_BODY = "INVALID_REQUEST_BODY"
diff --git a/static/js/search.js b/static/js/search.js
index 8192c15..47bfcc1 100644
--- a/static/js/search.js
+++ b/static/js/search.js
@@ -6,8 +6,33 @@ import { showProjects } from './projects.js';
// ==================== Search ====================
+const SEARCH_LIMIT = 50;
+
let lastSearchRequestId = 0;
+export function highlightSnippet(snippet, query) {
+ if (!snippet) return '';
+ if (!query) return esc(snippet);
+ const chars = [...snippet];
+ const needle = [...query].map((ch) => ch.toLowerCase());
+ const hay = chars.map((ch) => ch.toLowerCase());
+ for (let i = 0; i <= hay.length - needle.length; i += 1) {
+ let matched = true;
+ for (let j = 0; j < needle.length; j += 1) {
+ if (hay[i + j] !== needle[j]) {
+ matched = false;
+ break;
+ }
+ }
+ if (!matched) continue;
+ const before = chars.slice(0, i).join('');
+ const match = chars.slice(i, i + needle.length).join('');
+ const after = chars.slice(i + needle.length).join('');
+ return esc(before) + '' + esc(match) + '' + esc(after);
+ }
+ return esc(snippet);
+}
+
export function showSearchPage() {
setHamburgerVisible(false);
setWorkspaceMode(false);
@@ -21,6 +46,14 @@ export function showSearchPage() {
+ By default, search covers the last 30 days. Chats without a parseable date may still appear. + Results are capped at ${SEARCH_LIMIT} (up to 500 via the API). Check + + to include older messages. +