diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6611ad2..044a787 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,7 +38,7 @@ jobs: run: | python - <<'PY' from app import create_app - app = create_app() + app = create_app(testing=True) client = app.test_client() assert client.get("/").status_code == 200 PY diff --git a/api/search.py b/api/search.py index e16427f..6bfd5a0 100644 --- a/api/search.py +++ b/api/search.py @@ -3,6 +3,7 @@ from __future__ import annotations import logging +import os from datetime import datetime, timezone from typing import Any @@ -11,17 +12,16 @@ from api._flask_types import FlaskReturn, json_response from api.error_codes import ErrorCode, error_response from models.search import SearchHitDict -from models.session import MessageDict from utils.exclusion_rules import is_session_excluded from utils.search_index import ( index_is_usable, index_search_enabled, + message_searchable_text, query_index_hits, resolve_search_since_ms, search_snippet, timestamp_in_search_window_iso, timestamp_to_ms, - tool_result_searchable_text, ) from utils.session_cache import get_cached_session from utils.session_path import get_claude_projects_dir, list_projects, list_sessions @@ -32,6 +32,7 @@ _DEFAULT_LIMIT = 50 _MAX_LIMIT = 500 +_MAX_QUERY_LEN = 500 _MAX_SEARCH_SINCE_DAYS = 36_500 @@ -53,10 +54,10 @@ def _parse_since_days(raw: str | None) -> int | None: return None try: days = int(str(raw).strip()) - except ValueError: - return None + except ValueError as exc: + raise ValueError("Invalid since_days: must be a positive integer") from exc if days <= 0: - return None + raise ValueError("Invalid since_days: must be a positive integer") return min(days, _MAX_SEARCH_SINCE_DAYS) @@ -71,16 +72,44 @@ def _rank_search_hits(results: list[SearchHitDict]) -> list[SearchHitDict]: ) -def _message_searchable_text(msg: MessageDict) -> str: - text = msg.get("text", "") or msg.get("content", "") - if not isinstance(text, str): - text = "" - tool_result = msg.get("tool_result") - if isinstance(tool_result, dict): - tool_text = tool_result_searchable_text(tool_result) - if tool_text: - text = f"{text}\n{tool_text}" if text else tool_text - return text +def _hit_dedup_key(hit: SearchHitDict) -> tuple[str, str, str | None, str]: + ts = hit.get("timestamp") + return ( + hit["project"], + hit["session_id"], + ts if isinstance(ts, str) else None, + hit["role"], + ) + + +def _merge_search_hits( + primary: list[SearchHitDict], + extra: list[SearchHitDict], + *, + max_results: int, +) -> list[SearchHitDict]: + seen = {_hit_dedup_key(hit) for hit in primary} + merged = list(primary) + for hit in extra: + key = _hit_dedup_key(hit) + if key in seen: + continue + merged.append(hit) + seen.add(key) + if len(merged) >= max_results: + break + return _rank_search_hits(merged)[:max_results] + + +def _projects_dir_inaccessible(projects_dir: str) -> bool: + """True when the projects path exists but cannot be listed (503 case).""" + try: + if not os.path.isdir(projects_dir): + return False + os.listdir(projects_dir) + return False + except OSError: + return True def _index_hit_excluded( @@ -104,7 +133,7 @@ def _index_hit_excluded( file_path, exc_info=True, ) - return False + return True return is_session_excluded(rules, session, project_name) @@ -116,13 +145,14 @@ def _search_via_index( *, since_ms: int | None, max_results: int, -) -> list[SearchHitDict] | None: +) -> tuple[list[SearchHitDict] | None, bool]: if not index_search_enabled() or not index_is_usable(projects_dir, rules): - return None + return None, False rules_fp = rules_fingerprint(rules) results: list[SearchHitDict] = [] sql_offset = 0 + fts_exhausted = False while len(results) < max_results: need = max_results - len(results) indexed = query_index_hits( @@ -131,9 +161,18 @@ def _search_via_index( max_results=need, sql_offset=sql_offset, ) + if indexed["index_locked"]: + _logger.warning( + "Search index locked during query; %d hit(s) collected so far", + len(results), + ) + if results: + return _rank_search_hits(results)[:max_results], False + return None, False if not indexed["query_ok"]: - return None + return None, False if indexed["sql_rows_fetched"] == 0: + fts_exhausted = indexed["sql_exhausted"] break for hit in indexed["hits"]: @@ -160,8 +199,50 @@ def _search_via_index( sql_offset += indexed["sql_rows_fetched"] if indexed["sql_exhausted"]: + fts_exhausted = True break - return _rank_search_hits(results)[:max_results] + return _rank_search_hits(results)[:max_results], fts_exhausted + + +def _resolve_search_results( + base: str, + rules: list[Any], + query: str, + query_lower: str, + *, + since_ms: int | None, + max_results: int, +) -> list[SearchHitDict]: + indexed, _fts_exhausted = _search_via_index( + base, + rules, + query, + query_lower, + since_ms=since_ms, + max_results=max_results, + ) + if indexed is None: + return _search_live_scan( + base, + rules, + query, + query_lower, + since_ms=since_ms, + max_results=max_results, + ) + + if len(indexed) >= max_results: + return indexed + + live = _search_live_scan( + base, + rules, + query, + query_lower, + since_ms=since_ms, + max_results=max_results, + ) + return _merge_search_hits(indexed, live, max_results=max_results) def _search_live_scan( @@ -192,7 +273,7 @@ def _search_live_scan( continue for msg in session["messages"]: - text = _message_searchable_text(msg) + text = message_searchable_text(msg) if not text or query_lower not in text.lower(): continue if not timestamp_in_search_window_iso( @@ -215,9 +296,20 @@ def _search_live_scan( @search_bp.route("/api/search") def search() -> FlaskReturn: - query = request.args.get("q", "").strip() + raw_query = request.args.get("q", "") + query = raw_query.strip() if not query: - return json_response([]) + return error_response( + ErrorCode.SEARCH_EMPTY_QUERY, + "Search query is required", + 400, + ) + if len(query) > _MAX_QUERY_LEN: + return error_response( + ErrorCode.SEARCH_QUERY_TOO_LONG, + f"Search query must be at most {_MAX_QUERY_LEN} characters", + 400, + ) try: max_results = _parse_limit(request.args.get("limit")) @@ -228,35 +320,49 @@ def search() -> FlaskReturn: 400, ) + since_days_raw = request.args.get("since_days") + try: + since_days = _parse_since_days(since_days_raw) + except ValueError: + return error_response( + ErrorCode.SEARCH_INVALID_SINCE_DAYS, + "Invalid since_days: must be a positive integer", + 400, + ) + query_lower = query.lower() all_history = request.args.get("all_history") in ("1", "true") since_ms = resolve_search_since_ms( all_history=all_history, - since_days=_parse_since_days(request.args.get("since_days")), + since_days=since_days, now=datetime.now(timezone.utc), ) base = current_app.config.get("CLAUDE_PROJECTS_DIR") or get_claude_projects_dir() - rules = current_app.config.get("EXCLUSION_RULES") or [] + if _projects_dir_inaccessible(base): + return error_response( + ErrorCode.SEARCH_PROJECTS_UNAVAILABLE, + "Claude projects directory is not accessible", + 503, + ) - indexed = _search_via_index( - base, - rules, - query, - query_lower, - since_ms=since_ms, - max_results=max_results, - ) - if indexed is not None: - return json_response(indexed) + rules = current_app.config.get("EXCLUSION_RULES") or [] - return json_response( - _search_live_scan( - base, - rules, - query, - query_lower, - since_ms=since_ms, - max_results=max_results, + try: + return json_response( + _resolve_search_results( + base, + rules, + query, + query_lower, + since_ms=since_ms, + max_results=max_results, + ) + ) + except Exception: + _logger.exception("Unexpected error during search") + return error_response( + ErrorCode.INTERNAL_ERROR, + "Search failed", + 500, ) - ) diff --git a/app.py b/app.py index 67b5b21..0144681 100644 --- a/app.py +++ b/app.py @@ -92,8 +92,11 @@ def validate_startup_cli(args: argparse.Namespace) -> None: def create_app( base_dir: str | None = None, exclusion_rules_path: str | None = None, + *, + testing: bool = False, ) -> Flask: app = Flask(__name__) + app.config["TESTING"] = testing app.config["CLAUDE_PROJECTS_DIR"] = base_dir resolved = resolve_exclusion_rules_path(exclusion_rules_path) @@ -106,7 +109,7 @@ def create_app( app.register_blueprint(export_bp) app.register_blueprint(schema_report_bp) - if not app.config.get("TESTING"): + if not testing: try: from utils.search_index import start_search_index_background diff --git a/docs/api-reference.md b/docs/api-reference.md index d979cf3..47d1e77 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -56,6 +56,10 @@ Extra fields may appear for specific codes (for example `since` on invalid bulk- | `code` | HTTP | Routes | Meaning | |--------|------|--------|---------| | `SEARCH_INVALID_LIMIT` | 400 | `GET /api/search` | Query param `limit` is not a positive integer | +| `SEARCH_EMPTY_QUERY` | 400 | `GET /api/search` | Query param `q` is empty or whitespace | +| `SEARCH_QUERY_TOO_LONG` | 400 | `GET /api/search` | Query param `q` exceeds 500 characters | +| `SEARCH_INVALID_SINCE_DAYS` | 400 | `GET /api/search` | Query param `since_days` is not a positive integer | +| `SEARCH_PROJECTS_UNAVAILABLE` | 503 | `GET /api/search` | Claude projects directory exists but is not readable | | `INVALID_PATH` | 400 | Session, stats, export session | Path traversal or rejected URL segment | | `SESSION_NOT_FOUND` | 404 | Session, stats, export session | File missing on disk or session excluded by rules | | `INVALID_REQUEST_BODY` | 400 | `POST /api/export` | Body is not a JSON object | @@ -280,14 +284,18 @@ curl -s "http://127.0.0.1:5000/api/sessions/F--boost-capy/session_abc123/stats" **Source:** [`api/search.py`](../api/search.py) -Case-insensitive substring search across all non-excluded messages in all projects. Linear scan — suitable for local history size, not indexed search. +Case-insensitive substring search across all non-excluded messages in all projects. Uses a local FTS5 index when available; falls back to a live JSONL scan when the index is missing, stale, or the query cannot be served from SQLite. #### Query parameters | Name | Type | Default | Description | |------|------|---------|-------------| -| `q` | string | `""` | Search string; whitespace stripped; empty → `[]` | +| `q` | string | — | Search string (required); whitespace stripped; max 500 characters | | `limit` | integer | `50` | Max results; must be ≥ 1; **capped at 500** | +| `all_history` | flag | off | When `1` or `true`, search all history (no time window) | +| `since_days` | integer | `30` | When `all_history` is off, include messages from the last *N* days (positive integer; capped at 36 500) | + +By default, messages older than 30 days are excluded. Sessions without a parseable timestamp may still match when windowed. #### Response — `200 OK` @@ -306,10 +314,16 @@ Case-insensitive substring search across all non-excluded messages in all projec | Status | `code` | When | |--------|--------|------| +| 400 | `SEARCH_EMPTY_QUERY` | `q` is empty or whitespace | +| 400 | `SEARCH_QUERY_TOO_LONG` | `q` exceeds 500 characters | | 400 | `SEARCH_INVALID_LIMIT` | `limit` not a positive integer (e.g. `abc`, `0`, `1.5`) | +| 400 | `SEARCH_INVALID_SINCE_DAYS` | `since_days` not a positive integer | +| 503 | `SEARCH_PROJECTS_UNAVAILABLE` | Projects directory exists but is not readable | +| 500 | `INTERNAL_ERROR` | Unexpected server failure | ```bash curl -s "http://127.0.0.1:5000/api/search?q=parser&limit=10" | jq '.[0]' +curl -s "http://127.0.0.1:5000/api/search?q=" # → 400 SEARCH_EMPTY_QUERY curl -s "http://127.0.0.1:5000/api/search?q=test&limit=abc" # → 400 ``` diff --git a/models/error_codes.py b/models/error_codes.py index 1d81781..4295b68 100644 --- a/models/error_codes.py +++ b/models/error_codes.py @@ -7,6 +7,10 @@ class ErrorCode(StrEnum): SEARCH_INVALID_LIMIT = "SEARCH_INVALID_LIMIT" + SEARCH_EMPTY_QUERY = "SEARCH_EMPTY_QUERY" + SEARCH_QUERY_TOO_LONG = "SEARCH_QUERY_TOO_LONG" + SEARCH_INVALID_SINCE_DAYS = "SEARCH_INVALID_SINCE_DAYS" + SEARCH_PROJECTS_UNAVAILABLE = "SEARCH_PROJECTS_UNAVAILABLE" INVALID_PATH = "INVALID_PATH" SESSION_NOT_FOUND = "SESSION_NOT_FOUND" INVALID_REQUEST_BODY = "INVALID_REQUEST_BODY" diff --git a/static/js/search.js b/static/js/search.js index 8192c15..47bfcc1 100644 --- a/static/js/search.js +++ b/static/js/search.js @@ -6,8 +6,33 @@ import { showProjects } from './projects.js'; // ==================== Search ==================== +const SEARCH_LIMIT = 50; + let lastSearchRequestId = 0; +export function highlightSnippet(snippet, query) { + if (!snippet) return ''; + if (!query) return esc(snippet); + const chars = [...snippet]; + const needle = [...query].map((ch) => ch.toLowerCase()); + const hay = chars.map((ch) => ch.toLowerCase()); + for (let i = 0; i <= hay.length - needle.length; i += 1) { + let matched = true; + for (let j = 0; j < needle.length; j += 1) { + if (hay[i + j] !== needle[j]) { + matched = false; + break; + } + } + if (!matched) continue; + const before = chars.slice(0, i).join(''); + const match = chars.slice(i, i + needle.length).join(''); + const after = chars.slice(i + needle.length).join(''); + return esc(before) + '' + esc(match) + '' + esc(after); + } + return esc(snippet); +} + export function showSearchPage() { setHamburgerVisible(false); setWorkspaceMode(false); @@ -21,6 +46,14 @@ export function showSearchPage() {

Search

+

+ By default, search covers the last 30 days. Chats without a parseable date may still appear. + Results are capped at ${SEARCH_LIMIT} (up to 500 via the API). Check + + to include older messages. +