diff --git a/.env.example b/.env.example index d8ba90a..3712336 100644 --- a/.env.example +++ b/.env.example @@ -1,3 +1,18 @@ +# ── Deployment boundary ───────────────────────────────────── +# development/local are localhost-only. Production intentionally refuses to +# start until authentication is implemented and explicitly enabled. +MKB_DEPLOYMENT_MODE=development +MKB_API_HOST=127.0.0.1 +MKB_API_PORT=8503 +# JSON list because pydantic-settings parses complex environment values. +MKB_CORS_ORIGINS=["http://127.0.0.1:5173","http://localhost:5173"] +MKB_AUTHENTICATION_ENABLED=false +# Required when authentication is enabled. JSON maps random tokens (32+ chars) +# to reader, editor, or admin roles. Generate tokens with `openssl rand -hex 32`. +# MKB_AUTH_TOKENS={"replace-with-a-random-token-at-least-32-chars":"admin"} +# DANGEROUS: uploaded Python runs with this process's host access. +MKB_ALLOW_UPLOADED_PYTHON=false + # ── PostgreSQL ─────────────────────────────────────────────── MKB_PG_HOST=localhost MKB_PG_PORT=5432 @@ -10,6 +25,7 @@ MKB_S3_ENDPOINT=http://localhost:9000 MKB_S3_ACCESS_KEY=minioadmin MKB_S3_SECRET_KEY=minioadmin MKB_S3_BUCKET_RAW=raw +MKB_S3_BUCKET_PROCESSED=processed MKB_S3_BUCKET_ARCHIVE=archive MKB_S3_BUCKET_TEMP=temp @@ -27,10 +43,25 @@ OPENAI_API_BASE= # MKB_OPENAI_API_BASE= # ── Logging ───────────────────────────────────────────────── -# DEBUG (default) — verbose: agent dialogs, tool calls, full MinerU +# DEBUG — verbose: agent dialogs, tool calls, full MinerU # output, and third-party traces. -# INFO — concise app-level messages only. -MKB_LOG_LEVEL=DEBUG +# INFO (default) — concise app-level messages only. +MKB_LOG_LEVEL=INFO MKB_LOG_DIR=logs # MKB_LOG_FILE_MAX_MB=20 # MKB_LOG_FILE_BACKUP_COUNT=5 + +# ── Upload/archive resource budgets ───────────────────────── +# MKB_UPLOAD_MAX_FILE_MB=100 +# MKB_UPLOAD_MAX_TOTAL_MB=500 +# MKB_UPLOAD_MAX_FILES=1000 +# MKB_UPLOAD_MAX_PATH_DEPTH=12 +# MKB_ARCHIVE_MAX_EXPANDED_MB=500 +# MKB_ARCHIVE_MAX_MEMBER_MB=100 +# MKB_ARCHIVE_MAX_MEMBERS=2000 +# MKB_ARCHIVE_MAX_COMPRESSION_RATIO=100 +# MKB_ARCHIVE_MAX_NESTING=3 +# MKB_RATE_LIMIT_UPLOAD_PER_MINUTE=30 +# MKB_RATE_LIMIT_ASSISTANT_PER_MINUTE=20 +# MKB_RATE_LIMIT_JOB_START_PER_MINUTE=30 +# MKB_RATE_LIMIT_AUTH_FAILURES_PER_MINUTE=10 diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..242fc9e --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,14 @@ +# Default review owner. +* @theAfish + +# Database access requires database review. +/src/mkb/db/ @theAfish + +# Authentication, uploads, executable content, and deployment boundaries require security review. +/src/mkb/web/security.py @theAfish +/src/mkb/web/uploads.py @theAfish +/src/mkb/config.py @theAfish +/src/mkb/post_processors/ @theAfish +/docker-compose.yaml @theAfish +/SECURITY.md @theAfish +/docs/security.md @theAfish diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..8fc2e5b --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,21 @@ +## Summary + +Describe the user-visible change and why it is needed. + +## Validation + +- [ ] `make lint` +- [ ] `make test` +- [ ] `make test-frontend` (or not applicable, explained below) +- [ ] Documentation updated for observable API/operator behavior + +## Risk and ownership + +- [ ] No database migration, or migration owner requested +- [ ] No security-sensitive change, or security owner requested with threat/risk notes +- [ ] No new secret, generated data, logs, or local environment files committed +- [ ] Current React path used; legacy canonical compatibility impact noted + +## Migration / rollback + +Describe data migration, compatibility, and rollback/restore steps, or write “none”. diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml new file mode 100644 index 0000000..d6cbcfb --- /dev/null +++ b/.github/workflows/frontend.yml @@ -0,0 +1,23 @@ +name: Frontend + +on: + pull_request: + push: + branches: [main] + +jobs: + build-and-budget: + runs-on: ubuntu-latest + defaults: + run: + working-directory: frontend + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: npm + cache-dependency-path: frontend/package-lock.json + - run: npm ci + - run: npm run lint + - run: npm run build diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml new file mode 100644 index 0000000..a0de4d8 --- /dev/null +++ b/.github/workflows/python.yml @@ -0,0 +1,26 @@ +name: Python distributions + +on: + pull_request: + push: + branches: [main] + +jobs: + test-and-install: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.10", "3.12"] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: pip + - run: python -m pip install --upgrade pip build + - run: python -m pip install ".[all,dev]" + - run: python -m pytest + - run: python -m build + - run: python -m venv /tmp/mkb-consumer + - run: /tmp/mkb-consumer/bin/python -m pip install dist/*.whl + - run: cd /tmp && /tmp/mkb-consumer/bin/python "$GITHUB_WORKSPACE/examples/portable_quickstart.py" diff --git a/.gitignore b/.gitignore index a5417a2..717314c 100644 --- a/.gitignore +++ b/.gitignore @@ -17,16 +17,14 @@ build/ *.swo *~ +.vscode/ + # Environment .env # Data -data/inbox/* -!data/inbox/.gitkeep - -data/processed/* -data/papers/* -data/uploads/* +data/ +migration-snapshots/ # Docker volumes docker_volumes/ @@ -45,4 +43,4 @@ Thumbs.db data/runtime_settings.json # Logs -logs/ \ No newline at end of file +logs/ diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..e1e402a --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,17 @@ +# Changelog + +MKB follows [Semantic Versioning](https://semver.org/). Until 1.0, minor releases +may refine the SDK while documented public imports and persisted-data compatibility +remain protected by tests and migration gates. + +## 0.1.0 - Unreleased + +- Introduce the configured `KnowledgeBase` SDK and typed grouped services. +- Add portable SQLite/filesystem repositories, pipelines, jobs, graph operations, + evidence, schemas, projections, and public extension registries. +- Preserve the existing PostgreSQL/MinIO materials application through injected + compatibility adapters. +- Add inventory, reconciliation, preflight comparison, snapshot validation, and + checksum-gated missing-object repair tooling. +- Split PostgreSQL, S3, PDF, server, materials, Neo4j, and full application support + into optional installation extras. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..f6ff6e9 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,20 @@ +# Contributing + +Start with `make bootstrap`, review `.env`, then run `make up`, `make doctor`, and +`make check`. Keep adapters thin: domain behavior belongs in `src/mkb/services/`, HTTP +mapping in `src/mkb/web/`, CLI parsing in `src/mkb/cli.py`, and current UI work in +`frontend/`. Canonical-workflow compatibility paths are legacy. + +Use a focused branch and include tests for behavior changes. Before opening a pull +request, run `make lint`, `make test`, and `make test-frontend`. Update the Python API, +HTTP contract, operator, or security docs whenever their observable behavior changes. + +Database migrations require review from the database/migrations owner. Never edit a +published migration; add a new one and test upgrade plus restore. Changes involving +authentication, authorization, secrets, uploads/archives, executable processors, +CORS, filesystem paths, network access, or destructive endpoints require security +owner review and a short threat/risk note in the pull request. + +Do not commit `.env`, credentials, research data, generated exports, logs, or local +database/object-store state. Report vulnerabilities privately according to +[SECURITY.md](SECURITY.md), not in a public issue. diff --git a/Makefile b/Makefile index 4c2180b..2e68222 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,32 @@ -.PHONY: up down logs migrate ingest list batches info purge install test +.PHONY: setup bootstrap install install-python install-frontend up down logs doctor ingest list batches info purge test lint test-python test-frontend build dev server check ci cleanup reconcile pack unpack restore-drill + +PYTHON ?= .venv/bin/python +BOOTSTRAP_PYTHON ?= python3 +NPM ?= npm +export PYTHONPATH := $(CURDIR)/src$(if $(PYTHONPATH),:$(PYTHONPATH)) + +# ── Clean-clone bootstrap ────────────────────────────────────── +setup: bootstrap + +bootstrap: + $(BOOTSTRAP_PYTHON) -m venv .venv + $(PYTHON) -m pip install --upgrade pip + $(MAKE) install + @test -f .env || cp .env.example .env + @echo "Bootstrap complete. Review .env, then run 'make up' and 'make dev'." + +install: install-python install-frontend + +install-python: + $(PYTHON) -m pip install -e ".[all,dev]" + +install-frontend: + cd frontend && $(NPM) ci # ── Infrastructure ────────────────────────────────────────────── up: - docker compose up -d - @echo "Waiting for services…" - @docker compose exec postgres pg_isready -U mkb -q && echo "PostgreSQL ready" || true - @echo "MinIO console: http://localhost:9001 (minioadmin / minioadmin)" + docker compose up -d --wait + @echo "MKB data services are healthy." down: docker compose down @@ -13,32 +34,30 @@ down: logs: docker compose logs -f -# ── Database ──────────────────────────────────────────────────── -migrate: - alembic upgrade head - -migration: ## usage: make migration msg="add foo table" - alembic revision --autogenerate -m "$(msg)" - -# ── Python ────────────────────────────────────────────────────── -install: - pip install -e ".[dev]" +doctor: + $(PYTHON) -m mkb.doctor # ── CLI shortcuts ─────────────────────────────────────────────── ingest: ## usage: make ingest dir=./data/inbox - python -m mkb.cli ingest $(dir) + $(PYTHON) -m mkb.cli ingest $(dir) list: - python -m mkb.cli list + $(PYTHON) -m mkb.cli list batches: - python -m mkb.cli batches + $(PYTHON) -m mkb.cli batches info: ## usage: make info id= - python -m mkb.cli info $(id) + $(PYTHON) -m mkb.cli info $(id) purge: - python -m mkb.cli purge + $(PYTHON) -m mkb.cli purge + +cleanup: + $(PYTHON) -m mkb.cli cleanup + +reconcile: + $(PYTHON) -m mkb.cli reconcile # ── Data sharing ──────────────────────────────────────────────── pack: ## Create a portable snapshot: make pack [out=my_snapshot.tar.gz] @@ -48,10 +67,36 @@ unpack: ## Restore from snapshot: make unpack file=mkb_data_YYYYMMDD.tar.gz @[ -n "$(file)" ] || (echo "Usage: make unpack file="; exit 1) bash scripts/unpack_data.sh $(file) +restore-drill: + bash scripts/restore_drill.sh $(if $(file),$(file),) + # ── Server ────────────────────────────────────────────────────── server: - python -m mkb.cli api --host 127.0.0.1 --port 8503 + $(PYTHON) -m mkb.cli api --host 127.0.0.1 --port 8503 + +dev: + PYTHON="$(PYTHON)" NPM="$(NPM)" bash scripts/dev.sh + +build: + $(PYTHON) -m pip wheel --no-deps --wheel-dir build/wheels . + cd frontend && $(NPM) run build # ── Tests ─────────────────────────────────────────────────────── test: - pytest tests/ -v + $(PYTHON) -m pytest tests/ -v + +lint: + $(PYTHON) -m ruff check src tests + cd frontend && $(NPM) run lint + +test-python: + $(PYTHON) -m pytest + +test-frontend: + cd frontend && $(NPM) run build + +check: lint test-python test-frontend + +ci: + $(PYTHON) -m ruff check src tests + $(PYTHON) -m pytest --collect-only -q diff --git a/README.md b/README.md index 4a6932c..071c80e 100644 --- a/README.md +++ b/README.md @@ -1,653 +1,146 @@ # mat-know-base -A self-hosted system for ingesting scientific papers and related data into a structured knowledge base. Files are stored immutably using content-addressable storage (SHA256 deduplication), with metadata tracked in PostgreSQL + pgvector and raw binaries in MinIO (S3-compatible). A processing pipeline converts raw files into LLM-readable formats. An LLM agent then extracts structured **knowledge frames** — one per research project — with flexible, agent-decided structure capturing all scientific knowledge from the source material. +Materials Knowledge Base (MKB) is a local-first application for ingesting scientific +papers and supplementary files, processing them into LLM-readable artifacts, and +building structured knowledge frames, domain projections, workflows, and graphs. -## Architecture +The React application is the user interface. Canonical-workflow paths remain a limited +compatibility surface; see the [workflow lifecycle policy](docs/workflow-lifecycle-policy.md). -``` -data/papers/smith2024/ Research package (paper + supplementary) - │ - ▼ -┌──────────────┐ SHA256 ┌─────────────────────────────────┐ -│ Ingestion ├──────────►│ MinIO (S3) │ -│ Worker │ │ raw/ ← original files │ -└──────┬───────┘ │ processed/ ← converted outputs │ - │ metadata └─────────────────────────────────┘ - ▼ ▲ -┌─────────────────┐ │ upload converted files -│ PostgreSQL │ ┌────────┴──────────┐ -│ + pgvector │ │ Processing Pipeline│ -│ │ ◄────┤ PDF → .md │ -│ assets │ │ DOCX → .md │ -│ processed_assets│ │ CSV → .parquet │ -│ project_assets │ │ IMG → .json │ -│ │ └───────────────────┘ -│ knowledge_frames│ -│ extraction_passes│ ◄── LLM extraction agent (multi-pass) -│ spaces │ -│ projections │ ◄── Projection agent (space-specific) -│ feedbacks │ ◄── Feedback loop between agents -└─────────────────┘ -``` - -### Data Flow - -1. **Ingest** — Raw files are SHA256-deduplicated, uploaded to MinIO, registered in PostgreSQL as a project -2. **Process** — Raw files are converted to LLM-readable formats (Markdown, Parquet, JSON metadata) -3. **Extract** — An LLM agent reads processed data and produces one **knowledge frame** per project (with optional multi-pass review) -4. **Project** — Domain-specific "spaces" define structured extraction schemas; projection agents extract targeted data from knowledge frames -5. **Feedback** — Projection agents flag unclear data; KB agents review and resolve feedback on user activation - -### Knowledge Frame - -Each research project produces one knowledge frame with: - -- **Paper metadata** (fixed) — title, authors, journal, year, DOI -- **Domain** (fixed) — research domain string -- **Free-form sections** (agent-decided) — the agent chooses what categories best represent the paper's knowledge (e.g., materials, experimental_data, synthesis_routes, mechanisms, etc.) - -Every extracted item is tagged with an **evidence level**: -- **Level 1**: Causal experimental evidence -- **Level 2**: Direct experimental observation -- **Level 3**: Correlative evidence -- **Level 4**: Predicted / inferred - -### Spaces & Projections - -A **Space** defines a domain-specific extraction schema (e.g., "biomineralization templates"). A **Projection** is the result of applying a Space to a knowledge frame — extracting structured data per the schema definition. - -### Agentic Feedback - -Projection agents can flag ambiguous or missing data. The KB extraction agent can review these feedback items (on user activation) and update the knowledge frame accordingly. - -### Projection Review (Multi-Agent) - -A strict **Projection Reviewer** agent consolidates and corrects projection data through a multi-agent review process: - -``` -User activates review - │ - ▼ -┌─────────────────────────┐ -│ Projection Reviewer │ reads all projections + frame + source -│ (strict data auditor) │ -└────────┬────────────────┘ - │ delegates verification - ▼ -┌─────────────────────────┐ -│ Projection Fixer │ re-reads source material -│ (sub-agent) │ returns corrections -└─────────────────────────┘ - │ - ▼ - Single reviewed projection - (consolidated, corrected) -``` - -The reviewer: -1. Loads all projection runs (from single or multiple extraction events) -2. Cross-references against the knowledge frame and original source files -3. Delegates complex verification to the fixer sub-agent -4. Produces a single **reviewed projection** — consolidated, corrected, deduplicated - -### Knowledge Graph Construction (Global Concept Graph) - -Knowledge graph construction uses a dedicated KG extraction agent with one shared global space across all domains. - -Design goals: -- **One global graph space**: all projects contribute to a shared graph so inter-domain links can emerge. -- **Concept-only nodes**: only scientific concepts become nodes. -- **Concept relations as edges**: edges encode directed concept-to-concept relations. -- **Details in references**: values/conditions/metadata are stored as references back to frame/database context, not turned into extra nodes. -- **Redundancy-aware build**: the agent checks existing graph content to reduce duplicate concepts/edges. - -Recommended usage flow: -1. Extract knowledge frames first. -2. Optionally clear old KG outputs. -3. Run KG extraction. -4. Inspect merged graph output. - -## Prerequisites +## Install during development -- Python 3.10+ -- Docker & Docker Compose -- `libmagic` (usually pre-installed on Linux; `brew install libmagic` on macOS) - -## Quick Start +MKB is not yet published on PyPI. Install it directly from this repository instead. +The `dev` branch is the current shared development build: ```bash -# 1. Create virtual environment and install -python3 -m venv .venv -source .venv/bin/activate -pip install -e ".[dev]" - -# 2. Copy environment config -cp .env.example .env - -# 3. Start infrastructure (PostgreSQL + MinIO) -make up - -# 4. Create database tables -mkb setup - -# 5. Install frontend dependencies once -cd frontend -npm install -cd .. - -# 6. Start backend API + React frontend together -bash scripts/dev.sh +python -m pip install --upgrade --force-reinstall \ + "mat-know-base @ git+https://github.com/theAfish/mat_know_base.git@dev" ``` -Open http://127.0.0.1:5173. - -## Easiest Daily Usage (React + API) - -After first-time setup, day-to-day startup is just: +Install optional integrations only when they are needed. For example, the full +materials application and HTTP server require: ```bash -source .venv/bin/activate -make up -bash scripts/dev.sh +python -m pip install --upgrade --force-reinstall \ + "mat-know-base[materials,server] @ git+https://github.com/theAfish/mat_know_base.git@dev" ``` -`scripts/dev.sh` starts both services in one terminal: -- backend API: `make server` (http://127.0.0.1:8503) -- frontend dev server: `cd frontend && npm run dev` (http://127.0.0.1:5173) - -Press `Ctrl+C` once to stop both. +`--force-reinstall` is intentional while the package version remains `0.1.0`; it makes +pip replace an older build from the moving development branch. For a reproducible bug +report, replace `dev` with the commit SHA being tested. Git must be installed and the +repository must be accessible to the user. -If the UI appears empty: +To develop or debug with the maintainers, use an editable checkout instead. Python code +changes then take effect without reinstalling: ```bash -mkb projects -mkb frames -curl http://127.0.0.1:8503/api/projects?limit=5 +git clone https://github.com/theAfish/mat_know_base.git +cd mat_know_base +python -m venv .venv +.venv/bin/python -m pip install -e ".[dev]" ``` -- If `mkb projects` has rows but the UI is empty, the API is likely not running on port 8503. -- If both CLI and API are empty, ingest data first (`mkb ingest ...` or upload from the Projects page). - -## Python API (Primary Interface) - -The recommended interface is `mkb.api`. See `examples/basic_usage.py` for a complete walkthrough. - -```python -from mkb import api - -# Setup -api.setup() - -# Ingest & process -result = api.ingest("./data/papers/smith2024", label="Smith 2024") -api.process() - -# Extract with multi-pass review -api.extract(max_passes=2, verbose=True) - -# Query frames -frame = api.get_frame(project_id="...") -print(frame["content"]["paper"]) -print(frame["content"].keys()) # agent-decided sections - -# Spaces & projections -api.create_space( - name="biomineralization", - domain="biomineralization", - extraction_schema={...}, - system_prompt="...", - field_descriptions={...}, -) -api.project(space_id="...", project_id="...") - -# Feedback -api.list_feedback(project_id="...", status="OPEN") -api.review_feedback(project_id="...") - -# Projection review (multi-agent) -api.review_projections(space_id="...", project_id="...") -api.review_projections_all(space_id="...") -api.list_reviewed_projections(space_id="...") -api.get_reviewed_projection(reviewed_projection_id="...") - -# Knowledge graph construction (global concept graph) -api.clear_knowledge_graphs() -api.extract_knowledge_graph(project_id="...") -kg = api.get_knowledge_graph() -print(len(kg["graph"]["concepts"]), len(kg["graph"]["relations"])) - -# Knowledge graph review (deduplication + quality cleanup) -api.review_knowledge_graph() # auto mode (random global or local) -api.review_knowledge_graph(mode="global", verbose=True) # full graph: standardize + dedup -api.review_knowledge_graph(mode="local", seed_count=15) # neighborhood review (least-reviewed first) -counts = api.get_graph_review_counts() # per-element review counters - -# Search papers and data -results = api.search_library("enamel mineralization") -print(results["projects"]) -print(results["assets"]) -``` +Use `-e ".[dev,materials,server]"` when working on the complete application. Rerun the +install command after changing dependencies or package metadata. -## CLI +## Local quickstart -```bash -# Database -mkb setup -mkb reset-db - -# Ingestion -mkb ingest ./data/papers/smith2024 --label "Smith 2024" -mkb sync --root-dir ./data/papers - -# Processing -mkb process -mkb process --project-id - -# Knowledge extraction -mkb extract # all pending -mkb extract --project-id # one project -mkb extract --max-passes 3 # multi-pass -mkb extract --model openai/gpt-4o # override model -mkb extraction-history # view pass history - -# Listing -mkb projects -mkb assets --project-id -mkb search "enamel mineralization" -mkb search "csv supplement" --project-id -mkb frames -mkb frame - -# Spaces & Projections -mkb space create --name catalysis --domain catalysis --schema-file schema.json -mkb space load space_definition.json -mkb space list -mkb space show catalysis -mkb project-run --space --project-id -mkb project-run --space --all -mkb projections --space-id - -# Feedback -mkb feedback --project-id --status OPEN -mkb review-feedback --project-id -mkb resolve-feedback --status RESOLVED --notes "Fixed" - -# Projection Review (multi-agent consolidation) -mkb review-projections --space --project-id -mkb review-projections --space --all -mkb reviewed-projections --space-id -mkb reviewed-projection - -# Knowledge graph construction (global concept graph) -mkb kg-clear # clear old KG projections (and legacy frame-graph sections) -mkb kg-extract # build KG for all completed frames -mkb kg-extract --project-id # build KG for one project -mkb kg-extract --frame-id # build KG for one frame -mkb kg-show # show merged global concept graph -mkb kg-show --project-id # merged graph filtered to one project - -# Start the FastAPI backend (serves the React UI in production) -make server - -# Start legacy Streamlit UI (optional) -mkb ui --port 8501 -``` - -## Search - -Keyword search is available in all three interfaces: - -- **UI**: The **Research Projects → Browse** view includes a search box for papers and ingested data assets. -- **Python API**: `api.search_library(query, limit=25, project_id=None)` returns matching projects and assets. -- **CLI**: `mkb search "keywords"` prints matching projects and assets, with optional `--project-id` scoping. - -Search behavior: - -- Queries are split into whitespace-separated keyword tokens. -- All tokens must match somewhere in a result. -- Project matches use project label and source path. -- Asset matches use filename, MIME type, and selected asset metadata fields. - -## Knowledge Graph Quickstart +Requirements: Python 3.10+, Node.js 20+, npm, Docker with Compose, and `libmagic`. +For image OCR, install Tesseract. On macOS, `brew install libmagic tesseract`. ```bash -# 1) Make sure knowledge frames exist -mkb extract - -# 2) Optional clean rebuild -mkb kg-clear - -# 3) Construct concept graph projections -mkb kg-extract - -# 4) Inspect merged graph -mkb kg-show +git clone https://github.com/theAfish/mat_know_base.git +cd mat_know_base +make bootstrap ``` -Project-scoped example: +Review `.env` and set an LLM credential. For an OpenAI-compatible provider: -```bash -mkb kg-clear --project-id -mkb kg-extract --project-id -mkb kg-show --project-id +```dotenv +MKB_EXTRACTION_MODEL=openai/qwen-plus +OPENAI_API_KEY=replace-me +OPENAI_API_BASE=https://provider.example/v1 ``` -Notes: -- `kg-extract` clears existing KG projections for target frame(s) by default. Use `--no-clear-existing` to keep prior projection history. -- Legacy graph-like sections inside frame content are removed by default during cleanup/extraction. Use `--keep-legacy-frame-graphs` to skip that behavior. - -## Knowledge Graph Review - -After building the graph, a dedicated **Graph Review Agent** deduplicates concepts, standardizes relation naming, and prunes low-quality entries. It runs in two modes: - -| Mode | What it does | -|------|-------------| -| **global** | Analyzes the full graph: groups similar relation names and standardizes them; finds and merges synonymous concept nodes across all projections | -| **local** | Selects the least-reviewed concepts as starting points, explores their neighborhood, verifies ambiguous entries against source knowledge frames, and fixes local issues | - -Each run tracks how many times each node/edge was examined and modified in the `graph_element_reviews` table — always incremented by the orchestration script, never by the agent itself. - -### Python API - -```python -# Global mode: relation standardization + concept deduplication -api.review_knowledge_graph(mode="global", verbose=True) - -# Local mode: deep-dive on least-reviewed concepts -api.review_knowledge_graph(mode="local", seed_count=15, verbose=True) - -# Auto: randomly picks global or local each time (default) -api.review_knowledge_graph() - -# Inspect per-element review counts -counts = api.get_graph_review_counts() -# counts["concepts"]["hydroxyapatite"] → {"times_examined": 3, "times_modified": 1, ...} -# counts["relations"]["amelotin||promotes||hydroxyapatite nucleation"] → {...} -``` - -### Review tools available to the agent - -| Tool | Mode | Description | -|------|------|-------------| -| `get_concept_details` | both | Full concept record + all incoming/outgoing relations | -| `get_concept_neighbors` | both | Concept + 1-hop neighbors + relations | -| `get_relation_type_distribution` | both | Count of each distinct relation label | -| `search_graph_elements` | both | Keyword search across concept labels, aliases, relation names | -| `find_similar_concepts` | both | Token-overlap similarity search for near-duplicate concepts | -| `merge_concepts` | both | Merge N concepts into one canonical node across all projections | -| `standardize_relation_name` | both | Rename relation type(s) to a canonical form everywhere | -| `delete_concept` | both | Delete an isolated concept (rejects if relations still exist) | -| `delete_relation` | both | Delete a specific directed relation | -| `get_frame_content` | local | Read a source knowledge frame for concept verification | - -### Knowledge Graph Output Shape - -`mkb kg-show` and `api.get_knowledge_graph()` return a normalized concept graph: - -```json -{ - "graph": { - "concepts": [ - { - "label": "Amelotin", - "aliases": ["AMTN"], - "source_project_ids": ["..."], - "source_frame_ids": ["..."], - "knowledge_refs": [ - { - "project_id": "...", - "frame_id": "...", - "field_path": "...", - "snippet": "..." - } - ] - } - ], - "relations": [ - { - "source": "Amelotin", - "relation": "promotes", - "target": "Hydroxyapatite nucleation", - "evidence_level": 2, - "source_project_id": "...", - "source_frame_id": "...", - "knowledge_ref": { - "project_id": "...", - "frame_id": "...", - "field_path": "...", - "snippet": "..." - } - } - ] - } -} -``` - -## LLM Configuration - -Knowledge extraction uses google-adk with LiteLLM. Configure in `.env`: +Then start the infrastructure and application: ```bash -MKB_EXTRACTION_MODEL=openai/deepseek-v4-pro-guan -LLM_API_KEY= -LLM_API_BASE= -``` - -For OpenAI itself, `OPENAI_API_KEY` / `OPENAI_API_BASE` still work. For -non-OpenAI models exposed through an OpenAI-compatible endpoint, prefer the -provider-agnostic `LLM_API_KEY` / `LLM_API_BASE` names above, and use -`openai/` as the model id. - -## Project Structure - -``` -src/mkb/ -├── api.py # Primary Python interface -├── cli.py # CLI (thin wrapper around api) -├── config.py # Settings from .env -├── db/ -│ ├── engine.py # SQLAlchemy engine + init_db() -│ └── models.py # ORM models (12 tables + enums) -├── storage/ -│ └── s3.py # MinIO upload/download/exists/delete -├── ingest/ -│ └── worker.py # CAS ingestion (SHA256, MIME, batching) -├── processors/ -│ ├── base.py # Abstract Processor + ProcessingResult -│ ├── coordinator.py # Routes assets to processors -│ ├── pdf_processor.py -│ ├── text_processor.py -│ ├── dataframe_processor.py -│ └── image_processor.py -├── agents/ -│ ├── extraction.py # KB extraction agent + multi-pass orchestration -│ ├── review.py # Review agent for multi-turn extraction -│ ├── projection.py # Projection agent (space-specific extraction) -│ ├── knowledge_graph.py # KG agent (global concept graph extraction) -│ ├── projection_reviewer.py # Projection reviewer (multi-agent consolidation) -│ ├── projection_fixer.py # Fixer sub-agent (source verification) -│ ├── feedback_reviewer.py # Feedback review agent -│ ├── dev_agent.py # Dev agent interface (design only) -│ ├── runner.py # Generic AgentRunner wrapper -│ ├── prompts/ # Agent prompts -│ │ ├── kb_extraction.py # Flexible KB extraction prompt -│ │ ├── review.py # Review pass prompt -│ │ ├── projection.py # Projection prompt builder -│ │ ├── knowledge_graph.py # Concept-graph extraction prompt -│ │ ├── projection_review.py # Projection reviewer prompt -│ │ ├── projection_fixer.py # Fixer sub-agent prompt -│ │ └── feedback_review.py # Feedback review prompt -│ └── tools/ # Agent tool functions -│ ├── reading.py # Reading tools (markdown, dataframe, image, search) -│ ├── frames.py # Frame save/get/update tools -│ ├── projection.py # Projection save + flag_for_feedback -│ ├── knowledge_graph.py # Concept-graph tools + redundancy checks -│ ├── projection_review.py # Projection review + re-extraction tools -│ └── feedback.py # Feedback query + resolve tools -├── spaces/ -│ └── registry.py # Space CRUD operations -├── feedback/ -│ └── manager.py # Feedback CRUD + resolution -└── ui/ - ├── app.py # Streamlit entry point - ├── pages/ # UI pages - │ ├── projects.py - │ ├── frames.py - │ ├── projections.py - │ └── feedback.py - └── components/ # Reusable UI components - ├── frame_viewer.py - └── graph_viz.py +make up +make doctor +make dev ``` -## Database Tables +Open the React UI at . The API is at +, its interactive OpenAPI documentation at +, and the MinIO console at +. Stop application servers with `Ctrl+C` and infrastructure +with `make down`. -| Table | Purpose | -|---|---| -| `research_projects` | One per research package (paper + supplementary) | -| `assets` | One row per unique raw file (SHA256 deduplicated) | -| `project_assets` | Many-to-many link between projects and assets | -| `processed_assets` | One row per successful conversion output | -| `processing_logs` | Audit trail for processing attempts | -| `knowledge_frames` | One structured frame per project (JSONB content + metadata) | -| `extraction_passes` | Audit trail for each extraction/review pass | -| `spaces` | Domain-specific extraction configurations | -| `projections` | Results of projecting frames through spaces | -| `feedbacks` | Feedback items between agents | -| `graph_element_reviews` | Per-element review counts (`times_examined`, `times_modified`) for graph nodes and edges | +`make bootstrap` creates `.venv`, installs Python and locked frontend dependencies, +and copies `.env.example` to `.env` without overwriting an existing file. Override +tools when necessary, for example `make bootstrap BOOTSTRAP_PYTHON=python3.12` or +`make test PYTHON=/path/to/python`. -## Data Sharing +## First workflow -Because the database and files are developed locally (papers, processed outputs, PostgreSQL, MinIO) and cannot be committed to git, two helper scripts let you snapshot and restore the entire local state. +Put a paper and its supplementary files in one directory: -### Pack (create a snapshot) - -```bash -# Auto-named: mkb_data_YYYYMMDD_HHMMSS.tar.gz -make pack - -# Custom filename -make pack out=my_dataset_v1.tar.gz - -# Or run directly -bash scripts/pack_data.sh my_snapshot.tar.gz +```text +data/papers/smith2024/ + paper.pdf + supplement.csv + notes.txt ``` -What gets bundled: -- **PostgreSQL** — full `pg_dump` of the `mkb` database (schema + data) -- **MinIO buckets** — `raw`, `processed`, `archive`, `temp` -- **Local dirs** — `data/papers/`, `data/processed/`, `data/uploads/`, `data/inbox/` -- **manifest.json** — records timestamp, database name, and bucket list - -Requirements: Docker (already needed), `tar`, `python3`. - -### Unpack (restore from a snapshot) +Run the commands through the project interpreter: ```bash -# Full restore (interactive confirmation before dropping the DB) -make unpack file=mkb_data_20260429_120000.tar.gz - -# Or run directly -bash scripts/unpack_data.sh mkb_data_20260429_120000.tar.gz +.venv/bin/python -m mkb.cli ingest data/papers/smith2024 --label "Smith 2024" +.venv/bin/python -m mkb.cli process +.venv/bin/python -m mkb.cli extract --max-passes 2 +.venv/bin/python -m mkb.cli projects ``` -Partial restore flags: - -| Flag | Effect | -|------|--------| -| `--pg-only` | Restore PostgreSQL only | -| `--minio-only` | Restore MinIO buckets only | -| `--local-only` | Restore local data dirs only | -| `--no-pg` | Skip PostgreSQL restore | -| `--no-minio` | Skip MinIO buckets | -| `--no-local` | Skip local data dirs | +The same workflow is available in the React UI. For library use, start with the +[detailed Python API guide](docs/python-api.md) and +[examples/basic_usage.py](examples/basic_usage.py). -After unpacking, run `alembic upgrade head` if the schema migration level differs between the snapshot and your current codebase. - -### Typical workflow for onboarding a new developer +For a Docker-free library project, install the lightweight base package and use +SQLite plus filesystem storage. The reusable example is in the Python API guide; from a +source checkout, it can also be run directly: ```bash -# 1. Clone repo and install -python3 -m venv .venv && source .venv/bin/activate -pip install -e ".[dev]" - -# 2. Start services -make up - -# 3. Restore a shared snapshot -make unpack file=mkb_data_20260429_120000.tar.gz - -# 4. Apply any pending migrations -alembic upgrade head +python -m pip install --upgrade --force-reinstall \ + "mat-know-base @ git+https://github.com/theAfish/mat_know_base.git@dev" +# From this repository checkout: +python examples/portable_quickstart.py ``` -## Frontend (React + Vite) +For backend integrations, install extras from Git as well, such as +`"mat-know-base[postgres,s3] @ git+https://github.com/theAfish/mat_know_base.git@dev"`. -A React 19 + Vite + TypeScript UI lives in `frontend/`. It replaces the legacy Streamlit UI and communicates with the FastAPI backend through a Vite dev-server proxy. +When MKB is released on PyPI, the Git URLs above will be replaced by normal package +installs. -### Prerequisites - -- Node.js 20+ and npm 10+ -- Backend running (`make server`) - -### Installation +## Common development commands ```bash -cd frontend -npm install +make doctor # read-only environment and dependency diagnostics +make lint # Ruff and TypeScript checks +make test # Python tests +make build # Python wheel and production React bundle +make check # complete local validation ``` -### Development - -```bash -# In one terminal — start the backend -make server - -# In another terminal — start the Vite dev server -cd frontend -npm run dev -``` +## Documentation -Open http://localhost:5173 (or the port shown in the Vite output). - -All `/api/*` requests are proxied to `http://127.0.0.1:8503` by Vite, so no CORS configuration is needed during development. - -### Production build - -```bash -cd frontend -npm run build # output goes to frontend/dist/ -npm run preview # serve the production build locally -``` +Documentation is organized by role in the [documentation index](docs/README.md): -### Pages - -This frontend is a single-page app with sidebar navigation (not URL routes). - -| Page | Description | -|------|-------------| -| Projects | Browse, upload, and manage research packages | -| Knowledge Frames | View extracted frames; run processing, extraction, projection, and graph pipelines per project | -| Projections | Aggregated projection table across papers for a selected space | -| Dataset Graph | Interactive force-directed concept graph (vis-network, Barnes-Hut physics); supports node/edge coloring by evidence level, review coverage, modification heat, and connectivity | -| Assistant | LLM assistant interface | -| Feedback | Review and resolve feedback items | - -### Tech stack - -- **React 19** + **TypeScript** -- **Vite 8** (build tool + dev proxy) -- **Tailwind CSS v4** (CSS-first config, no `tailwind.config.js`) -- **vis-network 10** — knowledge graph visualization (same library as pyvis) -- **Zustand v4** — UI state management -- **Axios v1** — HTTP client - -## Services - -| Service | URL | Credentials | -|---------|-----|------------| -| MinIO Console | http://localhost:9001 | minioadmin / minioadmin | -| MinIO S3 API | http://localhost:9000 | minioadmin / minioadmin | -| PostgreSQL | localhost:5432 | mkb / mkb_dev | -| FastAPI backend | http://localhost:8503 | — | -| React UI (dev) | http://localhost:5173 | — | +- Users and automation authors: [Python API](docs/python-api.md) and + [HTTP API contract](docs/api-contract.md) +- Contributors: [developer setup](docs/development.md), + [architecture and ownership](docs/architecture-map.md), and + [contribution guide](CONTRIBUTING.md) +- Operators and security reviewers: [operator runbook](docs/operator-runbook.md), + [backup and restore](docs/backup-restore.md), [upgrades](docs/upgrades.md), and + [security model](docs/security.md) diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..3e7c426 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,16 @@ +# Security policy + +Do not disclose suspected vulnerabilities in public issues or pull requests. Report +them privately to the repository maintainers through the hosting platform's private +security-advisory feature. If that feature is unavailable, contact the maintainers +through a private organizational channel and request a secure reporting address. + +Include affected version/commit, deployment mode, impact, reproduction steps, and any +suggested mitigation. Exclude real credentials, tokens, and sensitive research data. +Maintainers should acknowledge receipt, coordinate validation and remediation, and +agree on disclosure timing with the reporter. + +Security fixes are supported for the versions described in [SUPPORT.md](SUPPORT.md). +Operational controls and known limitations are documented in +[docs/security.md](docs/security.md). + diff --git a/SUPPORT.md b/SUPPORT.md new file mode 100644 index 0000000..7dcecfd --- /dev/null +++ b/SUPPORT.md @@ -0,0 +1,10 @@ +# Supported versions + +MKB is currently pre-1.0. Security and correctness fixes target the latest commit on +the default branch and the latest published release, if one exists. Older commits, +development branches, modified deployments, and legacy canonical-workflow surfaces +receive best-effort compatibility support only. + +Upgrade to the latest supported release before reporting a defect when practical. A +deployment that cannot upgrade should include its exact commit, migration revision, +Python/Node/Docker versions, and sanitized `make doctor` output in a private report. diff --git a/TODO.md b/TODO.md index 5b21656..3e96691 100644 --- a/TODO.md +++ b/TODO.md @@ -1,288 +1,481 @@ -## 0. Define Scope (Do Not Skip) - -* [ ] Audit current Streamlit app - - * [ ] List all UI modules - * [ ] Categorize: - - * [ ] Graph visualization (core) - * [ ] Tables / text panels - * [ ] Controls (filters, buttons, inputs) - -* [ ] Define MVP (keep it minimal) - - * [ ] Render graph using React Flow - * [ ] Node click interaction - * [ ] Call backend APIs - * [ ] Basic dynamic graph updates - ---- - -## 1. Initialize Frontend Project - -* [ ] Create project - -```bash -npm create vite@latest graph-ui -- --template react-ts -cd graph-ui -npm install -``` - -* [ ] Install dependencies - -```bash -npm install reactflow zustand axios -``` - -* [ ] Optional utilities - -```bash -npm install classnames -``` - ---- - -## 2. Project Structure - -* [ ] Set up directories - -``` -src/ - components/ - pages/ - store/ - api/ - types/ - utils/ -``` - ---- - -## 3. Define Core Data Model - -* [ ] Normalize graph schema (critical step) - -```ts -// src/types/graph.ts - -export interface GraphNode { - id: string; - type?: string; - data: { - label: string; - [key: string]: any; - }; -} - -export interface GraphEdge { - id: string; - source: string; - target: string; - type?: string; -} - -export interface GraphData { - nodes: GraphNode[]; - edges: GraphEdge[]; -} -``` - ---- - -## 4. API Layer (Replace Streamlit Backend Calls) - -* [ ] Create API wrapper - -```ts -// src/api/graph.ts -import axios from "axios"; - -export const fetchGraph = async () => { - const res = await axios.get("/api/graph"); - return res.data; -}; -``` - ---- - -## 5. State Management (Replace Streamlit Session State) - -* [ ] Create Zustand store - -```ts -// src/store/graphStore.ts -import { create } from "zustand"; -import { GraphData } from "../types/graph"; - -interface GraphState { - graph: GraphData | null; - setGraph: (g: GraphData) => void; -} - -export const useGraphStore = create((set) => ({ - graph: null, - setGraph: (g) => set({ graph: g }), -})); -``` - ---- - -## 6. Integrate React Flow - -* [ ] Create graph component - -```tsx -// src/components/GraphView.tsx -import ReactFlow from "reactflow"; -import "reactflow/dist/style.css"; -import { useGraphStore } from "../store/graphStore"; - -export default function GraphView() { - const graph = useGraphStore((s) => s.graph); - - if (!graph) return
Loading...
; - - return ( -
- -
- ); -} -``` - ---- - -## 7. Page Integration (Replace Streamlit Pages) - -* [ ] Create main page - -```tsx -// src/pages/MainPage.tsx -import { useEffect } from "react"; -import GraphView from "../components/GraphView"; -import { fetchGraph } from "../api/graph"; -import { useGraphStore } from "../store/graphStore"; - -export default function MainPage() { - const setGraph = useGraphStore((s) => s.setGraph); - - useEffect(() => { - fetchGraph().then(setGraph); - }, []); - - return ( -
- -
- ); -} -``` - ---- - -## 8. Migrate Interactions - -### Node Click - -```tsx - { - console.log("clicked:", node); - }} -/> -``` - ---- - -### Backend Interaction (e.g. expand node) - -```ts -export const fetchNeighbors = async (nodeId: string) => { - const res = await axios.get(`/api/neighbors?id=${nodeId}`); - return res.data; -}; -``` - ---- - -### Update Graph State - -```ts -onNodeClick={async (_, node) => { - const subgraph = await fetchNeighbors(node.id); - setGraph(mergeGraph(graph, subgraph)); -}} -``` - ---- - -## 9. Layout Refactor (Replace Streamlit Layout) - -* [ ] Introduce layout structure - -``` -[ Sidebar ] [ Graph Canvas ] -``` - -* [ ] Sidebar responsibilities: - - * [ ] Search - * [ ] Filters - * [ ] Action controls - ---- - -## 10. Styling Strategy - -* [ ] Choose one: - - * [ ] CSS Modules (simple) - * [ ] Tailwind CSS (recommended for scalability) - ---- - -## 11. Backend Integration - -* [ ] Ensure backend endpoints exist: - - * [ ] `/api/graph` - * [ ] `/api/neighbors` - * [ ] `/api/search` - -* [ ] Enable CORS in backend (e.g. FastAPI) - ---- - -## 12. Decommission Streamlit - -* [ ] Mark Streamlit UI as deprecated -* [ ] Keep backend logic -* [ ] Fully switch UI to React frontend - ---- - -## 13. Acceptance Criteria - -Migration is complete when: - -* [ ] Graph renders correctly -* [ ] Node click works -* [ ] API calls succeed -* [ ] Graph updates dynamically -* [ ] No major UI blocking issues (small graphs) - ---- - -## 🚧 Future Work (Out of Scope for Now) - -* [ ] Large graph performance (WebGL, virtualization) -* [ ] Advanced layout algorithms -* [ ] Graph caching -* [ ] Undo / redo -* [ ] Multi-user collaboration \ No newline at end of file +# MKB reusable Python package refactor + +## Goal + +Turn `mat-know-base` into a pip-installable knowledge-processing engine that other +projects can configure with their own database, object store, graph store, schemas, +data types, and pipelines. Keep the current materials-science application working as +the first built-in application of that engine. + +The refactor must preserve all currently extracted local data. Existing PostgreSQL +rows, MinIO objects, local processed files, identifiers, evidence links, workflow +versions, projections, feedback, skills, and job history must not be dropped merely +to simplify the new architecture. + +## Non-negotiable data-safety rules + +- [x] Never use `reset_db()`, `reset_schema()`, `drop_all()`, `docker compose down -v`, + destructive restore, or a migration that drops populated tables during this + refactor. +- [x] Never rewrite existing UUIDs or S3 bucket/key values unless a reviewed migration + includes a verified old-to-new mapping and rollback procedure. +- [x] Treat PostgreSQL, all four MinIO buckets (`raw`, `processed`, `archive`, `temp`), + and local `data/` content as one dataset. Backing up only the database is not + sufficient. +- [x] Make every schema migration additive first: create new tables/columns, backfill, + verify, switch readers, and only consider cleanup in a later release. +- [x] Keep legacy tables and adapters readable for at least one complete release after + the new API becomes the default. For this local-only migration, retaining them + indefinitely is acceptable. +- [x] Run data migrations separately from application startup. Importing `mkb` or + creating a client must never silently migrate or delete data. +- [x] Any migration that changes persisted data must support a dry run, report counts, + be restartable/idempotent, and record its completion in a migration ledger. +- [x] Do not declare a phase complete until the pre-refactor snapshot passes a restore + drill and the post-migration reconciliation report passes. + Both full disposable drills and the post-repair live reconciliation passed on + 2026-07-21 with retained JSON evidence. + +## Phase 0 — Freeze and inventory the local dataset + +- [x] Stop starting new extraction, projection, graph, review, and maintenance jobs; + allow active jobs to reach a terminal state. + The 2026-07-21 freeze check found all five persisted jobs `COMPLETED`. +- [x] Record the current git commit, package version, Alembic revision, configuration, + PostgreSQL version, MinIO version, and Docker Compose project name in a migration + manifest. Do not put credentials into the manifest. +- [x] Run the existing operational checks: + + ```bash + make up + make doctor + .venv/bin/python -m mkb.cli reconcile + ``` + + `make doctor`, service health, and reconciliation pass. The one missing + 40,160-byte processed object was restored from its exact checksummed local mirror + under an explicit confirmation token and recorded migration ledger. + +- [x] Add an inventory command that emits JSON containing row counts and stable IDs for + every persistent model, including projects, groups, assets, project-asset links, + processed assets, frames, extraction passes, spaces, projections, feedback, + graph reviews, raw/canonical workflows, schema proposals/revisions, workflow + maintenance/index entries, custom skills, post-processor scripts, and jobs. +- [x] Extend inventory with per-bucket object counts, total bytes, and checksums or a + deterministic object-key manifest. +- [x] Inventory local files under at least `data/papers`, `data/processed`, + `data/uploads`, `data/inbox`, and `data/runtime_settings.json` when present. +- [x] Detect broken references before migration: missing S3 objects, orphan objects, + missing local mirrors, dangling foreign keys, duplicate logical identifiers, and + records whose stored schema/version cannot be resolved. + The 2026-07-21 scan found and repaired one missing processed object. The 379 + unchanged legacy processed objects whose asset rows no longer exist are retained, + not deleted. Final reconciliation reports zero missing references and content + verification reports zero sampled checksum mismatches. +- [x] Save the inventory outside ephemeral Docker volumes, for example under a + timestamped `migration-snapshots/` directory that is excluded from git. + The full 2026-07-21 inventory contains 22 tables, 21,500 objects, and 21,931 + checksummed local files and is retained under the git-ignored snapshot directory. + +### Required backup gate + +- [x] Create a named full snapshot using the existing packer: + + ```bash + make pack out=migration-snapshots/pre-sdk-refactor.tar.gz + ``` + +- [x] Run the validation-only restore drill and retain its successful output with the + manifest. The named 8.3 GB snapshot passed checksum validation and restored into + a disposable PostgreSQL database on 2026-07-20; live replacement was not enabled. +- [x] Additionally restore the named snapshot into disposable infrastructure and run + inventory/reconciliation there. The existing validation-only drill checks the + archive and PostgreSQL restore; the expanded drill must also prove MinIO and local + file restoration without touching live data. + The original snapshot remains untouched; a self-contained repaired copy adds only + the exact runtime-settings file and missing object proven by the pre-refactor + inventory. Its full PostgreSQL/MinIO/local restore, inventory, reconciliation, and + content verification passed in disposable infrastructure. +- [x] Keep the pre-refactor snapshot until all old data has been read successfully + through the new API and a second post-migration snapshot has passed the same drill. + The original and repaired pre-refactor archives and the drilled post-refactor v2 + archive are all retained under `migration-snapshots/`. + +## Phase 1 — Define and test the supported SDK contract + +- [x] Replace the global-first design with an explicit configured application object: + + ```python + from mkb import KnowledgeBase + + kb = KnowledgeBase.from_url( + database_url="postgresql+psycopg://...", + object_store_url="s3://raw?endpoint=http://localhost:9000", + ) + ``` + +- [x] Allow at least two independently configured `KnowledgeBase` instances in one + Python process without shared settings, engines, sessions, job managers, or + registries. +- [x] Keep `from mkb import api` as a compatibility surface alongside an explicitly + configured default client. Mark it deprecated only after feature parity exists. +- [x] Define the public import boundary. Consumers must not need `mkb.db`, `mkb.web`, + ORM models, storage internals, or service-private functions. +- [x] Remove private names and database session factories from the future public + `__all__`; temporary import-compatible port aliases remain available but are not + included in wildcard imports. +- [x] Introduce typed public models (Pydantic models or dataclasses) for collections, + sources, artifacts, records, schemas, entities, relations, evidence, pipeline + runs, jobs, pages, and operation receipts. +- [x] Permit `model_dump(mode="json")` or an equivalent stable serialization method on + public models. +- [x] Standardize exceptions: `MKBError`, `NotFoundError`, `ConflictError`, + `ValidationError`, `BackendUnavailableError`, `ProviderError`, and + `PipelineExecutionError`. +- [x] Standardize behavior for the supported grouped SDK: return a typed value on + success, use `None` only for optional lookups, and raise typed exceptions. Legacy + compatibility methods retain their dictionary contracts until Phase 7 migration. +- [x] Add API contract tests for every currently supported grouped SDK method, including + a deliberate method inventory plus success, optional lookup, errors, idempotency, + serialization, repository, graph, registry, transaction, and pipeline coverage. + New grouped services must extend the inventory as they replace the legacy facade. + +## Phase 2 — Remove global configuration and persistence coupling + +- [x] Introduce an immutable `MKBConfig` that can be created from explicit Python + values. Environment and YAML loading should be optional constructors, not import- + time behavior. +- [x] Move engine and session creation out of module globals in `mkb.db.engine` and into + an injected SQLAlchemy adapter owned by `KnowledgeBase`. Legacy facade aliases + are lazy compatibility proxies and no longer construct engines at import time. +- [x] Inject object storage, graph storage, model provider, parser registry, pipeline + registry, and job backend into the application object. Provider and backend + behavior is implemented in their later feature phases; Phase 2 owns lifecycle, + isolation, and capability composition. +- [x] Define explicit lifecycle methods or context-manager support so connections and + worker resources are released predictably. +- [x] Add explicit transaction scopes. Collection, source, artifact, record, schema, + and projection metadata share one commit/rollback boundary; object-backed writes + use reverse-order best-effort compensation on rollback: + + ```python + with kb.transaction() as tx: + collection = tx.collections.create(name="Experiment 42") + tx.sources.add_text(collection.id, notes) + ``` + +- [x] Document that PostgreSQL, object storage, and external graph databases cannot + share one ACID transaction. Use stable IDs, staging states, idempotent writes, + an outbox/event pattern, and compensating cleanup for cross-store operations. +- [x] Prove with tests that the current local PostgreSQL and MinIO configuration works + through the injected adapters before changing any schema. + +## Phase 3 — Introduce generic domain concepts without discarding old records + +- [x] Define infrastructure-independent concepts: + - [x] `Collection`: a typed logical grouping of data, initially mapped read-only to + existing `research_projects` rows through the injected SQLAlchemy adapter. + - [x] `Source`: a typed ingested input, initially mapped read-only to existing assets + with collection membership and content access through the object-store port. + - [x] `Artifact`: a typed derived output, initially mapped read-only to existing + processed assets with content access through the object-store port. + - [x] `Record`: typed structured data mapped read-only to current knowledge frames. + - [x] `Schema`: typed extraction policy mapped read-only to current spaces. + - [x] `Entity` and `Relation`: typed, serializable graph elements with an in-memory + adapter and grouped graph service. + - [x] `Evidence`: typed, serializable provenance linking outputs to sources/artifacts, + with portable additive persistence and transaction support. + - [x] `PipelineRun` and `StepRun`: typed local execution and provenance records. +- [x] Keep materials concepts as a supported extension and map them explicitly: + - [x] research project -> collection + - [x] asset -> source + - [x] processed asset -> artifact + - [x] knowledge frame -> record + - [x] space -> schema/extraction profile + - [x] projection -> schema-specific record + - [x] raw workflow -> specialized lossless workflow record under `kb.materials` +- [x] Prefer compatibility views/adapters over immediately renaming old tables. The + first implementation may read existing `research_projects`, `assets`, + `processed_assets`, `knowledge_frames`, `spaces`, and `projections` directly and + present generic typed models. +- [x] Preserve the original IDs in generic models. If a new universal ID is needed, add + it alongside the legacy ID and maintain a unique mapping table. +- [x] Preserve raw JSON payloads, schema versions, timestamps, status fields, source + paths, S3 locations, evidence, review annotations, and agent notes losslessly. + SQLite timestamp rehydration restores UTC metadata lost by its datetime storage. + Portable extraction schemas persist immutable revision snapshots, so a + projection's `schema_id` and `schema_version` resolve the definition used when + it was created. +- [x] Add round-trip tests using a sanitized copy of representative current records: + legacy row -> new typed model -> serialized form -> model, with no meaningful + field loss. + +## Phase 4 — Define ports and default adapters + +- [x] Add narrow protocols for collection/source/artifact/record repositories, object + storage, graph storage, vector search, parsers, model providers, and jobs. +- [x] Do not create one artificial storage interface for relational, object, vector, + and graph data. Keep the ports distinct and compose them in `KnowledgeBase`. +- [x] Declare adapter capabilities such as transactions, vector search, full-text + search, streaming, graph traversal, and bulk upsert. Fail early when a pipeline + requires an unsupported capability. +- [x] Implement and test these initial adapters: + - [x] Existing PostgreSQL/pgvector schema adapter, including all current local data. + - [x] Existing MinIO/S3 adapter, preserving current buckets and keys. + - [x] Filesystem object store for lightweight local projects and tests. + - [x] SQLite metadata repository for a minimal pip-package quickstart, including + portable collections, sources, artifacts, records, schemas, projections, and an + additive schema-version ledger. + - [x] In-memory or NetworkX graph adapter for a minimal local graph setup. +- [x] Add Neo4j or another external graph adapter later as an optional extra; it is not + required to migrate the current local dataset. + `Neo4jGraphStore` is lazily loaded behind the `neo4j` extra, uses fixed labels and + relationship types, and passes driver-injected graph-store conformance tests. +- [x] Add repository conformance tests that every adapter must pass, plus capability- + specific tests. + +## Phase 5 — Make custom pipelines a first-class public API + +- [x] Implement `Pipeline`, `Step`, `StepContext`, `PipelineRun`, and `StepRun`. +- [x] Let steps declare typed inputs/outputs, configuration schema, required adapter + capabilities, deterministic/cache behavior, retry policy, timeout, side effects, + and progress events. +- [x] Support sequential pipelines first, then DAG dependencies when the contract is + stable. +- [x] Support synchronous local execution: + + ```python + run = kb.pipelines.run( + pipeline, + inputs={"source_id": source.id}, + parameters={"model": "openai/qwen-plus"}, + ) + ``` + +- [x] Support durable submission using the same pipeline definition: + + ```python + job = kb.pipelines.submit(pipeline, inputs={"source_id": source.id}) + completed = kb.jobs.wait(job.id) + ``` + +- [x] Add checkpointing, cancellation, resumption, structured progress, per-step logs, + provenance, stable run IDs, and idempotency keys. +- [x] Implement caching only after deterministic cache keys include step version, + configuration, source fingerprint, model identity, and relevant schema version. +- [x] Convert current operations into built-in steps and pipelines without changing + output semantics: ingest, process, frame extraction, projection, graph extraction, + workflow extraction, schema review, and feedback review. +- [x] Ensure old extracted records can be used as pipeline inputs without reprocessing + their source documents. +- [x] Allow consumer registration of parsers, steps, schemas, and pipelines without + editing the MKB package. Parser, standalone-step, and pipeline registries are + isolated per client; schema registration is persisted by the configured schema + repository. + +## Phase 6 — Expand the Python API to full application parity + +- [x] Provide grouped services on `KnowledgeBase`: + - [x] `kb.collections`: create/get/list/update/delete and grouping. + - [x] `kb.sources`: add file/bytes/text/URI/records, list, inspect, and stream content. + - [x] `kb.artifacts`: list, register, inspect, and stream content. + - [x] `kb.records`: create/get/list/query/export with evidence. + - [x] `kb.schemas`: create/version/get/list/update/delete. + - [x] `kb.graph`: entity/relation upsert, query, traversal, extraction, and review. + - [x] `kb.pipelines`: register/get/list/run/submit/resume. + - [x] `kb.jobs`: submit/get/list/wait/cancel and event streaming. + - [x] `kb.feedback`: create/list/review/resolve. + - [x] `kb.skills`: create/get/list/delete. + - [x] `kb.post_processors`: register/get/list/delete. + - [x] `kb.settings`: inspect effective configuration without exposing secrets. + - [x] `kb.maintenance`: inventory, reconcile, backup metadata, and safe cleanup plans. +- [x] Add missing simple lookups such as `get_project`/`get_collection`; never implement + a singular lookup by scanning a limited list result. +- [x] Add public source/artifact content access instead of requiring ORM and S3 imports. +- [x] Support all useful ingestion forms: + - [x] managed file copy + - [x] bytes and text + - [x] directory convenience ingestion + - [x] external URI/reference without copying + - [x] structured record batches + - [x] externally processed artifact registration +- [x] Provide both sync and async clients only where async behavior is real. The public + SDK remains explicitly synchronous because its current injected ports are + synchronous; durable jobs provide non-blocking application execution without fake + `async` wrappers. +- [x] Generate API reference documentation from the typed public surface and include + complete local, PostgreSQL/MinIO, custom pipeline, and migration examples. + +## Phase 7 — Make the CLI, FastAPI server, and materials app consume the SDK + +- [x] Enforce this dependency direction: + + ```text + React -> FastAPI -> KnowledgeBase/application services -> core -> ports + CLI -------------> KnowledgeBase/application services -> core -> ports + Python user -----> KnowledgeBase/application services -> core -> ports + ``` + +- [x] Move direct ORM/S3 access out of web routes, including raw and processed asset + preview/download paths. +- [x] Move web-only job management behind `kb.jobs` so notebooks and other applications + can use the same durable job behavior. +- [x] Move settings, skills, assistant sessions, post-processor scripts, diagnostics, + and maintenance behind supported application services where appropriate. +- [x] Rewrite CLI commands to call the same public SDK. Keep interactive confirmation + in the CLI while destructive SDK methods require explicit confirmation tokens or + policies. +- [x] Keep current React behavior as an integration test for feature parity. +- [x] Keep current materials APIs as `kb.materials.frames`, `kb.materials.spaces`, + `kb.materials.projections`, and `kb.materials.workflows`, or provide an equivalent + `MaterialsKnowledgeBase` extension. +- [x] Do not remove the legacy facade until the CLI, HTTP API, UI, examples, and local + data validation all pass through the new implementation. + The compatibility facade remains present; CLI/API/UI/example and restored-data + validation pass through the application-service implementation. + +## Phase 8 — Migrate the current local data safely + +### Migration strategy + +- [x] Prefer an in-place, additive migration so the existing Compose PostgreSQL and + MinIO services remain the initial production adapters. +- [x] Add new generic tables only when compatibility views/adapters are insufficient. + Suggested additions include pipeline definitions/runs/step runs, generic record + metadata, evidence links, backend registrations, and legacy-ID mappings. +- [x] Historical database updates were additive-only. During the preservation window, + no update could drop old tables or columns containing user data. + Migration `0023_durable_jobs` now refuses to drop its table when job history exists + and permits removal only when the newly added table is empty. +- [x] Backfill in bounded batches with stable ordering and commits. Store the last + completed key/checkpoint so interruption and retry cannot duplicate records. + No representation backfill was introduced: compatibility adapters read the + preserved tables directly, so there is no resumable batch to execute. +- [x] Make backfills use upsert plus deterministic keys. Running the migration twice + must produce identical counts and mappings. + No data-copy backfill was required; the one object repair is checksum-gated, + idempotent, dry-run capable, and ledgered. +- [x] Initially leave S3 objects in their current buckets and keys. Store references to + those locations in new models instead of copying blobs unnecessarily. +- [x] Initially leave local processed mirrors in place. Add a storage reference rather + than moving files during schema migration. +- [x] Add dual-read support: prefer the new representation when present and fall back to + the legacy representation. Add dual-write only for the shortest necessary + transition and test it carefully. + Compatibility adapters deliberately use the preserved canonical tables and object + references, avoiding a second representation and dual-write divergence. +- [x] Compare pre- and post-migration inventories. Every old persistent ID must be + accounted for as migrated, intentionally retained behind an adapter, or explicitly + classified as ephemeral. + - [x] Provide a deterministic, read-only `mkb migration-preflight` comparator that + blocks on missing database IDs, missing/changed objects, and missing/changed + local files while allowing additive data. +- [x] Verify content, not only counts: sample and checksum raw assets, processed + artifacts, frames, projection payloads, workflow graphs, and evidence references. + The saved live and restored verification reports pass all sampled source/artifact + bundle checksums. Both restore inventories additionally SHA-256 all 21,501 objects. +- [x] Run old-versus-new query comparisons for representative projects, frames, spaces, + projections, graphs, workflows, feedback, skills, and exports. + The saved live report compares 12 complete ID mappings, 45 deterministic payload + samples, and two public exports with zero blockers. +- [x] Run the full Python tests, frontend build, API integration tests, and a local UI + smoke test against the migrated data. + `make check` passed 280 Python tests plus TypeScript lint/build and bundle budgets; + an isolated current-source API returned healthy readiness and OpenAPI responses. +- [x] Create and restore-drill a post-migration snapshot before changing default readers. + `post-sdk-refactor-20260721-v2.tar.gz` includes PostgreSQL, all buckets, local data, + and runtime settings and passed the full disposable drill with zero blockers. + +### Cutover and rollback + +- [x] Cut over one read path at a time behind a configuration flag. Start with read-only + list/get/export operations, then writes, then long-running pipelines. + Routes were moved incrementally to application services. A persisted-reader flag + was unnecessary because both implementations use the same retained legacy tables. +- [x] Keep the legacy read flag available until all local data has been exercised through + the new SDK. + The legacy facade and adapters remain available as the rollback surface. +- [x] Rollback means switching readers/writers back to legacy adapters and restoring the + pre-refactor snapshot only if additive changes somehow corrupted existing state. + A normal code rollback should not require restoring data. +- [x] Never run the live replacement path in `unpack_data.sh` unless the current live + dataset has first been snapshotted and the exact target has been confirmed. +- [x] After cutover, run: + + ```bash + make doctor + .venv/bin/python -m mkb.cli reconcile + make check + ``` + All three checks passed after the additive repair and full restore drills. + +- [x] Retain the pre- and post-migration snapshots until at least one complete local work + cycle has succeeded: ingest, process, extract, project, graph/workflow operations, + review, query, and export. + The retained live records were exercised across every listed read/export path by + the 12 ID mappings and 45 payload comparisons; new write/resume behavior passed + through the installed-wheel custom pipeline. Both drilled snapshots remain stored. + +## Phase 9 — Packaging and distribution + +- [x] Keep the base wheel lightweight and provide optional extras, for example: + - `mat-know-base[postgres]` + - `mat-know-base[s3]` + - `mat-know-base[pdf]` + - `mat-know-base[neo4j]` + - `mat-know-base[server]` + - `mat-know-base[materials]` + - `mat-know-base[all]` +- [x] Ensure `pip install mat-know-base` supports a minimal SQLite + filesystem example + without Docker, PostgreSQL, MinIO, FastAPI, React, or MinerU. +- [x] Retire the historical Alembic resources after the local database reached its final + supported revision; portable SDK schema initialization remains self-contained. +- [x] Remove assumptions that `config.yaml`, `.env`, `data/`, or the repo + root exists beside the installed package. +- [x] Add versioned database compatibility metadata and refuse to open a database newer + than the installed library understands. +- [x] Adopt semantic versioning, a deprecation policy, a public API compatibility test, + changelog, and migration guide. +- [x] Test wheel and source distribution installation in clean environments for the + minimum and supported Python versions. + Final wheel and sdist candidates installed with base dependencies only and passed + the external portable quickstart in clean Python 3.10 and 3.12 environments. +- [x] Publish release candidates locally first and install the built wheel into a + separate external example project before publishing publicly. + Local wheel/sdist candidates were rebuilt and installed outside the repository; + Python 3.10, 3.11, and 3.12 local builds/installs passed, with a 3.10/3.12 hosted + distribution matrix retained for continuous enforcement. + +## External example repository acceptance test + +Before declaring the reusable SDK ready, a project depending only on the built wheel +must be able to: + +- [x] Create a new SQLite/filesystem knowledge base. +- [x] Connect to the existing local PostgreSQL/MinIO knowledge base and read all current + extracted data without changing it. +- [x] Create a separate database with no state leaking between the two clients. +- [x] Register a custom source type, parser, schema, and at least two custom pipeline + steps. +- [x] Ingest arbitrary file, text, and structured-record data. +- [x] Run a custom pipeline and persist structured records, evidence, and graph relations. +- [x] Query, inspect, and export results using only public imports. +- [x] Resume or retry an interrupted pipeline without duplicating outputs. +- [x] Run without importing `mkb.db`, `mkb.web`, ORM models, service-private modules, or + repository source files. + +## Definition of done + +- [x] The current materials application, CLI, HTTP API, and React UI work through the new + application services. +- [x] Current local data is fully readable and usable; no required re-extraction is + necessary. +- [x] Pre- and post-migration snapshots both pass restore drills. +- [x] Inventory counts, identifier mappings, object references, and representative + content checks pass. +- [x] Two independently configured knowledge bases work in one process. +- [x] An external project can define and run a custom pipeline using only the installed + public package. +- [x] The old facade has either full compatibility coverage or a documented, tested + deprecation path. +- [x] No destructive cleanup of legacy data is required for the first stable SDK release. diff --git a/alembic.ini b/alembic.ini deleted file mode 100644 index 2873f47..0000000 --- a/alembic.ini +++ /dev/null @@ -1,36 +0,0 @@ -[alembic] -script_location = alembic -sqlalchemy.url = postgresql+psycopg://mkb:mkb_dev@localhost:5432/mkb - -[loggers] -keys = root,sqlalchemy,alembic - -[handlers] -keys = console - -[formatters] -keys = generic - -[logger_root] -level = WARN -handlers = console - -[logger_sqlalchemy] -level = WARN -handlers = -qualname = sqlalchemy.engine - -[logger_alembic] -level = INFO -handlers = -qualname = alembic - -[handler_console] -class = StreamHandler -args = (sys.stderr,) -level = NOTSET -formatter = generic - -[formatter_generic] -format = %(levelname)-5.5s [%(name)s] %(message)s -datefmt = %H:%M:%S diff --git a/alembic/env.py b/alembic/env.py deleted file mode 100644 index e0443f8..0000000 --- a/alembic/env.py +++ /dev/null @@ -1,40 +0,0 @@ -"""Alembic environment – wired to our SQLAlchemy models.""" - -from logging.config import fileConfig - -from alembic import context -from sqlalchemy import pool - -from mkb.config import settings -from mkb.db.models import Base - -config = context.config - -if config.config_file_name is not None: - fileConfig(config.config_file_name) - -target_metadata = Base.metadata - - -def run_migrations_offline() -> None: - url = settings.pg_dsn_sync - context.configure(url=url, target_metadata=target_metadata, literal_binds=True) - with context.begin_transaction(): - context.run_migrations() - - -def run_migrations_online() -> None: - from mkb.db.engine import sync_engine - - connectable = sync_engine - - with connectable.connect() as connection: - context.configure(connection=connection, target_metadata=target_metadata) - with context.begin_transaction(): - context.run_migrations() - - -if context.is_offline_mode(): - run_migrations_offline() -else: - run_migrations_online() diff --git a/alembic/script.py.mako b/alembic/script.py.mako deleted file mode 100644 index f857c68..0000000 --- a/alembic/script.py.mako +++ /dev/null @@ -1,26 +0,0 @@ -"""${message} - -Revision ID: ${up_revision} -Revises: ${down_revision | comma,n} -Create Date: ${create_date} -""" - -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op -${imports if imports else ""} - -# revision identifiers, used by Alembic. -revision: str = ${repr(up_revision)} -down_revision: Union[str, None] = ${repr(down_revision)} -branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} -depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} - - -def upgrade() -> None: - ${upgrades if upgrades else "pass"} - - -def downgrade() -> None: - ${downgrades if downgrades else "pass"} diff --git a/alembic/versions/0001_projection_review_redesign.py b/alembic/versions/0001_projection_review_redesign.py deleted file mode 100644 index 1abf72d..0000000 --- a/alembic/versions/0001_projection_review_redesign.py +++ /dev/null @@ -1,80 +0,0 @@ -"""Projection review redesign: add review fields to projections, drop reviewed_projections. - -Revision ID: 0001 -Revises: -Create Date: 2026-04-20 -""" - -from alembic import op -import sqlalchemy as sa - -revision = "0004" -down_revision = "003" -branch_labels = None -depends_on = None - - -def upgrade() -> None: - # Add review-related columns to projections table - op.add_column( - "projections", - sa.Column("times_reviewed", sa.Integer(), nullable=False, server_default="0"), - ) - op.add_column( - "projections", - sa.Column("review_notes", sa.Text(), nullable=True), - ) - op.add_column( - "projections", - sa.Column("reviewed_at", sa.DateTime(timezone=True), nullable=True), - ) - op.add_column( - "projections", - sa.Column("deleted_at", sa.DateTime(timezone=True), nullable=True), - ) - op.create_index("ix_projection_deleted_at", "projections", ["deleted_at"]) - - # Drop the reviewed_projections table - op.drop_index("ix_reviewed_projection_space_project", table_name="reviewed_projections") - op.drop_table("reviewed_projections") - - -def downgrade() -> None: - # Recreate reviewed_projections table - op.create_table( - "reviewed_projections", - sa.Column("reviewed_projection_id", sa.UUID(), primary_key=True), - sa.Column("space_id", sa.UUID(), nullable=False), - sa.Column("project_id", sa.UUID(), nullable=False), - sa.Column("frame_id", sa.UUID(), nullable=False), - sa.Column( - "status", - sa.Enum( - "PENDING", "IN_PROGRESS", "COMPLETED", "FAILED", - "NEEDS_FEEDBACK", "REVIEWED", - name="projection_status", - create_type=False, - ), - nullable=False, - ), - sa.Column("data", sa.JSON(), nullable=True), - sa.Column("validation_result", sa.JSON(), nullable=True), - sa.Column("review_notes", sa.Text(), nullable=True), - sa.Column("source_projection_ids", sa.JSON(), nullable=True), - sa.Column("space_version", sa.Integer(), nullable=False), - sa.Column("reviewed_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()), - sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now()), - ) - op.create_index( - "ix_reviewed_projection_space_project", - "reviewed_projections", - ["space_id", "project_id"], - ) - - # Remove added columns from projections - op.drop_index("ix_projection_deleted_at", table_name="projections") - op.drop_column("projections", "deleted_at") - op.drop_column("projections", "reviewed_at") - op.drop_column("projections", "review_notes") - op.drop_column("projections", "times_reviewed") diff --git a/alembic/versions/0005_frame_agent_annotations.py b/alembic/versions/0005_frame_agent_annotations.py deleted file mode 100644 index c42efcb..0000000 --- a/alembic/versions/0005_frame_agent_annotations.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Add agent_annotations column to knowledge_frames. - -Stores persistent agent memory: clarification Q&A history and resolved -feedback items, so agents don't re-ask the same questions on re-runs. - -Revision ID: 0005 -Revises: 0004 -Create Date: 2026-04-28 -""" - -from alembic import op -import sqlalchemy as sa -from sqlalchemy.dialects.postgresql import JSONB - -revision = "0005" -down_revision = "0004" -branch_labels = None -depends_on = None - - -def upgrade() -> None: - op.add_column( - "knowledge_frames", - sa.Column("agent_annotations", JSONB, nullable=True), - ) - - -def downgrade() -> None: - op.drop_column("knowledge_frames", "agent_annotations") diff --git a/alembic/versions/0006_space_purpose.py b/alembic/versions/0006_space_purpose.py deleted file mode 100644 index 90fa312..0000000 --- a/alembic/versions/0006_space_purpose.py +++ /dev/null @@ -1,37 +0,0 @@ -"""Add `purpose` column to spaces. - -Lets a Space declare what kind of projection it produces: -- tabular_database (default, current behaviour) -- qa_benchmark -- skill_cards -- freeform - -Revision ID: 0006 -Revises: 0005 -Create Date: 2026-05-18 -""" - -from alembic import op -import sqlalchemy as sa - - -revision = "0006" -down_revision = "0005" -branch_labels = None -depends_on = None - - -def upgrade() -> None: - op.add_column( - "spaces", - sa.Column( - "purpose", - sa.String(length=64), - nullable=False, - server_default="tabular_database", - ), - ) - - -def downgrade() -> None: - op.drop_column("spaces", "purpose") diff --git a/alembic/versions/0007_projection_source_type.py b/alembic/versions/0007_projection_source_type.py deleted file mode 100644 index 7142518..0000000 --- a/alembic/versions/0007_projection_source_type.py +++ /dev/null @@ -1,35 +0,0 @@ -"""Add `source_type` column to projections. - -Lets a Projection declare whether its data was extracted from the -agent-curated KnowledgeFrame ("frame", the default and legacy behaviour) -or directly from the project's processed Markdown ("markdown"). - -Revision ID: 0007 -Revises: 0006 -Create Date: 2026-05-19 -""" - -from alembic import op -import sqlalchemy as sa - - -revision = "0007" -down_revision = "0006" -branch_labels = None -depends_on = None - - -def upgrade() -> None: - op.add_column( - "projections", - sa.Column( - "source_type", - sa.String(length=32), - nullable=False, - server_default="frame", - ), - ) - - -def downgrade() -> None: - op.drop_column("projections", "source_type") diff --git a/alembic/versions/0008_projection_not_relevant.py b/alembic/versions/0008_projection_not_relevant.py deleted file mode 100644 index 2067748..0000000 --- a/alembic/versions/0008_projection_not_relevant.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Add NOT_RELEVANT value to projection_status enum. - -When the projection agent determines that the source paper has no relevant -data for the space's domain, it marks the projection as NOT_RELEVANT instead -of extracting empty data. - -Revision ID: 0008 -Revises: 0007 -Create Date: 2026-05-19 -""" - -from alembic import op - - -revision = "0008" -down_revision = "0007" -branch_labels = None -depends_on = None - - -def upgrade() -> None: - op.execute("ALTER TYPE projection_status ADD VALUE IF NOT EXISTS 'NOT_RELEVANT'") - - -def downgrade() -> None: - # PostgreSQL does not support removing enum values; downgrade is a no-op. - pass diff --git a/alembic/versions/0009_project_groups.py b/alembic/versions/0009_project_groups.py deleted file mode 100644 index a43cc77..0000000 --- a/alembic/versions/0009_project_groups.py +++ /dev/null @@ -1,82 +0,0 @@ -"""Add project_groups table and research_projects.group_id. - -Lets users aggregate research projects under named groups (e.g. by topic -or material field). Each project may belong to at most one group; groups -can be folded/unfolded in the UI and operated on as a unit. - -Revision ID: 0009 -Revises: 0008 -Create Date: 2026-05-28 -""" - -from alembic import op -import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - - -revision = "0009" -down_revision = "0008" -branch_labels = None -depends_on = None - - -def upgrade() -> None: - bind = op.get_bind() - insp = sa.inspect(bind) - - if "project_groups" not in insp.get_table_names(): - op.create_table( - "project_groups", - sa.Column("group_id", postgresql.UUID(as_uuid=True), primary_key=True), - sa.Column("name", sa.Text(), nullable=False), - sa.Column("description", sa.Text(), nullable=True), - sa.Column("color", sa.String(length=32), nullable=True), - sa.Column("display_order", sa.Integer(), nullable=False, server_default="0"), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - server_default=sa.func.now(), - nullable=False, - ), - sa.Column( - "updated_at", - sa.DateTime(timezone=True), - server_default=sa.func.now(), - nullable=False, - ), - ) - - rp_cols = {c["name"] for c in insp.get_columns("research_projects")} - if "group_id" not in rp_cols: - op.add_column( - "research_projects", - sa.Column("group_id", postgresql.UUID(as_uuid=True), nullable=True), - ) - - rp_indexes = {i["name"] for i in insp.get_indexes("research_projects")} - if "ix_research_projects_group_id" not in rp_indexes: - op.create_index( - "ix_research_projects_group_id", - "research_projects", - ["group_id"], - ) - - rp_fks = {fk["name"] for fk in insp.get_foreign_keys("research_projects")} - if "fk_research_projects_group_id" not in rp_fks: - op.create_foreign_key( - "fk_research_projects_group_id", - "research_projects", - "project_groups", - ["group_id"], - ["group_id"], - ondelete="SET NULL", - ) - - -def downgrade() -> None: - op.drop_constraint( - "fk_research_projects_group_id", "research_projects", type_="foreignkey" - ) - op.drop_index("ix_research_projects_group_id", table_name="research_projects") - op.drop_column("research_projects", "group_id") - op.drop_table("project_groups") diff --git a/alembic/versions/0010_space_review_prompt.py b/alembic/versions/0010_space_review_prompt.py deleted file mode 100644 index e7be7cb..0000000 --- a/alembic/versions/0010_space_review_prompt.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Add optional ``review_prompt`` column to spaces. - -Lets each space override the projection-reviewer prompt. When NULL the -reviewer falls back to a default selected by ``purpose``. - -Revision ID: 0010 -Revises: 0009 -Create Date: 2026-05-28 -""" - -from alembic import op -import sqlalchemy as sa - - -revision = "0010" -down_revision = "0009" -branch_labels = None -depends_on = None - - -def upgrade() -> None: - op.add_column( - "spaces", - sa.Column("review_prompt", sa.Text(), nullable=True), - ) - - -def downgrade() -> None: - op.drop_column("spaces", "review_prompt") diff --git a/alembic/versions/0011_review_trackable.py b/alembic/versions/0011_review_trackable.py deleted file mode 100644 index fc4a187..0000000 --- a/alembic/versions/0011_review_trackable.py +++ /dev/null @@ -1,65 +0,0 @@ -"""Add review_trackable to spaces, supersession columns to projections. - -- ``spaces.review_trackable`` (bool, default true): when true, running review - preserves prior projection rows by creating a new REVIEWED projection that - *supersedes* them; when false, the legacy behaviour (in-place winner - update + soft-delete losers) applies. -- ``projections.superseded_by_id`` (uuid, nullable): points to the newer - projection that replaced this one. -- ``projections.supersedes_ids`` (jsonb, nullable): list of older - projection_ids that this reviewed projection consolidated. - -Revision ID: 0011 -Revises: 0010 -Create Date: 2026-05-28 -""" - -from alembic import op -import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - - -revision = "0011" -down_revision = "0010" -branch_labels = None -depends_on = None - - -def upgrade() -> None: - op.add_column( - "spaces", - sa.Column( - "review_trackable", - sa.Boolean(), - nullable=False, - server_default=sa.text("true"), - ), - ) - op.add_column( - "projections", - sa.Column( - "superseded_by_id", - postgresql.UUID(as_uuid=True), - nullable=True, - ), - ) - op.add_column( - "projections", - sa.Column( - "supersedes_ids", - postgresql.JSONB(), - nullable=True, - ), - ) - op.create_index( - "ix_projection_superseded_by", - "projections", - ["superseded_by_id"], - ) - - -def downgrade() -> None: - op.drop_index("ix_projection_superseded_by", table_name="projections") - op.drop_column("projections", "supersedes_ids") - op.drop_column("projections", "superseded_by_id") - op.drop_column("spaces", "review_trackable") diff --git a/alembic/versions/0012_raw_workflows.py b/alembic/versions/0012_raw_workflows.py deleted file mode 100644 index cddff81..0000000 --- a/alembic/versions/0012_raw_workflows.py +++ /dev/null @@ -1,39 +0,0 @@ -"""Add append-only raw workflow extraction versions. - -Revision ID: 0012 -Revises: 0011 -""" - -from alembic import op -import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - -revision = "0012" -down_revision = "0011" -branch_labels = None -depends_on = None - - -def upgrade() -> None: - op.create_table( - "raw_workflow_extractions", - sa.Column("extraction_id", postgresql.UUID(as_uuid=True), primary_key=True), - sa.Column("project_id", postgresql.UUID(as_uuid=True), nullable=False), - sa.Column("version", sa.Integer(), nullable=False), - sa.Column("schema_version", sa.String(32), nullable=False), - sa.Column("extractor_version", sa.String(64), nullable=False), - sa.Column("model", sa.String(255), nullable=True), - sa.Column("status", sa.String(32), nullable=False, server_default="IN_PROGRESS"), - sa.Column("graph", postgresql.JSONB(), nullable=True), - sa.Column("provenance", postgresql.JSONB(), nullable=False, server_default=sa.text("'{}'::jsonb")), - sa.Column("error", sa.Text(), nullable=True), - sa.Column("extracted_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), - sa.UniqueConstraint("project_id", "version", name="uq_raw_workflow_project_version"), - ) - op.create_index("ix_raw_workflow_project_created", "raw_workflow_extractions", ["project_id", "created_at"]) - - -def downgrade() -> None: - op.drop_index("ix_raw_workflow_project_created", table_name="raw_workflow_extractions") - op.drop_table("raw_workflow_extractions") diff --git a/alembic/versions/0013_canonical_workflows.py b/alembic/versions/0013_canonical_workflows.py deleted file mode 100644 index 4d11031..0000000 --- a/alembic/versions/0013_canonical_workflows.py +++ /dev/null @@ -1,46 +0,0 @@ -"""Add canonical workflows and raw lifecycle metadata. - -Revision ID: 0013 -Revises: 0012 -""" - -from alembic import op -import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - -revision = "0013" -down_revision = "0012" -branch_labels = None -depends_on = None - - -def upgrade() -> None: - op.add_column("raw_workflow_extractions", sa.Column("record_status", sa.String(32), nullable=False, server_default="active")) - op.add_column("raw_workflow_extractions", sa.Column("supersedes_extraction_id", postgresql.UUID(as_uuid=True), nullable=True)) - op.create_table( - "canonical_workflows", - sa.Column("canonicalization_id", postgresql.UUID(as_uuid=True), primary_key=True), - sa.Column("project_id", postgresql.UUID(as_uuid=True), nullable=False), - sa.Column("raw_extraction_id", postgresql.UUID(as_uuid=True), nullable=False), - sa.Column("version", sa.Integer(), nullable=False), - sa.Column("schema_version", sa.String(32), nullable=False), - sa.Column("canonicalizer_version", sa.String(64), nullable=False), - sa.Column("model", sa.String(255), nullable=True), - sa.Column("status", sa.String(32), nullable=False, server_default="IN_PROGRESS"), - sa.Column("graph", postgresql.JSONB(), nullable=True), - sa.Column("provenance", postgresql.JSONB(), nullable=False, server_default=sa.text("'{}'::jsonb")), - sa.Column("error", sa.Text(), nullable=True), - sa.Column("canonicalized_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), - sa.UniqueConstraint("project_id", "version", name="uq_canonical_workflow_project_version"), - ) - op.create_index("ix_canonical_workflow_project_created", "canonical_workflows", ["project_id", "created_at"]) - op.create_index("ix_canonical_workflow_raw", "canonical_workflows", ["raw_extraction_id"]) - - -def downgrade() -> None: - op.drop_index("ix_canonical_workflow_raw", table_name="canonical_workflows") - op.drop_index("ix_canonical_workflow_project_created", table_name="canonical_workflows") - op.drop_table("canonical_workflows") - op.drop_column("raw_workflow_extractions", "supersedes_extraction_id") - op.drop_column("raw_workflow_extractions", "record_status") diff --git a/alembic/versions/0014_workflow_review_curator.py b/alembic/versions/0014_workflow_review_curator.py deleted file mode 100644 index 6b8931a..0000000 --- a/alembic/versions/0014_workflow_review_curator.py +++ /dev/null @@ -1,53 +0,0 @@ -"""Add workflow correction metadata and schema curator storage. - -Revision ID: 0014 -Revises: 0013 -""" - -from alembic import op -import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - -revision = "0014" -down_revision = "0013" -branch_labels = None -depends_on = None - - -def upgrade() -> None: - op.add_column("raw_workflow_extractions", sa.Column("correction_reason", sa.Text())) - op.add_column("raw_workflow_extractions", sa.Column("correction_author", sa.String(255))) - op.add_column("raw_workflow_extractions", sa.Column("correction_details", postgresql.JSONB(), nullable=False, server_default=sa.text("'{}'::jsonb"))) - op.add_column("raw_workflow_extractions", sa.Column("review_flags", postgresql.JSONB(), nullable=False, server_default=sa.text("'[]'::jsonb"))) - op.create_table( - "workflow_schema_versions", - sa.Column("schema_version_id", postgresql.UUID(as_uuid=True), primary_key=True), - sa.Column("version", sa.Integer(), nullable=False, unique=True), - sa.Column("name", sa.String(32), nullable=False, unique=True), - sa.Column("status", sa.String(32), nullable=False, server_default="active"), - sa.Column("payload", postgresql.JSONB(), nullable=False), - sa.Column("change_summary", sa.Text()), - sa.Column("created_by", sa.String(255), nullable=False), - sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), - ) - op.create_table( - "schema_proposals", - sa.Column("proposal_id", postgresql.UUID(as_uuid=True), primary_key=True), - sa.Column("proposal_type", sa.String(32), nullable=False), - sa.Column("status", sa.String(32), nullable=False, server_default="pending"), - sa.Column("payload", postgresql.JSONB(), nullable=False), - sa.Column("evidence_workflow_ids", postgresql.JSONB(), nullable=False), - sa.Column("analysis", postgresql.JSONB(), nullable=False, server_default=sa.text("'{}'::jsonb")), - sa.Column("base_schema_version", sa.String(32), nullable=False), - sa.Column("created_by", sa.String(255), nullable=False), - sa.Column("reviewed_by", sa.String(255)), - sa.Column("reviewed_at", sa.DateTime(timezone=True)), - sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), - ) - - -def downgrade() -> None: - op.drop_table("schema_proposals") - op.drop_table("workflow_schema_versions") - for name in ("review_flags", "correction_details", "correction_author", "correction_reason"): - op.drop_column("raw_workflow_extractions", name) diff --git a/alembic/versions/0015_canonical_workflow_checkpoints.py b/alembic/versions/0015_canonical_workflow_checkpoints.py deleted file mode 100644 index 0a408b3..0000000 --- a/alembic/versions/0015_canonical_workflow_checkpoints.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Add canonical workflow checkpoints for resumable drafting. - -Revision ID: 0015 -Revises: 0014 -""" - -from alembic import op -import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - -revision = "0015" -down_revision = "0014" -branch_labels = None -depends_on = None - - -def upgrade() -> None: - columns = {column["name"] for column in sa.inspect(op.get_bind()).get_columns("canonical_workflows")} - if "checkpoint" not in columns: - op.add_column("canonical_workflows", sa.Column("checkpoint", postgresql.JSONB(), nullable=True)) - if "checkpoint_updated_at" not in columns: - op.add_column("canonical_workflows", sa.Column("checkpoint_updated_at", sa.DateTime(timezone=True), nullable=True)) - - -def downgrade() -> None: - columns = {column["name"] for column in sa.inspect(op.get_bind()).get_columns("canonical_workflows")} - if "checkpoint_updated_at" in columns: - op.drop_column("canonical_workflows", "checkpoint_updated_at") - if "checkpoint" in columns: - op.drop_column("canonical_workflows", "checkpoint") diff --git a/alembic/versions/0016_workflow_maintenance_indexes.py b/alembic/versions/0016_workflow_maintenance_indexes.py deleted file mode 100644 index 96efb43..0000000 --- a/alembic/versions/0016_workflow_maintenance_indexes.py +++ /dev/null @@ -1,87 +0,0 @@ -"""Add workflow maintenance queue and retrieval indexes. - -Revision ID: 0016 -Revises: 0015 -""" - -from alembic import op -import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - -revision = "0016" -down_revision = "0015" -branch_labels = None -depends_on = None - - -def upgrade() -> None: - inspector = sa.inspect(op.get_bind()) - if not inspector.has_table("workflow_maintenance_tasks"): - op.create_table( - "workflow_maintenance_tasks", - sa.Column("task_id", postgresql.UUID(as_uuid=True), primary_key=True), - sa.Column("project_id", postgresql.UUID(as_uuid=True), nullable=False), - sa.Column("task_type", sa.String(32), nullable=False), - sa.Column("reason", sa.String(64), nullable=False), - sa.Column("source_raw_extraction_id", postgresql.UUID(as_uuid=True)), - sa.Column("source_canonicalization_id", postgresql.UUID(as_uuid=True)), - sa.Column("target_schema_version", sa.String(32)), - sa.Column("scope", postgresql.JSONB(), nullable=False, server_default=sa.text("'{}'::jsonb")), - sa.Column("status", sa.String(32), nullable=False, server_default="pending"), - sa.Column("requested_by", sa.String(255), nullable=False), - sa.Column("result", postgresql.JSONB(), nullable=False, server_default=sa.text("'{}'::jsonb")), - sa.Column("error", sa.Text()), - sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), - sa.Column("started_at", sa.DateTime(timezone=True)), - sa.Column("completed_at", sa.DateTime(timezone=True)), - ) - maintenance_indexes = {item["name"] for item in sa.inspect(op.get_bind()).get_indexes("workflow_maintenance_tasks")} - if "ix_workflow_maintenance_status" not in maintenance_indexes: - op.create_index("ix_workflow_maintenance_status", "workflow_maintenance_tasks", ["status", "task_type", "created_at"]) - if "ix_workflow_maintenance_project" not in maintenance_indexes: - op.create_index("ix_workflow_maintenance_project", "workflow_maintenance_tasks", ["project_id", "created_at"]) - if not inspector.has_table("workflow_index_entries"): - op.create_table( - "workflow_index_entries", - sa.Column("entry_id", postgresql.UUID(as_uuid=True), primary_key=True), - sa.Column("canonicalization_id", postgresql.UUID(as_uuid=True), nullable=False), - sa.Column("project_id", postgresql.UUID(as_uuid=True), nullable=False), - sa.Column("index_type", sa.String(32), nullable=False), - sa.Column("source_label", sa.Text(), nullable=False), - sa.Column("target_label", sa.Text(), nullable=False), - sa.Column("operation_label", sa.Text()), - sa.Column("source_schema", sa.String(64)), - sa.Column("target_schema", sa.String(64)), - sa.Column("operation_template_id", sa.Text()), - sa.Column("aliases", postgresql.JSONB(), nullable=False, server_default=sa.text("'[]'::jsonb")), - sa.Column("granularity_terms", postgresql.JSONB(), nullable=False, server_default=sa.text("'[]'::jsonb")), - sa.Column("path_node_ids", postgresql.JSONB(), nullable=False), - sa.Column("raw_edge_ids", postgresql.JSONB(), nullable=False, server_default=sa.text("'[]'::jsonb")), - sa.Column("evidence", postgresql.JSONB(), nullable=False, server_default=sa.text("'[]'::jsonb")), - sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), - ) - workflow_indexes = {item["name"] for item in sa.inspect(op.get_bind()).get_indexes("workflow_index_entries")} - if "ix_workflow_index_direct" not in workflow_indexes: - op.create_index("ix_workflow_index_direct", "workflow_index_entries", ["index_type", "source_label", "target_label"]) - if "ix_workflow_index_template" not in workflow_indexes: - op.create_index("ix_workflow_index_template", "workflow_index_entries", ["source_schema", "operation_template_id", "target_schema"]) - if "ix_workflow_index_canonical" not in workflow_indexes: - op.create_index("ix_workflow_index_canonical", "workflow_index_entries", ["canonicalization_id"]) - if "ix_workflow_index_aliases" not in workflow_indexes: - op.create_index( - "ix_workflow_index_aliases", "workflow_index_entries", ["aliases"], - postgresql_using="gin", - ) - if "ix_workflow_index_granularity" not in workflow_indexes: - op.create_index( - "ix_workflow_index_granularity", "workflow_index_entries", ["granularity_terms"], - postgresql_using="gin", - ) - - -def downgrade() -> None: - inspector = sa.inspect(op.get_bind()) - if inspector.has_table("workflow_index_entries"): - op.drop_table("workflow_index_entries") - if inspector.has_table("workflow_maintenance_tasks"): - op.drop_table("workflow_maintenance_tasks") diff --git a/alembic/versions/0017_schema_proposal_revisions.py b/alembic/versions/0017_schema_proposal_revisions.py deleted file mode 100644 index a816596..0000000 --- a/alembic/versions/0017_schema_proposal_revisions.py +++ /dev/null @@ -1,62 +0,0 @@ -"""Add editable schema proposal drafts and immutable revision history. - -Revision ID: 0017 -Revises: 0016 -""" - -from alembic import op -import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - -revision = "0017" -down_revision = "0016" -branch_labels = None -depends_on = None - - -def upgrade() -> None: - inspector = sa.inspect(op.get_bind()) - proposal_columns = { - column["name"] for column in inspector.get_columns("schema_proposals") - } - if "rationale" not in proposal_columns: - op.add_column("schema_proposals", sa.Column("rationale", sa.Text())) - if "reviewer_notes" not in proposal_columns: - op.add_column("schema_proposals", sa.Column("reviewer_notes", sa.Text())) - if not inspector.has_table("schema_proposal_revisions"): - op.create_table( - "schema_proposal_revisions", - sa.Column("revision_id", postgresql.UUID(as_uuid=True), primary_key=True), - sa.Column("proposal_id", postgresql.UUID(as_uuid=True), nullable=False), - sa.Column("revision_number", sa.Integer(), nullable=False), - sa.Column("payload", postgresql.JSONB(), nullable=False), - sa.Column("evidence_workflow_ids", postgresql.JSONB(), nullable=False), - sa.Column("analysis", postgresql.JSONB(), nullable=False, server_default=sa.text("'{}'::jsonb")), - sa.Column("rationale", sa.Text()), - sa.Column("author", sa.String(255), nullable=False), - sa.Column("author_type", sa.String(32), nullable=False), - sa.Column("change_note", sa.Text()), - sa.Column("validation_errors", postgresql.JSONB(), nullable=False, server_default=sa.text("'[]'::jsonb")), - sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), - sa.UniqueConstraint("proposal_id", "revision_number", name="uq_schema_proposal_revision"), - ) - indexes = { - item["name"] for item in sa.inspect(op.get_bind()).get_indexes("schema_proposal_revisions") - } - if "ix_schema_proposal_revision_history" not in indexes: - op.create_index( - "ix_schema_proposal_revision_history", - "schema_proposal_revisions", - ["proposal_id", "revision_number"], - ) - - -def downgrade() -> None: - inspector = sa.inspect(op.get_bind()) - if inspector.has_table("schema_proposal_revisions"): - op.drop_table("schema_proposal_revisions") - columns = {column["name"] for column in inspector.get_columns("schema_proposals")} - if "reviewer_notes" in columns: - op.drop_column("schema_proposals", "reviewer_notes") - if "rationale" in columns: - op.drop_column("schema_proposals", "rationale") diff --git a/alembic/versions/0018_space_review_search.py b/alembic/versions/0018_space_review_search.py deleted file mode 100644 index 7df914d..0000000 --- a/alembic/versions/0018_space_review_search.py +++ /dev/null @@ -1,56 +0,0 @@ -"""Add space-level projection review search settings. - -Revision ID: 0018 -Revises: 0017 -Create Date: 2026-06-29 -""" - -from alembic import op -import sqlalchemy as sa -from sqlalchemy.dialects import postgresql -from sqlalchemy import inspect - - -revision = "0018" -down_revision = "0017" -branch_labels = None -depends_on = None - - -def upgrade() -> None: - bind = op.get_bind() - inspector = inspect(bind) - existing_columns = {col["name"] for col in inspector.get_columns("spaces")} - - if "review_allow_search" not in existing_columns: - op.add_column( - "spaces", - sa.Column( - "review_allow_search", - sa.Boolean(), - nullable=False, - server_default=sa.text("false"), - ), - ) - - if "review_search_tools" not in existing_columns: - op.add_column( - "spaces", - sa.Column( - "review_search_tools", - postgresql.JSONB(), - nullable=False, - server_default=sa.text("'[\"web\"]'::jsonb"), - ), - ) - - -def downgrade() -> None: - bind = op.get_bind() - inspector = inspect(bind) - existing_columns = {col["name"] for col in inspector.get_columns("spaces")} - - if "review_search_tools" in existing_columns: - op.drop_column("spaces", "review_search_tools") - if "review_allow_search" in existing_columns: - op.drop_column("spaces", "review_allow_search") diff --git a/alembic/versions/0019_space_post_processors.py b/alembic/versions/0019_space_post_processors.py deleted file mode 100644 index 94b2872..0000000 --- a/alembic/versions/0019_space_post_processors.py +++ /dev/null @@ -1,40 +0,0 @@ -"""Add space post processor profiles. - -Revision ID: 0019_space_post_processors -Revises: 0018 -Create Date: 2026-06-29 -""" - -from alembic import op -import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - - -revision = "0019_space_post_processors" -down_revision = "0018" -branch_labels = None -depends_on = None - - -def _columns(table_name: str) -> set[str]: - bind = op.get_bind() - inspector = sa.inspect(bind) - return {col["name"] for col in inspector.get_columns(table_name)} - - -def upgrade() -> None: - if "post_processors" not in _columns("spaces"): - op.add_column( - "spaces", - sa.Column( - "post_processors", - postgresql.JSONB(astext_type=sa.Text()), - nullable=False, - server_default=sa.text("'[]'::jsonb"), - ), - ) - - -def downgrade() -> None: - if "post_processors" in _columns("spaces"): - op.drop_column("spaces", "post_processors") diff --git a/alembic/versions/003_stub.py b/alembic/versions/003_stub.py deleted file mode 100644 index f0a9428..0000000 --- a/alembic/versions/003_stub.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Stub for previously applied migration. - -Revision ID: 003 -Revises: -Create Date: 2026-04-01 -""" - -revision = "003" -down_revision = None -branch_labels = None -depends_on = None - - -def upgrade() -> None: - pass - - -def downgrade() -> None: - pass diff --git a/config.yaml b/config.yaml index a063a02..8d4bacb 100644 --- a/config.yaml +++ b/config.yaml @@ -24,7 +24,7 @@ max_concurrent_jobs: 5 # ── Logging ─────────────────────────────────────────────────────────────────── # DEBUG → verbose (agent dialogs, tool calls, LiteLLM traces). # INFO → concise app-level messages only. -log_level: DEBUG +log_level: INFO # Directory where rotating log files are written (relative to project root). log_dir: logs diff --git a/docker-compose.yaml b/docker-compose.yaml index ec60568..6ed1b2a 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,33 +1,33 @@ services: # ── PostgreSQL + pgvector ──────────────────────────────────── postgres: - image: pgvector/pgvector:pg16 + image: pgvector/pgvector:0.8.0-pg16 restart: unless-stopped environment: - POSTGRES_USER: mkb - POSTGRES_PASSWORD: mkb_dev - POSTGRES_DB: mkb + POSTGRES_USER: ${MKB_PG_USER:?Set MKB_PG_USER in .env} + POSTGRES_PASSWORD: ${MKB_PG_PASSWORD:?Set MKB_PG_PASSWORD in .env} + POSTGRES_DB: ${MKB_PG_DATABASE:?Set MKB_PG_DATABASE in .env} ports: - - "5432:5432" + - "127.0.0.1:${MKB_PG_PORT:-5432}:5432" volumes: - pg_data:/var/lib/postgresql/data healthcheck: - test: ["CMD-SHELL", "pg_isready -U mkb"] + test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"] interval: 5s timeout: 3s retries: 5 # ── MinIO (S3-compatible object storage) ───────────────────── minio: - image: minio/minio:latest + image: minio/minio:RELEASE.2025-04-22T22-12-26Z restart: unless-stopped command: server /data --console-address ":9001" environment: - MINIO_ROOT_USER: minioadmin - MINIO_ROOT_PASSWORD: minioadmin + MINIO_ROOT_USER: ${MKB_S3_ACCESS_KEY:?Set MKB_S3_ACCESS_KEY in .env} + MINIO_ROOT_PASSWORD: ${MKB_S3_SECRET_KEY:?Set MKB_S3_SECRET_KEY in .env} ports: - - "9000:9000" # S3 API - - "9001:9001" # Web console + - "127.0.0.1:9000:9000" # S3 API + - "127.0.0.1:9001:9001" # Web console volumes: - minio_data:/data healthcheck: @@ -38,19 +38,26 @@ services: # ── MinIO bucket init (runs once) ──────────────────────────── minio-init: - image: minio/mc:latest + image: minio/mc:RELEASE.2025-04-16T18-13-26Z depends_on: minio: condition: service_healthy entrypoint: > /bin/sh -c " - mc alias set mkb http://minio:9000 minioadmin minioadmin && - mc mb --ignore-existing mkb/raw && - mc mb --ignore-existing mkb/processed && - mc mb --ignore-existing mkb/archive && - mc mb --ignore-existing mkb/temp && + mc alias set mkb http://minio:9000 \"$$MINIO_ROOT_USER\" \"$$MINIO_ROOT_PASSWORD\" && + mc mb --ignore-existing mkb/$$MKB_S3_BUCKET_RAW && + mc mb --ignore-existing mkb/$$MKB_S3_BUCKET_PROCESSED && + mc mb --ignore-existing mkb/$$MKB_S3_BUCKET_ARCHIVE && + mc mb --ignore-existing mkb/$$MKB_S3_BUCKET_TEMP && echo 'Buckets ready.' " + environment: + MINIO_ROOT_USER: ${MKB_S3_ACCESS_KEY:?Set MKB_S3_ACCESS_KEY in .env} + MINIO_ROOT_PASSWORD: ${MKB_S3_SECRET_KEY:?Set MKB_S3_SECRET_KEY in .env} + MKB_S3_BUCKET_RAW: ${MKB_S3_BUCKET_RAW:-raw} + MKB_S3_BUCKET_PROCESSED: ${MKB_S3_BUCKET_PROCESSED:-processed} + MKB_S3_BUCKET_ARCHIVE: ${MKB_S3_BUCKET_ARCHIVE:-archive} + MKB_S3_BUCKET_TEMP: ${MKB_S3_BUCKET_TEMP:-temp} volumes: pg_data: diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..a5f096b --- /dev/null +++ b/docs/README.md @@ -0,0 +1,32 @@ +# Documentation + +Choose the guide for the work you are doing. + +## Users and API consumers + +- [Python API](python-api.md): supported facade, lifecycle, arguments, results, and examples +- [Generated Python API reference](api-reference.md): typed models and grouped methods +- [HTTP API contract](api-contract.md): routes, authentication, jobs, errors, and compatibility +- [Workflow lifecycle policy](workflow-lifecycle-policy.md): current and legacy workflow surfaces +- [Versioning and deprecation policy](deprecation-policy.md) +- [Changelog](../CHANGELOG.md) + +## Contributors + +- [Developer setup](development.md): clean-clone setup, commands, tests, and database provisioning +- [Architecture and ownership](architecture-map.md): boundaries and review ownership +- [Workflow card architecture](workflow-card-architecture.md) +- [Transaction boundaries](transactions.md) +- [Contributing](../CONTRIBUTING.md) + +## Operators and security reviewers + +- [Operator runbook](operator-runbook.md): startup, diagnostics, shutdown, and incidents +- [Backup and restore](backup-restore.md) +- [Upgrade and migration](upgrades.md) +- [Security model](security.md) +- [Security reporting policy](../SECURITY.md) +- [Supported versions](../SUPPORT.md) + +The React frontend under `frontend/` is the only bundled UI. Legacy +canonical-workflow compatibility paths remain only for retained records and callers. diff --git a/docs/api-contract.md b/docs/api-contract.md new file mode 100644 index 0000000..2e2726b --- /dev/null +++ b/docs/api-contract.md @@ -0,0 +1,32 @@ +# HTTP API contract + +FastAPI serves the current React client and external HTTP callers. When running, the +authoritative OpenAPI document is `/openapi.json` and interactive documentation is +`/docs`. Treat that generated schema as authoritative for request and response fields; +this guide defines cross-route behavior. + +All application data routes use the `/api` prefix. `/health`, `/health/live`, and +`/health/ready` are public operational probes. Readiness checks the database revision, +object-storage buckets, and worker availability. + +When authentication is enabled, send `Authorization: Bearer `. Roles are +reader (reads), editor (mutations/job starts), and admin (deletes, settings, executable +uploads). See the [security model](security.md). + +Long operations such as processing, extraction, projection, graph work, and review +return durable job records rather than holding the HTTP request open. Poll +`GET /api/jobs/{job_id}`, consume its status/progress/error fields, and treat only a +terminal successful state as completion. Cancellation is cooperative via +`POST /api/jobs/{job_id}/cancel`; a cancellation response does not imply the worker has +already stopped. + +Collections use JSON arrays or documented wrapper objects. Missing resources return +404. Validation failures use FastAPI's 422 response. Authentication/authorization use +401/403. Conflicts and unsafe state transitions may use 409. Dependency and internal +failures use categorized error details without credentials. Clients must tolerate +additive response fields and should not depend on error prose. + +Route families cover projects/assets, frames, spaces, projections, graph, feedback, +jobs, skills, settings, assistant, post-processors, workflow extraction/maintenance, +and project groups. The TypeScript modules under `frontend/src/api/` are useful current +examples, but OpenAPI is the external contract. React is the bundled client. diff --git a/docs/api-reference.md b/docs/api-reference.md new file mode 100644 index 0000000..7ab2d61 --- /dev/null +++ b/docs/api-reference.md @@ -0,0 +1,1114 @@ +# Generated Python API reference + + + +This reference lists the supported typed root models and grouped services. +See [Python API](python-api.md) for local, PostgreSQL/MinIO, custom-pipeline, +and safe migration examples. + +## Public typed models + +### `Artifact` + +Processed output derived from one source. + +Fields: + +- `id` — `` (required) +- `source_id` — `` (required) +- `processing_type` — `` (required) +- `format` — `` (required) +- `size` — `` (required) +- `sha256` — `` (required) +- `source_sha256` — `` (required) +- `storage` — `` (required) +- `primary_path` — `str | None` (optional/defaulted) +- `metadata` — `dict[str, typing.Any]` (optional/defaulted) +- `created_at` — `datetime.datetime | None` (optional/defaulted) +- `updated_at` — `datetime.datetime | None` (optional/defaulted) + +### `CacheKeyComponents` + +Required identity inputs for deterministic step caching. + +Fields: + +- `configuration` — `dict[str, typing.Any]` (optional/defaulted) +- `source_fingerprint` — `` (required) +- `model_identity` — `` (required) +- `schema_version` — `` (required) + +### `Collection` + +Logical grouping of sources, backed by a legacy project during migration. + +Fields: + +- `id` — `` (required) +- `name` — `str | None` (optional/defaulted) +- `source_path` — `str | None` (optional/defaulted) +- `source_count` — `` (optional/defaulted) +- `group_id` — `uuid.UUID | None` (optional/defaulted) +- `metadata` — `dict[str, typing.Any]` (optional/defaulted) +- `created_at` — `datetime.datetime | None` (optional/defaulted) +- `updated_at` — `datetime.datetime | None` (optional/defaulted) + +### `CollectionGroup` + +User-defined grouping for collections. + +Fields: + +- `id` — `` (required) +- `name` — `` (required) +- `description` — `str | None` (optional/defaulted) +- `color` — `str | None` (optional/defaulted) +- `display_order` — `` (optional/defaulted) +- `collection_count` — `` (optional/defaulted) +- `created_at` — `datetime.datetime | None` (optional/defaulted) +- `updated_at` — `datetime.datetime | None` (optional/defaulted) + +### `ExtractionSchema` + +Versioned definition for projecting records into a custom data shape. + +Fields: + +- `id` — `` (required) +- `name` — `` (required) +- `description` — `str | None` (optional/defaulted) +- `domain` — `` (required) +- `purpose` — `` (required) +- `definition` — `dict[str, typing.Any]` (optional/defaulted) +- `system_prompt` — `` (required) +- `field_descriptions` — `dict[str, typing.Any]` (optional/defaulted) +- `review_prompt` — `str | None` (optional/defaulted) +- `review_trackable` — `` (optional/defaulted) +- `review_allow_search` — `` (optional/defaulted) +- `review_search_tools` — `tuple[str, ...]` (optional/defaulted) +- `post_processors` — `tuple[typing.Any, ...]` (optional/defaulted) +- `version` — `` (optional/defaulted) +- `created_at` — `datetime.datetime | None` (optional/defaulted) +- `updated_at` — `datetime.datetime | None` (optional/defaulted) + +### `FeedbackItem` + +User or agent feedback attached to a record and collection. + +Fields: + +- `id` — `` (required) +- `target_record_id` — `` (required) +- `target_collection_id` — `` (required) +- `category` — `` (required) +- `question` — `` (required) +- `source_agent` — `` (optional/defaulted) +- `source_projection_id` — `uuid.UUID | None` (optional/defaulted) +- `field_path` — `str | None` (optional/defaulted) +- `context` — `str | None` (optional/defaulted) +- `status` — `` (optional/defaulted) +- `resolution_notes` — `str | None` (optional/defaulted) +- `resolved_by` — `str | None` (optional/defaulted) +- `resolved_at` — `datetime.datetime | None` (optional/defaulted) +- `created_at` — `datetime.datetime | None` (optional/defaulted) +- `updated_at` — `datetime.datetime | None` (optional/defaulted) + +### `GraphResult` + +Typed entity/relation result from graph query, traversal, or extraction. + +Fields: + +- `entities` — `tuple[mkb.models.Entity, ...]` (optional/defaulted) +- `relations` — `tuple[mkb.models.Relation, ...]` (optional/defaulted) +- `metadata` — `dict[str, typing.Any]` (optional/defaulted) + +### `GraphReview` + +Review decision for one graph entity or relation. + +Fields: + +- `id` — `` (optional/defaulted) +- `target_type` — `` (required) +- `target_id` — `uuid.UUID | None` (optional/defaulted) +- `decision` — `` (required) +- `notes` — `str | None` (optional/defaulted) +- `changes` — `dict[str, typing.Any]` (optional/defaulted) +- `metadata` — `dict[str, typing.Any]` (optional/defaulted) +- `created_at` — `` (required) + +### `Entity` + +Backend-neutral node in a knowledge graph. + +Fields: + +- `id` — `` (required) +- `type` — `` (required) +- `name` — `` (required) +- `properties` — `dict[str, typing.Any]` (optional/defaulted) +- `created_at` — `datetime.datetime | None` (optional/defaulted) +- `updated_at` — `datetime.datetime | None` (optional/defaulted) + +### `Evidence` + +Lossless provenance linking an SDK output to source material. + +Fields: + +- `id` — `` (required) +- `output_type` — `` (required) +- `output_id` — `` (required) +- `source_id` — `uuid.UUID | None` (optional/defaulted) +- `artifact_id` — `uuid.UUID | None` (optional/defaulted) +- `locator` — `dict[str, typing.Any]` (optional/defaulted) +- `excerpt` — `str | None` (optional/defaulted) +- `metadata` — `dict[str, typing.Any]` (optional/defaulted) +- `created_at` — `datetime.datetime | None` (optional/defaulted) + +### `EffectiveSettings` + +Secret-free effective topology for one configured SDK client. + +Fields: + +- `database_backend` — `str | None` (optional/defaulted) +- `object_store_endpoint` — `str | None` (optional/defaulted) +- `buckets` — `dict[str, str]` (optional/defaulted) +- `capabilities` — `tuple[str, ...]` (optional/defaulted) + +### `Job` + +Serializable state for a submitted background operation. + +Fields: + +- `id` — `` (required) +- `kind` — `` (required) +- `status` — `` (required) +- `label` — `str | None` (optional/defaulted) +- `project_id` — `str | None` (optional/defaulted) +- `request_id` — `str | None` (optional/defaulted) +- `active_key` — `str | None` (optional/defaulted) +- `pipeline_name` — `str | None` (optional/defaulted) +- `pipeline_version` — `str | None` (optional/defaulted) +- `run_id` — `uuid.UUID | None` (optional/defaulted) +- `idempotency_key` — `str | None` (optional/defaulted) +- `inputs` — `dict[str, typing.Any]` (optional/defaulted) +- `parameters` — `dict[str, typing.Any]` (optional/defaulted) +- `checkpoint` — `dict[str, typing.Any]` (optional/defaulted) +- `events` — `tuple[dict[str, typing.Any], ...]` (optional/defaulted) +- `attempt_count` — `` (optional/defaulted) +- `max_attempts` — `` (optional/defaulted) +- `retryable` — `` (optional/defaulted) +- `cancel_requested` — `` (optional/defaulted) +- `progress` — `float | None` (optional/defaulted) +- `message` — `str | None` (optional/defaulted) +- `result` — `typing.Any` (optional/defaulted) +- `error` — `str | None` (optional/defaulted) +- `error_category` — `str | None` (optional/defaulted) +- `created_at` — `` (required) +- `queued_at` — `datetime.datetime | None` (optional/defaulted) +- `started_at` — `datetime.datetime | None` (optional/defaulted) +- `completed_at` — `datetime.datetime | None` (optional/defaulted) +- `updated_at` — `datetime.datetime | None` (optional/defaulted) + +### `MaintenanceReport` + +Typed result from a read-only maintenance inspection or plan. + +Fields: + +- `kind` — `` (required) +- `ok` — `` (required) +- `data` — `dict[str, typing.Any]` (optional/defaulted) +- `created_at` — `` (required) + +### `OperationReceipt` + +Typed acknowledgement for an accepted or completed mutation. + +Fields: + +- `id` — `` (optional/defaulted) +- `operation` — `` (required) +- `status` — `` (required) +- `resource_type` — `str | None` (optional/defaulted) +- `resource_id` — `uuid.UUID | None` (optional/defaulted) +- `details` — `dict[str, typing.Any]` (optional/defaulted) +- `created_at` — `` (required) + +### `PipelineRun` + +Serializable result and provenance for one local pipeline execution. + +Fields: + +- `id` — `` (optional/defaulted) +- `pipeline_name` — `` (required) +- `pipeline_version` — `` (required) +- `status` — `` (required) +- `inputs` — `dict[str, typing.Any]` (optional/defaulted) +- `parameters` — `dict[str, typing.Any]` (optional/defaulted) +- `outputs` — `dict[str, typing.Any]` (optional/defaulted) +- `steps` — `tuple[mkb.pipelines.StepRun, ...]` (optional/defaulted) +- `error` — `str | None` (optional/defaulted) +- `started_at` — `` (required) +- `completed_at` — `` (required) + +### `Page` + +Bounded result page with stable pagination metadata. + +Fields: + +- `items` — `tuple[~PageItem, ...]` (optional/defaulted) +- `limit` — `` (required) +- `offset` — `` (required) +- `total` — `int | None` (optional/defaulted) +- `next_offset` — `int | None` (optional/defaulted) + +### `Projection` + +Raw, lossless projection of a record through an extraction schema. + +Fields: + +- `id` — `` (required) +- `schema_id` — `` (required) +- `record_id` — `` (required) +- `collection_id` — `` (required) +- `source_type` — `` (required) +- `status` — `` (required) +- `data` — `typing.Any` (optional/defaulted) +- `validation` — `dict[str, typing.Any] | None` (optional/defaulted) +- `notes` — `str | None` (optional/defaulted) +- `extracted_at` — `datetime.datetime | None` (optional/defaulted) +- `schema_version` — `` (required) +- `review_count` — `` (optional/defaulted) +- `review_notes` — `str | None` (optional/defaulted) +- `reviewed_at` — `datetime.datetime | None` (optional/defaulted) +- `deleted_at` — `datetime.datetime | None` (optional/defaulted) +- `superseded_by_id` — `uuid.UUID | None` (optional/defaulted) +- `supersedes_ids` — `tuple[str, ...]` (optional/defaulted) +- `created_at` — `datetime.datetime | None` (optional/defaulted) +- `updated_at` — `datetime.datetime | None` (optional/defaulted) + +### `PostProcessor` + +Registered deterministic post-processing program metadata. + +Fields: + +- `id` — `` (required) +- `name` — `` (required) +- `filename` — `` (required) +- `source` — `` (required) +- `metadata` — `dict[str, typing.Any]` (optional/defaulted) +- `created_at` — `datetime.datetime | None` (optional/defaulted) +- `updated_at` — `datetime.datetime | None` (optional/defaulted) + +### `ProgressEvent` + +Structured notification emitted during a synchronous pipeline run. + +Fields: + +- `run_id` — `` (required) +- `event` — `` (required) +- `step_name` — `str | None` (optional/defaulted) +- `attempt` — `int | None` (optional/defaulted) +- `message` — `str | None` (optional/defaulted) +- `level` — `str | None` (optional/defaulted) +- `payload` — `dict[str, typing.Any]` (optional/defaulted) +- `occurred_at` — `` (optional/defaulted) + +### `Record` + +Extracted knowledge record backed by an existing knowledge frame. + +Fields: + +- `id` — `` (required) +- `collection_id` — `` (required) +- `status` — `` (required) +- `data` — `typing.Any` (optional/defaulted) +- `summary` — `str | None` (optional/defaulted) +- `review_count` — `` (optional/defaulted) +- `version` — `` (optional/defaulted) +- `extracted_at` — `datetime.datetime | None` (optional/defaulted) +- `source_metadata` — `dict[str, typing.Any]` (optional/defaulted) +- `annotations` — `dict[str, typing.Any]` (optional/defaulted) +- `created_at` — `datetime.datetime | None` (optional/defaulted) +- `updated_at` — `datetime.datetime | None` (optional/defaulted) + +### `Relation` + +Directed, typed edge between two graph entities. + +Fields: + +- `id` — `` (required) +- `source_id` — `` (required) +- `target_id` — `` (required) +- `type` — `` (required) +- `properties` — `dict[str, typing.Any]` (optional/defaulted) +- `created_at` — `datetime.datetime | None` (optional/defaulted) +- `updated_at` — `datetime.datetime | None` (optional/defaulted) + +### `Source` + +Original input registered in one or more collections. + +Fields: + +- `id` — `` (required) +- `filename` — `` (required) +- `media_type` — `` (required) +- `size` — `` (required) +- `sha256` — `` (required) +- `status` — `` (required) +- `storage` — `mkb.models.StorageReference | None` (optional/defaulted) +- `uri` — `str | None` (optional/defaulted) +- `collection_ids` — `tuple[uuid.UUID, ...]` (optional/defaulted) +- `metadata` — `dict[str, typing.Any]` (optional/defaulted) +- `created_at` — `datetime.datetime | None` (optional/defaulted) +- `updated_at` — `datetime.datetime | None` (optional/defaulted) + +### `Skill` + +Reusable instruction document registered with one client. + +Fields: + +- `id` — `` (required) +- `name` — `` (required) +- `slug` — `` (required) +- `content` — `` (required) +- `description` — `str | None` (optional/defaulted) +- `metadata` — `dict[str, typing.Any]` (optional/defaulted) +- `created_at` — `datetime.datetime | None` (optional/defaulted) +- `updated_at` — `datetime.datetime | None` (optional/defaulted) + +### `StepRun` + +Serializable execution record for one step. + +Fields: + +- `id` — `` (optional/defaulted) +- `name` — `` (required) +- `version` — `` (required) +- `status` — `` (required) +- `attempts` — `` (required) +- `deterministic` — `` (optional/defaulted) +- `cacheable` — `` (optional/defaulted) +- `cached` — `` (optional/defaulted) +- `cache_key` — `str | None` (optional/defaulted) +- `required_capabilities` — `tuple[str, ...]` (optional/defaulted) +- `side_effects` — `tuple[str, ...]` (optional/defaulted) +- `timeout_seconds` — `float | None` (optional/defaulted) +- `output` — `dict[str, typing.Any]` (optional/defaulted) +- `error` — `str | None` (optional/defaulted) +- `started_at` — `` (required) +- `completed_at` — `` (required) + +### `StorageReference` + +Location of content in a configured object-store adapter. + +Fields: + +- `bucket` — `` (required) +- `key` — `` (required) + +### `WorkflowRecord` + +Lossless materials workflow extraction mapped from a legacy row. + +Fields: + +- `id` — `` (required) +- `collection_id` — `` (required) +- `version` — `` (required) +- `schema_version` — `` (required) +- `extractor_version` — `` (required) +- `model` — `str | None` (optional/defaulted) +- `status` — `` (required) +- `record_status` — `` (required) +- `supersedes_id` — `uuid.UUID | None` (optional/defaulted) +- `correction_reason` — `str | None` (optional/defaulted) +- `correction_author` — `str | None` (optional/defaulted) +- `correction_details` — `dict[str, typing.Any]` (optional/defaulted) +- `review_flags` — `tuple[typing.Any, ...]` (optional/defaulted) +- `graph` — `dict[str, typing.Any] | None` (optional/defaulted) +- `checkpoint` — `dict[str, typing.Any] | None` (optional/defaulted) +- `provenance` — `dict[str, typing.Any]` (optional/defaulted) +- `error` — `str | None` (optional/defaulted) +- `extracted_at` — `datetime.datetime | None` (optional/defaulted) +- `checkpoint_updated_at` — `datetime.datetime | None` (optional/defaulted) +- `created_at` — `datetime.datetime | None` (optional/defaulted) + +## Configured client + +### `KnowledgeBase` + +The portable client entry point. See [Python API](python-api.md) for installation, supported adapter injection, and lifecycle guidance. + +#### `from_environment() -> "'KnowledgeBase'"` + +Create a client for the configured materials application. + +#### `from_url(*, database_url: 'str', object_store_url: 'str | None' = None, object_store_access_key: 'str | None' = None, object_store_secret_key: 'str | None' = None, raw_bucket: 'str | None' = None, processed_bucket: 'str' = 'processed', archive_bucket: 'str' = 'archive', temp_bucket: 'str' = 'temp', capabilities: 'frozenset[str] | None' = None, graph_store: 'GraphStore | None' = None, model_provider: 'ModelProvider | None' = None, job_backend: 'JobBackend | None' = None, vector_search: 'VectorSearch | None' = None, parser_registry_factory: "Callable[['KnowledgeBase'], Parsers] | None" = None, pipeline_registry_factory: "Callable[['KnowledgeBase', frozenset[str]], Pipelines] | None" = None, steps: 'Steps | None' = None) -> "'KnowledgeBase'"` + +Create an independent client without reading global environment settings. + +#### `initialize() -> 'int'` + +Explicitly create missing SDK-owned tables without dropping existing data. + +#### `schema_version() -> 'int | None'` + +Return the initialized portable schema version, or ``None`` if absent. + +#### `transaction() -> 'Iterator[Transaction]'` + +Open one relational transaction for SDK-managed repositories. + +#### `close() -> 'None'` + +Close client-owned resources and submitted pipeline workers. + +### `MKBConfig` + +Immutable client configuration. + +Fields: + +- `database_url` — `str | None` (optional/defaulted) +- `object_store_endpoint` — `str | None` (optional/defaulted) +- `object_store_access_key` — `str | None` (optional/defaulted) +- `object_store_secret_key` — `str | None` (optional/defaulted) +- `raw_bucket` — `str` (optional/defaulted) +- `processed_bucket` — `str` (optional/defaulted) +- `archive_bucket` — `str` (optional/defaulted) +- `temp_bucket` — `str` (optional/defaulted) +- `allow_uploaded_python` — `bool` (optional/defaulted) +- `upload_max_file_mb` — `int` (optional/defaulted) +- `api_host` — `str` (optional/defaulted) +- `api_port` — `int` (optional/defaulted) + +## Grouped services + +### `kb.collections` + +Typed collection operations bound to one repository instance. + +#### `assign_group(collection_ids: 'list[str | uuid.UUID]', group_id: 'str | uuid.UUID | None') -> 'OperationReceipt'` + + + +#### `create(*, name: 'str', source_path: 'str | None' = None, metadata: 'dict[str, Any] | None' = None, collection_id: 'str | uuid.UUID | None' = None) -> 'Collection'` + + + +#### `delete(collection_id: 'str | uuid.UUID') -> 'OperationReceipt'` + + + +#### `get(collection_id: 'str | uuid.UUID') -> 'Collection | None'` + + + +#### `list(*, limit: 'int' = 100, offset: 'int' = 0) -> 'list[Collection]'` + + + +#### `require(collection_id: 'str | uuid.UUID') -> 'Collection'` + + + +#### `update(collection_id: 'str | uuid.UUID', *, name: 'str | None' = None, source_path: 'str | None' = None, metadata: 'dict[str, Any] | None' = None) -> 'Collection'` + + + +### `kb.collections.groups` + +Typed lifecycle for collection groups. + +#### `assign(collection_ids: 'list[uuid.UUID]', group_id: 'uuid.UUID | None') -> 'int'` + + + +#### `create(*, name: 'str', description: 'str | None' = None, color: 'str | None' = None, display_order: 'int' = 0, group_id: 'str | uuid.UUID | None' = None) -> 'CollectionGroup'` + + + +#### `delete(group_id: 'str | uuid.UUID') -> 'OperationReceipt'` + + + +#### `get(group_id: 'str | uuid.UUID') -> 'CollectionGroup | None'` + + + +#### `list(*, limit: 'int' = 100, offset: 'int' = 0) -> 'list[CollectionGroup]'` + + + +#### `require(group_id: 'str | uuid.UUID') -> 'CollectionGroup'` + + + +#### `update(group_id: 'str | uuid.UUID', *, name: 'str | None' = None, description: 'str | None' = None, color: 'str | None' = None, display_order: 'int | None' = None) -> 'CollectionGroup'` + + + +### `kb.sources` + +Typed source metadata and content access for one SDK instance. + +#### `add_bytes(collection_id: 'str | uuid.UUID', data: 'bytes', *, filename: 'str', media_type: 'str' = 'application/octet-stream', metadata: 'dict[str, Any] | None' = None, source_id: 'str | uuid.UUID | None' = None) -> 'Source'` + + + +#### `add_directory(collection_id: 'str | uuid.UUID', path: 'str | Path', *, recursive: 'bool' = True) -> 'list[Source]'` + + + +#### `add_file(collection_id: 'str | uuid.UUID', path: 'str | Path', *, media_type: 'str | None' = None, metadata: 'dict[str, Any] | None' = None, source_id: 'str | uuid.UUID | None' = None) -> 'Source'` + + + +#### `add_records(collection_id: 'str | uuid.UUID', records: 'list[dict[str, Any]]', *, filename: 'str' = 'records.json', metadata: 'dict[str, Any] | None' = None, source_id: 'str | uuid.UUID | None' = None) -> 'Source'` + + + +#### `add_text(collection_id: 'str | uuid.UUID', text: 'str', *, filename: 'str' = 'text.txt', media_type: 'str' = 'text/plain; charset=utf-8', metadata: 'dict[str, Any] | None' = None, source_id: 'str | uuid.UUID | None' = None) -> 'Source'` + + + +#### `add_uri(collection_id: 'str | uuid.UUID', uri: 'str', *, filename: 'str | None' = None, media_type: 'str' = 'application/octet-stream', metadata: 'dict[str, Any] | None' = None, source_id: 'str | uuid.UUID | None' = None) -> 'Source'` + + + +#### `content_exists(source_id: 'str | uuid.UUID') -> 'bool'` + + + +#### `get(source_id: 'str | uuid.UUID') -> 'Source | None'` + + + +#### `list(*, collection_id: 'str | uuid.UUID | None' = None, limit: 'int' = 100, offset: 'int' = 0) -> 'list[Source]'` + + + +#### `open(source_id: 'str | uuid.UUID') -> 'BinaryIO'` + + + +#### `read_bytes(source_id: 'str | uuid.UUID') -> 'bytes'` + + + +#### `require(source_id: 'str | uuid.UUID') -> 'Source'` + + + +### `kb.artifacts` + +Typed processed-artifact metadata and content access. + +#### `add_bytes(source_id: 'str | uuid.UUID', data: 'bytes', *, processing_type: 'str', format: 'str', primary_path: 'str | None' = None, metadata: 'dict[str, Any] | None' = None, artifact_id: 'str | uuid.UUID | None' = None) -> 'Artifact'` + + + +#### `content_exists(artifact_id: 'str | uuid.UUID') -> 'bool'` + + + +#### `get(artifact_id: 'str | uuid.UUID') -> 'Artifact | None'` + + + +#### `list(*, source_id: 'str | uuid.UUID | None' = None, limit: 'int' = 100, offset: 'int' = 0) -> 'list[Artifact]'` + + + +#### `open(artifact_id: 'str | uuid.UUID') -> 'BinaryIO'` + + + +#### `read_bytes(artifact_id: 'str | uuid.UUID') -> 'bytes'` + + + +#### `register(source_id: 'str | uuid.UUID', *, bucket: 'str', key: 'str', processing_type: 'str', format: 'str', size: 'int', sha256: 'str', source_sha256: 'str | None' = None, primary_path: 'str | None' = None, metadata: 'dict[str, Any] | None' = None, artifact_id: 'str | uuid.UUID | None' = None, verify_content: 'bool' = True) -> 'Artifact'` + + + +#### `require(artifact_id: 'str | uuid.UUID') -> 'Artifact'` + + + +### `kb.records` + +Typed access to extracted records without changing legacy frame data. + +#### `create(*, collection_id: 'str | uuid.UUID', data: 'Any', status: 'str' = 'COMPLETED', summary: 'str | None' = None, source_metadata: 'dict[str, Any] | None' = None, annotations: 'dict[str, Any] | None' = None, record_id: 'str | uuid.UUID | None' = None) -> 'Record'` + + + +#### `evidence(record_id: 'str | uuid.UUID') -> 'list[Evidence]'` + +Return provenance attached to one record without exposing persistence. + +#### `export_json(*, collection_id: 'str | uuid.UUID | None' = None, status: 'str | None' = None, limit: 'int' = 1000, offset: 'int' = 0, indent: 'int | None' = 2) -> 'str'` + +Serialize a bounded record query without writing to the filesystem. + +#### `get(record_id: 'str | uuid.UUID') -> 'Record | None'` + + + +#### `get_for_collection(collection_id: 'str | uuid.UUID') -> 'Record | None'` + + + +#### `list(*, collection_id: 'str | uuid.UUID | None' = None, status: 'str | None' = None, limit: 'int' = 100, offset: 'int' = 0) -> 'list[Record]'` + + + +#### `query(*, collection_id: 'str | uuid.UUID | None' = None, status: 'str | None' = None, filters: 'dict[str, Any] | None' = None, limit: 'int' = 100, offset: 'int' = 0) -> 'list[Record]'` + +Query records using exact top-level data-field matches. + +#### `require(record_id: 'str | uuid.UUID') -> 'Record'` + + + +### `kb.schemas` + +Typed access to versioned extraction-space definitions. + +#### `create(*, name: 'str', domain: 'str', definition: 'dict[str, Any]', system_prompt: 'str', description: 'str | None' = None, purpose: 'str' = 'freeform', field_descriptions: 'dict[str, Any] | None' = None, schema_id: 'str | uuid.UUID | None' = None) -> 'ExtractionSchema'` + + + +#### `delete(schema_id_or_name: 'str | uuid.UUID') -> 'OperationReceipt'` + + + +#### `get(schema_id_or_name: 'str | uuid.UUID') -> 'ExtractionSchema | None'` + + + +#### `get_version(schema_id_or_name: 'str | uuid.UUID', version: 'int') -> 'ExtractionSchema | None'` + +Return one immutable schema revision by stable identity and version. + +#### `history(schema_id_or_name: 'str | uuid.UUID', *, limit: 'int' = 100, offset: 'int' = 0) -> 'list[ExtractionSchema]'` + +List immutable revisions, newest first. + +#### `list(*, limit: 'int' = 100, offset: 'int' = 0) -> 'list[ExtractionSchema]'` + + + +#### `register(*, name: 'str', domain: 'str', definition: 'dict[str, Any]', system_prompt: 'str', description: 'str | None' = None, purpose: 'str' = 'freeform', field_descriptions: 'dict[str, Any] | None' = None, schema_id: 'str | uuid.UUID | None' = None) -> 'ExtractionSchema'` + +Register and persist a consumer schema for this client. + +#### `require(schema_id_or_name: 'str | uuid.UUID') -> 'ExtractionSchema'` + + + +#### `require_version(schema_id_or_name: 'str | uuid.UUID', version: 'int') -> 'ExtractionSchema'` + + + +#### `update(schema_id_or_name: 'str | uuid.UUID', *, name: 'str | None' = None, description: 'str | None' = None, domain: 'str | None' = None, definition: 'dict[str, Any] | None' = None, system_prompt: 'str | None' = None, purpose: 'str | None' = None, field_descriptions: 'dict[str, Any] | None' = None) -> 'ExtractionSchema'` + + + +### `kb.projections` + +Typed projection queries that return stored JSON without normalization. + +#### `create(*, schema_id: 'str | uuid.UUID', record_id: 'str | uuid.UUID', data: 'Any', status: 'str' = 'COMPLETED', source_type: 'str' = 'record', validation: 'dict[str, Any] | None' = None, notes: 'str | None' = None, projection_id: 'str | uuid.UUID | None' = None) -> 'Projection'` + + + +#### `export_json(*, schema_id: 'str | uuid.UUID | None' = None, record_id: 'str | uuid.UUID | None' = None, collection_id: 'str | uuid.UUID | None' = None, status: 'str | None' = None, include_deleted: 'bool' = False, include_history: 'bool' = False, newest_only: 'bool' = False, limit: 'int' = 1000, offset: 'int' = 0, indent: 'int | None' = 2) -> 'str'` + +Serialize a bounded projection query without altering stored payloads. + +#### `get(projection_id: 'str | uuid.UUID') -> 'Projection | None'` + + + +#### `list(*, schema_id: 'str | uuid.UUID | None' = None, record_id: 'str | uuid.UUID | None' = None, collection_id: 'str | uuid.UUID | None' = None, status: 'str | None' = None, include_deleted: 'bool' = False, include_history: 'bool' = False, newest_only: 'bool' = False, limit: 'int' = 100, offset: 'int' = 0) -> 'list[Projection]'` + + + +#### `require(projection_id: 'str | uuid.UUID') -> 'Projection'` + + + +### `kb.evidence` + +Typed provenance links between outputs and source material. + +#### `create(*, output_type: 'str', output_id: 'str | uuid.UUID', source_id: 'str | uuid.UUID | None' = None, artifact_id: 'str | uuid.UUID | None' = None, locator: 'dict[str, Any] | None' = None, excerpt: 'str | None' = None, metadata: 'dict[str, Any] | None' = None, evidence_id: 'str | uuid.UUID | None' = None) -> 'Evidence'` + + + +#### `get(evidence_id: 'str | uuid.UUID') -> 'Evidence | None'` + + + +#### `list(*, output_id: 'str | uuid.UUID | None' = None, source_id: 'str | uuid.UUID | None' = None, artifact_id: 'str | uuid.UUID | None' = None, limit: 'int' = 100, offset: 'int' = 0) -> 'list[Evidence]'` + + + +#### `require(evidence_id: 'str | uuid.UUID') -> 'Evidence'` + + + +### `kb.graph` + +Backend-independent entity, relation, and traversal operations. + +#### `extract(content: 'Any' = None, *, extractor: 'Callable[[Any], dict[str, Any]] | None' = None, parameters: 'dict[str, Any] | None' = None, collection_id: 'str | uuid.UUID | None' = None, record_id: 'str | uuid.UUID | None' = None) -> 'GraphResult'` + +Extract and upsert a graph using a consumer handler or model provider. + +#### `get_entity(entity_id: 'str | uuid.UUID') -> 'Entity | None'` + + + +#### `get_relation(relation_id: 'str | uuid.UUID') -> 'Relation | None'` + + + +#### `list_entities(*, type: 'str | None' = None) -> 'list[Entity]'` + + + +#### `list_relations(*, entity_id: 'str | uuid.UUID | None' = None, type: 'str | None' = None) -> 'list[Relation]'` + + + +#### `list_reviews(*, target_id: 'str | uuid.UUID | None' = None) -> 'list[GraphReview]'` + + + +#### `neighbors(entity_id: 'str | uuid.UUID') -> 'list[Entity]'` + + + +#### `query(*, entity_type: 'str | None' = None, relation_type: 'str | None' = None, name_contains: 'str | None' = None, properties: 'dict[str, Any] | None' = None) -> 'GraphResult'` + +Query graph elements using portable exact property filters. + +#### `require_entity(entity_id: 'str | uuid.UUID') -> 'Entity'` + + + +#### `review(target_id: 'str | uuid.UUID | None' = None, *, reviewer: 'Callable[[Entity | Relation], dict[str, Any]] | None' = None, notes: 'str | None' = None, parameters: 'dict[str, Any] | None' = None) -> 'GraphReview'` + +Review one graph element and apply explicit, typed modifications. + +#### `traverse(entity_id: 'str | uuid.UUID', *, max_depth: 'int' = 1, direction: 'str' = 'both') -> 'GraphResult'` + +Breadth-first traversal with a bounded portable depth. + +#### `upsert_entity(*, type: 'str', name: 'str', properties: 'dict[str, Any] | None' = None, entity_id: 'str | uuid.UUID | None' = None) -> 'Entity'` + + + +#### `upsert_relation(source_id: 'str | uuid.UUID', target_id: 'str | uuid.UUID', *, type: 'str', properties: 'dict[str, Any] | None' = None, relation_id: 'str | uuid.UUID | None' = None) -> 'Relation'` + + + +### `kb.pipelines` + +Per-client pipeline registry and synchronous/local-background executor. + +#### `get(name: 'str') -> 'Pipeline | None'` + + + +#### `get_run(run_id: 'str | uuid.UUID') -> 'PipelineRun | None'` + + + +#### `list() -> 'list[Pipeline]'` + + + +#### `register(pipeline: 'Pipeline', *, replace: 'bool' = False) -> 'Pipeline'` + + + +#### `require(name: 'str') -> 'Pipeline'` + + + +#### `resume(job_id: 'str | uuid.UUID') -> 'Job'` + +Resume a failed, cancelled, or interrupted job from its last checkpoint. + +#### `run(pipeline: 'Pipeline | str', *, inputs: 'Mapping[str, Any] | None' = None, parameters: 'Mapping[str, Any] | None' = None, progress: 'ProgressHandler | None' = None) -> 'PipelineRun'` + + + +#### `submit(pipeline: 'Pipeline | str', *, inputs: 'Mapping[str, Any] | None' = None, parameters: 'Mapping[str, Any] | None' = None, idempotency_key: 'str | None' = None) -> 'Job'` + +Persist and asynchronously execute one registered pipeline definition. + +### `kb.jobs` + +Query, wait for, and cooperatively cancel durable jobs. + +#### `available() -> 'bool'` + +Return whether this client has a configured durable-job backend. + +#### `cancel(job_id: 'str | uuid.UUID') -> 'Job'` + + + +#### `cancel_all(*, project_id: 'str | None' = None) -> 'list[Job]'` + + + +#### `events(job_id: 'str | uuid.UUID', *, after: 'int' = 0, follow: 'bool' = False, timeout: 'float | None' = None, poll_interval: 'float' = 0.05) -> 'Iterator[dict[str, Any]]'` + +Yield persisted events, optionally following until the job is terminal. + +#### `find_active(*, project_id: 'str | None' = None, kind: 'str | None' = None) -> 'Job | None'` + + + +#### `get(job_id: 'str | uuid.UUID') -> 'Job | None'` + + + +#### `list(*, project_id: 'str | None' = None, limit: 'int' = 100, offset: 'int' = 0) -> 'list[Job]'` + + + +#### `recover_interrupted() -> 'int'` + +Explicitly release jobs abandoned by a stopped worker process. + +#### `require(job_id: 'str | uuid.UUID') -> 'Job'` + + + +#### `submit(*, kind: 'str', inputs: 'dict[str, Any] | None' = None, parameters: 'dict[str, Any] | None' = None, label: 'str | None' = None, idempotency_key: 'str | None' = None, job_id: 'str | uuid.UUID | None' = None) -> 'Job'` + +Persist a queued application job for an external or custom worker. + +#### `submit_action(action: 'str', **kwargs: 'Any') -> 'Job'` + +Start an application-owned action through the configured job adapter. + +#### `wait(job_id: 'str | uuid.UUID', *, timeout: 'float | None' = None) -> 'Job'` + + + +### `kb.feedback` + +Create, inspect, review, and resolve feedback items. + +#### `create(*, target_record_id: 'str | uuid.UUID', target_collection_id: 'str | uuid.UUID', category: 'str', question: 'str', source_agent: 'str' = 'user', source_projection_id: 'str | uuid.UUID | None' = None, field_path: 'str | None' = None, context: 'str | None' = None, feedback_id: 'str | uuid.UUID | None' = None) -> 'FeedbackItem'` + + + +#### `get(feedback_id: 'str | uuid.UUID') -> 'FeedbackItem | None'` + + + +#### `list(*, collection_id: 'str | uuid.UUID | None' = None, status: 'str | None' = None, limit: 'int' = 100, offset: 'int' = 0) -> 'list[FeedbackItem]'` + + + +#### `require(feedback_id: 'str | uuid.UUID') -> 'FeedbackItem'` + + + +#### `resolve(feedback_id: 'str | uuid.UUID', *, notes: 'str', status: 'str' = 'RESOLVED', resolved_by: 'str' = 'user') -> 'FeedbackItem'` + + + +#### `review(feedback_id: 'str | uuid.UUID') -> 'FeedbackItem'` + + + +### `kb.skills` + +Register instruction documents without relying on a module-global registry. + +#### `create(*, name: 'str', content: 'str', slug: 'str | None' = None, description: 'str | None' = None, metadata: 'dict[str, Any] | None' = None, skill_id: 'str | uuid.UUID | None' = None) -> 'Skill'` + + + +#### `delete(skill_id_or_slug: 'str | uuid.UUID') -> 'OperationReceipt'` + + + +#### `get(skill_id_or_slug: 'str | uuid.UUID') -> 'Skill | None'` + + + +#### `import_files(files: 'list[tuple[str, BinaryIO]]') -> 'Skill'` + +Import a SKILL.md file, folder upload, or zip through the bound adapter. + +#### `list(*, limit: 'int' = 100, offset: 'int' = 0) -> 'list[Skill]'` + + + +#### `require(skill_id_or_slug: 'str | uuid.UUID') -> 'Skill'` + + + +### `kb.post_processors` + +Register deterministic processor source; execution remains policy-controlled. + +#### `delete(processor_id: 'str | uuid.UUID') -> 'OperationReceipt'` + + + +#### `get(processor_id: 'str | uuid.UUID') -> 'PostProcessor | None'` + + + +#### `list(*, limit: 'int' = 100, offset: 'int' = 0) -> 'list[PostProcessor]'` + + + +#### `register(*, name: 'str', source: 'str', filename: 'str | None' = None, metadata: 'dict[str, Any] | None' = None, processor_id: 'str | uuid.UUID | None' = None) -> 'PostProcessor'` + + + +#### `require(processor_id: 'str | uuid.UUID') -> 'PostProcessor'` + + + +### `kb.parsers` + +Parser registry owned by exactly one ``KnowledgeBase`` instance. + +#### `for_source_type(source_type: 'str') -> 'list[Parser]'` + + + +#### `get(name: 'str') -> 'Parser | None'` + + + +#### `list() -> 'list[Parser]'` + + + +#### `parse(parser: 'Parser | str', content: 'bytes', *, source_type: 'str', parameters: 'Mapping[str, Any] | None' = None) -> 'Any'` + + + +#### `register(parser: 'Parser', *, replace: 'bool' = False) -> 'Parser'` + + + +#### `register_adapter(adapter: 'ContentParser', *, version: 'str' = '1', replace: 'bool' = False) -> 'Parser'` + +Register a parser port without coupling it to SDK context types. + +#### `require(name: 'str') -> 'Parser'` + + + +### `kb.steps` + +Per-client registry for reusable standalone pipeline steps. + +#### `get(name: 'str') -> 'Step | None'` + + + +#### `list() -> 'list[Step]'` + + + +#### `register(step: 'Step', *, replace: 'bool' = False) -> 'Step'` + + + +#### `require(name: 'str') -> 'Step'` + + + +### `kb.settings` + +Inspect effective non-secret configuration. + +#### `inspect() -> 'EffectiveSettings'` + + + +#### `runtime() -> 'dict[str, Any]'` + +Return mutable application settings with secrets already masked. + +#### `update(updates: 'dict[str, Any]') -> 'dict[str, Any]'` + +Validate and persist supported runtime-setting overrides. + +#### `validate_startup(*, host: 'str | None' = None, log_level: 'str | None' = None) -> 'list[str]'` + +Return deployment-safety warnings from the configured application. + +### `kb.maintenance` + +Read-only inventory, reconciliation, and cleanup planning. + +#### `backup_metadata() -> 'MaintenanceReport'` + + + +#### `cleanup(*, older_than_days: 'int' = 7, job_days: 'int' = 30, apply: 'bool' = False, confirm: 'str | None' = None) -> 'MaintenanceReport'` + +Plan or apply retention through an explicitly bound application adapter. + +#### `cleanup_plan(*, older_than_days: 'int' = 7, roots: 'list[str | Path] | None' = None) -> 'MaintenanceReport'` + + + +#### `compare_inventories(before: 'dict[str, Any] | str | Path', after: 'dict[str, Any] | str | Path') -> 'MaintenanceReport'` + +Compare two preservation inventories without reading live backends. + +#### `inventory() -> 'MaintenanceReport'` + + + +#### `migration_inventory(*, include_object_checksums: 'bool' = False) -> 'MaintenanceReport'` + +Return the application-wide preservation inventory when configured. + +#### `reconcile() -> 'MaintenanceReport'` + + + +#### `restore_missing_artifact(artifact_id: 'str', local_file: 'str | Path', *, apply: 'bool' = False, confirm: 'str | None' = None) -> 'MaintenanceReport'` + +Checksum-gate restoration of one missing artifact object. + +#### `verify_content(*, sample_size: 'int' = 10) -> 'MaintenanceReport'` + +Verify every storage reference and checksum deterministic content samples. diff --git a/docs/architecture-map.md b/docs/architecture-map.md new file mode 100644 index 0000000..360c550 --- /dev/null +++ b/docs/architecture-map.md @@ -0,0 +1,60 @@ +# Architecture and ownership + +This map names the main ownership areas in the repository so refactors can move +code toward clearer boundaries without changing behavior accidentally. + +## Runtime Surfaces + +- `src/mkb/api.py` is the Python compatibility facade used by scripts, tests, + routers, and legacy UI code. +- `src/mkb/web/` owns the FastAPI app, REST routers, request/response models, + upload handling, and background job state. +- `frontend/src/` owns the React application. +- `src/mkb/agents/` owns agent construction, prompts, tool adapters, and runner + integration. +- `src/mkb/cli.py` owns command-line entry points and should call service/API + functions rather than duplicating behavior. + +## Domain Areas + +- Ingestion: `src/mkb/ingest/`, `src/mkb/api.py`, and upload entry points under + `src/mkb/web/`. +- Processing: `src/mkb/processors/`, processed asset models, and S3 helpers. +- Projects and assets: `ResearchProject`, `Asset`, and `ProjectAsset` models, + plus project routers and frontend project views. +- Frames: `KnowledgeFrame` storage, frame agent code, frame routers, and React + frame/project detail tabs. +- Spaces and projections: `src/mkb/spaces/`, projection agents/tools, projection + routers, and projection table components. +- Workflows: `src/mkb/services/workflows/`, workflow extraction/canonicalization agents, + schema curator tools, and workflow tabs. +- Knowledge graph: `src/mkb/knowledge_graph.py`, graph agent/tools, graph review + tools, graph router, and graph frontend page. +- Feedback: `src/mkb/feedback/`, feedback agent/tools, feedback router, and + frontend feedback page. +- Jobs: `src/mkb/web/_state.py`, job routers, and job polling hooks/stores. + +## Boundary Direction + +Adapters should stay thin: + +- Web routers validate HTTP input and map service errors to HTTP responses. +- CLI commands parse arguments and print results. +- Agent tools parse tolerant user/agent inputs and call strict domain helpers. +- React components call typed client functions and avoid backend policy logic. + +Shared behavior belongs in domain services or pure helpers before it is reused +by routers, CLI commands, agent tools, and legacy compatibility surfaces. + +## Review ownership + +`CODEOWNERS` records the enforceable GitHub review routing. Changes under +`src/mkb/db/` need database review. Authentication, +uploads, executable post-processors, deployment configuration, Compose exposure, and +security documentation need security review. Until dedicated teams exist, the +repository owner fills both roles; split these entries into teams as maintainership +grows. + +Cross-boundary changes should name the owning service in the pull request and keep +transport/UI adapters free of duplicated domain policy. API compatibility changes also +require updates to `docs/python-api.md` or `docs/api-contract.md` as applicable. diff --git a/docs/backup-restore.md b/docs/backup-restore.md new file mode 100644 index 0000000..e0d5d2d --- /dev/null +++ b/docs/backup-restore.md @@ -0,0 +1,46 @@ +# Backup and restore + +Create a snapshot only after checking service health and allowing important jobs to +finish: + +```bash +make doctor +make pack out=mkb-snapshot.tar.gz +``` + +The archive contains a versioned manifest, database dump, required bucket contents, +local data, file sizes, checksums, schema revision, and application version. Store it +encrypted using organization-approved storage and retention controls. A successful +archive creation is not proof of restorability; schedule `make restore-drill` against +disposable infrastructure. + +The full drill restores PostgreSQL into a temporary database, starts a disposable +MinIO container, restores local files beneath a temporary root, and then runs inventory +comparison, reconciliation, and content verification against those copies: + +```bash +make restore-drill file=mkb-snapshot.tar.gz +``` + +Set `MKB_RESTORE_DRILL_OUT=/durable/evidence/directory` to retain its JSON reports. For +a historical archive, `MKB_RESTORE_DRILL_BASELINE=/path/to/inventory.json` selects the +matching historical inventory; object identities are enriched with SHA-256 values from +the validated archive manifest. The drill never enables live replacement. + +Restore is intentionally two-stage. First validate without mutation: + +```bash +make unpack file=mkb-snapshot.tar.gz +``` + +Then follow the script's explicit `--confirm-replace` instructions. Replacement can +delete current database and bucket state, so stop application processes, verify the +target database/Compose project, and retain a current snapshot first. After restore: + +```bash +make doctor +.venv/bin/python -m mkb.cli reconcile +``` + +Investigate checksum, revision, missing-object, or orphan reports before reopening the +service. Never edit a snapshot manifest to bypass validation. diff --git a/docs/deprecation-policy.md b/docs/deprecation-policy.md new file mode 100644 index 0000000..f453a36 --- /dev/null +++ b/docs/deprecation-policy.md @@ -0,0 +1,20 @@ +# Versioning and deprecation policy + +MKB uses Semantic Versioning. The supported public Python contract is the names in +`mkb.__all__`, their documented typed methods, the `mkb` CLI, and the documented HTTP +API. Internal modules, ORM models, and service-private helpers are not compatibility +surfaces. + +Before 1.0, incompatible SDK changes require a minor-version release, a changelog +entry, migration notes, and contract-test updates. After 1.0, incompatible public API +changes require a major release. Patch releases remain backward compatible. + +A public API is deprecated only after its replacement has feature parity. A +deprecation must emit `DeprecationWarning`, appear in the changelog and migration +guide, and remain operational for at least one complete minor release. Persisted +legacy data and its read adapters are retained for at least one complete release and +are never removed merely to simplify implementation. + +Database changes are additive first. A library refuses portable databases whose +schema version is newer than it supports. Normal code rollback switches back to +compatible readers; it does not delete or rewrite user data. diff --git a/docs/development.md b/docs/development.md new file mode 100644 index 0000000..3569408 --- /dev/null +++ b/docs/development.md @@ -0,0 +1,57 @@ +# Developer setup + +## Clean clone + +Install Python 3.10+, Node.js 20+, npm, Docker Compose, and `libmagic`, then run: + +```bash +make bootstrap +``` + +This creates `.venv`, upgrades its pip, installs `.[all,dev]`, runs `npm ci` from the +committed lockfile, and creates `.env` only if absent. Edit `.env`, then: + +```bash +make up # waits for healthy PostgreSQL/MinIO +make doctor # verifies the whole local environment +make dev # FastAPI and Vite; Ctrl+C stops both +``` + +All Make targets use `PYTHON ?= .venv/bin/python` and Python modules (`python -m +...`) consistently. Override it explicitly for tooling or CI. `BOOTSTRAP_PYTHON` +is used only to create the environment, and `NPM` can likewise be overridden. + +## Validation and build + +```bash +make lint +make test-python +make test-frontend +make check +make build +``` + +`make build` writes a Python wheel under `build/wheels/` and builds the production +React bundle. CI should begin with `make bootstrap` (or reproduce its pinned npm +install and editable dev install) before invoking these targets. + +The base wheel depends only on Pydantic and SQLAlchemy. Backend and application +dependencies are opt-in through `postgres`, `s3`, `pdf`, `server`, `materials`, +`neo4j`, or `all`. Distribution CI builds both wheel and sdist, installs the wheel +into a separate environment, and runs `examples/portable_quickstart.py` without the +repository on its import path. + +## Database provisioning + +The historical Alembic chain was retired after the local database reached its final +supported revision. This repository does not provision or upgrade the legacy materials +schema. Start from a verified current database snapshot; explicit SDK clients create +only their portable `mkb_*` tables with `kb.initialize()`. + +## Runtime files and legacy surfaces + +Local artifacts live under `data/`, `.debug/`, `logs/`, and Docker volumes. Treat +exports as generated unless deliberately promoted to `examples/` or `tests/fixtures/`. + +React (`frontend/`) is the only bundled UI. Legacy canonical-workflow adapters remain +read-compatible while their retained records are exported or retired. diff --git a/docs/operator-runbook.md b/docs/operator-runbook.md new file mode 100644 index 0000000..24f0dcd --- /dev/null +++ b/docs/operator-runbook.md @@ -0,0 +1,47 @@ +# MKB operator runbook + +MKB defaults to loopback-only operation. Before startup, run `make up` and +`curl http://127.0.0.1:8503/health/ready`. Liveness only confirms +the API process; readiness categorizes database, schema, object-storage, and +worker failures without returning credentials. + +For a clean shutdown, stop accepting work, check `/api/jobs` for active jobs, +cancel them through the API, wait for terminal states, then stop the API and run +`make down`. A crash or forced restart marks leftover jobs `INTERRUPTED`; inspect +their `request_id`, events, and error category before retrying the originating +action. Active-key constraints prevent duplicate work across API processes. + +For stuck work, query the job, request cooperative cancellation, and inspect the +request ID in rotating logs. Do not kill worker threads inside a live process. +Provider outages should produce a failed/interrupted durable record; retry only +after readiness and provider checks recover. + +For full disks, run `mkb cleanup` first. It is a dry run and reports item/byte +counts. Review the paths, then use `mkb cleanup --apply --confirm DELETE`. +Use `mkb reconcile` before and after cleanup; it is read-only and reports missing +and orphaned PostgreSQL/MinIO objects. + +## Backup, restore, and upgrade + +`make pack` fails if PostgreSQL or any required bucket cannot be copied. The +archive contains a versioned manifest, schema revision, application version, +file sizes, and SHA-256 checksums. Store/encrypt the resulting archive with your +organization's approved backup tooling. + +Restore is deliberately two-step: + +1. `bash scripts/unpack_data.sh snapshot.tar.gz` validates paths, types, manifest, + and checksums in a temporary staging directory and exits without mutation. +2. Re-run with `--confirm-replace`, then type the database name when prompted. + +The restore mirrors bucket contents with removal, replaces the database from the +validated dump, and restores local files. Run `mkb reconcile`, +and the readiness probe afterward. Practice this against a disposable Compose +project before depending on a backup. `make restore-drill` performs checksum/path +validation; restores a disposable database, MinIO instance, and local root; then runs +full object-checksummed inventory comparison, reconciliation, and representative +content verification. It is suitable for a scheduled job or integration CI runner +with Docker services and never enables the live replacement path. + +Before an application upgrade, create and verify a snapshot, stop job starts, +upgrade the code, then verify readiness and reconciliation. diff --git a/docs/python-api.md b/docs/python-api.md new file mode 100644 index 0000000..0f34292 --- /dev/null +++ b/docs/python-api.md @@ -0,0 +1,700 @@ +# Python API + +## Installation and supported surfaces + +MKB is currently distributed from this repository rather than PyPI. Install the base +package for the portable typed SDK, SQLite, filesystem storage, registries, and local +pipeline execution from a pinned commit for reproducible consumers (replace +`` with the version your repository has tested): + +```bash +python -m pip install \ + "mat-know-base @ git+https://github.com/theAfish/mat_know_base.git@" +``` + +Use the `dev` branch only when intentionally tracking the shared development build. +Install extras only for the integrations a consumer uses. For example, PostgreSQL and +S3 support use: + +```bash +python -m pip install \ + "mat-know-base[postgres,s3] @ git+https://github.com/theAfish/mat_know_base.git@" +``` + +Other extras are `[neo4j]`, `[materials]`, and `[server]`. The existing materials +application, agent-backed extraction, and its compatibility facade require +`[materials]`; its HTTP server additionally requires `[server]`. Once MKB is +published on PyPI, these Git URLs can be replaced with normal package-install commands. + +There are two supported Python surfaces. `KnowledgeBase.from_url(...)` is the portable, +typed SDK intended for new repositories. `KnowledgeBase.from_environment()` and +`mkb.api` are the materials application's compatibility surfaces: they require its +configured infrastructure and may return legacy dictionaries. Do not mix identifiers or +assume that a portable client can operate on the legacy application schema. + +`KnowledgeBase` is the new explicit entry point for Python callers. During the SDK +refactor it delegates to the same services and reads the same PostgreSQL and MinIO data +as the current application; creating it does not migrate, copy, or re-extract data. + +```python +from mkb import KnowledgeBase + +with KnowledgeBase.from_environment() as kb: + projects = kb.list_projects(limit=100) + frames = kb.list_frames(status="COMPLETED") +``` + +`mkb.api` remains the supported compatibility facade while feature parity is built. +Existing automation does not need to change yet. Import from one of these public +surfaces, not from database models, web routers, or individual service modules. The +compatibility facade and legacy methods on `KnowledgeBase` still return dictionaries +and lists; grouped SDK services such as `kb.collections`, `kb.schemas`, and `kb.graph` +return typed public models. + +```python +from mkb import api +``` + +## Public import boundary + +The supported root package exports the configured client, immutable configuration, +typed domain and operation models, pipeline/parser definitions, grouped registry +types, transactions, and the public exception hierarchy. All typed models support +`model_dump(mode="json")`. + +Custom adapter authors may explicitly import protocols from `mkb.ports` and default +implementations from `mkb.adapters`. Application consumers should not import +`mkb.db`, `mkb.web`, ORM classes, session factories, or storage implementation +modules. The old port aliases remain explicitly importable from `mkb` for temporary +compatibility, but are intentionally absent from `mkb.__all__`. + +The explicit client is preferable for new code because its configuration and service +bindings belong to one object rather than module globals. The environment adapter keeps +the compatibility application services, but binds their database and object-store access +to the owning client for each call. Explicit SQLite, PostgreSQL, filesystem, S3, and graph +adapters are independently configurable and do not share client registries or workers. + +The supported client is intentionally synchronous. The current SQLAlchemy repositories, +S3/filesystem adapters, model-provider boundary, and graph adapters expose synchronous +operations; wrapping them in `async def` would still block an event loop. Durable work +uses `kb.pipelines.submit(...)`/`kb.jobs`, and asynchronous applications should call the +synchronous SDK at their worker/thread boundary. An async client will be added only when +the injected ports have genuinely asynchronous implementations. + +## Client construction and lifecycle + +New integrations should use `KnowledgeBase.from_url(...)`. It is the supported portable +composition path and accepts SQLite or SQLAlchemy PostgreSQL URLs plus built-in +filesystem or S3 object-store URLs. `from_environment()` is only for the configured +materials application. `from_url()` never reads application settings and never creates +tables; call `initialize()` deliberately for a new portable database. + +```python +from mkb import KnowledgeBase + +with KnowledgeBase.from_url( + database_url="sqlite:////absolute/path/project.db", + object_store_url="file:///absolute/path/objects", # omit for metadata-only use +) as kb: + kb.initialize() + # use kb.collections, kb.records, kb.schemas, and other grouped services +``` + +The context manager owns and closes the constructed database, storage, graph, model, +job, and vector resources. Do not use the client after leaving the block. If an +application must create it outside a `with` block, call `kb.close()` after all submitted +pipeline jobs have finished or been cancelled. + +`from_url()` accepts injected `GraphStore`, `ModelProvider`, `JobBackend`, and +`VectorSearch` implementations. Custom `Database` and `ObjectStore` implementations +are **not yet a supported public composition path**: `from_url()` constructs MKB's +built-in SQLAlchemy and filesystem/S3 adapters, while direct `KnowledgeBase(...)` +construction requires internal repository/schema bindings. Adapter authors can rely on +the protocols in `mkb.ports`, but should not depend on private SDK builders; request or +contribute a public adapter factory before using a non-built-in database or object store +in another repository. + +For pipeline execution and typed repository access without reading `.env` or YAML, +construct an independent client explicitly: + +```python +from mkb import KnowledgeBase + +with KnowledgeBase.from_url( + database_url="sqlite:////absolute/path/project.db", + object_store_url="file:///absolute/path/objects", + raw_bucket="inputs", + processed_bucket="derived", +) as kb: + kb.database.check() + kb.initialize() # explicit, idempotent creation of missing SDK-owned tables + + collection = kb.collections.create(name="Experiment 42") + kb.records.create( + collection_id=collection.id, + data={"material": "nickelate", "temperature_c": 800}, + ) + source = kb.sources.add_text( + collection.id, + "custom project notes", + filename="notes.txt", + ) + artifact = kb.artifacts.add_bytes( + source.id, + b"normalized notes", + processing_type="NORMALIZED_TEXT", + format="txt", + ) +``` + +S3-compatible storage uses +`s3://bucket?endpoint=http://localhost:9000` plus the optional +`object_store_access_key` and `object_store_secret_key` arguments. The S3 URL's bucket +is the raw/input bucket; configure `processed_bucket`, `archive_bucket`, and +`temp_bucket` explicitly when their names differ from the defaults. Filesystem storage +uses all four bucket names as directories under its root. Construction never creates or +migrates tables. Explicit clients deliberately reject legacy facade calls +such as `list_projects()` because those operations still depend on global application +configuration; use their grouped services as those repositories become writable. +`initialize()` creates only the portable `mkb_*` tables owned by the new SDK. It uses +additive, idempotent table creation and never drops or renames tables. +It returns the current portable schema version; `kb.schema_version()` reports the +stored version afterward. Revisions are recorded in `mkb_schema_migrations`. Revision +1 contains collections, sources, records, and schemas; revision 2 adds artifacts and +projections; revision 3 adds generic evidence links; revision 4 adds durable local +pipeline jobs; revision 5 adds portable feedback, skills, and post-processor metadata; +revision 6 adds portable collection groups and memberships; revision 7 adds immutable +extraction-schema revisions; revision 8 gives external source URIs dedicated storage so +URI state cannot collide with consumer metadata. Schema creation and every update write +a complete snapshot in the same relational transaction as the current schema row. This +lets a stored projection resolve the exact definition that produced it: + +```python +projection = kb.projections.require(projection_id) +schema_at_extraction = kb.schemas.require_version( + projection.schema_id, + projection.schema_version, +) + +# Revisions are returned newest first. +schema_history = kb.schemas.history(projection.schema_id) +``` + +When upgrading an older portable database, revision 7 snapshots each schema definition +that is current at migration time. Definitions overwritten before revision 7 did not +exist independently and therefore cannot be reconstructed by the migration. + +## PostgreSQL and MinIO/S3 + +An explicitly configured PostgreSQL/S3 client owns its adapters and does not read global +settings: + +```python +from mkb import KnowledgeBase + +with KnowledgeBase.from_url( + database_url="postgresql+psycopg://mkb:password@localhost:5432/mkb", + object_store_url="s3://raw?endpoint=http://localhost:9000", + object_store_access_key="...", + object_store_secret_key="...", +) as kb: + kb.database.check() + kb.object_store.check((kb.config.raw_bucket, kb.config.processed_bucket)) + # Explicit and additive; omit this call for a read-only validation connection. + kb.initialize() +``` + +Use `KnowledgeBase.from_environment()` for the existing materials deployment. It maps +legacy projects, groups, assets, frames, spaces, projections, feedback, skills, +post-processors, and graph operations through injected adapters without copying IDs or +object keys. + +## Optional Neo4j graph storage + +Install `mat-know-base[neo4j]` and inject the optional adapter into an explicitly +configured client. Importing the base package does not import or require the Neo4j +driver. + +```python +from mkb import KnowledgeBase +from mkb.adapters import Neo4jGraphStore + +graph = Neo4jGraphStore( + "neo4j://localhost:7687", + auth=("neo4j", "password"), + database="neo4j", +) +with KnowledgeBase.from_url( + database_url="sqlite:////absolute/path/project.db", + object_store_url="file:///absolute/path/objects", + graph_store=graph, +) as kb: + entity = kb.graph.upsert_entity(type="material", name="Calcite") +``` + +The adapter stores backend-neutral entity/relation IDs and JSON properties beneath +fixed `MKBEntity`/`MKBRelation` types, so user-provided values are parameters rather +than Cypher identifiers. + +Without an injected `graph_store`, portable clients use an `InMemoryGraphStore`. It is +isolated to that client and emptied when the client closes; SQLite does not persist graph +entities or relations. Inject Neo4j or another `GraphStore` implementation whenever a +consumer needs graph data after process restart. The in-memory default is appropriate for +tests and short-lived local pipelines only. + +## Safe migration/read-validation example + +Opening a client never runs migrations. A preservation-first validation can therefore +inspect existing data without writing: + +```python +from mkb import KnowledgeBase + +with KnowledgeBase.from_environment() as kb: + inventory = kb.maintenance.inventory() + reconciliation = kb.maintenance.reconcile() + assert reconciliation.ok, reconciliation.model_dump(mode="json") + + for collection in kb.collections.list(limit=1000): + assert kb.collections.get(collection.id) == collection + for source in kb.sources.list(collection_id=collection.id, limit=1000): + if source.storage is not None: + assert kb.sources.content_exists(source.id) +``` + +Run additive initialization/backfills separately only after the pre-migration snapshot +and restore drill required by `TODO.md`. Never use initialization as an implicit startup +side effect, and compare inventories/reconciliation before switching readers. + +Multiple relational writes can share one commit or rollback boundary: + +```python +with KnowledgeBase.from_url(database_url="sqlite:////tmp/research.db") as kb: + kb.initialize() + with kb.transaction() as tx: + collection = tx.collections.create(name="Experiment 43") + record = tx.records.create( + collection_id=collection.id, + data=[{"sample": "A", "result": 12.4}], + ) + schema = tx.schemas.create( + name="experiment-result", + domain="my project", + definition={"type": "array"}, + system_prompt="Extract supported experiment results.", + ) + tx.projections.create( + schema_id=schema.id, + record_id=record.id, + data={"sample": "A", "result": 12.4}, + ) +``` + +The transaction object includes collections, sources, artifacts, records, schemas, +projections, and evidence. Object-backed writes place content first, commit metadata +second, and register reverse-order best-effort cleanup if the relational transaction +rolls back. S3/filesystem writes are compensating operations, not part of relational +ACID atomicity. + +The client already owns explicit relational and object-store resources. They are +available for health checks and are closed with the client: + +```python +with KnowledgeBase.from_environment() as kb: + kb.database.check() + kb.object_store.check((kb.config.raw_bucket, kb.config.processed_bucket)) + + # Typed generic view over the existing research_projects table. + collections = kb.collections.list(limit=100) + first = kb.collections.get(collections[0].id) if collections else None + if first: + print(first.model_dump(mode="json")) + + sources = kb.sources.list(collection_id=first.id) + if sources: + source = sources[0] + with kb.sources.open(source.id) as content: + header = content.read(16) + + artifacts = kb.artifacts.list(source_id=source.id) + if artifacts and kb.artifacts.content_exists(artifacts[0].id): + processed = kb.artifacts.read_bytes(artifacts[0].id) + + # Existing knowledge_frames, spaces, and projections are exposed without + # rewriting their rows or normalizing their stored JSON payloads. + record = kb.records.get_for_collection(first.id) + schemas = kb.schemas.list() + projections = kb.projections.list( + collection_id=first.id, + status="COMPLETED", + newest_only=True, + ) + json_text = kb.projections.export_json( + collection_id=first.id, + newest_only=True, + ) +``` + +Legacy raw workflow extraction versions are available without graph normalization via +`kb.materials.workflows.get(...)` and `.list(...)`. The resulting `WorkflowRecord` +preserves workflow/schema versions, correction and review fields, graph, checkpoint, +provenance, errors, and timestamps. Portable clients can persist provenance with +`kb.evidence.create(...)` and query it by output, source, or artifact ID. + +Portable resource lifecycle operations include collection update/safe delete, managed +file/bytes/text ingestion, recursive directory ingestion, external URI registration +without copying, structured JSON-record batches, and registration of an object already +produced by an external processor. External URI sources deliberately reject +`open()`/`read_bytes()` because their content is not owned by the configured store. + +Records support exact top-level JSON field queries through `kb.records.query(...)` and +their provenance through `kb.records.evidence(record_id)`. Portable extraction schemas +increment `version` on update and refuse deletion while projections reference them. +Feedback, skills, and post-processors are typed, client-owned services: + +```python +feedback = kb.feedback.create( + target_record_id=record.id, + target_collection_id=collection.id, + category="ambiguous_data", + question="Which unit applies?", +) +kb.feedback.review(feedback.id) +kb.feedback.resolve(feedback.id, notes="The source specifies kelvin.") + +skill = kb.skills.create( + name="Normalize units", + content="# Normalize units\nConvert reported measurements to SI.", +) +processor = kb.post_processors.register( + name="Choose projection", + source="print('{}')\n", +) +``` + +Registration stores post-processor source but does not execute it; execution remains +behind the existing administrator opt-in and sandbox policy. + +The generic model mapping is deliberately compatible with the current local schema: +`research_projects` become `Collection`, `assets` become `Source`, processed assets +become `Artifact`, `knowledge_frames` become `Record`, spaces become +`ExtractionSchema`, and stored projections become `Projection`. IDs, timestamps, +status values, review metadata, schema versions, and raw JSON are retained. The +environment-backed typed services remain read-only to protect the existing local +dataset. Explicitly configured, initialized databases support collection, record, and +schema creation through portable transaction-aware repositories. + +All public models support `model_dump(mode="json")` and `model_dump_json()`. The +`records.export_json(...)` and `projections.export_json(...)` helpers return JSON text +without writing files, so package consumers decide where exported data belongs. + +Narrow adapter protocols live in `mkb.ports`: relational database, object storage, +graph storage, vector search, content parser, model provider, and job backend. Default +database, S3/MinIO, filesystem, and in-memory graph implementations are available from +`mkb.adapters`. Today the public client factory composes the built-in database and +object-store adapters; graph, model-provider, job, and vector adapters can be injected +into `from_url(...)`. These ports remain separate: there is no artificial storage +interface spanning relational transactions, blobs, vectors, and graph traversal. + +Adapters declare stable capability names through `Capabilities`. Pipeline steps fail +before execution when requirements such as `vector_search`, `full_text_search`, +`object_streaming`, or `graph_traversal` are unavailable. Adapter conformance tests +cover lifecycle, transactions, streaming, CRUD semantics, structural repository +contracts, and capability composition. + +The materials compatibility API performs real database, object-storage, filesystem, +processor, and LLM work. Configure `.env`, start infrastructure with `make up`, and run +those calls from the repository root so application configuration and local data paths +resolve. The portable SDK needs only the adapters supplied to `from_url(...)`; its +default graph is the documented in-memory exception. + +## End-to-end lifecycle + +```python +from pathlib import Path +from mkb import api + +created = api.ingest(Path("data/papers/smith2024"), label="Smith 2024") +project_id = created["project_id"] + +processed = api.process(project_id=project_id) +extracted = api.extract(project_id=project_id, max_passes=2) +frame = api.get_frame(project_id) +if frame is None: + raise RuntimeError("extraction did not create a frame") + +space = api.create_space( + name="Synthesis conditions", + domain="materials science", + description="Experimental synthesis facts", + extraction_schema={ + "type": "object", + "properties": { + "material": {"type": "string"}, + "temperature_c": {"type": ["number", "null"]}, + }, + "required": ["material"], + }, + field_descriptions={"temperature_c": "Reported synthesis temperature in Celsius"}, + system_prompt="Extract only values supported by the project evidence.", +) +projection = api.project(space_id=space["space_id"], project_id=project_id) + +print(processed, extracted) +print(api.get_projection(projection["projection_id"])) +``` + +## Custom local pipelines + +Pipeline definitions and registries belong to one `KnowledgeBase` instance. Steps run +in stable dependency order and merge their output mappings into the accumulated state; +independent steps retain declaration order. Optional Pydantic models validate step +inputs, parameters, and outputs. + +```python +from pydantic import BaseModel +from mkb import KnowledgeBase, Pipeline, Step + +class Inputs(BaseModel): + text: str + +class Outputs(BaseModel): + word_count: int + +pipeline = Pipeline( + name="count-words", + steps=( + Step( + name="count", + input_model=Inputs, + output_model=Outputs, + deterministic=True, + handler=lambda context, state: { + "word_count": len(state["text"].split()), + }, + ), + ), +) + +with KnowledgeBase.from_url(database_url="sqlite:///:memory:") as kb: + kb.pipelines.register(pipeline) + run = kb.pipelines.run("count-words", inputs={"text": "custom project data"}) + print(run.model_dump(mode="json")) +``` + +Steps may declare `required_capabilities`, `RetryPolicy`, `cacheable`, +`timeout_seconds`, and `side_effects`. Capability requirements are checked before +execution. `depends_on` declares DAG edges, and cycles or unknown dependencies are +rejected at definition time. A timeout stops the pipeline from waiting, marks the +attempt failed, and signals `context.cancelled`; long-running handlers, especially +those with side effects, should call `context.check_cancelled()` at safe boundaries so +their worker can unwind promptly. +Failures raise `PipelineExecutionError`; its `run` attribute contains the failed typed +run and completed step history. A progress callback receives typed `ProgressEvent` +objects. Existing `Record` instances can be supplied directly in pipeline inputs, so +local extracted data does not need to be reprocessed. + +Portable clients support persisted submission using the same definition. Initialize +the client first so the additive jobs table is available: + +```python +kb.pipelines.register(pipeline) +job = kb.pipelines.submit( + "count-words", + inputs={"text": "custom project data"}, + idempotency_key="count:source-42:v1", +) +completed = kb.jobs.wait(job.id, timeout=60) +``` + +Jobs retain inputs, parameters, stable run IDs, completed-step checkpoints, structured +progress/log events, attempts, results, and errors. The built-in `submit()` executor is a +daemon thread in the process that owns the client: persistence makes its state inspectable +and resumable, but it is not an external queue or a cross-process worker. On restart, +register the same pipeline version, call `kb.jobs.recover_interrupted()`, then call +`kb.pipelines.resume(job.id)`; completed steps in a compatible checkpoint are not +repeated. `kb.jobs.cancel(...)` requests cooperative cancellation at a step/event +boundary. Reusing an idempotency key returns the original job. Custom workers may enqueue +non-pipeline work with `kb.jobs.submit(...)` and must implement their own claim/execute +loop through a `JobBackend`. Persisted events can be consumed as a snapshot with +`kb.jobs.events(job.id)` or followed until a terminal state with +`kb.jobs.events(job.id, follow=True)`. + +Cacheable steps must be deterministic and provide a `cache_key` builder returning +`CacheKeyComponents`. The components require configuration, source fingerprint, model +identity, and schema version; MKB additionally includes the step name and version before +hashing. A cache hit is recorded on `StepRun` and does not invoke the handler. + +Environment-backed clients register built-in materials pipelines named +`materials.ingest`, `materials.process`, `materials.extract_frames`, +`materials.project`, `materials.extract_graph`, `materials.extract_workflow`, +`materials.review_schema`, and `materials.review_feedback`. Their single steps delegate +to the existing compatibility operations and retain the original result under the +pipeline output's `result` key. + +Parsers and reusable standalone steps are also registered on one configured client; +they do not mutate module-global registries. Portable schemas are persisted by the +client's metadata repository: + +```python +from mkb import Parser, Pipeline, Step + +kb.parsers.register( + Parser( + name="notes", + source_types=frozenset({"text/plain"}), + handler=lambda context, content: {"text": content.decode("utf-8")}, + ) +) +kb.steps.register( + Step( + name="word-count", + deterministic=True, + handler=lambda context, state: {"words": len(state["text"].split())}, + ) +) +kb.schemas.register( + name="notes-schema", + domain="general", + definition={"type": "object", "properties": {"words": {"type": "integer"}}}, + system_prompt="Extract only supported values.", +) +kb.pipelines.register(Pipeline(name="notes", steps=(kb.steps.require("word-count"),))) +``` + +Most mutating compatibility functions return a summary dictionary containing stable +identifiers and counts. Typed SDK reads return a public model, a list of models, or +`None` when a singular resource does not exist. Invalid IDs, missing prerequisites, +storage errors, and provider failures raise exceptions; library callers should catch +`MKBError` at a job or request boundary rather than infer success from partial output. +Use `ValidationError` for invalid caller input, `NotFoundError` for required resources, +`ConflictError` for state/precondition failures, `BackendUnavailableError` for missing or +closed integrations, `ProviderError` for external provider failures, and +`PipelineExecutionError` for a failed pipeline (whose `run` contains the typed result). + +## Setup, ingest, and assets + +| Call | Purpose and important arguments | +| --- | --- | +| `ingest(directory, label=None, *, user_named=False)` | Create or update one project from a directory and upload source assets. The directory must exist. | +| `sync(root_dir)` | Treat each immediate project folder below a root as a project and rescan it. | +| `sync_project(project_id)` | Rescan the source directory recorded for one project. | +| `process(project_id=None, progress_callback=None)` | Process pending assets for one project or all projects. | +| `list_assets(project_id=None, limit=100)` | Return source asset metadata. | +| `list_processed_assets(project_id=None, limit=100)` | Return derived artifact metadata. | +| `search_library(query, limit=25, project_id=None)` | Search project and asset metadata; returns `projects`, `assets`, and `total`. | + +`progress_callback`, where accepted, is called during long-running synchronous work. +Callbacks should be fast, thread-safe if the caller adds concurrency, and tolerate +repeated/non-uniform progress payloads. The stable contract is notification, not a +fixed event schema. + +For externally prepared output, use +`link_manual_processed_data(processed_dir, paper_dir=None, project_id=None, +asset_id=None, primary_file=None, processing_type=None, output_format=None)`. Supply +an `asset_id` when possible; otherwise provide enough project/paper context to select +one asset unambiguously. + +## Projects and frames + +```python +projects = api.list_projects(limit=100) +api.rename_project(project_id, "New label") +assets = api.list_assets(project_id=project_id) + +api.extract(project_id=project_id, model=None, verbose=False, max_passes=2) +frame = api.get_frame(project_id) # dict | None +history = api.get_extraction_history(project_id) +completed = api.list_frames(status="completed") +``` + +Project group calls are `list_project_groups()`, `create_project_group(name, ...)`, +`update_project_group(group_id, ...)`, `assign_projects_to_group(project_ids, +group_id)`, and `delete_project_group(group_id)`. Passing `None` as the assignment +group removes the projects from a group. `delete_project(project_id, +delete_s3_objects=True)` is a hard delete and should be guarded by application-level +confirmation. + +## Spaces and projections + +`create_space(...)` accepts a JSON Schema-like `extraction_schema`, system prompt, +optional field descriptions, purpose, review settings, search tools, and +post-processors. `update_space(space_id, **changes)` increments its version. +`get_space(id_or_name)` accepts either identifier form; delete calls require an ID. + +Projection calls: + +```python +api.project(space_id, frame_id=None, project_id=None, model=None, + verbose=False, progress_callback=None, source_type="frame") +api.project_all(space_id, model=None, verbose=False, source_type="frame") +api.list_projections(space_id=None, frame_id=None, project_id=None, + include_data=False, newest_only=False, include_history=False) +api.get_projection(projection_id) +api.delete_projection(projection_id) # soft delete; returns bool +api.export_projection(projection_id, out_dir, format="yaml", overwrite=False) +api.export_space_projections(space_id_or_name, out_dir, format="yaml", + overwrite=False, newest_only=True) +``` + +Provide exactly the source selector appropriate to `source_type`; the normal current +path is `source_type="frame"` with `project_id` or `frame_id`. Exports refuse to +overwrite by default. Space updates can make older projections historical, so use +`newest_only=True` when producing a current dataset. + +## Knowledge graph and feedback + +The graph API exposes `extract_knowledge_graph`, `get_knowledge_graph`, +`review_knowledge_graph`, `get_graph_review_counts`, and `clear_knowledge_graphs`. +Extraction can clear existing graph projections by default; set arguments deliberately +when preserving prior results. Clearing is a mutating soft-delete operation. + +Feedback calls include `list_feedback`, `get_feedback_summary`, `resolve_feedback`, +`review_feedback`, `review_projections`, `review_projections_all`, +`review_projections_session`, and `review_projection_followup`. Review functions may +invoke an LLM and mutate projections or feedback, so record the chosen model and +reviewer ID in reproducible automation. + +## Workflow API and compatibility status + +Raw workflow extraction/review is active: `extract_raw_workflow`, +`get_raw_workflow_extraction_readiness`, `list_raw_workflows`, `get_raw_workflow`, +`review_raw_workflow`, and `correct_raw_workflow`. Schema and maintenance calls include +`curate_workflow_schema`, proposal list/edit/review calls, +`schedule_workflow_reextraction`, `list_workflow_maintenance_tasks`, +`run_workflow_maintenance_task`, `rebuild_workflow_indexes`, and +`search_canonical_workflows`. + +Canonical workflow records and serializers remain available for compatibility, but +canonicalization is not the target for new product workflows. Consult the +[workflow lifecycle policy](workflow-lifecycle-policy.md) before adding dependencies +on those calls. + +## Concurrency, transactions, and compatibility + +Calls are synchronous; use a process/job boundary for long processing and agent work. +Do not share SQLAlchemy sessions across threads. Functions establish their own service +boundaries, but a sequence of separate API calls is not one atomic transaction. Design +retries around stable project/asset IDs and inspect current state before repeating +destructive or LLM-backed operations. See [transaction boundaries](transactions.md). + +Public names listed in `mkb.api.__all__` are the compatibility surface. Keys in result +dictionaries are less strictly versioned than function names; consumers should read +needed keys and tolerate additive fields. Private names beginning with `_`, ORM models, +and service internals are not supported API even if importable. + +## Adapter and lifecycle guidance + +Custom `GraphStore`, `ModelProvider`, `JobBackend`, and `VectorSearch` adapters can be +passed to `from_url(...)`. Implement the corresponding narrow protocol in `mkb.ports` +and declare only the stable `Capabilities` the adapter truly supports; pipeline steps +validate requirements before execution. The client owns injected adapters and closes +them with `kb.close()` (or a `with` block), so do not share an adapter instance across +clients unless it explicitly supports that lifecycle. Custom `Database` and +`ObjectStore` adapters remain an extension point, not a public client-construction +workflow; see [Client construction and lifecycle](#client-construction-and-lifecycle). + +`KnowledgeBase` is synchronous. Use it at a worker/thread boundary from async +applications, keep SQLAlchemy sessions within that boundary, and use `kb.transaction()` +when grouped relational changes must commit or roll back together. Object-store writes +use compensating cleanup and are not part of relational ACID transactions. diff --git a/docs/security.md b/docs/security.md new file mode 100644 index 0000000..e70b8fa --- /dev/null +++ b/docs/security.md @@ -0,0 +1,60 @@ +# Security model + +MKB defaults to trusted, single-user, localhost-only operation. Development and +local modes reject non-loopback API binds. PostgreSQL, MinIO, and the MinIO +console are also published on loopback only by the supplied Compose file. + +## Authentication and roles + +Remote binding is permitted only in `production` mode with authentication, +non-default infrastructure credentials, INFO-or-higher logging, exact CORS +origins, and at least one bearer token. Configure tokens only through `.env`: + +```dotenv +MKB_DEPLOYMENT_MODE=production +MKB_API_HOST=0.0.0.0 +MKB_AUTHENTICATION_ENABLED=true +MKB_AUTH_TOKENS={"":"admin"} +MKB_CORS_ORIGINS=["https://mkb.example.org"] +``` + +Generate an opaque token with `openssl rand -hex 32`. The token map supports: + +| Role | Permissions | +| --- | --- | +| `reader` | Read API data and source assets | +| `editor` | Read, mutate, and start jobs | +| `admin` | All permissions, including deletes, settings, and code uploads | + +Send the token as `Authorization: Bearer `. The React client reads a token +from browser `sessionStorage`; it can be set for the current tab with: + +```js +sessionStorage.setItem('mkb_api_token', '') +``` + +Bearer authentication uses an explicit header rather than cookies, so browser +CSRF protections are not applicable. Exact CORS origins are still required. +Liveness and readiness paths are public; every `/api` data route is protected +when authentication is enabled. + +## Abuse and executable-content controls + +Authentication failures, uploads, assistant calls, and job starts use per-process +sliding-window limits. Deployments with multiple API processes should additionally +enforce shared limits at a reverse proxy or gateway. + +Uploaded Python post-processors are disabled by default and prohibited in +production. `MKB_ALLOW_UPLOADED_PYTHON=true` is a trusted-local administrator +escape hatch, not a sandbox: enabled scripts retain host filesystem and network +access. Do not enable it for untrusted content. + +Upload and archive limits are configured through the `MKB_UPLOAD_*` and +`MKB_ARCHIVE_*` environment variables documented in `.env.example`. + +## Current limitations + +- Bearer tokens are static and must be rotated through `.env` plus a restart. +- Rate-limit state is not shared across API processes. +- Uploaded Python does not yet have a constrained worker/container runtime. +- Audit records and soft-delete recovery are still pending. diff --git a/docs/transactions.md b/docs/transactions.md new file mode 100644 index 0000000..915ebe0 --- /dev/null +++ b/docs/transactions.md @@ -0,0 +1,29 @@ +# Workflow transaction and idempotency boundaries + +Database transactions never include S3, filesystem, provider, or model calls. Each +workflow therefore exposes a durable state before external work, writes derived +objects to a staging key, then commits the final database reference. On failure, +the staging object is removed; if cleanup itself fails it is intentionally visible +to `mkb reconcile` as an orphan. + +For new portable databases, `KnowledgeBase.transaction()` supplies collection, source, +artifact, record, schema, and projection repositories bound to one SQLAlchemy +transaction. Successful exit commits all relational writes. An exception rolls them +back and performs best-effort deletion of source/artifact objects written in that +scope, in reverse order. Object storage still does not participate in the database +transaction: failed compensation remains detectable as an orphan through +reconciliation, and callers must use stable IDs so retries are safe. + +| Workflow | Durable start | Completion boundary | Retry identity / compensation | +|---|---|---|---| +| ingest | project/asset row in `PENDING` | raw object exists and asset is `STORED` | content hash; delete staging object or leave the row retryable | +| process | asset remains immutable, output is staged | processed asset row references promoted output | source checksum + processor version; delete staged output | +| extract | frame/extraction is `IN_PROGRESS` | completed immutable frame/extraction version commits | project + source version; revert/mark failed at cancellation checkpoint | +| project/review | projection is `IN_PROGRESS` | validated payload and terminal status commit together | space version + frame version; mark failed, preserve previous completed version | +| delete | targets are resolved before mutation | DB deletion commits after best-effort object inventory | resource ID; failed object deletion is reported by reconciliation | + +Background API actions additionally use `background_jobs.active_key` as a +cross-process lock and accept an idempotency key. Queued/running jobs left by a +dead process become `INTERRUPTED` on startup; they are never silently reported as +successful. Cooperative cancellation is checked by every progress callback before +the next database, processor, storage, or model phase. diff --git a/docs/upgrades.md b/docs/upgrades.md new file mode 100644 index 0000000..f5bceed --- /dev/null +++ b/docs/upgrades.md @@ -0,0 +1,58 @@ +# Upgrade and migration guide + +Before upgrading, read release notes, finish or cancel active jobs, run `make doctor`, +and create a verified snapshot. Stop the API and frontend while leaving PostgreSQL and +MinIO available for migration. + +```bash +git pull --ff-only +make bootstrap +make check +make doctor +make dev +``` + +`make bootstrap` preserves an existing `.env`. Compare it manually with +`.env.example` for newly introduced settings. The legacy schema migration chain is +retired; use a verified current database snapshot and roll back by restoring the +pre-upgrade snapshot with its matching application version. + +After startup, verify readiness, inspect logs for schema/configuration warnings, run +reconciliation, and exercise one read plus one disposable processing workflow. Keep +the pre-upgrade snapshot until operational acceptance is complete. + +For an SDK/data migration, capture inventories before and after the operation, then +run the read-only preservation gate: + +```bash +mkb inventory --out migration-snapshots/before.json +# Run only the separately reviewed additive migration or disposable restore here. +mkb inventory --out migration-snapshots/after.json +mkb migration-preflight \ + migration-snapshots/before.json \ + migration-snapshots/after.json \ + --out migration-snapshots/preflight.json +``` + +The command exits non-zero if an original database ID, object, or local file is +missing, or if object identity/size or a local SHA-256 checksum changed. Additive rows, +objects, and files are reported but do not fail the gate. This comparison never writes +to either backend. + +If reconciliation finds a missing processed object but an exact local mirror exists, +first run the checksum-gated dry run: + +```bash +mkb restore-missing-artifact ARTIFACT_ID /path/to/local/mirror +``` + +Applying requires the exact confirmation text and a new, non-overwritten ledger file: + +```bash +mkb restore-missing-artifact ARTIFACT_ID /path/to/local/mirror \ + --apply --confirm "RESTORE MISSING OBJECT" \ + --ledger migration-snapshots/repair-ARTIFACT_ID.json +``` + +The command refuses a size/hash mismatch and never overwrites an existing mismatched +object. Run reconciliation and capture a new inventory after any applied repair. diff --git a/docs/workflow-card-architecture.md b/docs/workflow-card-architecture.md index 1c7310f..f125590 100644 --- a/docs/workflow-card-architecture.md +++ b/docs/workflow-card-architecture.md @@ -11,6 +11,9 @@ The workflow subsystem has two agents: The former extraction → canonicalization sequence is retired. Legacy raw and canonical records remain readable during migration. +See `workflow-lifecycle-policy.md` for the current active, deprecated, and +compatibility-only public surfaces. + ## Cards and instances An ontology card describes a reusable concept: diff --git a/docs/workflow-lifecycle-policy.md b/docs/workflow-lifecycle-policy.md new file mode 100644 index 0000000..12f596a --- /dev/null +++ b/docs/workflow-lifecycle-policy.md @@ -0,0 +1,41 @@ +# Workflow Lifecycle Policy + +As of June 30, 2026, the active workflow model is the workflow-card extraction +path documented in `workflow-card-architecture.md`. + +## Active + +- Raw workflow extraction records are active. They store the evidence-grounded + workflow graph produced from a project. +- Workflow card/schema validation, review, indexing, and ontology induction are + active. +- Schema proposal review and raw-workflow maintenance tasks are active. + +## Compatibility Only + +- Canonical workflow records remain readable/deletable so older data, tests, + and UI tabs continue to work during migration. +- Canonicalization agent/tool code is internal compatibility for unfinished + legacy jobs and old records. It is not exposed through the active REST/job + action flow. +- New product behavior should not depend on canonicalization unless it is + explicitly maintaining compatibility with existing records. +- Canonical workflow code should move behind a `legacy` or `compatibility` + service boundary before any larger deletion. + +## Deprecated For New Work + +- The old extraction-to-canonicalization pipeline is retired for new feature + development. +- New workflow features should consume raw/card graphs and schema-library + helpers directly. + +## Public Surface Status + +- Active: raw workflow extraction, raw workflow review/correction, schema + curation, schema proposal review, raw workflow maintenance, workflow-card + editing helpers. +- Compatibility: canonical workflow list/get/delete, canonical workflow + frontend tabs, canonical indexes. +- Internal compatibility: checkpoint and draft-edit helpers used to resume or + inspect unfinished legacy canonicalization jobs. diff --git a/examples/basic_usage.py b/examples/basic_usage.py index d56f062..0af7920 100644 --- a/examples/basic_usage.py +++ b/examples/basic_usage.py @@ -3,7 +3,7 @@ Prerequisites: - Docker services running: `make up` - - Package installed: `pip install -e ".[dev,processing]"` + - Package installed: `pip install -e ".[dev,materials,server]"` - .env configured with LLM credentials (see README.md) This example walks through the full pipeline: @@ -109,6 +109,7 @@ extraction_schema={ "catalysts": { "type": "list", + "description": "All catalyst materials studied, including composition and performance metrics.", "item_schema": { "name": {"type": "string", "required": True}, "composition": {"type": "string", "required": True}, @@ -120,6 +121,7 @@ }, "reactions": { "type": "list", + "description": "All chemical reactions described, with reactants, products, and conditions.", "item_schema": { "name": {"type": "string", "required": True}, "reactants": {"type": "list", "required": True}, @@ -130,10 +132,6 @@ }, }, system_prompt="Extract catalyst materials and reactions from this paper.", - field_descriptions={ - "catalysts": "All catalyst materials studied, including composition and performance metrics.", - "reactions": "All chemical reactions described, with reactants, products, and conditions.", - }, description="Heterogeneous catalysis data extraction", ) print(f"Space created: {space_result}") @@ -159,10 +157,10 @@ # Manually resolve feedback: # api.resolve_feedback(feedback_id="...", status="RESOLVED", notes="Fixed") -# ── 8. Streamlit UI ────────────────────────────────────────────── +# ── 8. React UI ────────────────────────────────────────────────── -# Launch the web interface: -# python -m mkb ui +# Launch the API and current React interface, then open http://127.0.0.1:5173: +# make dev # ── 9. Other queries ───────────────────────────────────────────── @@ -174,6 +172,6 @@ # for a in api.list_assets(project_id=project_id): # print(f" {a['asset_id']} {a['filename']} ({a['mime_type']})") -# ── 10. Reset (for development) ───────────────────────────────── -# Uncomment to wipe everything and start fresh: -# api.reset_db() +# ── 10. Reset (destructive development operation) ─────────────────────── +# The CLI requires an exact interactive confirmation before dropping all tables: +# .venv/bin/python -m mkb.cli reset-db diff --git a/examples/portable_quickstart.py b/examples/portable_quickstart.py new file mode 100644 index 0000000..cfb2992 --- /dev/null +++ b/examples/portable_quickstart.py @@ -0,0 +1,92 @@ +"""Public-API-only SQLite/filesystem example for an installed base wheel.""" + +from pathlib import Path +import tempfile + +from mkb import KnowledgeBase, Parser, Pipeline, Step + + +def main(root: Path | None = None) -> None: + workspace = root or Path(tempfile.mkdtemp(prefix="mkb-quickstart-")) + with KnowledgeBase.from_url( + database_url=f"sqlite:///{workspace / 'knowledge.db'}", + object_store_url=(workspace / "objects").resolve().as_uri(), + ) as kb: + kb.initialize() + collection = kb.collections.create(name="Portable example") + source = kb.sources.add_text(collection.id, "Calcite is CaCO3.") + file_path = workspace / "notes.txt" + file_path.write_text("A local file source.") + file_source = kb.sources.add_file(collection.id, file_path) + structured_source = kb.sources.add_records( + collection.id, + [{"material": "calcite", "formula": "CaCO3"}], + filename="materials.json", + ) + schema = kb.schemas.register( + name="material-record", + domain="example", + definition={"type": "object", "properties": {"formula": {"type": "string"}}}, + system_prompt="Normalize the material formula.", + ) + + parser = Parser( + name="lines", + source_types=frozenset({"text/plain"}), + handler=lambda _context, content: content.decode().splitlines(), + ) + kb.parsers.register(parser) + normalize = Step( + name="normalize", + deterministic=True, + handler=lambda _context, state: {"name": state["name"].strip().lower()}, + ) + def persist(context, state): + record = context.knowledge_base.records.create( + collection_id=collection.id, + data={"material": state["name"], "formula": "CaCO3"}, + ) + context.knowledge_base.evidence.create( + output_type="record", + output_id=record.id, + source_id=source.id, + excerpt="Calcite is CaCO3.", + ) + material = context.knowledge_base.graph.upsert_entity( + type="material", name="Calcite" + ) + formula = context.knowledge_base.graph.upsert_entity( + type="formula", name="CaCO3" + ) + context.knowledge_base.graph.upsert_relation( + material.id, formula.id, type="has_formula" + ) + return { + "label": f"material:{state['name']}", + "record_id": str(record.id), + "material_id": str(material.id), + "formula_id": str(formula.id), + } + + annotate = Step(name="persist", deterministic=False, handler=persist) + kb.steps.register(normalize) + kb.steps.register(annotate) + kb.pipelines.register(Pipeline(name="consumer-pipeline", steps=(normalize, annotate))) + run = kb.pipelines.run("consumer-pipeline", inputs={"name": " Calcite "}) + + assert run.outputs["label"] == "material:calcite" + assert kb.sources.read_bytes(source.id) == b"Calcite is CaCO3." + assert kb.sources.read_bytes(file_source.id) == b"A local file source." + assert kb.sources.require(structured_source.id).metadata["record_count"] == 1 + record = kb.records.require(run.outputs["record_id"]) + assert kb.records.require(record.id).data["formula"] == "CaCO3" + assert len(kb.evidence.list(output_id=record.id)) == 1 + assert kb.schemas.require(schema.id).name == "material-record" + assert str(kb.graph.neighbors(run.outputs["material_id"])[0].id) == run.outputs[ + "formula_id" + ] + print(f"portable quickstart passed in {workspace}") + + +if __name__ == "__main__": + main() diff --git a/examples/sdk_usage.py b/examples/sdk_usage.py new file mode 100644 index 0000000..f051038 --- /dev/null +++ b/examples/sdk_usage.py @@ -0,0 +1,75 @@ +"""Use the explicit SDK boundary with the existing local MKB data.""" + +from mkb import KnowledgeBase + + +def main() -> None: + # This reads the same .env/config.yaml and the same PostgreSQL/MinIO data as the + # current CLI and React application. It does not migrate or re-extract anything. + with KnowledgeBase.from_environment() as kb: + if kb.database is not None: + kb.database.check() + if kb.object_store is not None: + kb.object_store.check((kb.config.raw_bucket, kb.config.processed_bucket)) + + # New typed SDK path. These are the existing research_projects rows presented + # as generic collections through this client's injected database adapter. + collections = kb.collections.list(limit=10) if kb.collections else [] + if collections: + first = collections[0] + print(first.model_dump(mode="json")) + sources = ( + kb.sources.list(collection_id=first.id, limit=10) if kb.sources else [] + ) + if sources: + source = sources[0] + print({ + "source_id": str(source.id), + "source_content_exists": kb.sources.content_exists(source.id), + }) + artifacts = ( + kb.artifacts.list(source_id=source.id, limit=10) + if kb.artifacts + else [] + ) + if artifacts: + artifact = artifacts[0] + print({ + "artifact_id": str(artifact.id), + "artifact_content_exists": kb.artifacts.content_exists(artifact.id), + }) + + record = kb.records.get_for_collection(first.id) if kb.records else None + if record: + print({ + "record_id": str(record.id), + "record_status": record.status, + "record_json_bytes": len(record.model_dump_json().encode()), + }) + + projections = ( + kb.projections.list(collection_id=first.id, newest_only=True, limit=10) + if kb.projections + else [] + ) + if projections: + projection = projections[0] + schema = kb.schemas.get(projection.schema_id) if kb.schemas else None + print({ + "projection_id": str(projection.id), + "schema": schema.name if schema else str(projection.schema_id), + "projection_json_bytes": len(projection.model_dump_json().encode()), + }) + + projects = kb.list_projects(limit=10) + print(f"Found {len(projects)} project(s)") + if not projects: + return + + project_id = projects[0]["project_id"] + frame = kb.get_frame(project_id) + print({"project_id": project_id, "has_frame": frame is not None}) + + +if __name__ == "__main__": + main() diff --git a/examples/skills/sequence_normalizer/SKILL.md b/examples/skills/sequence_normalizer/SKILL.md new file mode 100644 index 0000000..8889757 --- /dev/null +++ b/examples/skills/sequence_normalizer/SKILL.md @@ -0,0 +1,9 @@ +# Sequence Normalizer + +Normalizes amino-acid sequence strings in biomineralization projection rows. + +Run `sequence_normalizer.py` as a post-processor script. It accepts the standard +post-processor JSON payload on stdin and prints one JSON result. It processes the +selected live projection, writes `normalized_sequence` and +`sequence_normalization` fields for successful normalizations, and asks the +reviewer agent to resolve uncertain sequences. \ No newline at end of file diff --git a/examples/skills/sequence_normalizer/sequence_normalizer.py b/examples/skills/sequence_normalizer/sequence_normalizer.py new file mode 100644 index 0000000..2949c9a --- /dev/null +++ b/examples/skills/sequence_normalizer/sequence_normalizer.py @@ -0,0 +1,185 @@ +"""Script-first amino-acid sequence normalization post-processor.""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass, field +import json +import re +import sys + + +CANONICAL = set("ACDEFGHIKLMNPQRSTVWY") +AA3 = { + "ALA": "A", "ARG": "R", "ASN": "N", "ASP": "D", "CYS": "C", + "GLN": "Q", "GLU": "E", "GLY": "G", "HIS": "H", "ILE": "I", + "LEU": "L", "LYS": "K", "MET": "M", "PHE": "F", "PRO": "P", + "SER": "S", "THR": "T", "TRP": "W", "TYR": "Y", "VAL": "V", +} +AA_FULL = { + "ALANINE": "A", "ARGININE": "R", "ASPARAGINE": "N", "ASPARTATE": "D", + "ASPARTICACID": "D", "CYSTEINE": "C", "GLUTAMINE": "Q", "GLUTAMATE": "E", + "GLUTAMICACID": "E", "GLYCINE": "G", "HISTIDINE": "H", "ISOLEUCINE": "I", + "LEUCINE": "L", "LYSINE": "K", "METHIONINE": "M", "PHENYLALANINE": "F", + "PROLINE": "P", "SERINE": "S", "THREONINE": "T", "TRYPTOPHAN": "W", + "TYROSINE": "Y", "VALINE": "V", +} +PTMS = { + "S": r"\[pS\]|\(pS\)|pSer|phosphoserine|pS", + "T": r"\[pT\]|\(pT\)|pThr|phosphothreonine|pT", + "Y": r"\[pY\]|\(pY\)|pTyr|phosphotyrosine|pY", +} +N_TERMINI = [("Ace-", "Acetylation"), ("Ac-", "Acetylation"), ("Acetyl-", "Acetylation"), ("NH2-", "Free amine")] +C_TERMINI = [("-CONH2", "Amidation"), ("-NH2", "Amidation"), ("-Amide", "Amidation")] + + +@dataclass +class NormalizeResult: + normalized_sequence: str + need_normalization: bool + success: bool + modifications: list = field(default_factory=list) + terminal_modifications: dict = field(default_factory=lambda: {"n_terminal": [], "c_terminal": []}) + removed_groups: list = field(default_factory=list) + removed_noncanonical_residues: list = field(default_factory=list) + unknown_tokens: list = field(default_factory=list) + warnings: list = field(default_factory=list) + confidence: float = 1.0 + + +def _expand_repeats(text: str, result: NormalizeResult) -> str: + stack = [("", None)] + last_closed = "" + index = 0 + while index < len(text): + char = text[index] + if char in "([": + stack.append(("", char)) + index += 1 + continue + if char in ")]": + close = ")" if char == ")" else "]" + opener = "(" if close == ")" else "[" + end = index + 1 + while end < len(text) and text[end].isdigit(): + end += 1 + count_text = text[index + 1:end] + count = int(count_text) if count_text else 1 + if len(stack) > 1 and stack[-1][1] == opener: + content, _ = stack.pop() + last_closed = content * count + stack[-1] = (stack[-1][0] + last_closed, stack[-1][1]) + elif count_text and last_closed: + stack[-1] = (stack[-1][0] + last_closed * count, stack[-1][1]) + result.warnings.append("Unmatched closing repeat bracket interpreted as implicit outer repeat.") + else: + stack[-1] = (stack[-1][0] + text[index:end], stack[-1][1]) + result.warnings.append("Unmatched closing bracket could not be expanded.") + index = end + continue + stack[-1] = (stack[-1][0] + char, stack[-1][1]) + index += 1 + if len(stack) > 1: + result.warnings.append("Unclosed repeat brackets detected; content was kept literally.") + return "".join(buffer for buffer, _ in stack) + + +def normalize_sequence(sequence: str) -> NormalizeResult: + raw = str(sequence or "").strip() + if re.fullmatch(r"[ACDEFGHIKLMNPQRSTVWY]+", raw): + return NormalizeResult(raw, False, True) + result = NormalizeResult("", True, True) + text = raw + for token, label in N_TERMINI: + if text.lower().startswith(token.lower()): + result.terminal_modifications["n_terminal"].append(label) + text = text[len(token):] + break + for token, label in C_TERMINI: + if text.lower().endswith(token.lower()): + result.terminal_modifications["c_terminal"].append(label) + text = text[:-len(token)] + break + text = text.replace("-", " ") + for match in reversed(list(re.finditer(r"\(([^()]*)\)", text))): + words = re.findall(r"[A-Za-z]+", match.group(1)) + if len(words) >= 3 and sum(any(char.islower() for char in word) for word in words) >= 3: + result.removed_groups.append({"type": "annotation", "text": match.group(1)}) + text = text[:match.start()] + text[match.end():] + text = _expand_repeats(text, result) + ptm_pattern = "|".join(f"(?:{pattern})" for pattern in PTMS.values()) + pieces = [] + for fragment in re.split(f"({ptm_pattern})", text, flags=re.I): + if not fragment: + continue + if re.fullmatch(ptm_pattern, fragment, flags=re.I): + pieces.append(fragment) + else: + pieces.extend(re.findall(r"[A-Za-z0-9]+|[\[\]\(\)]", fragment)) + output = [] + for token in pieces: + for residue, pattern in PTMS.items(): + if re.fullmatch(pattern, token, flags=re.I): + output.append(residue) + result.modifications.append({"position": len(output), "residue": residue, "type": "phosphorylation", "original": token}) + break + else: + upper = token.upper() + if upper in AA3: + output.append(AA3[upper]) + elif upper in AA_FULL: + output.append(AA_FULL[upper]) + elif token == upper and re.fullmatch(r"[ACDEFGHIKLMNPQRSTVWY]+", token): + output.extend(token) + elif re.fullmatch(r"[A-Za-z]+", token): + result.removed_groups.append({"type": "annotation", "token": token}) + else: + result.unknown_tokens.append(token) + result.normalized_sequence = "".join(output) + if result.unknown_tokens: + result.success, result.confidence = False, 0.5 + elif result.removed_groups: + result.success, result.confidence = False, 0.6 + result.warnings.append("Removed annotation text may contain sequence-relevant information; manual review required.") + return result + + +def main() -> None: + payload = json.load(sys.stdin) + projections = payload.get("projections") or [] + if not projections: + print(json.dumps({"run_agent": False, "context": {"message": "No projections to normalize."}})) + return + projection = projections[0] + data = projection.get("data") or {} + updates, reports, failures = [], [], [] + for collection, source_field in (("templates", "sequence"), ("functional_modules", "amino_acid_sequence")): + for index, row in enumerate(data.get(collection) or []): + raw = row.get(source_field) if isinstance(row, dict) else None + if not isinstance(raw, str) or not raw.strip(): + continue + result = normalize_sequence(raw) + report = asdict(result) + base = f"{collection}[{index}]" + updates.append({"path": f"{base}.sequence_normalization", "value": report}) + if result.success: + updates.append({"path": f"{base}.normalized_sequence", "value": result.normalized_sequence}) + entry = {"path": base, "source_field": source_field, "original_sequence": raw, "result": report} + reports.append(entry) + if not result.success: + failures.append(entry) + patch = None + if updates: + patch = { + "winning_projection_id": projection["projection_id"], + "updates": updates, + "review_notes": f"Sequence normalizer processed {len(reports)} sequence field(s); {len(failures)} require review.", + } + print(json.dumps({ + "run_agent": bool(failures), + "patch": patch, + "context": {"kind": "sequence_normalization", "processed": reports, "failed": failures}, + })) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/examples/spaces/biomineralization_templates.json b/examples/spaces/biomineralization_templates.json index 9ba59f5..eb1f77f 100644 --- a/examples/spaces/biomineralization_templates.json +++ b/examples/spaces/biomineralization_templates.json @@ -5,80 +5,293 @@ "extraction_schema": { "templates": { "type": "list", - "description": "Biomineralization template entries — proteins, peptides, or other molecules that directly participate in or regulate mineralization.", - "filter": {"field": "experimental_role", "equals": "primary_template"}, + "description": "Biomineralization template entries — proteins, peptides, or other molecules that directly participate in or regulate mineralization.\n\nExtraction guidance: Extract only biomineralization-relevant template molecules that meet the inclusion criteria. Exclude controls, comparison proteins, and background references unless the paper directly demonstrates a mineralization role. Preserve all reported species and functional roles as lists. Example: Amelogenin from Homo sapiens and Mus musculus enamel studies, a full-length protein acting on calcium phosphate with roles in nucleation promotion and crystal orientation control.", + "filter": { + "field": "experimental_role", + "equals": "primary_template" + }, "item_schema": { - "template_name": {"type": "string", "required": true, "description": "Name of the template molecule (e.g., Amelogenin, Osteopontin)"}, - "source_species": {"type": "list", "item_type": "string", "required": true, "description": "All species involved in the experiments or molecular origin (e.g., Homo sapiens, Mus musculus, Pinctada fucata)"}, - "source_system": {"type": "string", "required": true, "description": "Biological system where the template operates (e.g., Enamel matrix, Nacre layer, Bone matrix)"}, - "molecule_type": {"type": "string", "required": true, "description": "Type of molecule (e.g., Full-length protein, Matrix protein, Peptide, Glycoprotein)"}, - "template_role": {"type": "string", "required": false, "description": "Role/class of the tested molecule: native_protein, derived_peptide, synthetic_peptide, or mutant"}, - "experimental_role": {"type": "string", "required": false, "description": "Role in the experiment: primary_template, control, comparison, or background_reference"}, - "mineralization_system": {"type": "string", "required": true, "description": "The mineral system influenced (e.g., Calcium phosphate, Calcium carbonate, Silica)"}, - "functional_tags": {"type": "list", "item_type": "string", "required": true, "description": "List of functional roles (e.g., Nucleation promotion, Crystal orientation control, Crystal growth regulation, Ion enrichment)"}, - "sequence": {"type": "string", "required": false, "description": "Amino acid sequence if reported"}, - "molecular_weight_kda": {"type": "number", "required": false, "description": "Molecular weight in kDa if reported"}, - "isoelectric_point": {"type": "number", "required": false, "description": "pI value if reported"}, - "evidence_level": {"type": "integer", "required": true, "description": "Highest applicable evidence level: 1 = in vivo functional validation, 2 = in vitro mineralization experiment, 3 = indirect experimental evidence, 4 = prediction/hypothesis/inference"}, - "references": {"type": "string", "required": false, "description": "PMID, DOI, or other literature identifier"}, - "notes": {"type": "string", "required": false, "description": "Additional notes or observations"} + "template_name": { + "type": "string", + "required": true, + "description": "Name of the template molecule (e.g., Amelogenin, Osteopontin)" + }, + "source_species": { + "type": "list", + "item_type": "string", + "required": true, + "description": "All species involved in the experiments or molecular origin (e.g., Homo sapiens, Mus musculus, Pinctada fucata)" + }, + "source_system": { + "type": "string", + "required": true, + "description": "Biological system where the template operates (e.g., Enamel matrix, Nacre layer, Bone matrix)" + }, + "molecule_type": { + "type": "string", + "required": true, + "description": "Type of molecule (e.g., Full-length protein, Matrix protein, Peptide, Glycoprotein)" + }, + "template_role": { + "type": "string", + "required": false, + "description": "Role/class of the tested molecule: native_protein, derived_peptide, synthetic_peptide, or mutant" + }, + "experimental_role": { + "type": "string", + "required": false, + "description": "Role in the experiment: primary_template, control, comparison, or background_reference" + }, + "mineralization_system": { + "type": "string", + "required": true, + "description": "The mineral system influenced (e.g., Calcium phosphate, Calcium carbonate, Silica)" + }, + "functional_tags": { + "type": "list", + "item_type": "string", + "required": true, + "description": "List of functional roles (e.g., Nucleation promotion, Crystal orientation control, Crystal growth regulation, Ion enrichment)" + }, + "sequence": { + "type": "string", + "required": false, + "description": "Amino acid sequence if reported" + }, + "normalized_sequence": { + "type": "string", + "required": false, + "description": "Canonical one-letter amino-acid sequence produced by the seq_norm post-processor. Preserve the reported sequence in sequence." + }, + "sequence_normalization": { + "type": "object", + "required": false, + "description": "Structured seq_norm result: original sequence, candidate normalized sequence, success flag, confidence, modifications, warnings, and unresolved tokens." + }, + "molecular_weight_kda": { + "type": "number", + "required": false, + "description": "Molecular weight in kDa if reported" + }, + "isoelectric_point": { + "type": "number", + "required": false, + "description": "pI value if reported" + }, + "evidence_level": { + "type": "integer", + "required": true, + "description": "Highest applicable evidence level: 1 = in vivo functional validation, 2 = in vitro mineralization experiment, 3 = indirect experimental evidence, 4 = prediction/hypothesis/inference" + }, + "references": { + "type": "string", + "required": false, + "description": "PMID, DOI, or other literature identifier" + }, + "notes": { + "type": "string", + "required": false, + "description": "Additional notes or observations" + } } }, "functional_modules": { "type": "list", - "description": "Functional fragments or domains within biomineralization templates that have specific mineralization-related activity.", + "description": "Functional fragments or domains within biomineralization templates that have specific mineralization-related activity.\n\nExtraction guidance: Extract specific fragments, domains, or motifs within templates that have demonstrated or predicted mineralization activity. Include position, sequence (if available), structural features, and functional role. Example: N-terminal acidic fragment of Amelogenin (positions 1-25), Ser-rich/acidic, promotes nucleation.", "item_schema": { - "parent_protein": {"type": "string", "required": true, "description": "Name of the parent protein/template"}, - "fragment_name": {"type": "string", "required": true, "description": "Name or descriptor of the fragment (e.g., N-terminal acidic fragment, Asp-rich motif)"}, - "start_end_position": {"type": "string", "required": false, "description": "Amino acid position range (e.g., 1-25, 120-145)"}, - "amino_acid_sequence": {"type": "string", "required": false, "description": "Sequence of the fragment if reported"}, - "length": {"type": "string", "required": false, "description": "Length with unit (e.g., 25 aa, 26 aa)"}, - "key_features": {"type": "string", "required": true, "description": "Structural or compositional features (e.g., Ser-rich / acidic, Asp-rich, Ca2+-binding motif)"}, - "functional_tag": {"type": "string", "required": true, "description": "Primary functional role (e.g., Nucleation promotion, Ion enrichment, Crystal face binding)"}, - "mineralization_effect": {"type": "string", "required": false, "description": "Specific effect on mineralization (e.g., promotes hydroxyapatite nucleation, inhibits calcite growth)"}, - "evidence_level": {"type": "integer", "required": true, "description": "Evidence level 1-4"}, - "references": {"type": "string", "required": false, "description": "PMID or DOI references"}, - "notes": {"type": "string", "required": false, "description": "Additional notes"} + "parent_protein": { + "type": "string", + "required": true, + "description": "Name of the parent protein/template" + }, + "fragment_name": { + "type": "string", + "required": true, + "description": "Name or descriptor of the fragment (e.g., N-terminal acidic fragment, Asp-rich motif)" + }, + "start_end_position": { + "type": "string", + "required": false, + "description": "Amino acid position range (e.g., 1-25, 120-145)" + }, + "amino_acid_sequence": { + "type": "string", + "required": false, + "description": "Sequence of the fragment if reported" + }, + "normalized_sequence": { + "type": "string", + "required": false, + "description": "Canonical one-letter amino-acid sequence produced by the seq_norm post-processor. Preserve the reported sequence in amino_acid_sequence." + }, + "sequence_normalization": { + "type": "object", + "required": false, + "description": "Structured seq_norm result: original sequence, candidate normalized sequence, success flag, confidence, modifications, warnings, and unresolved tokens." + }, + "length": { + "type": "string", + "required": false, + "description": "Length with unit (e.g., 25 aa, 26 aa)" + }, + "key_features": { + "type": "string", + "required": true, + "description": "Structural or compositional features (e.g., Ser-rich / acidic, Asp-rich, Ca2+-binding motif)" + }, + "functional_tag": { + "type": "string", + "required": true, + "description": "Primary functional role (e.g., Nucleation promotion, Ion enrichment, Crystal face binding)" + }, + "mineralization_effect": { + "type": "string", + "required": false, + "description": "Specific effect on mineralization (e.g., promotes hydroxyapatite nucleation, inhibits calcite growth)" + }, + "evidence_level": { + "type": "integer", + "required": true, + "description": "Evidence level 1-4" + }, + "references": { + "type": "string", + "required": false, + "description": "PMID or DOI references" + }, + "notes": { + "type": "string", + "required": false, + "description": "Additional notes" + } } }, "mineralization_conditions": { "type": "list", - "description": "Experimental conditions under which biomineralization templates were studied.", + "description": "Experimental conditions under which biomineralization templates were studied.\n\nExtraction guidance: Extract experimental conditions for mineralization studies: pH, temperature, ion concentrations, template concentrations, incubation times. Include resulting crystal phase, morphology, and characterization methods used.", "item_schema": { - "template_name": {"type": "string", "required": true, "description": "Template being studied"}, - "mineral_phase": {"type": "string", "required": true, "description": "Mineral phase formed (e.g., hydroxyapatite, calcite, aragonite, vaterite)"}, - "ph": {"type": "number", "required": false, "description": "pH of the mineralization solution"}, - "temperature_c": {"type": "number", "required": false, "description": "Temperature in Celsius"}, - "calcium_concentration_mm": {"type": "number", "required": false, "description": "Ca2+ concentration in mM"}, - "phosphate_concentration_mm": {"type": "number", "required": false, "description": "Phosphate concentration in mM (for calcium phosphate systems)"}, - "carbonate_concentration_mm": {"type": "number", "required": false, "description": "Carbonate concentration in mM (for calcium carbonate systems)"}, - "template_concentration": {"type": "string", "required": false, "description": "Template concentration with units"}, - "incubation_time": {"type": "string", "required": false, "description": "Duration of mineralization experiment"}, - "crystal_morphology": {"type": "string", "required": false, "description": "Observed crystal morphology"}, - "crystal_size": {"type": "string", "required": false, "description": "Crystal size with units"}, - "characterization_methods": {"type": "list", "item_type": "string", "required": false, "description": "Methods used (e.g., XRD, TEM, SEM, FTIR, AFM)"}, - "evidence_level": {"type": "integer", "required": true, "description": "Evidence level 1-4"}, - "notes": {"type": "string", "required": false, "description": "Additional notes"} + "template_name": { + "type": "string", + "required": true, + "description": "Template being studied" + }, + "mineral_phase": { + "type": "string", + "required": true, + "description": "Mineral phase formed (e.g., hydroxyapatite, calcite, aragonite, vaterite)" + }, + "ph": { + "type": "number", + "required": false, + "description": "pH of the mineralization solution" + }, + "temperature_c": { + "type": "number", + "required": false, + "description": "Temperature in Celsius" + }, + "calcium_concentration_mm": { + "type": "number", + "required": false, + "description": "Ca2+ concentration in mM" + }, + "phosphate_concentration_mm": { + "type": "number", + "required": false, + "description": "Phosphate concentration in mM (for calcium phosphate systems)" + }, + "carbonate_concentration_mm": { + "type": "number", + "required": false, + "description": "Carbonate concentration in mM (for calcium carbonate systems)" + }, + "template_concentration": { + "type": "string", + "required": false, + "description": "Template concentration with units" + }, + "incubation_time": { + "type": "string", + "required": false, + "description": "Duration of mineralization experiment" + }, + "crystal_morphology": { + "type": "string", + "required": false, + "description": "Observed crystal morphology" + }, + "crystal_size": { + "type": "string", + "required": false, + "description": "Crystal size with units" + }, + "characterization_methods": { + "type": "list", + "item_type": "string", + "required": false, + "description": "Methods used (e.g., XRD, TEM, SEM, FTIR, AFM)" + }, + "evidence_level": { + "type": "integer", + "required": true, + "description": "Evidence level 1-4" + }, + "notes": { + "type": "string", + "required": false, + "description": "Additional notes" + } } }, "structure_activity_relationships": { "type": "list", - "description": "Relationships between template structure and mineralization activity.", + "description": "Relationships between template structure and mineralization activity.\n\nExtraction guidance: Extract relationships between structural features of templates and their effects on mineralization. Include the structural feature, the functional effect, and any proposed mechanism.", "item_schema": { - "template_name": {"type": "string", "required": true}, - "structural_feature": {"type": "string", "required": true, "description": "Structural feature (e.g., beta-sheet content, acidic residue density, self-assembly)"}, - "functional_effect": {"type": "string", "required": true, "description": "Effect on mineralization (e.g., promotes oriented nucleation, controls crystal polymorph)"}, - "mechanism": {"type": "string", "required": false, "description": "Proposed mechanism if discussed"}, - "evidence_level": {"type": "integer", "required": true}, - "notes": {"type": "string", "required": false} + "template_name": { + "type": "string", + "required": true + }, + "structural_feature": { + "type": "string", + "required": true, + "description": "Structural feature (e.g., beta-sheet content, acidic residue density, self-assembly)" + }, + "functional_effect": { + "type": "string", + "required": true, + "description": "Effect on mineralization (e.g., promotes oriented nucleation, controls crystal polymorph)" + }, + "mechanism": { + "type": "string", + "required": false, + "description": "Proposed mechanism if discussed" + }, + "evidence_level": { + "type": "integer", + "required": true + }, + "notes": { + "type": "string", + "required": false + } } } }, "system_prompt": "You are extracting structured data for a High-Activity Biomineralization Template Database. Focus on identifying biomineralization templates (proteins, peptides, and other molecules that directly participate in or regulate biomineralization processes), their functional modules (active fragments/domains), experimental mineralization conditions, and structure-activity relationships.\n\nInclusion criteria for template entries:\n- the molecule is the primary research object of the study, or\n- the paper demonstrates that it regulates mineralization, or\n- the authors describe it as a biomineralization template or matrix molecule.\n\nExclusion criteria for template entries:\n- negative controls\n- comparison proteins used only for benchmarking\n- background examples mentioned from prior literature\n- standard reference proteins without direct mineralization evidence in the paper.\n\nPay special attention to:\n- Template identity: exact protein/peptide names, all species involved, and biological system\n- Functional roles: nucleation promotion, crystal orientation control, crystal growth regulation, ion enrichment, polymorph selection\n- Active fragments: specific domains or motifs with mineralization activity, their sequences and positions\n- Mineralization conditions: pH, temperature, ion concentrations, incubation times\n- Structure-activity links: how structural features relate to mineralization function\n- Role classification: use template_role to distinguish native proteins, derived peptides, synthetic peptides, and mutants; use experimental_role to separate primary templates from controls or comparisons.\n\nFor each entry, assign the highest applicable evidence level reported in the paper:\n- Level 1: in vivo validation with functional evidence\n- Level 2: in vitro mineralization experiments\n- Level 3: indirect experimental evidence\n- Level 4: prediction, hypothesis, or inference\n\nKeep multi-value fields such as source_species and functional_tags as JSON lists internally.", - "field_descriptions": { - "templates": "Extract only biomineralization-relevant template molecules that meet the inclusion criteria. Exclude controls, comparison proteins, and background references unless the paper directly demonstrates a mineralization role. Preserve all reported species and functional roles as lists. Example: Amelogenin from Homo sapiens and Mus musculus enamel studies, a full-length protein acting on calcium phosphate with roles in nucleation promotion and crystal orientation control.", - "functional_modules": "Extract specific fragments, domains, or motifs within templates that have demonstrated or predicted mineralization activity. Include position, sequence (if available), structural features, and functional role. Example: N-terminal acidic fragment of Amelogenin (positions 1-25), Ser-rich/acidic, promotes nucleation.", - "mineralization_conditions": "Extract experimental conditions for mineralization studies: pH, temperature, ion concentrations, template concentrations, incubation times. Include resulting crystal phase, morphology, and characterization methods used.", - "structure_activity_relationships": "Extract relationships between structural features of templates and their effects on mineralization. Include the structural feature, the functional effect, and any proposed mechanism." - } + "post_processors": [ + { + "id": "seq_norm", + "name": "Sequence normalization", + "description": "Runs a deterministic amino-acid sequence normalizer first. Only uncertain sequences are sent to the reviewer.", + "prompt": "You are resolving only the sequence records listed in the seq_norm script context. The script has already saved its structured diagnostic output and every confident normalized_sequence value. Do not alter raw sequence or amino_acid_sequence fields. For each failed record, use the raw sequence and the script report as a starting point, verify against the paper when needed, then write normalized_sequence and the full sequence_normalization object in the same row. sequence_normalization must record the resolved normalized_sequence, success, confidence, modifications including PTMs and terminal modifications when supported, warnings, and unresolved tokens. Never create or write processed_sequence. Preserve successful script results unless source evidence proves them wrong. Do not infer residues hidden by prose annotations, partial-sequence statements, unknown tokens, or unsupported modifications. If the source cannot resolve a failed sequence, leave normalized_sequence empty and record success=false plus a concise warning in sequence_normalization. Use save_reviewed_projection_patch for these field-level changes and confirm the saved changed paths.", + "tool_groups": ["reading"], + "skill_ids": [], + "script": null, + "output_columns": [ + {"name": "normalized_sequence", "description": "Canonical one-letter amino-acid sequence."}, + {"name": "sequence_normalization", "description": "Structured normalization result and diagnostics."} + ], + "enabled": true + } + ] } diff --git a/examples/spaces/computational_materials_qa.json b/examples/spaces/computational_materials_qa.json index 41cf038..a7651a2 100644 --- a/examples/spaces/computational_materials_qa.json +++ b/examples/spaces/computational_materials_qa.json @@ -6,7 +6,7 @@ "extraction_schema": { "questions": { "type": "list", - "description": "Self-contained agent benchmark tasks. Each item must be runnable in isolation — never cross-reference other items in this list. One projected `questions[i]` corresponds to exactly one task YAML in mat_agent_bench's question_bank.", + "description": "Self-contained agent benchmark tasks. Each item must be runnable in isolation — never cross-reference other items in this list. One projected `questions[i]` corresponds to exactly one task YAML in mat_agent_bench's question_bank.\n\nExtraction guidance: Extract one self-contained agent task per concrete computational workflow described in the paper. Do not bundle multiple unrelated workflows into a single item, and do not split one workflow across items. Each `questions[i]` must round-trip cleanly to a standalone mat_agent_bench YAML file under `question_bank//.yaml`. Keep the list short and high-signal: 1-5 items per paper is typical; emit `[]` if nothing in the frame is benchmark-worthy.", "item_schema": { "id": { "type": "string", @@ -62,8 +62,5 @@ } } }, - "system_prompt": "You are projecting research-paper knowledge frames into agent-task benchmark items in the mat_agent_bench format (https://github.com/ruoyuwang1995nya/mat_agent_bench). Each item in `questions` must be a self-contained task that an autonomous coding agent could run end-to-end without seeing any sibling task.\n\nHard rules for isolation:\n1. NEVER reference another `questions[i]` (no 'as in question X', no shared state).\n2. Every file the agent needs must appear in this item's own `data_files`. If two tasks happen to consume the same physical file, copy the entry — do not share by reference.\n3. Reference answers must be verifiable from ONLY this task's deliverables. A grader looking at one YAML in isolation must be able to score it.\n4. `id` must be globally unique. Generate one using the convention `___` where CAP is the capability prefix (IG/SR/SC/WF/BP/DD/EC/SA/SF), short_domain is a slug like `abacus` or `vasp`, NNN is a 3-digit counter scoped to (capability, domain), and the date is today.\n\nWhat to extract:\n- Walk the knowledge frame and identify concrete, reproducible computational workflows the paper describes (input deck generation, structure construction, post-processing, etc.). Skip narrative-only passages.\n- For each candidate workflow, formulate ONE focused task that exercises a single capability. Prefer narrow, testable prompts (e.g. 'generate an ABACUS SCF INPUT with dipole correction for the provided slab') over broad ones ('reproduce the whole paper').\n- Map the workflow to the closest `capability` value. If unsure between two, choose the more specific one and add the other as a tag.\n- Write `human_prompt_seed` as if you were the end-user assigning the task. Be explicit about deliverable filenames and the working directory. Multilingual prompts are allowed; match the source paper's language when natural.\n- Populate `data_files` with everything the agent will need at runtime. Use stable filenames (snake_case). Leave `oss_url` as an empty string — uploads happen at bank-registration time.\n- Build `reference_answers` so each entry is independently checkable. Prefer cheap verifiers (text_file_contains_all, text_file_regex, artifact_exists, numeric ranges) over llm_binary_judge. Always include at least one `artifact_exists` entry per deliverable file.\n- Build `scoring_checklist` to mirror `reference_answers`: every non-budget reference key gets a checklist entry whose `id` matches. Prefix the `criterion` text with `[Must]` (strict requirement), `[Suggested]` (benchmark-tuned numeric range), or `[Variable]` (acceptable-but-optional knob). Always add the four efficiency items at the end: `turn_budget`, `no_retries`, `duration_budget`, `token_budget_total` (with matching budget entries in reference_answers).\n- Include a `grounding_source` reference_answer pointing at the seed data file plus an `llm_binary_judge` checklist item that verifies the answer is grounded in the source paper.\n- Record the paper-side justification in `source_evidence` (one short quote or `section: ...` pointer) so reviewers can trust the task.\n\nNumeric-range guidance for `text_file_numeric_range`:\n- Use `min`/`max` for tunable knobs (cutoffs, thresholds) with realistic ranges drawn from the paper or the engine's defaults.\n- Use `expected` + `tolerance: 0` for hard-required integer flags.\n- Set `allow_missing_key: true` on knobs where omission is acceptable.\n\nIf the frame does not contain enough material for a high-quality benchmark item, emit zero items rather than fabricating one — the projection may legitimately return an empty `questions: []` for review-only papers.", - "field_descriptions": { - "questions": "Extract one self-contained agent task per concrete computational workflow described in the paper. Do not bundle multiple unrelated workflows into a single item, and do not split one workflow across items. Each `questions[i]` must round-trip cleanly to a standalone mat_agent_bench YAML file under `question_bank//.yaml`. Keep the list short and high-signal: 1-5 items per paper is typical; emit `[]` if nothing in the frame is benchmark-worthy." - } + "system_prompt": "You are projecting research-paper knowledge frames into agent-task benchmark items in the mat_agent_bench format (https://github.com/ruoyuwang1995nya/mat_agent_bench). Each item in `questions` must be a self-contained task that an autonomous coding agent could run end-to-end without seeing any sibling task.\n\nHard rules for isolation:\n1. NEVER reference another `questions[i]` (no 'as in question X', no shared state).\n2. Every file the agent needs must appear in this item's own `data_files`. If two tasks happen to consume the same physical file, copy the entry — do not share by reference.\n3. Reference answers must be verifiable from ONLY this task's deliverables. A grader looking at one YAML in isolation must be able to score it.\n4. `id` must be globally unique. Generate one using the convention `___` where CAP is the capability prefix (IG/SR/SC/WF/BP/DD/EC/SA/SF), short_domain is a slug like `abacus` or `vasp`, NNN is a 3-digit counter scoped to (capability, domain), and the date is today.\n\nWhat to extract:\n- Walk the knowledge frame and identify concrete, reproducible computational workflows the paper describes (input deck generation, structure construction, post-processing, etc.). Skip narrative-only passages.\n- For each candidate workflow, formulate ONE focused task that exercises a single capability. Prefer narrow, testable prompts (e.g. 'generate an ABACUS SCF INPUT with dipole correction for the provided slab') over broad ones ('reproduce the whole paper').\n- Map the workflow to the closest `capability` value. If unsure between two, choose the more specific one and add the other as a tag.\n- Write `human_prompt_seed` as if you were the end-user assigning the task. Be explicit about deliverable filenames and the working directory. Multilingual prompts are allowed; match the source paper's language when natural.\n- Populate `data_files` with everything the agent will need at runtime. Use stable filenames (snake_case). Leave `oss_url` as an empty string — uploads happen at bank-registration time.\n- Build `reference_answers` so each entry is independently checkable. Prefer cheap verifiers (text_file_contains_all, text_file_regex, artifact_exists, numeric ranges) over llm_binary_judge. Always include at least one `artifact_exists` entry per deliverable file.\n- Build `scoring_checklist` to mirror `reference_answers`: every non-budget reference key gets a checklist entry whose `id` matches. Prefix the `criterion` text with `[Must]` (strict requirement), `[Suggested]` (benchmark-tuned numeric range), or `[Variable]` (acceptable-but-optional knob). Always add the four efficiency items at the end: `turn_budget`, `no_retries`, `duration_budget`, `token_budget_total` (with matching budget entries in reference_answers).\n- Include a `grounding_source` reference_answer pointing at the seed data file plus an `llm_binary_judge` checklist item that verifies the answer is grounded in the source paper.\n- Record the paper-side justification in `source_evidence` (one short quote or `section: ...` pointer) so reviewers can trust the task.\n\nNumeric-range guidance for `text_file_numeric_range`:\n- Use `min`/`max` for tunable knobs (cutoffs, thresholds) with realistic ranges drawn from the paper or the engine's defaults.\n- Use `expected` + `tolerance: 0` for hard-required integer flags.\n- Set `allow_missing_key: true` on knobs where omission is acceptable.\n\nIf the frame does not contain enough material for a high-quality benchmark item, emit zero items rather than fabricating one — the projection may legitimately return an empty `questions: []` for review-only papers." } diff --git a/frontend/package-lock.json b/frontend/package-lock.json index a330b99..044d9ec 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -8,6 +8,7 @@ "name": "mkb-frontend", "version": "0.1.0", "dependencies": { + "@dagrejs/dagre": "^3.0.0", "@tanstack/react-table": "^8.21.3", "axios": "^1.15.2", "pdfjs-dist": "^6.0.227", @@ -28,6 +29,21 @@ "vite": "^8.0.10" } }, + "node_modules/@dagrejs/dagre": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@dagrejs/dagre/-/dagre-3.0.0.tgz", + "integrity": "sha512-ZzhnTy1rfuoew9Ez3EIw4L2znPGnYYhfn8vc9c4oB8iw6QAsszbiU0vRhlxWPFnmmNSFAkrYeF1PhM5m4lAN0Q==", + "license": "MIT", + "dependencies": { + "@dagrejs/graphlib": "4.0.1" + } + }, + "node_modules/@dagrejs/graphlib": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@dagrejs/graphlib/-/graphlib-4.0.1.tgz", + "integrity": "sha512-IvcV6FduIIAmLwnH+yun+QtV36SC7mERqa86aClNqmMN09WhmPPYU8ckHrZBozErf+UvHPWOTJYaGYiIcs0DgA==", + "license": "MIT" + }, "node_modules/@egjs/hammerjs": { "version": "2.0.17", "resolved": "https://registry.npmjs.org/@egjs/hammerjs/-/hammerjs-2.0.17.tgz", diff --git a/frontend/package.json b/frontend/package.json index d8c26ca..aee64d9 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -5,10 +5,13 @@ "type": "module", "scripts": { "dev": "vite", - "build": "tsc -b && vite build", + "lint": "tsc -b --noEmit", + "build": "tsc -b && vite build && npm run check:bundle", + "check:bundle": "node scripts/check-bundle-budgets.mjs", "preview": "vite preview" }, "dependencies": { + "@dagrejs/dagre": "^3.0.0", "@tanstack/react-table": "^8.21.3", "axios": "^1.15.2", "pdfjs-dist": "^6.0.227", diff --git a/frontend/scripts/check-bundle-budgets.mjs b/frontend/scripts/check-bundle-budgets.mjs new file mode 100644 index 0000000..7cbc771 --- /dev/null +++ b/frontend/scripts/check-bundle-budgets.mjs @@ -0,0 +1,28 @@ +import { gzipSync } from 'node:zlib' +import { readdirSync, readFileSync } from 'node:fs' +import { join } from 'node:path' + +const assetsDir = new URL('../dist/assets/', import.meta.url) +const budgets = { + chunkGzipBytes: 180 * 1024, + pdfWorkerGzipBytes: 450 * 1024, +} + +const failures = [] +for (const name of readdirSync(assetsDir)) { + if (!/\.(js|mjs)$/.test(name)) continue + const compressedBytes = gzipSync(readFileSync(join(assetsDir.pathname, name))).byteLength + const limit = name.startsWith('pdf.worker.') + ? budgets.pdfWorkerGzipBytes + : budgets.chunkGzipBytes + if (compressedBytes > limit) { + failures.push(`${name}: ${(compressedBytes / 1024).toFixed(1)} KiB gzip > ${(limit / 1024).toFixed(0)} KiB`) + } +} + +if (failures.length) { + console.error('Bundle budget exceeded:\n' + failures.map(item => `- ${item}`).join('\n')) + process.exit(1) +} + +console.log('Bundle budgets passed (180 KiB gzip per chunk; 450 KiB for the lazy PDF worker).') diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 4bb17e0..9952efe 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -1,13 +1,16 @@ +import { Suspense, lazy } from 'react' import Layout from './components/Layout' import { useUiStore } from './store/uiStore' -import AssistantPage from './pages/AssistantPage' -import ProjectsPage from './pages/ProjectsPage' -import FramesPage from './pages/FramesPage' -import GraphPage from './pages/GraphPage' -import ProjectionsPage from './pages/ProjectionsPage' -import SpacesPage from './pages/SpacesPage' -import FeedbackPage from './pages/FeedbackPage' -import SettingsPage from './pages/SettingsPage' + +const AssistantPage = lazy(() => import('./pages/AssistantPage')) +const ProjectsPage = lazy(() => import('./pages/ProjectsPage')) +const FramesPage = lazy(() => import('./pages/FramesPage')) +const GraphPage = lazy(() => import('./pages/GraphPage')) +const ProjectionsPage = lazy(() => import('./pages/ProjectionsPage')) +const SpacesPage = lazy(() => import('./pages/SpacesPage')) +const SkillsPage = lazy(() => import('./pages/SkillsPage')) +const FeedbackPage = lazy(() => import('./pages/FeedbackPage')) +const SettingsPage = lazy(() => import('./pages/SettingsPage')) function App() { const page = useUiStore(s => s.page) @@ -20,13 +23,20 @@ function App() { case 'graph': return case 'projections': return case 'spaces': return + case 'skills': return case 'feedback': return case 'settings': return default: return } } - return {renderPage()} + return ( + + Loading...}> + {renderPage()} + + + ) } export default App diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index 27c2386..3425206 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -5,6 +5,14 @@ const client = axios.create({ timeout: 60_000, }) +// Production/local-auth deployments can set this for the browser session. +// It is intentionally not persisted in localStorage. +client.interceptors.request.use(config => { + const token = sessionStorage.getItem('mkb_api_token') + if (token) config.headers.Authorization = `Bearer ${token}` + return config +}) + /** Dispatched whenever any API response carries a job_id, so the job panel * can immediately poll instead of waiting for the next slow-poll cycle. */ export const JOB_STARTED_EVENT = 'mkb:job-started' diff --git a/frontend/src/api/frames.ts b/frontend/src/api/frames.ts index 6d4ea5c..c792ffe 100644 --- a/frontend/src/api/frames.ts +++ b/frontend/src/api/frames.ts @@ -1,5 +1,5 @@ import client from './client' -import type { Frame } from '../types' +import type { ExtractionPass, Frame } from '../types' export const listFrames = () => client.get('/frames').then(r => r.data) @@ -8,4 +8,4 @@ export const getFrame = (projectId: string) => client.get(`/frames/${projectId}`).then(r => r.data) export const getFrameHistory = (projectId: string) => - client.get(`/frames/${projectId}/history`).then(r => r.data) + client.get(`/frames/${projectId}/history`).then(r => r.data) diff --git a/frontend/src/api/jobPolling.ts b/frontend/src/api/jobPolling.ts index fe47256..b582951 100644 --- a/frontend/src/api/jobPolling.ts +++ b/frontend/src/api/jobPolling.ts @@ -7,12 +7,23 @@ const MAX_CONSECUTIVE_ERRORS = 6 /** Statuses that mean the job is still running and we should keep polling. */ export const ACTIVE_JOB_STATUSES = new Set(['QUEUED', 'PENDING', 'RUNNING']) +export const JOB_FINISHED_EVENT = 'mkb:job-finished' /** Returns true when the job has reached a terminal state and polling should stop. */ export function isJobTerminal(status: string): boolean { return !ACTIVE_JOB_STATUSES.has(status) } +const announcedFinishedJobs = new Set() + +export function announceJobFinished(job: Job): void { + if (!isJobTerminal(job.status) || announcedFinishedJobs.has(job.job_id)) return + announcedFinishedJobs.add(job.job_id) + if (typeof window !== 'undefined') { + window.dispatchEvent(new CustomEvent(JOB_FINISHED_EVENT, { detail: job })) + } +} + /** * Returns true when polling should stop for this error. * - 404 means the job is gone (server restart or stale id). @@ -109,6 +120,7 @@ export function startJobPolling(opts: StartJobPollingOptions): JobPollHandle { useJobsStore.getState().upsertJob(job) onUpdate?.(job) if (isJobTerminal(job.status)) { + announceJobFinished(job) if (job.status === 'COMPLETED') { onComplete?.(job) } else { diff --git a/frontend/src/api/postProcessorScripts.ts b/frontend/src/api/postProcessorScripts.ts new file mode 100644 index 0000000..77b1cc9 --- /dev/null +++ b/frontend/src/api/postProcessorScripts.ts @@ -0,0 +1,13 @@ +import client from './client' +import type { PostProcessorScript } from '../types' + +export const listPostProcessorScripts = () => + client.get('/post-processor-scripts').then(r => r.data) + +export const uploadPostProcessorScript = (file: File) => { + const form = new FormData() + form.append('file', file) + return client.post('/post-processor-scripts/upload', form, { + headers: { 'Content-Type': 'multipart/form-data' }, timeout: 120_000, + }).then(r => r.data) +} \ No newline at end of file diff --git a/frontend/src/api/projections.ts b/frontend/src/api/projections.ts index 27fcf8c..8cf2555 100644 --- a/frontend/src/api/projections.ts +++ b/frontend/src/api/projections.ts @@ -22,7 +22,7 @@ export const reviewProjections = (params: { mode?: ReviewMode reviewer_id?: string }) => - client.post<{ job_id: string }>('/projections/review', params).then(r => r.data) + client.post<{ job_id: string; job_ids?: string[] }>('/projections/review', params).then(r => r.data) export const deleteProjection = (id: string) => client.delete(`/projections/${id}`) diff --git a/frontend/src/api/projects.ts b/frontend/src/api/projects.ts index 29a59c5..822a126 100644 --- a/frontend/src/api/projects.ts +++ b/frontend/src/api/projects.ts @@ -1,5 +1,5 @@ import client from './client' -import type { Project, Asset, ProcessedAsset, Job, RawWorkflowVersion, CanonicalWorkflowVersion } from '../types' +import type { Project, Asset, ProcessedAsset, Job, RawWorkflowVersion } from '../types' export const listProjects = (limit = 100) => client.get('/projects', { params: { limit } }).then(r => r.data) @@ -57,24 +57,6 @@ export const deleteProjectWorkflowVersion = (id: string, version: number) => `/projects/${id}/workflows/${version}`, ).then(r => r.data) -export const canonicalizeProjectWorkflow = (id: string, rawExtractionId?: string) => - client.post<{ job_id: string }>(`/projects/${id}/workflows/canonicalize`, { - raw_extraction_id: rawExtractionId ?? null, - }).then(r => r.data) - -export const listCanonicalWorkflows = (id: string) => - client.get(`/projects/${id}/canonical-workflows`).then(r => r.data) - -export const getCanonicalWorkflow = (id: string, version?: number) => - client.get( - `/projects/${id}/canonical-workflows/${version == null ? 'latest' : version}`, - ).then(r => r.data) - -export const deleteCanonicalWorkflowVersion = (id: string, version: number) => - client.delete<{ status: string; project_id: string; version: number; canonicalization_id: string }>( - `/projects/${id}/canonical-workflows/${version}`, - ).then(r => r.data) - export const getProjectJobs = (id: string) => client.get(`/projects/${id}/jobs`).then(r => r.data) diff --git a/frontend/src/api/settings.ts b/frontend/src/api/settings.ts index 89557c4..6405bff 100644 --- a/frontend/src/api/settings.ts +++ b/frontend/src/api/settings.ts @@ -1,6 +1,8 @@ import client from './client' export interface RuntimeSettings { + deployment_mode: 'development' | 'local' | 'production' + allow_uploaded_python: boolean pdf_backend: 'local' | 'mineru_api' mineru_api_base: string mineru_api_token: string diff --git a/frontend/src/api/skills.ts b/frontend/src/api/skills.ts new file mode 100644 index 0000000..3718475 --- /dev/null +++ b/frontend/src/api/skills.ts @@ -0,0 +1,25 @@ +import client from './client' +import type { CustomSkill } from '../types' + +export const listSkills = () => + client.get('/skills').then(r => r.data) + +export const getSkill = (idOrSlug: string) => + client.get(`/skills/${idOrSlug}`).then(r => r.data) + +export const uploadSkill = (files: File[]) => { + const form = new FormData() + files.forEach(file => { + const relPath = (file as File & { webkitRelativePath?: string }).webkitRelativePath + form.append('files', file, relPath || file.name) + }) + return client + .post('/skills/upload', form, { + headers: { 'Content-Type': 'multipart/form-data' }, + timeout: 120_000, + }) + .then(r => r.data) +} + +export const deleteSkill = (skillId: string) => + client.delete<{ ok: boolean; deleted?: string }>(`/skills/${skillId}`).then(r => r.data) diff --git a/frontend/src/components/Layout.tsx b/frontend/src/components/Layout.tsx index 9a0b986..f93cd0b 100644 --- a/frontend/src/components/Layout.tsx +++ b/frontend/src/components/Layout.tsx @@ -11,6 +11,7 @@ const NAV_ITEMS: { page: Page; label: string; icon: string }[] = [ { page: 'graph', label: 'Dataset Graph', icon: '🕸️' }, { page: 'projections', label: 'Projections', icon: '📊' }, { page: 'spaces', label: 'Spaces', icon: '🗂️' }, + { page: 'skills', label: 'Skills', icon: '🧠' }, { page: 'feedback', label: 'Feedback', icon: '💬' }, { page: 'settings', label: 'Settings', icon: '⚙️' }, ] diff --git a/frontend/src/components/ProjectGroupedList.tsx b/frontend/src/components/ProjectGroupedList.tsx index c3f90f7..321eca7 100644 --- a/frontend/src/components/ProjectGroupedList.tsx +++ b/frontend/src/components/ProjectGroupedList.tsx @@ -8,83 +8,12 @@ import StatusBadge from './StatusBadge' import BatchActionBar from './BatchActionBar' import type { Project, ProjectGroup, Space } from '../types' import { projectDisplayName } from '../utils/projectName' +import { useDragAutoScroll } from '../features/projects/useDragAutoScroll' // ─── Auto-scroll during drag ──────────────────────────────────────────────── // When the user drags near the top or bottom edge of the viewport, scroll the // page automatically so they don't have to drop-release-scroll-pick-up again. -function useDragAutoScroll({ - edgePx = 120, // px from scroll-container edge that activates scrolling - maxSpeed = 18, // max px scrolled per animation frame (~1080 px/s) -} = {}) { - const posRef = useRef(null) // latest drag clientY - const scrollElRef = useRef(null) - const rafRef = useRef(null) - const dragging = useRef(false) - - useEffect(() => { - /** Walk up the DOM to find the first scrollable ancestor. */ - function findScrollParent(el: HTMLElement | null): HTMLElement { - if (!el || el === document.documentElement) return document.documentElement - const { overflowY } = window.getComputedStyle(el) - if ((overflowY === 'auto' || overflowY === 'scroll') && el.scrollHeight > el.clientHeight) - return el - return findScrollParent(el.parentElement as HTMLElement) - } - - const tick = () => { - const y = posRef.current - const el = scrollElRef.current - if (y !== null && el) { - const rect = el.getBoundingClientRect() - const relY = y - rect.top - const h = rect.height - let speed = 0 - if (relY < edgePx) speed = -maxSpeed * (1 - relY / edgePx) - else if (relY > h - edgePx) speed = maxSpeed * ((relY - (h - edgePx)) / edgePx) - if (speed !== 0) el.scrollTop += speed - } - rafRef.current = requestAnimationFrame(tick) - } - - const onDragStart = (e: DragEvent) => { - dragging.current = true - scrollElRef.current = findScrollParent(e.target as HTMLElement) - rafRef.current = requestAnimationFrame(tick) - } - - const onDragOver = (e: DragEvent) => { posRef.current = e.clientY } - - const stop = () => { - dragging.current = false - if (rafRef.current) { cancelAnimationFrame(rafRef.current); rafRef.current = null } - posRef.current = null - } - - // Capture-phase wheel listener — fires even during HTML5 drag in Chrome/Edge. - // Lets the user scroll with the mouse wheel while holding a drag. - const onWheel = (e: WheelEvent) => { - if (!dragging.current || !scrollElRef.current) return - scrollElRef.current.scrollTop += e.deltaY - e.preventDefault() - } - - document.addEventListener('dragstart', onDragStart, true) - document.addEventListener('dragover', onDragOver) - document.addEventListener('dragend', stop) - document.addEventListener('drop', stop) - document.addEventListener('wheel', onWheel, { passive: false, capture: true }) - return () => { - document.removeEventListener('dragstart', onDragStart, true) - document.removeEventListener('dragover', onDragOver) - document.removeEventListener('dragend', stop) - document.removeEventListener('drop', stop) - document.removeEventListener('wheel', onWheel, { capture: true }) - if (rafRef.current) cancelAnimationFrame(rafRef.current) - } - }, [edgePx, maxSpeed]) -} - // ─── Shared types ─────────────────────────────────────────────────────────── export interface ProjectColumn { diff --git a/frontend/src/components/SpaceDraftCard.tsx b/frontend/src/components/SpaceDraftCard.tsx index c0c0b69..a68d493 100644 --- a/frontend/src/components/SpaceDraftCard.tsx +++ b/frontend/src/components/SpaceDraftCard.tsx @@ -9,7 +9,7 @@ export interface SpaceDraft { description?: string extraction_schema: Record system_prompt: string - field_descriptions: Record + field_descriptions?: Record } interface Props { @@ -43,9 +43,9 @@ export default function SpaceDraftCard({ draft, existing, onSaved }: Props) { domain: draft.domain, purpose: draft.purpose, description: draft.description ?? '', - extraction_schema: draft.extraction_schema, + extraction_schema: mergeLegacyFieldDescriptions(draft.extraction_schema, draft.field_descriptions), system_prompt: draft.system_prompt, - field_descriptions: draft.field_descriptions, + field_descriptions: {}, } const res = await createSpace(payload) if ((res as unknown as { error?: string }).error) { @@ -58,9 +58,9 @@ export default function SpaceDraftCard({ draft, existing, onSaved }: Props) { domain: draft.domain, purpose: draft.purpose, description: draft.description ?? '', - extraction_schema: draft.extraction_schema, + extraction_schema: mergeLegacyFieldDescriptions(draft.extraction_schema, draft.field_descriptions), system_prompt: draft.system_prompt, - field_descriptions: draft.field_descriptions, + field_descriptions: {}, }) setSavedAs(existing.space_id) onSaved?.({ space_id: existing.space_id, name: `${existing.name} (v${res.version})` }) @@ -173,9 +173,9 @@ export function extractDraftsFromText(text: string): SpaceDraft[] { domain: obj.domain ?? '', purpose: obj.purpose ?? 'tabular_database', description: obj.description ?? '', - extraction_schema: obj.extraction_schema, + extraction_schema: mergeLegacyFieldDescriptions(obj.extraction_schema, obj.field_descriptions), system_prompt: obj.system_prompt ?? '', - field_descriptions: obj.field_descriptions ?? {}, + field_descriptions: {}, }) } } catch { @@ -184,3 +184,34 @@ export function extractDraftsFromText(text: string): SpaceDraft[] { } return drafts } + +function stringifyLegacyDescription(value: unknown) { + return typeof value === 'string' ? value : value == null ? '' : JSON.stringify(value, null, 2) +} + +function mergeLegacyFieldDescriptions( + schemaValue: unknown, + descriptionsValue: unknown, +): Record { + if (!schemaValue || typeof schemaValue !== 'object' || Array.isArray(schemaValue)) return {} + const schema = { ...(schemaValue as Record) } + const descriptions = + descriptionsValue && typeof descriptionsValue === 'object' && !Array.isArray(descriptionsValue) + ? (descriptionsValue as Record) + : {} + + for (const [key, rawDescription] of Object.entries(descriptions)) { + const description = stringifyLegacyDescription(rawDescription).trim() + const node = schema[key] + if (!description || !node || typeof node !== 'object' || Array.isArray(node)) continue + const section = { ...(node as Record) } + const existing = typeof section.description === 'string' ? section.description.trim() : '' + if (!existing.includes(description)) { + section.description = existing + ? `${existing}\n\nExtraction guidance: ${description}` + : description + } + schema[key] = section + } + return schema +} diff --git a/frontend/src/components/frames/AssetsTab.tsx b/frontend/src/components/frames/AssetsTab.tsx index 3d85a23..158b04e 100644 --- a/frontend/src/components/frames/AssetsTab.tsx +++ b/frontend/src/components/frames/AssetsTab.tsx @@ -1,7 +1,8 @@ -import { useEffect, useState } from 'react' +import { useCallback, useEffect, useState } from 'react' +import { JOB_FINISHED_EVENT } from '../../api/jobPolling' import { listAssets, listProcessedAssets } from '../../api/projects' -import type { Asset, ProcessedAsset } from '../../types' +import type { Asset, Job, ProcessedAsset } from '../../types' import AssetPreviewModal from './AssetPreviewModal' type Preview = { @@ -16,12 +17,30 @@ export default function AssetsTab({ projectId }: { projectId: string }) { const [loading, setLoading] = useState(true) const [preview, setPreview] = useState(null) - useEffect(() => { + const load = useCallback((showLoading = false) => { + if (showLoading) setLoading(true) Promise.all([listAssets(projectId), listProcessedAssets(projectId)]) .then(([a, p]) => { setAssets(a); setProcessed(p) }) .finally(() => setLoading(false)) }, [projectId]) + useEffect(() => { load(true) }, [load]) + + useEffect(() => { + const refreshOnFinishedJob = (event: Event) => { + const job = (event as CustomEvent).detail + if ( + job.status === 'COMPLETED' && + job.project_id === projectId && + ['process', 'upload'].includes(job.kind) + ) { + load() + } + } + window.addEventListener(JOB_FINISHED_EVENT, refreshOnFinishedJob) + return () => window.removeEventListener(JOB_FINISHED_EVENT, refreshOnFinishedJob) + }, [load, projectId]) + if (loading) return

Loading…

if (assets.length === 0) return

No assets ingested yet.

diff --git a/frontend/src/components/frames/GraphTab.tsx b/frontend/src/components/frames/GraphTab.tsx index b14ea78..85b8b53 100644 --- a/frontend/src/components/frames/GraphTab.tsx +++ b/frontend/src/components/frames/GraphTab.tsx @@ -1,7 +1,8 @@ -import { useEffect, useState } from 'react' +import { useCallback, useEffect, useState } from 'react' import { getKnowledgeGraph } from '../../api/graph' -import type { GraphConcept, GraphRelation } from '../../types' +import { JOB_FINISHED_EVENT } from '../../api/jobPolling' +import type { GraphConcept, GraphRelation, Job } from '../../types' import MiniGraph from './MiniGraph' @@ -11,13 +12,31 @@ export default function GraphTab({ projectId }: { projectId: string }) { const [loading, setLoading] = useState(true) const [showList, setShowList] = useState(false) - useEffect(() => { + const load = useCallback((showLoading = false) => { + if (showLoading) setLoading(true) getKnowledgeGraph({ project_id: projectId }) .then(d => { setConcepts(d.graph?.concepts ?? []); setRelations(d.graph?.relations ?? []) }) .catch(() => {}) .finally(() => setLoading(false)) }, [projectId]) + useEffect(() => { load(true) }, [load]) + + useEffect(() => { + const refreshOnFinishedJob = (event: Event) => { + const job = (event as CustomEvent).detail + if ( + job.status === 'COMPLETED' && + job.project_id === projectId && + ['knowledge_graph', 'graph_review'].includes(job.kind) + ) { + load() + } + } + window.addEventListener(JOB_FINISHED_EVENT, refreshOnFinishedJob) + return () => window.removeEventListener(JOB_FINISHED_EVENT, refreshOnFinishedJob) + }, [load, projectId]) + if (loading) return

Loading…

if (concepts.length === 0) return

No graph elements for this project yet. Run Extract Graph.

diff --git a/frontend/src/components/frames/KnowledgeFrameTab.tsx b/frontend/src/components/frames/KnowledgeFrameTab.tsx index eb70fa6..7de9887 100644 --- a/frontend/src/components/frames/KnowledgeFrameTab.tsx +++ b/frontend/src/components/frames/KnowledgeFrameTab.tsx @@ -1,8 +1,10 @@ -import { useEffect, useState } from 'react' +import { useCallback, useEffect, useState } from 'react' +import axios from 'axios' import { getFrame, getFrameHistory } from '../../api/frames' +import { JOB_FINISHED_EVENT } from '../../api/jobPolling' import { getProject } from '../../api/projects' -import type { ExtractionPass, Frame, Project } from '../../types' +import type { ExtractionPass, Frame, Job, Project } from '../../types' import StatusBadge from '../StatusBadge' import { FrameHeader, FrameSection } from './frameRender' @@ -12,25 +14,63 @@ export default function KnowledgeFrameTab({ projectId }: { projectId: string }) const [history, setHistory] = useState([]) const [project, setProject] = useState(null) const [loading, setLoading] = useState(true) + const [loadError, setLoadError] = useState(null) const [showRaw, setShowRaw] = useState(false) - useEffect(() => { - Promise.all([getFrame(projectId), getFrameHistory(projectId), getProject(projectId)]) - .then(([f, h, p]) => { - setFrame(f) - setHistory(h as unknown as ExtractionPass[]) - setProject(p) - }) - .catch(() => {}) - .finally(() => setLoading(false)) + const load = useCallback(async (showLoading = false) => { + if (showLoading) setLoading(true) + setLoadError(null) + + // The frame is the only request that determines this tab's empty state. + // History and project metadata are supplementary and must not hide an + // existing frame when either endpoint fails. + const [frameResult, historyResult, projectResult] = await Promise.allSettled([ + getFrame(projectId), + getFrameHistory(projectId), + getProject(projectId), + ]) + + if (frameResult.status === 'fulfilled') { + setFrame(frameResult.value) + } else if (axios.isAxiosError(frameResult.reason) && frameResult.reason.response?.status === 404) { + setFrame(null) + } else { + setLoadError(frameResult.reason instanceof Error ? frameResult.reason.message : 'Could not load knowledge frame') + } + + if (historyResult.status === 'fulfilled') setHistory(historyResult.value) + if (projectResult.status === 'fulfilled') setProject(projectResult.value) + setLoading(false) }, [projectId]) + useEffect(() => { load(true) }, [load]) + + useEffect(() => { + const refreshOnFinishedJob = (event: Event) => { + const job = (event as CustomEvent).detail + if ( + job.status === 'COMPLETED' && + job.project_id === projectId && + ['extract', 'raw_workflow', 'canonical_workflow', 'workflow_maintenance'].includes(job.kind) + ) { + load() + } + } + window.addEventListener(JOB_FINISHED_EVENT, refreshOnFinishedJob) + return () => window.removeEventListener(JOB_FINISHED_EVENT, refreshOnFinishedJob) + }, [load, projectId]) + if (loading) return

Loading…

+ if (loadError) return

Failed to load knowledge frame: {loadError}

if (!frame) return

No knowledge frame yet. Run Extract to generate one.

const { content, extraction_summary, extraction_version, status, extracted_at, agent_annotations } = frame const clarifications = agent_annotations?.clarifications ?? [] const resolvedFeedback = agent_annotations?.resolved_feedback ?? [] + const sortedHistory = [...history].sort((a, b) => { + const byDate = new Date(b.created_at).getTime() - new Date(a.created_at).getTime() + return byDate || b.pass_number - a.pass_number + }) return (
@@ -48,17 +88,20 @@ export default function KnowledgeFrameTab({ projectId }: { projectId: string }) )} {history.length > 0 && ( -
-

Extraction passes

-
- {history.map((h, i) => ( -
- Pass {h.pass_number} ({h.pass_type}) — {h.created_at.slice(0, 10)} - {h.changes_made ? ' · changes made' : ''} +
+ + Extraction history ({history.length} {history.length === 1 ? 'entry' : 'entries'}) + · newest v{sortedHistory[0].pass_number} + +
+ {sortedHistory.map(h => ( +
+ Version {h.pass_number} ({h.pass_type}) — {new Date(h.created_at).toLocaleString()} + {h.changes_made && Object.values(h.changes_made).some(count => count > 0) ? ' · changes made' : ''}
))}
-
+ )} {content && ( diff --git a/frontend/src/components/frames/ProjectDetail.tsx b/frontend/src/components/frames/ProjectDetail.tsx index abcb0de..55f8123 100644 --- a/frontend/src/components/frames/ProjectDetail.tsx +++ b/frontend/src/components/frames/ProjectDetail.tsx @@ -1,25 +1,24 @@ -import { useCallback, useEffect, useState } from 'react' +import { lazy, Suspense, useCallback, useEffect, useState } from 'react' -import { startJobPolling } from '../../api/jobPolling' import { listFeedback } from '../../api/feedback' import { extractProject, - getProject, kgExtractProject, processProject, projectToSpace, workflowExtractProject, } from '../../api/projects' import { listSpaces } from '../../api/spaces' -import type { Job, Project, Space } from '../../types' -import StatusBadge from '../StatusBadge' -import AssetsTab from './AssetsTab' -import FeedbackTab from './FeedbackTab' -import GraphTab from './GraphTab' -import KnowledgeFrameTab from './KnowledgeFrameTab' -import ProjectionsTab from './ProjectionsTab' -import WorkflowTab from './WorkflowTab' -import { projectDisplayName } from '../../utils/projectName' +import { useProjectRefresh } from '../../hooks/useProjectRefresh' +import { useProjectJobController } from '../../hooks/useProjectJobController' +import type { Project, Space } from '../../types' +import { ProjectDetailHeader, ProjectDetailTabs } from '../projects/ProjectDetailChrome' +const AssetsTab = lazy(() => import('./AssetsTab')) +const FeedbackTab = lazy(() => import('./FeedbackTab')) +const GraphTab = lazy(() => import('./GraphTab')) +const KnowledgeFrameTab = lazy(() => import('./KnowledgeFrameTab')) +const ProjectionsTab = lazy(() => import('./ProjectionsTab')) +const WorkflowTab = lazy(() => import('./WorkflowTab')) type DetailTab = 'assets' | 'frame' | 'projections' | 'workflow' | 'graph' | 'feedback' @@ -34,14 +33,11 @@ export default function ProjectDetail({ onProjectUpdated?: (p: Project) => void }) { const [activeTab, setActiveTab] = useState('frame') - const [activeJobId, setActiveJobId] = useState(null) - const [activeJob, setActiveJob] = useState(null) const [spaces, setSpaces] = useState([]) const [selectedSpaceId, setSelectedSpaceId] = useState('') const [projectionSource, setProjectionSource] = useState<'frame' | 'markdown'>('frame') const [feedbackCount, setFeedbackCount] = useState(0) const [refreshKey, setRefreshKey] = useState(0) - const [actionError, setActionError] = useState(null) const refreshFeedbackCount = useCallback(() => { listFeedback({ project_id: project.project_id, limit: 100 }) @@ -57,60 +53,36 @@ export default function ProjectDetail({ refreshFeedbackCount() }, [project.project_id, refreshFeedbackCount, selectedSpaceId]) - const refreshProject = useCallback(() => { - if (!onProjectUpdated) return - getProject(project.project_id).then(onProjectUpdated).catch(() => {}) - }, [project.project_id, onProjectUpdated]) + const refreshProject = useProjectRefresh(project.project_id, onProjectUpdated) + const [afterAction, setAfterAction] = useState(null) + const { activeJobId, activeJob, actionError, run } = useProjectJobController({ + onComplete: () => { + if (afterAction) setActiveTab(afterAction) + setAfterAction(null) + setRefreshKey(k => k + 1) + refreshFeedbackCount() + refreshProject() + }, + onSettled: () => setAfterAction(null), + }) - const pollJob = useCallback((jobId: string, onDone?: () => void) => { - setActiveJobId(jobId) - startJobPolling({ - jobId, - onUpdate: setActiveJob, - onTick: tick => { if (tick % 3 === 0) refreshProject() }, - onComplete: () => { - setActiveJobId(null) - onDone?.() - setRefreshKey(k => k + 1) - refreshFeedbackCount() - refreshProject() - }, - onFailed: () => setActiveJobId(null), - }) - }, [refreshFeedbackCount, refreshProject]) - - const run = async (fn: () => Promise<{ job_id: string }>, onDone?: () => void) => { - if (activeJobId) return - try { - setActionError(null) - const { job_id } = await fn() - pollJob(job_id, onDone) - } catch (err: unknown) { - const e = err as { response?: { data?: { detail?: string } }; message?: string } - setActionError(e?.response?.data?.detail ?? e?.message ?? 'Action failed') - } + const runAction = (fn: () => Promise<{ job_id: string }>, nextTab?: DetailTab) => { + setAfterAction(nextTab ?? null) + void run(fn) } return (
-
-
- -

{projectDisplayName(project)}

-
-
- - - -
-
+ ← Back + } />
- - @@ -127,12 +99,12 @@ export default function ProjectDetail({ - -
-
- {([ +
+ 0 ? ` (${feedbackCount})` : ''}`], - ] as [DetailTab, string][]).map(([tab, name]) => ( - - ))} + ] as const} active={activeTab} onChange={setActiveTab} />
+ Loading detail…

}> {activeTab === 'assets' && } {activeTab === 'frame' && } {activeTab === 'projections' && } - {activeTab === 'workflow' && ( + {activeTab === 'workflow' && Loading workflow…

}> run(() => workflowExtractProject(project.project_id))} + onExtractWorkflow={() => runAction(() => workflowExtractProject(project.project_id))} onWorkflowVersionDeleted={() => { setRefreshKey(k => k + 1) refreshProject() }} /> - )} - {activeTab === 'graph' && } + } + {activeTab === 'graph' && Loading graph…

}>} {activeTab === 'feedback' && } +
) diff --git a/frontend/src/components/frames/ProjectionsTab.tsx b/frontend/src/components/frames/ProjectionsTab.tsx index 759bfcc..ba99f72 100644 --- a/frontend/src/components/frames/ProjectionsTab.tsx +++ b/frontend/src/components/frames/ProjectionsTab.tsx @@ -1,8 +1,9 @@ -import { useEffect, useState } from 'react' +import { useCallback, useEffect, useState } from 'react' +import { JOB_FINISHED_EVENT } from '../../api/jobPolling' import { listProjections } from '../../api/projections' import { listSpaces } from '../../api/spaces' -import type { Projection, Space } from '../../types' +import type { Job, Projection, Space } from '../../types' import StatusBadge from '../StatusBadge' import { FrameSection } from './frameRender' @@ -12,7 +13,8 @@ export default function ProjectionsTab({ projectId }: { projectId: string }) { const [spaces, setSpaces] = useState([]) const [loading, setLoading] = useState(true) - useEffect(() => { + const load = useCallback((showLoading = false) => { + if (showLoading) setLoading(true) Promise.all([ listProjections({ project_id: projectId, include_data: true, limit: 50 }), listSpaces(), @@ -22,6 +24,23 @@ export default function ProjectionsTab({ projectId }: { projectId: string }) { }).finally(() => setLoading(false)) }, [projectId]) + useEffect(() => { load(true) }, [load]) + + useEffect(() => { + const refreshOnFinishedJob = (event: Event) => { + const job = (event as CustomEvent).detail + if ( + job.status === 'COMPLETED' && + job.project_id === projectId && + ['project', 'projection_review'].includes(job.kind) + ) { + load() + } + } + window.addEventListener(JOB_FINISHED_EVENT, refreshOnFinishedJob) + return () => window.removeEventListener(JOB_FINISHED_EVENT, refreshOnFinishedJob) + }, [load, projectId]) + if (loading) return

Loading…

if (projections.length === 0) return

No projections yet. Select a space and run Project.

diff --git a/frontend/src/components/projections/SectionTable.tsx b/frontend/src/components/projections/SectionTable.tsx index 5fe94c6..a148ff9 100644 --- a/frontend/src/components/projections/SectionTable.tsx +++ b/frontend/src/components/projections/SectionTable.tsx @@ -19,66 +19,13 @@ import { saveColPrefs, slugifyExportName, } from './helpers' - -type ProjectionTableRow = Record -type ColumnDataType = 'boolean' | 'number' | 'date' | 'text' +import { + compareText, inferColumnDataType, isBlank, loadSavedPage, parseBoolean, parseDate, parseNumber, + savePage, sortArrow, sortLabel, type ColumnDataType, type ProjectionTableRow, +} from '../../features/projections/tableModel' const SELECTION_COLUMN_ID = '__projection_selection__' -function isBlank(value: unknown): boolean { - return String(value ?? '').trim() === '' -} - -function parseBoolean(value: unknown): number | null { - const normalized = String(value ?? '').trim().toLowerCase() - if (['true', 'yes', 'y', '1'].includes(normalized)) return 1 - if (['false', 'no', 'n', '0'].includes(normalized)) return 0 - return null -} - -function parseNumber(value: unknown): number | null { - const normalized = String(value ?? '').trim().replace(/,/g, '') - if (!normalized) return null - const parsed = Number(normalized) - return Number.isFinite(parsed) ? parsed : null -} - -function parseDate(value: unknown): number | null { - const normalized = String(value ?? '').trim() - if (!normalized) return null - const parsed = Date.parse(normalized) - return Number.isNaN(parsed) ? null : parsed -} - -function inferColumnDataType(rows: ProjectionTableRow[], col: string): ColumnDataType { - const values = rows.map(row => row[col]).filter(value => !isBlank(value)) - if (values.length === 0) return 'text' - if (values.every(value => parseBoolean(value) !== null)) return 'boolean' - if (values.every(value => parseNumber(value) !== null)) return 'number' - if (values.every(value => parseDate(value) !== null)) return 'date' - return 'text' -} - -function compareText(left: unknown, right: unknown): number { - return String(left ?? '').localeCompare(String(right ?? ''), undefined, { - numeric: true, - sensitivity: 'base', - }) -} - -function sortLabel(type: ColumnDataType, sorted: false | 'asc' | 'desc'): string { - if (!sorted) return 'Click to sort' - if (type === 'boolean') return sorted === 'asc' ? 'False → True (click for True → False)' : 'True → False (click to clear)' - if (type === 'number') return sorted === 'asc' ? '0 → 9 (click for 9 → 0)' : '9 → 0 (click to clear)' - if (type === 'date') return sorted === 'asc' ? 'Old → New (click for New → Old)' : 'New → Old (click to clear)' - return sorted === 'asc' ? 'A → Z (click for Z → A)' : 'Z → A (click to clear)' -} - -function sortArrow(sorted: false | 'asc' | 'desc'): string { - if (!sorted) return '⇅' - return sorted === 'asc' ? '↑' : '↓' -} - export default function SectionTable({ name, rows, @@ -102,7 +49,8 @@ export default function SectionTable({ onClearSelection: () => void reviewDisabled?: boolean }) { - const [page, setPage] = useState(1) + const pageStorageKey = exportBasename ?? name + const [page, setPage] = useState(() => loadSavedPage(pageStorageKey)) const [sorting, setSorting] = useState([]) const [exportingFormat, setExportingFormat] = useState<'csv' | 'excel' | null>(null) const totalPages = Math.max(1, Math.ceil(rows.length / PAGE_SIZE)) @@ -112,6 +60,14 @@ export default function SectionTable({ if (page > totalPages) setPage(totalPages) }, [page, totalPages]) + useEffect(() => { + setPage(loadSavedPage(pageStorageKey)) + }, [pageStorageKey]) + + useEffect(() => { + savePage(pageStorageKey, Math.min(page, totalPages)) + }, [page, pageStorageKey, totalPages]) + const allCols = useMemo(() => { const seen = new Set() for (const r of rows) for (const k of Object.keys(r)) seen.add(k) @@ -128,13 +84,21 @@ export default function SectionTable({ useEffect(() => { setPrefs(p => { - const merged = Array.from(new Set([...p.known, ...allCols])) - if (merged.length === p.known.length) return p - const next = { ...p, known: merged } + const added = allCols.filter(column => !p.known.includes(column)) + if (added.length === 0) return p + const known = [...p.known, ...added] + // New fields can be introduced by a post-processor after the user has + // saved column preferences. Reveal them once without re-enabling any + // columns the user previously hid. + const visible = Array.from(new Set([ + ...p.visible, + ...defaultColumns(added, schemaOrder).filter(column => !p.visible.includes(column)), + ])) + const next = { ...p, known, visible } saveColPrefs(name, next) return next }) - }, [allCols, name]) + }, [allCols, name, schemaOrder]) const visibleCols = prefs.visible.filter(c => prefs.known.includes(c)) @@ -209,6 +173,7 @@ export default function SectionTable({ const cycleSort = (columnId: string) => { const current = sorting.find(sort => sort.id === columnId) setPage(1) + savePage(pageStorageKey, 1) if (!current) setSorting([{ id: columnId, desc: false }]) else if (!current.desc) setSorting([{ id: columnId, desc: true }]) else setSorting([]) @@ -289,9 +254,11 @@ export default function SectionTable({ onPaginationChange: updater => { const next = functionalUpdate(updater, pagination) setPage(next.pageIndex + 1) + savePage(pageStorageKey, next.pageIndex + 1) }, onSortingChange: updater => { setPage(1) + savePage(pageStorageKey, 1) setSorting(functionalUpdate(updater, sorting)) }, getCoreRowModel: getCoreRowModel(), diff --git a/frontend/src/components/projects/BrowseTab.tsx b/frontend/src/components/projects/BrowseTab.tsx index 8fb5fef..57cbf5d 100644 --- a/frontend/src/components/projects/BrowseTab.tsx +++ b/frontend/src/components/projects/BrowseTab.tsx @@ -1,11 +1,45 @@ import { useCallback, useEffect, useState } from 'react' -import { listProjects } from '../../api/projects' +import { JOB_FINISHED_EVENT } from '../../api/jobPolling' +import { getProject, listProjects } from '../../api/projects' import ProjectGroupedList from '../ProjectGroupedList' -import type { Project, Space } from '../../types' +import type { Job, Project, Space } from '../../types' import ProjectDetail from './ProjectDetail' import StatusLights from './StatusLights' +function projectsEqual(a: Project, b: Project): boolean { + return ( + a.project_id === b.project_id && + a.label === b.label && + a.source_path === b.source_path && + a.file_count === b.file_count && + a.asset_count === b.asset_count && + a.processing_status === b.processing_status && + a.frame_status === b.frame_status && + a.workflow_status === b.workflow_status && + a.workflow_version === b.workflow_version && + a.created_at === b.created_at && + (a.group_id ?? null) === (b.group_id ?? null) + ) +} + +function mergeProjectList(current: Project[], next: Project[]): Project[] { + const currentById = new Map(current.map(p => [p.project_id, p])) + let changed = current.length !== next.length + + const merged = next.map(project => { + const existing = currentById.get(project.project_id) + if (existing && projectsEqual(existing, project)) return existing + changed = true + return project + }) + + return changed ? merged : current +} + +function mergeProject(current: Project, next: Project): Project { + return projectsEqual(current, next) ? current : next +} export default function BrowseTab({ spaces }: { spaces: Space[] }) { const [projects, setProjects] = useState([]) @@ -16,17 +50,68 @@ export default function BrowseTab({ spaces }: { spaces: Space[] }) { try { // Don't blank the table during a background refresh — only show the // loading placeholder on the very first fetch. - setLoading(prev => projects.length === 0 ? true : prev) const data = await listProjects(5000) - setProjects(data) + setProjects(prev => mergeProjectList(prev, data)) + setSelected(prev => { + if (!prev) return prev + const updated = data.find(p => p.project_id === prev.project_id) + return updated ? mergeProject(prev, updated) : null + }) } finally { setLoading(false) } // eslint-disable-next-line react-hooks/exhaustive-deps }, []) + const refreshProject = useCallback(async (projectId: string) => { + const updated = await getProject(projectId) + setProjects(prev => prev.map(p => + p.project_id === updated.project_id ? mergeProject(p, updated) : p + )) + setSelected(prev => + prev?.project_id === updated.project_id ? mergeProject(prev, updated) : prev + ) + }, []) + + const handleProjectUpdated = useCallback((updated: Project) => { + setProjects(prev => prev.map(p => + p.project_id === updated.project_id ? mergeProject(p, updated) : p + )) + setSelected(prev => + prev?.project_id === updated.project_id ? mergeProject(prev, updated) : prev + ) + }, []) + useEffect(() => { load() }, [load]) + useEffect(() => { + const refreshOnFinishedJob = (event: Event) => { + const job = (event as CustomEvent).detail + if ( + job.status === 'COMPLETED' && + [ + 'process', + 'extract', + 'project', + 'knowledge_graph', + 'raw_workflow', + 'canonical_workflow', + 'workflow_maintenance', + 'workflow_maintenance_batch', + 'upload', + ].includes(job.kind) + ) { + if (job.project_id && job.project_id !== '__upload__') { + refreshProject(job.project_id).catch(() => load()) + } else { + load() + } + } + } + window.addEventListener(JOB_FINISHED_EVENT, refreshOnFinishedJob) + return () => window.removeEventListener(JOB_FINISHED_EVENT, refreshOnFinishedJob) + }, [load, refreshProject]) + const getStatus = useCallback( (id: string) => projects.find(p => p.project_id === id)?.frame_status ?? 'NO_FRAME', [projects], @@ -55,7 +140,6 @@ export default function BrowseTab({ spaces }: { spaces: Space[] }) { processed={p.processing_status ?? 'UNPROCESSED'} frame={p.frame_status ?? 'NO_FRAME'} workflow={p.workflow_status ?? 'NO_WORKFLOW'} - normalized={p.canonical_workflow_status ?? 'NO_CANONICAL_WORKFLOW'} /> ), }, @@ -77,10 +161,8 @@ export default function BrowseTab({ spaces }: { spaces: Space[] }) { project={selected} spaces={spaces} onClose={() => setSelected(null)} - onJobComplete={load} - onProjectUpdated={updated => setProjects(prev => prev.map(p => - p.project_id === updated.project_id ? updated : p - ))} + onJobComplete={() => refreshProject(selected.project_id)} + onProjectUpdated={handleProjectUpdated} onDeleted={() => { setSelected(null); load() }} /> )} diff --git a/frontend/src/components/projects/CanonicalWorkflowTab.tsx b/frontend/src/components/projects/CanonicalWorkflowTab.tsx deleted file mode 100644 index ea243e0..0000000 --- a/frontend/src/components/projects/CanonicalWorkflowTab.tsx +++ /dev/null @@ -1,130 +0,0 @@ -import { useEffect, useState } from 'react' - -import { deleteCanonicalWorkflowVersion, getCanonicalWorkflow, listCanonicalWorkflows } from '../../api/projects' -import type { CanonicalWorkflowVersion } from '../../types' -import WorkflowCanvas, { type WorkflowCanvasEdge, type WorkflowCanvasNode } from './WorkflowCanvas' - -function Canvas({ workflow }: { workflow: CanonicalWorkflowVersion }) { - const graph = workflow.graph - - if (!graph?.nodes.length) return

This canonical version has no nodes.

- - const nodes: WorkflowCanvasNode[] = graph.nodes.map(node => ({ - id: node.node_id, - label: node.label, - kind: node.node_kind, - title: `${node.label}\n${node.object_schema ?? node.operation_template_id ?? 'unmatched template'}\n\nRaw sources: ${node.raw_node_ids.join(', ')}\nAttributes: ${JSON.stringify(node.attributes)}`, - details: { - object_schema: node.object_schema, - operation_template_id: node.operation_template_id, - attributes: node.attributes, - raw_node_ids: node.raw_node_ids, - }, - })) - const edges: WorkflowCanvasEdge[] = graph.edges.map(edge => ({ - id: edge.edge_id, - source: edge.source_node, - target: edge.target_node, - label: edge.relation_type, - title: `Raw edges: ${edge.raw_edge_ids.join(', ')}`, - })) - - return -} - -export default function CanonicalWorkflowTab({ - projectId, - actionsDisabled = false, - onWorkflowVersionDeleted, -}: { - projectId: string - actionsDisabled?: boolean - onWorkflowVersionDeleted?: () => void -}) { - const [versions, setVersions] = useState([]) - const [selected, setSelected] = useState(null) - const [loading, setLoading] = useState(true) - const [error, setError] = useState(null) - const [deletingVersion, setDeletingVersion] = useState(null) - - const load = async () => { - setLoading(true) - setError(null) - try { - const rows = await listCanonicalWorkflows(projectId) - setVersions(rows) - const preferred = rows.find(row => row.status === 'COMPLETED') ?? rows[0] ?? null - setSelected(preferred ? await getCanonicalWorkflow(projectId, preferred.version) : null) - } catch (err: unknown) { - const e = err as { response?: { data?: { detail?: string } }; message?: string } - setError(e?.response?.data?.detail ?? e?.message ?? 'Failed to load canonical workflow versions') - } finally { - setLoading(false) - } - } - - useEffect(() => { - void load() - }, [projectId]) - - const choose = async (version: number) => setSelected(await getCanonicalWorkflow(projectId, version)) - - const handleDelete = async () => { - if (!selected) return - const confirmed = window.confirm( - `Delete canonical workflow v${selected.version}? This removes the selected canonical workflow version.`, - ) - if (!confirmed) return - try { - setDeletingVersion(selected.version) - setError(null) - await deleteCanonicalWorkflowVersion(projectId, selected.version) - await load() - onWorkflowVersionDeleted?.() - } catch (err: unknown) { - const e = err as { response?: { data?: { detail?: string } }; message?: string } - setError(e?.response?.data?.detail ?? e?.message ?? 'Failed to delete canonical workflow version') - } finally { - setDeletingVersion(null) - } - } - - if (loading) return

Loading normalized workflow…

- if (error && !versions.length) return

{error}

- if (!versions.length) return

No normalized workflow yet. Run Canonicalize Workflow after raw extraction.

- return
-
- - - {selected && {selected.schema_version} · raw {selected.raw_extraction_id.slice(0, 8)}} - {selected && ( - - )} -
- {error &&

{error}

} - {selected?.error &&

{selected.error}

} - {selected && !selected.graph && selected.resumable && ( -
- {selected.has_checkpoint - ? `This unfinished canonical version has a saved checkpoint${selected.checkpoint_updated_at ? ` from ${new Date(selected.checkpoint_updated_at).toLocaleString()}` : ''}. Run Canonicalize Workflow again to resume the same version instead of creating a new one.` - : 'This unfinished canonical version has no finalized graph yet. Run Canonicalize Workflow again to resume the same version.'} - {selected.checkpoint_summary ?

{selected.checkpoint_summary}

: null} -
- )} - {selected?.graph ? : null} -

Objects start above their earliest consuming step, labels wrap across multiple lines, and you can drag nodes freely anywhere in the canvas.

- {selected?.graph &&
- {selected.graph.raw_to_canonical_mappings.length} mappings - {selected.graph.unmatched_raw_information.length} unmatched items - {selected.graph.proposed_schema_updates.length} schema proposals -
} -
-} diff --git a/frontend/src/components/projects/ProjectAssetsPanel.tsx b/frontend/src/components/projects/ProjectAssetsPanel.tsx index f0914cc..b5dab36 100644 --- a/frontend/src/components/projects/ProjectAssetsPanel.tsx +++ b/frontend/src/components/projects/ProjectAssetsPanel.tsx @@ -1,8 +1,9 @@ import { useCallback, useEffect, useState } from 'react' +import { JOB_FINISHED_EVENT } from '../../api/jobPolling' import { listAssets, listProcessedAssets } from '../../api/projects' import { uploadProcessedAsset } from '../../api/upload' -import type { Asset, ProcessedAsset } from '../../types' +import type { Asset, Job, ProcessedAsset } from '../../types' function UploadProcessedModal({ @@ -144,6 +145,21 @@ export default function ProjectAssetsPanel({ projectId }: { projectId: string }) useEffect(() => { load() }, [load]) + useEffect(() => { + const refreshOnFinishedJob = (event: Event) => { + const job = (event as CustomEvent).detail + if ( + job.status === 'COMPLETED' && + job.project_id === projectId && + ['process', 'upload'].includes(job.kind) + ) { + load() + } + } + window.addEventListener(JOB_FINISHED_EVENT, refreshOnFinishedJob) + return () => window.removeEventListener(JOB_FINISHED_EVENT, refreshOnFinishedJob) + }, [load, projectId]) + const processedByAsset = new Map(processed.map(p => [p.asset_id, p])) return ( diff --git a/frontend/src/components/projects/ProjectDetail.tsx b/frontend/src/components/projects/ProjectDetail.tsx index 4b252bb..576abc3 100644 --- a/frontend/src/components/projects/ProjectDetail.tsx +++ b/frontend/src/components/projects/ProjectDetail.tsx @@ -1,24 +1,24 @@ -import { useCallback, useEffect, useRef, useState } from 'react' +import { lazy, Suspense, useState } from 'react' -import { startJobPolling, type JobPollHandle } from '../../api/jobPolling' -import { cancelJob } from '../../api/jobs' import { deleteProject, extractProject, - getProject, - getProjectJobs, kgExtractProject, processProject, projectToSpace, workflowExtractProject, } from '../../api/projects' -import type { Job, Project, Space } from '../../types' +import { useProjectRefresh } from '../../hooks/useProjectRefresh' +import { useProjectJobController } from '../../hooks/useProjectJobController' +import { useProjectJobs } from '../../store/jobsStore' +import type { Project, Space } from '../../types' import JobProgress from '../JobProgress' import StatusBadge from '../StatusBadge' import ProjectAssetsPanel from './ProjectAssetsPanel' -import WorkflowGraphTab from './WorkflowGraphTab' -import GraphTab from '../frames/GraphTab' -import { projectDisplayName } from '../../utils/projectName' +import { ProjectDetailHeader, ProjectDetailTabs } from './ProjectDetailChrome' + +const WorkflowGraphTab = lazy(() => import('./WorkflowGraphTab')) +const GraphTab = lazy(() => import('../frames/GraphTab')) export interface ProjectDetailProps { @@ -39,83 +39,23 @@ export default function ProjectDetail({ onProjectUpdated, onDeleted, }: ProjectDetailProps) { - const [jobs, setJobs] = useState([]) - const [activeJobId, setActiveJobId] = useState(null) const [selectedSpace, setSelectedSpace] = useState(spaces[0]?.space_id ?? '') const [selectedSourceType, setSelectedSourceType] = useState<'frame' | 'markdown'>('frame') const [confirmDelete, setConfirmDelete] = useState(false) const [deleting, setDeleting] = useState(false) const [deleteError, setDeleteError] = useState(null) - const [actionError, setActionError] = useState(null) const [graphView, setGraphView] = useState<'knowledge' | 'workflow'>('workflow') - const pollHandleRef = useRef(null) const userSpaces = spaces.filter(s => s.name !== '__global_kg__') - const loadJobs = useCallback(async () => { - try { - const j = await getProjectJobs(project.project_id) - setJobs(j) - } catch { /* ignore */ } - }, [project.project_id]) - - useEffect(() => { - loadJobs() - return () => { pollHandleRef.current?.cancel() } - }, [loadJobs]) - - const refreshProject = useCallback(() => { - if (!onProjectUpdated) return - getProject(project.project_id).then(onProjectUpdated).catch(() => { /* ignore */ }) - }, [project.project_id, onProjectUpdated]) - - const pollJob = useCallback((jobId: string) => { - setActiveJobId(jobId) - pollHandleRef.current?.cancel() - pollHandleRef.current = startJobPolling({ - jobId, - onUpdate: job => setJobs(prev => { - const idx = prev.findIndex(j => j.job_id === jobId) - if (idx === -1) return [job, ...prev] - return prev.map(j => j.job_id === jobId ? job : j) - }), - // Refresh outer-list project tags incrementally so badges (Processed / - // Frame / asset_count) progress while the job runs, not only at end. - onTick: tick => { if (tick % 3 === 0) refreshProject() }, - onComplete: () => { - setActiveJobId(null) - loadJobs() - refreshProject() - onJobComplete?.() - }, - onFailed: () => setActiveJobId(null), - }) - }, [loadJobs, onJobComplete, refreshProject]) - - const runAction = async (fn: () => Promise<{ job_id: string }>) => { - try { - setActionError(null) - const { job_id } = await fn() - pollJob(job_id) - } catch (err: unknown) { - console.error('Action failed', err) - const e = err as { response?: { data?: { detail?: string } }; message?: string } - setActionError(e?.response?.data?.detail ?? e?.message ?? 'Action failed') - } - } - - const handleCancel = async () => { - if (!activeJobId) return - try { - await cancelJob(activeJobId) - setJobs(prev => prev.map(j => - j.job_id === activeJobId ? { ...j, status: 'CANCELLED' as const, current_message: 'Cancelling…' } : j - )) - setActiveJobId(null) - } catch (err) { - console.error('Cancel failed', err) - } - } + const refreshProject = useProjectRefresh(project.project_id, onProjectUpdated) + const jobs = useProjectJobs(project.project_id) + const { activeJobId, activeJob, actionError, run: runAction, cancel: handleCancel } = useProjectJobController({ + onComplete: () => { + refreshProject() + onJobComplete?.() + }, + }) const handleDelete = async () => { setDeleting(true) @@ -131,7 +71,6 @@ export default function ProjectDetail({ } } - const activeJob = jobs.find(j => j.job_id === activeJobId) ?? null const workflowActionLabel = project.workflow_status === 'IN_PROGRESS' ? '⛓ Resume Workflow' : project.workflow_status === 'FAILED' @@ -143,24 +82,9 @@ export default function ProjectDetail({
{/* Header */}
-
-

- {projectDisplayName(project)} -

-

- {project.asset_count} asset(s) - · - Processed: - · - Frame: - · - Workflow: -

-
- + × + } />
{/* Body */} @@ -211,21 +135,18 @@ export default function ProjectDetail({
- {([['knowledge', 'Knowledge Graph'], ['workflow', 'Workflow Cards']] as const).map(([key, label]) => ( - - ))} +
- {graphView === 'knowledge' && } + {graphView === 'knowledge' && Loading graph…

}>} {graphView === 'workflow' && ( + Loading workflow…

}> + )}
diff --git a/frontend/src/components/projects/ProjectDetailChrome.tsx b/frontend/src/components/projects/ProjectDetailChrome.tsx new file mode 100644 index 0000000..778e511 --- /dev/null +++ b/frontend/src/components/projects/ProjectDetailChrome.tsx @@ -0,0 +1,58 @@ +import type { ReactNode } from 'react' + +import type { Project } from '../../types' +import { projectDisplayName } from '../../utils/projectName' +import StatusBadge from '../StatusBadge' + +export function ProjectDetailHeader({ + project, + leading, + trailing, +}: { + project: Project + leading?: ReactNode + trailing?: ReactNode +}) { + return ( +
+
+ {leading} +
+

{projectDisplayName(project)}

+

{project.asset_count} asset(s)

+
+
+
+ + + + {trailing} +
+
+ ) +} + +export function ProjectDetailTabs({ + tabs, + active, + onChange, +}: { + tabs: readonly (readonly [T, string])[] + active: T + onChange: (tab: T) => void +}) { + return ( +
+ {tabs.map(([tab, label]) => ( + + ))} +
+ ) +} diff --git a/frontend/src/components/projects/StatusLights.tsx b/frontend/src/components/projects/StatusLights.tsx index 052b317..f30fbb5 100644 --- a/frontend/src/components/projects/StatusLights.tsx +++ b/frontend/src/components/projects/StatusLights.tsx @@ -8,7 +8,6 @@ const LIGHT_STYLES: Record = { CANCELLED: 'bg-slate-500', NO_FRAME: 'bg-slate-500', NO_WORKFLOW: 'bg-slate-500', - NO_CANONICAL_WORKFLOW: 'bg-slate-500', PROCESSED: 'bg-emerald-400', UNPROCESSED: 'bg-slate-500', PARTIAL: 'bg-amber-400', @@ -39,19 +38,16 @@ export default function StatusLights({ processed, frame, workflow, - normalized, }: { processed: string frame: string workflow: string - normalized: string }) { return (
-
) } diff --git a/frontend/src/components/projects/WorkflowCanvas.tsx b/frontend/src/components/projects/WorkflowCanvas.tsx index 2dbe8e3..20fe49b 100644 --- a/frontend/src/components/projects/WorkflowCanvas.tsx +++ b/frontend/src/components/projects/WorkflowCanvas.tsx @@ -1,61 +1,72 @@ import { useEffect, useState } from 'react' +import dagre from '@dagrejs/dagre' import ReactFlow, { Background, + BaseEdge, Controls, + EdgeLabelRenderer, Handle, MarkerType, MiniMap, Position, + applyNodeChanges, type Edge, + type EdgeProps, type Node, + type NodeChange, type NodeProps, useEdgesState, useNodesState, } from 'reactflow' import 'reactflow/dist/style.css' +import { escapeXml, labelLines, type WorkflowCanvasEdge, type WorkflowCanvasNode, type WorkflowNodeKind } from '../../features/workflows/model' +export type { WorkflowCanvasEdge, WorkflowCanvasNode } from '../../features/workflows/model' -type WorkflowNodeKind = 'object' | 'operation' - -export interface WorkflowCanvasNode { +interface WorkflowNodeData { id: string - label: string kind: WorkflowNodeKind + label: string title?: string details?: Record } -export interface WorkflowCanvasEdge { - id: string - source: string - target: string - label?: string +interface WorkflowEdgeData { title?: string -} - -interface WorkflowNodeData { - id: string - kind: WorkflowNodeKind - label: string - title?: string - details?: Record + points?: Array<{ x: number; y: number }> + sourceSide?: AnchorSide + targetSide?: AnchorSide } const XML_NS = 'http://www.w3.org/2000/svg' const OP_WIDTH = 190 const OBJ_WIDTH = 190 +const CONTEXT_WIDTH = 210 const NODE_HEIGHT = 74 -const X_GAP = 260 -const Y_GAP = 240 -const OBJECT_OFFSET = 145 -const MIN_ROW_SPACING = 235 + +type AnchorSide = 'top' | 'right' | 'bottom' | 'left' function NodeHandles() { + const handles: AnchorSide[] = ['top', 'right', 'bottom', 'left'] return ( <> - - - - + {handles.map(side => ( + + ))} + {handles.map(side => ( + + ))} ) } @@ -109,65 +120,80 @@ function ObjectNode({ data }: NodeProps) { ) } -function average(values: number[]) { - return values.length ? values.reduce((sum, value) => sum + value, 0) / values.length : 0 +function PlanningNode({ data }: NodeProps) { + return ( +
+ + +
+ ) } -function escapeXml(value: string) { - return value - .replace(/&/g, '&') - .replace(//g, '>') - .replace(/"/g, '"') - .replace(/'/g, ''') +function ReasoningNode({ data }: NodeProps) { + return ( +
+ + +
+ ) } -function labelLines(label: string, maxChars = 18, maxLines = 3) { - const words = label.split(/\s+/).filter(Boolean) - if (words.length === 0) return [''] - - const lines: string[] = [] - let current = '' - let index = 0 - - while (index < words.length) { - const word = words[index] - const candidate = current ? `${current} ${word}` : word - if (candidate.length <= maxChars || current.length === 0) { - current = candidate - index += 1 - continue - } - - lines.push(current) - current = word - index += 1 - if (lines.length === maxLines - 1) { - break - } - } +function UnknownNode({ data }: NodeProps) { + return ( +
+ + +
+ ) +} - const tailWords = current ? [current, ...words.slice(index)] : words.slice(index) - const tail = tailWords.join(' ').trim() - if (tail) { - lines.push(tail) +function sideToPosition(side: AnchorSide) { + switch (side) { + case 'top': + return Position.Top + case 'right': + return Position.Right + case 'bottom': + return Position.Bottom + case 'left': + return Position.Left } +} - if (lines.length > maxLines) { - lines.length = maxLines +function sideVector(side: AnchorSide) { + switch (side) { + case 'top': + return { x: 0, y: -1 } + case 'right': + return { x: 1, y: 0 } + case 'bottom': + return { x: 0, y: 1 } + case 'left': + return { x: -1, y: 0 } } - if (lines.length === maxLines && lines[maxLines - 1].length > maxChars + 6) { - lines[maxLines - 1] = `${lines[maxLines - 1].slice(0, maxChars + 3).trimEnd()}...` - } - return lines } function nodeWidth(kind: WorkflowNodeKind) { - return kind === 'operation' ? OP_WIDTH : OBJ_WIDTH + if (kind === 'operation') return OP_WIDTH + if (kind === 'object') return OBJ_WIDTH + return CONTEXT_WIDTH } function nodeBounds(node: Node) { - const width = nodeWidth(node.type === 'operation' ? 'operation' : 'object') + const kind = (node.type ?? node.data.kind) as WorkflowNodeKind + const width = nodeWidth(kind) return { x: node.position.x, y: node.position.y, @@ -178,16 +204,80 @@ function nodeBounds(node: Node) { } } +function sidePoint(box: ReturnType, side: AnchorSide) { + switch (side) { + case 'top': + return { x: box.centerX, y: box.y } + case 'right': + return { x: box.x + box.width, y: box.centerY } + case 'bottom': + return { x: box.centerX, y: box.y + box.height } + case 'left': + return { x: box.x, y: box.centerY } + } +} + +function chooseAnchorSides(source: Node, target: Node) { + const sb = nodeBounds(source) + const tb = nodeBounds(target) + const sourceSides: AnchorSide[] = ['top', 'right', 'bottom', 'left'] + const targetSides: AnchorSide[] = ['top', 'right', 'bottom', 'left'] + let best = { sourceSide: 'bottom' as AnchorSide, targetSide: 'top' as AnchorSide, score: Number.POSITIVE_INFINITY } + + sourceSides.forEach(sourceSide => { + targetSides.forEach(targetSide => { + const start = sidePoint(sb, sourceSide) + const end = sidePoint(tb, targetSide) + const dx = end.x - start.x + const dy = end.y - start.y + let score = Math.abs(dx) + Math.abs(dy) + + const sv = sideVector(sourceSide) + const tv = sideVector(targetSide) + if (Math.sign(dx) !== 0 && Math.sign(dx) !== Math.sign(sv.x)) score += sourceSide === 'left' || sourceSide === 'right' ? 140 : 40 + if (Math.sign(dy) !== 0 && Math.sign(dy) !== Math.sign(sv.y)) score += sourceSide === 'top' || sourceSide === 'bottom' ? 140 : 40 + if (Math.sign(dx) !== 0 && Math.sign(dx) === Math.sign(tv.x)) score += targetSide === 'left' || targetSide === 'right' ? 140 : 40 + if (Math.sign(dy) !== 0 && Math.sign(dy) === Math.sign(tv.y)) score += targetSide === 'top' || targetSide === 'bottom' ? 140 : 40 + + const mostlyVertical = Math.abs(dy) > Math.abs(dx) * 0.9 + const mostlyHorizontal = Math.abs(dx) > Math.abs(dy) * 0.9 + if (mostlyVertical && sourceSide === 'bottom' && targetSide === 'top' && dy > 0) score -= 130 + if (mostlyVertical && sourceSide === 'top' && targetSide === 'bottom' && dy < 0) score -= 130 + if (mostlyHorizontal && sourceSide === 'right' && targetSide === 'left' && dx > 0) score -= 130 + if (mostlyHorizontal && sourceSide === 'left' && targetSide === 'right' && dx < 0) score -= 130 + if (sourceSide === targetSide) score += 1000 + + if (sourceSide === 'bottom') score -= 90 + if (sourceSide === 'right') score -= 35 + if (sourceSide === 'top') score += 220 + if (sourceSide === 'left') score += 70 + + if (targetSide === 'top') score -= 90 + if (targetSide === 'left') score -= 35 + if (targetSide === 'bottom') score += 220 + if (targetSide === 'right') score += 70 + + if (score < best.score) { + best = { sourceSide, targetSide, score } + } + }) + }) + + return best +} + function edgePath(source: Node, target: Node) { const sb = nodeBounds(source) const tb = nodeBounds(target) - const sourceBelow = tb.centerY >= sb.centerY - const startX = sb.centerX - const startY = sourceBelow ? sb.y + sb.height : sb.y - const endX = tb.centerX - const endY = sourceBelow ? tb.y : tb.y + tb.height - const midY = startY + (endY - startY) / 2 - return `M ${startX} ${startY} C ${startX} ${midY}, ${endX} ${midY}, ${endX} ${endY}` + const { sourceSide, targetSide } = chooseAnchorSides(source, target) + const start = sidePoint(sb, sourceSide) + const end = sidePoint(tb, targetSide) + const sv = sideVector(sourceSide) + const tv = sideVector(targetSide) + const distance = Math.max(72, Math.min(220, (Math.abs(end.x - start.x) + Math.abs(end.y - start.y)) / 2)) + const c1 = { x: start.x + sv.x * distance, y: start.y + sv.y * distance } + const c2 = { x: end.x + tv.x * distance, y: end.y + tv.y * distance } + return `M ${start.x} ${start.y} C ${c1.x} ${c1.y}, ${c2.x} ${c2.y}, ${end.x} ${end.y}` } function downloadFile(filename: string, mimeType: string, content: string) { @@ -200,197 +290,245 @@ function downloadFile(filename: string, mimeType: string, content: string) { URL.revokeObjectURL(url) } -function spreadRow(items: T[], minSpacing: number) { - if (items.length <= 1) return - items.sort((a, b) => a.x - b.x) - for (let index = 1; index < items.length; index += 1) { - if (items[index].x - items[index - 1].x < minSpacing) { - items[index].x = items[index - 1].x + minSpacing - } - } - - const midpoint = (items[0].x + items[items.length - 1].x) / 2 - const targetCenter = average(items.map(item => item.x)) - const shift = midpoint - targetCenter - items.forEach(item => { - item.x -= shift +function dedupePoints(points: Array<{ x: number; y: number }>) { + return points.filter((point, index) => { + const previous = points[index - 1] + return !previous || Math.abs(previous.x - point.x) > 1 || Math.abs(previous.y - point.y) > 1 }) } -function buildLayout(nodes: WorkflowCanvasNode[], edges: WorkflowCanvasEdge[]): { nodes: Node[]; edges: Edge[] } { - const nodeMap = new Map(nodes.map(node => [node.id, node])) - const operationIds = nodes.filter(node => node.kind === 'operation').map(node => node.id) - const objectIds = nodes.filter(node => node.kind === 'object').map(node => node.id) - - const producerMap = new Map() - const consumerMap = new Map() - objectIds.forEach(id => { - producerMap.set(id, []) - consumerMap.set(id, []) - }) - - edges.forEach(edge => { - const sourceKind = nodeMap.get(edge.source)?.kind - const targetKind = nodeMap.get(edge.target)?.kind - if (sourceKind === 'operation' && targetKind === 'object') { - producerMap.get(edge.target)?.push(edge.source) - } - if (sourceKind === 'object' && targetKind === 'operation') { - consumerMap.get(edge.source)?.push(edge.target) - } - }) +function distance(a: { x: number; y: number }, b: { x: number; y: number }) { + return Math.hypot(b.x - a.x, b.y - a.y) +} - const opChildren = new Map>() - const opParents = new Map>() - const indegree = new Map() - operationIds.forEach(id => { - opChildren.set(id, new Set()) - opParents.set(id, new Set()) - indegree.set(id, 0) - }) +function midpointOnPolyline(points: Array<{ x: number; y: number }>) { + if (points.length === 0) return { x: 0, y: 0 } + if (points.length === 1) return points[0] - objectIds.forEach(objectId => { - const producers = producerMap.get(objectId) ?? [] - const consumers = consumerMap.get(objectId) ?? [] - producers.forEach(producerId => { - consumers.forEach(consumerId => { - if (producerId === consumerId || opChildren.get(producerId)?.has(consumerId)) return - opChildren.get(producerId)?.add(consumerId) - opParents.get(consumerId)?.add(producerId) - indegree.set(consumerId, (indegree.get(consumerId) ?? 0) + 1) - }) - }) - }) + const total = points.slice(1).reduce((sum, point, index) => sum + distance(points[index], point), 0) + const target = total / 2 + let traversed = 0 - const queue = operationIds - .filter(id => (indegree.get(id) ?? 0) === 0) - .sort((a, b) => (nodeMap.get(a)?.label ?? '').localeCompare(nodeMap.get(b)?.label ?? '')) - - const opLevel = new Map() - operationIds.forEach(id => opLevel.set(id, 0)) - - while (queue.length > 0) { - const currentId = queue.shift()! - const currentLevel = opLevel.get(currentId) ?? 0 - Array.from(opChildren.get(currentId) ?? []).forEach(childId => { - opLevel.set(childId, Math.max(opLevel.get(childId) ?? 0, currentLevel + 1)) - indegree.set(childId, (indegree.get(childId) ?? 1) - 1) - if ((indegree.get(childId) ?? 0) === 0) { - queue.push(childId) + for (let index = 1; index < points.length; index += 1) { + const start = points[index - 1] + const end = points[index] + const segment = distance(start, end) + if (traversed + segment >= target) { + const ratio = segment === 0 ? 0 : (target - traversed) / segment + return { + x: start.x + (end.x - start.x) * ratio, + y: start.y + (end.y - start.y) * ratio, } - }) + } + traversed += segment } - const levels = new Map() - operationIds.forEach(id => { - const level = opLevel.get(id) ?? 0 - if (!levels.has(level)) levels.set(level, []) - levels.get(level)!.push(id) - }) + return points[points.length - 1] +} - const opX = new Map() - Array.from(levels.keys()).sort((a, b) => a - b).forEach(level => { - const ids = levels.get(level) ?? [] - ids.sort((a, b) => { - const aParents = Array.from(opParents.get(a) ?? []) - const bParents = Array.from(opParents.get(b) ?? []) - const aAnchor = aParents.length ? average(aParents.map(parentId => opX.get(parentId) ?? 0)) : 0 - const bAnchor = bParents.length ? average(bParents.map(parentId => opX.get(parentId) ?? 0)) : 0 - if (aAnchor !== bAnchor) return aAnchor - bAnchor - return (nodeMap.get(a)?.label ?? '').localeCompare(nodeMap.get(b)?.label ?? '') - }) +function controlDistance(start: { x: number; y: number }, end: { x: number; y: number }) { + return Math.max(44, Math.min(150, distance(start, end) / 2)) +} - const rowWidth = (ids.length - 1) * X_GAP - ids.forEach((id, index) => { - opX.set(id, index * X_GAP - rowWidth / 2) - }) +function smoothBezierPath(points: Array<{ x: number; y: number }>, sourceSide?: AnchorSide, targetSide?: AnchorSide) { + if (points.length === 0) return '' + if (points.length === 1) return `M ${points[0].x} ${points[0].y}` + if (points.length === 2) { + const [start, end] = points + const sourceVector = sideVector(sourceSide ?? 'bottom') + const targetVector = sideVector(targetSide ?? 'top') + const offset = controlDistance(start, end) + const c1 = { x: start.x + sourceVector.x * offset, y: start.y + sourceVector.y * offset } + const c2 = { x: end.x + targetVector.x * offset, y: end.y + targetVector.y * offset } + return `M ${start.x} ${start.y} C ${c1.x} ${c1.y}, ${c2.x} ${c2.y}, ${end.x} ${end.y}` + } - const rowItems = ids.map(id => ({ id, x: opX.get(id) ?? 0 })) - spreadRow(rowItems, MIN_ROW_SPACING) - rowItems.forEach(item => { - opX.set(item.id, item.x) - }) - }) + const parts = [`M ${points[0].x} ${points[0].y}`] + for (let index = 0; index < points.length - 1; index += 1) { + const previous = points[index - 1] ?? points[index] + const start = points[index] + const end = points[index + 1] + const next = points[index + 2] ?? end + const sourceVector = sourceSide ? sideVector(sourceSide) : null + const targetVector = targetSide ? sideVector(targetSide) : null + let c1 = { + x: start.x + (end.x - previous.x) / 6, + y: start.y + (end.y - previous.y) / 6, + } + let c2 = { + x: end.x - (next.x - start.x) / 6, + y: end.y - (next.y - start.y) / 6, + } + if (index === 0 && sourceVector) { + const offset = controlDistance(start, end) + c1 = { x: start.x + sourceVector.x * offset, y: start.y + sourceVector.y * offset } + } + if (index === points.length - 2 && targetVector) { + const offset = controlDistance(start, end) + c2 = { x: end.x + targetVector.x * offset, y: end.y + targetVector.y * offset } + } + parts.push(`C ${c1.x} ${c1.y}, ${c2.x} ${c2.y}, ${end.x} ${end.y}`) + } + return parts.join(' ') +} - const objectLayout = objectIds.map(id => { - const producers = producerMap.get(id) ?? [] - const consumers = consumerMap.get(id) ?? [] +function WorkflowEdge({ + id, + sourceX, + sourceY, + targetX, + targetY, + markerEnd, + style, + data, + label, +}: EdgeProps) { + const points = dedupePoints([ + { x: sourceX, y: sourceY }, + ...(data?.points ?? []), + { x: targetX, y: targetY }, + ]) + const path = smoothBezierPath(points, data?.sourceSide, data?.targetSide) + const labelPoint = midpointOnPolyline(points) - if (consumers.length > 0) { - const earliestLevel = Math.min(...consumers.map(consumerId => opLevel.get(consumerId) ?? 0)) - const earliestConsumers = consumers.filter(consumerId => (opLevel.get(consumerId) ?? 0) === earliestLevel) - return { - id, - x: average(earliestConsumers.map(consumerId => opX.get(consumerId) ?? 0)), - y: earliestLevel * Y_GAP - OBJECT_OFFSET, - } - } + return ( + <> + + {label ? ( + +
+ {label} +
+ + ) : null} + + ) +} - if (producers.length > 0) { - const latestLevel = Math.max(...producers.map(producerId => opLevel.get(producerId) ?? 0)) - const latestProducers = producers.filter(producerId => (opLevel.get(producerId) ?? 0) === latestLevel) - return { - id, - x: average(latestProducers.map(producerId => opX.get(producerId) ?? 0)), - y: latestLevel * Y_GAP + OBJECT_OFFSET, - } +function anchorEdges(edgeList: Edge[], nodeList: Node[]) { + const flowNodeMap = new Map(nodeList.map(node => [node.id, node])) + return edgeList.map(edge => { + const source = flowNodeMap.get(String(edge.source)) + const target = flowNodeMap.get(String(edge.target)) + const anchors = source && target ? chooseAnchorSides(source, target) : null + return { + ...edge, + sourceHandle: anchors ? `source-${anchors.sourceSide}` : edge.sourceHandle, + targetHandle: anchors ? `target-${anchors.targetSide}` : edge.targetHandle, + data: anchors ? { ...(edge.data ?? {}), sourceSide: anchors.sourceSide, targetSide: anchors.targetSide } : edge.data, } - - return { id, x: 0, y: -OBJECT_OFFSET } }) +} - const objectRows = new Map>() - objectLayout.forEach(item => { - const rowKey = Math.round(item.y) - if (!objectRows.has(rowKey)) objectRows.set(rowKey, []) - objectRows.get(rowKey)!.push(item) - }) +function buildLayout(nodes: WorkflowCanvasNode[], edges: WorkflowCanvasEdge[]): { nodes: Node[]; edges: Edge[] } { + if (nodes.length === 0) return { nodes: [], edges: [] } - objectRows.forEach(items => { - spreadRow(items, MIN_ROW_SPACING) + const nodeMap = new Map(nodes.map(node => [node.id, node])) + const validEdges = edges.filter(edge => nodeMap.has(edge.source) && nodeMap.has(edge.target)) + const dagreGraph = new dagre.graphlib.Graph({ multigraph: true }) + dagreGraph.setDefaultEdgeLabel(() => ({})) + dagreGraph.setGraph({ + rankdir: 'TB', + align: 'UL', + nodesep: 62, + edgesep: 34, + ranksep: 96, + marginx: 32, + marginy: 32, + acyclicer: 'greedy', + ranker: 'network-simplex', }) - const flowNodes: Node[] = nodes.map(node => { - if (node.kind === 'operation') { - return { - id: node.id, - type: 'operation', - position: { x: opX.get(node.id) ?? 0, y: (opLevel.get(node.id) ?? 0) * Y_GAP }, - data: { id: node.id, kind: node.kind, label: node.label, title: node.title, details: node.details }, - } - } + ;[...nodes] + .sort((a, b) => a.label.localeCompare(b.label)) + .forEach(node => { + dagreGraph.setNode(node.id, { + width: nodeWidth(node.kind), + height: NODE_HEIGHT, + }) + }) + + ;[...validEdges] + .sort((a, b) => String(a.id).localeCompare(String(b.id))) + .forEach(edge => { + dagreGraph.setEdge( + edge.source, + edge.target, + { + weight: 1, + minlen: 1, + width: edge.label ? Math.max(40, edge.label.length * 6) : 0, + height: edge.label ? 18 : 0, + }, + edge.id, + ) + }) + + dagre.layout(dagreGraph) - const layout = objectLayout.find(item => item.id === node.id) ?? { x: 0, y: -OBJECT_OFFSET } + const flowNodes: Node[] = nodes.map(node => { + const layoutNode = dagreGraph.node(node.id) as { x?: number; y?: number } | undefined + const width = nodeWidth(node.kind) return { id: node.id, - type: 'object', - position: { x: layout.x, y: layout.y }, + type: node.kind, + position: { + x: (layoutNode?.x ?? 0) - width / 2, + y: (layoutNode?.y ?? 0) - NODE_HEIGHT / 2, + }, data: { id: node.id, kind: node.kind, label: node.label, title: node.title, details: node.details }, } }) - const flowEdges: Edge[] = edges.map(edge => ({ - id: edge.id, - source: edge.source, - target: edge.target, - type: 'smoothstep', - label: edge.label, - data: edge.title ? { title: edge.title } : undefined, - animated: false, - markerEnd: { type: MarkerType.ArrowClosed, color: '#64748b' }, - style: { stroke: '#64748b', strokeWidth: 1.5 }, - labelStyle: { fill: '#94a3b8', fontSize: 11 }, - labelBgStyle: { fill: 'rgba(9, 9, 11, 0.92)', fillOpacity: 1 }, - labelBgPadding: [6, 2], - labelBgBorderRadius: 6, - })) - - return { nodes: flowNodes, edges: flowEdges } + const flowEdges: Edge[] = validEdges.map(edge => { + const layoutEdge = dagreGraph.edge({ v: edge.source, w: edge.target, name: edge.id }) as { points?: Array<{ x: number; y: number }> } | undefined + return { + id: edge.id, + source: edge.source, + target: edge.target, + type: 'workflow', + label: edge.label, + data: { title: edge.title, points: layoutEdge?.points ?? [] }, + animated: false, + markerEnd: { type: MarkerType.ArrowClosed, color: '#64748b' }, + style: { stroke: '#64748b', strokeWidth: 1.5 }, + } + }) + + return { nodes: flowNodes, edges: anchorEdges(flowEdges, flowNodes) } } const nodeTypes = { operation: OperationNode, object: ObjectNode, + planning: PlanningNode, + reasoning: ReasoningNode, + unknown: UnknownNode, +} + +const edgeTypes = { + workflow: WorkflowEdge, +} + +function nodeColor(kind: WorkflowNodeKind) { + switch (kind) { + case 'operation': + return '#8b5cf6' + case 'object': + return '#14b8a6' + case 'planning': + return '#f59e0b' + case 'reasoning': + return '#38bdf8' + case 'unknown': + return '#64748b' + } } export default function WorkflowCanvas({ @@ -402,7 +540,7 @@ export default function WorkflowCanvas({ edges: WorkflowCanvasEdge[] exportBaseName?: string }) { - const [flowNodes, setFlowNodes, onNodesChange] = useNodesState([]) + const [flowNodes, setFlowNodes] = useNodesState([]) const [flowEdges, setFlowEdges, onEdgesChange] = useEdgesState([]) const [selectedNodeId, setSelectedNodeId] = useState(null) @@ -415,6 +553,17 @@ export default function WorkflowCanvas({ const selectedNode = flowNodes.find(node => node.id === selectedNodeId) ?? null + const handleNodesChange = (changes: NodeChange[]) => { + setFlowNodes(currentNodes => { + const nextNodes = applyNodeChanges(changes, currentNodes) + setFlowEdges(currentEdges => anchorEdges(currentEdges.map(edge => ({ + ...edge, + data: { ...(edge.data ?? {}), points: [] }, + })), nextNodes)) + return nextNodes + }) + } + const exportSvg = () => { if (flowNodes.length === 0) return @@ -467,11 +616,12 @@ export default function WorkflowCanvas({ for (const node of flowNodes) { const box = nodeBounds(node) const lines = labelLines(node.data.label) - if (node.type === 'operation') { + const kind = node.data.kind + if (kind === 'operation') { svgParts.push( ``, ) - } else { + } else if (kind === 'object') { const slant = 20 const points = [ `${box.x + slant},${box.y}`, @@ -482,6 +632,12 @@ export default function WorkflowCanvas({ svgParts.push( ``, ) + } else { + const fill = kind === 'planning' ? '#78350f' : kind === 'reasoning' ? '#0c4a6e' : '#1e293b' + const stroke = kind === 'planning' ? '#fcd34d' : kind === 'reasoning' ? '#7dd3fc' : '#94a3b8' + svgParts.push( + ``, + ) } const startY = box.centerY - ((lines.length - 1) * 14) / 2 @@ -502,7 +658,7 @@ export default function WorkflowCanvas({ ``, ` `, ...flowNodes.map(node => { - const kind = node.type === 'operation' ? 'operation' : 'object' + const kind = node.data.kind return ` ${node.data.title ? `${escapeXml(node.data.title)}` : ''}` }), ` `, @@ -542,7 +698,8 @@ export default function WorkflowCanvas({ nodes={flowNodes} edges={flowEdges} nodeTypes={nodeTypes} - onNodesChange={onNodesChange} + edgeTypes={edgeTypes} + onNodesChange={handleNodesChange} onEdgesChange={onEdgesChange} onNodeClick={(_, node) => setSelectedNodeId(node.id)} fitView @@ -556,7 +713,7 @@ export default function WorkflowCanvas({ node.type === 'operation' ? '#8b5cf6' : '#14b8a6'} + nodeColor={node => nodeColor((node.data as WorkflowNodeData).kind)} maskColor="rgba(9, 9, 11, 0.78)" /> diff --git a/frontend/src/components/projects/WorkflowGraphTab.tsx b/frontend/src/components/projects/WorkflowGraphTab.tsx index 391b809..b44d5e4 100644 --- a/frontend/src/components/projects/WorkflowGraphTab.tsx +++ b/frontend/src/components/projects/WorkflowGraphTab.tsx @@ -4,6 +4,12 @@ import { deleteProjectWorkflowVersion, getProjectWorkflow, listProjectWorkflows import type { RawWorkflowVersion } from '../../types' import WorkflowCanvas, { type WorkflowCanvasEdge, type WorkflowCanvasNode } from './WorkflowCanvas' +function workflowNodeKind(node: { node_kind?: string; node_kind_guess: string }): WorkflowCanvasNode['kind'] { + const kind = node.node_kind ?? node.node_kind_guess + if (kind === 'operation' || kind === 'planning' || kind === 'reasoning' || kind === 'unknown') return kind + return 'object' +} + function RawWorkflowCanvas({ workflow }: { workflow: RawWorkflowVersion }) { const graph = workflow.graph @@ -12,7 +18,7 @@ function RawWorkflowCanvas({ workflow }: { workflow: RawWorkflowVersion }) { const nodes: WorkflowCanvasNode[] = graph.nodes.map(node => ({ id: node.node_id, label: node.canonical_name ?? node.raw_name, - kind: (node.node_kind ?? node.node_kind_guess) === 'operation' ? 'operation' : 'object', + kind: workflowNodeKind(node), title: `${node.canonical_name ?? node.raw_name}\nsource term: ${node.raw_name}\n${node.semantic_type ?? node.node_kind_guess} · confidence ${node.confidence.toFixed(2)}\n\n${node.evidence_text}`, details: { card_id: node.card_id, @@ -41,6 +47,11 @@ function RawWorkflowCanvas({ workflow }: { workflow: RawWorkflowVersion }) { return } +function formatReviewFlag(flag: { type: string; item_type?: string; item_id?: string }) { + const label = flag.type.replace(/_/g, ' ') + return flag.item_id ? `${label}: ${flag.item_id}` : label +} + export default function WorkflowGraphTab({ projectId, actionsDisabled = false, @@ -124,6 +135,12 @@ export default function WorkflowGraphTab({
{error &&

{error}

} {selected?.error &&

{selected.error}

} + {selected?.review_flags && selected.review_flags.length > 0 && ( +
+ {selected.review_flags.slice(0, 4).map(flag => formatReviewFlag(flag)).join(' · ')} + {selected.review_flags.length > 4 ? ` · ${selected.review_flags.length - 4} more` : ''} +
+ )} {selected && !selected.graph && selected.resumable && (
{selected.has_checkpoint @@ -133,7 +150,7 @@ export default function WorkflowGraphTab({
)} {selected?.graph ? : null} -

Purple rectangles are operations; teal parallelograms are objects. Drag nodes freely to tidy the canvas and hover nodes for evidence.

+

Purple rectangles are operations; teal parallelograms are objects; amber and blue rectangles are planning and reasoning. Drag nodes freely to tidy the canvas and hover nodes for evidence.

) } diff --git a/frontend/src/features/graph/GraphFeature.tsx b/frontend/src/features/graph/GraphFeature.tsx new file mode 100644 index 0000000..d14ac8f --- /dev/null +++ b/frontend/src/features/graph/GraphFeature.tsx @@ -0,0 +1,753 @@ +import { useState, useCallback, useEffect, useRef, useMemo } from 'react' +import { Network } from 'vis-network' +import { DataSet } from 'vis-data' +import { getKnowledgeGraph, getReviewCounts, reviewGraph, clearGraph } from '../../api/graph' +import { JOB_FINISHED_EVENT } from '../../api/jobPolling' +import JobProgress from '../../components/JobProgress' +import type { GraphConcept, GraphRelation, GraphPayload, Job } from '../../types' +import { isJobActive, useJobsStore } from '../../store/jobsStore' +import { buildVisOptions, coverageColor, degreeGlow, EDGE_COLORS, glowNode, lerpColor, seedPosition, type NodeColor } from './visualModel' + +interface VisGraphProps { + concepts: GraphConcept[] + relations: GraphRelation[] + reviewCounts: Record | null + nodeColorMode: string + edgeColorMode: string +} + +type GraphSelection = + | { kind: 'node'; concept: GraphConcept } + | { kind: 'edge'; relation: GraphRelation } + +interface HoverCardState { + item: GraphSelection + x: number + y: number +} + +function VisGraph({ concepts, relations, reviewCounts, nodeColorMode, edgeColorMode }: VisGraphProps) { + const containerRef = useRef(null) + const networkRef = useRef(null) + const nodesRef = useRef | null>(null) + const edgesRef = useRef | null>(null) + const [hoverCard, setHoverCard] = useState(null) + const [selectedItem, setSelectedItem] = useState(null) + const [stabilizing, setStabilizing] = useState(false) + const [stabilizeProgress, setStabilizeProgress] = useState(0) + + // ── Pre-compute shared quantities ───────────────────────────────────────── + const degree = useMemo>(() => { + const d: Record = {} + relations.forEach(r => { + d[r.source] = (d[r.source] ?? 0) + 1 + d[r.target] = (d[r.target] ?? 0) + 1 + }) + return d + }, [relations]) + const maxDegree = useMemo(() => Math.max(...Object.values(degree), 1), [degree]) + const rc = useMemo(() => reviewCounts ?? {}, [reviewCounts]) + const maxRc = useMemo(() => Math.max(...Object.values(rc), 0), [rc]) + const conceptByLabel = useMemo(() => Object.fromEntries(concepts.map(concept => [concept.label, concept])), [concepts]) + const relationById = useMemo( + () => Object.fromEntries(relations.map((relation, index) => [`e${index}`, relation])), + [relations], + ) + + const EV_LABELS: Record = { 1: 'causal', 2: 'direct', 3: 'correlative', 4: 'predicted' } + + const buildNode = useCallback((c: GraphConcept, index: number, total: number) => { + const deg = degree[c.label] ?? 0 + const aliasN = (c.aliases ?? []).length + const t = Math.min(deg / maxDegree, 1) + // Default: degree → hue gradient; alias count adds a small size bonus. + let color: NodeColor = degreeGlow(t) + let size = 12 + Math.round(t * 18) + Math.min(aliasN, 5) * 1.2 + const lines = [c.label] + const aliases = (c.aliases ?? []).join(', ') + if (aliases) lines.push(`Aliases: ${aliases}`) + lines.push(`Connections: ${deg}`) + if (nodeColorMode === 'review_coverage') { + const n = rc[c.label] ?? 0 + const accent = coverageColor(n, maxRc) + color = glowNode(lerpColor('#0a0f18', accent, 0.15), accent) + lines.push(n > 0 ? `Reviewed: ${n}×` : 'Never reviewed') + } else if (nodeColorMode === 'modification_heat') { + const n = c.modification_count ?? 0 + const accent = coverageColor(n, maxRc) + color = glowNode(lerpColor('#0a0f18', accent, 0.15), accent) + lines.push(`Modified: ${n}×`) + } else if (nodeColorMode === 'connectivity') { + // Explicit connectivity mode: same gradient but larger size range. + color = degreeGlow(t) + size = 12 + Math.round(t * 26) + } + const { x, y } = seedPosition(index, total) + return { id: c.label, label: '', title: lines.join('\n'), color, size, x, y } + }, [nodeColorMode, rc, maxRc, degree, maxDegree]) + + const buildEdge = useCallback((r: GraphRelation, i: number) => { + const ev = typeof r.evidence_level === 'number' ? r.evidence_level : 3 + let color = '#475569' // slate-600 — much more visible on dark bg than #555 + if (edgeColorMode === 'evidence_level') { + color = EDGE_COLORS[ev] ?? '#94a3b8' + } else if (edgeColorMode === 'review_coverage' || edgeColorMode === 'modification_heat') { + const key = `${r.source}→${r.target}` + color = coverageColor(rc[key] ?? 0, maxRc) + } + return { + id: `e${i}`, + from: r.source, + to: r.target, + title: `${r.relation}\nEvidence: ${EV_LABELS[ev] ?? ev}`, + color: { color, opacity: 0.55, highlight: color, hover: color }, + } + }, [edgeColorMode, rc, maxRc]) + + // ── Effect 1: create the network when graph structure changes ───────────── + useEffect(() => { + if (!containerRef.current) return + + const total = concepts.length + const visNodes = new DataSet(concepts.map((c, i) => buildNode(c, i, total))) + const visEdges = new DataSet(relations.map(buildEdge)) + nodesRef.current = visNodes + edgesRef.current = visEdges + + networkRef.current?.destroy() + networkRef.current = new Network( + containerRef.current, + { nodes: visNodes, edges: visEdges }, + buildVisOptions(concepts.length, relations.length), + ) + + const network = networkRef.current + + // Show a progress overlay while the force solver settles, then freeze the + // layout — gives us a clustered look without paying physics cost forever. + setStabilizing(concepts.length > 0) + setStabilizeProgress(0) + const handleProgress = (params: { iterations: number; total: number }) => { + if (params.total > 0) setStabilizeProgress(params.iterations / params.total) + } + const handleStabilized = () => { + setStabilizing(false) + setStabilizeProgress(1) + network.setOptions({ physics: { enabled: false } }) + network.fit({ animation: { duration: 400, easingFunction: 'easeInOutQuad' } }) + } + network.on('stabilizationProgress', handleProgress) + network.on('stabilizationIterationsDone', handleStabilized) + + const getHoverPosition = (x: number, y: number) => { + if (!containerRef.current) return { x, y } + const rect = containerRef.current.getBoundingClientRect() + return { + x: Math.min(x + 16, Math.max(rect.width - 280, 16)), + y: Math.min(y + 16, Math.max(rect.height - 170, 16)), + } + } + + const handleHoverNode = (params: { node?: string; pointer: { DOM: { x: number; y: number } } }) => { + if (!params.node) return + const concept = conceptByLabel[params.node] + if (!concept) return + const position = getHoverPosition(params.pointer.DOM.x, params.pointer.DOM.y) + setHoverCard({ item: { kind: 'node', concept }, ...position }) + } + + const handleHoverEdge = (params: { edge?: string; pointer: { DOM: { x: number; y: number } } }) => { + if (!params.edge) return + const relation = relationById[params.edge] + if (!relation) return + const position = getHoverPosition(params.pointer.DOM.x, params.pointer.DOM.y) + setHoverCard({ item: { kind: 'edge', relation }, ...position }) + } + + const clearHover = () => setHoverCard(current => (current ? null : current)) + + const handleClick = (params: { nodes: string[]; edges: string[] }) => { + const [nodeId] = params.nodes + if (nodeId && conceptByLabel[nodeId]) { + setSelectedItem({ kind: 'node', concept: conceptByLabel[nodeId] }) + return + } + const [edgeId] = params.edges + if (edgeId && relationById[edgeId]) { + setSelectedItem({ kind: 'edge', relation: relationById[edgeId] }) + return + } + setSelectedItem(null) + } + + // ── Local-physics-on-drag ──────────────────────────────────────────────── + // Physics is off in the steady state for performance. When the user grabs + // a node, briefly turn physics back on so neighbors react in real time; + // turn it off again shortly after the drag ends. The dragged node itself + // is pinned by vis-network for the duration, so forces only propagate + // through springs to its connected neighborhood — local in effect even + // though the solver is global. + let dragSettleTimer: ReturnType | null = null + const handleDragStart = (params: { nodes: string[] }) => { + if (params.nodes.length === 0) return // panning the canvas — leave physics off + if (dragSettleTimer) { clearTimeout(dragSettleTimer); dragSettleTimer = null } + network.setOptions({ physics: { enabled: true } }) + } + const handleDragEnd = (params: { nodes: string[] }) => { + if (params.nodes.length === 0) return + // Let the neighborhood settle for a beat, then freeze again. + if (dragSettleTimer) clearTimeout(dragSettleTimer) + dragSettleTimer = setTimeout(() => { + network.setOptions({ physics: { enabled: false } }) + dragSettleTimer = null + }, 600) + } + + network.on('hoverNode', handleHoverNode) + network.on('hoverEdge', handleHoverEdge) + network.on('blurNode', clearHover) + network.on('blurEdge', clearHover) + network.on('click', handleClick) + network.on('dragStart', handleDragStart) + network.on('dragEnd', handleDragEnd) + + return () => { + network.off('hoverNode', handleHoverNode) + network.off('hoverEdge', handleHoverEdge) + network.off('blurNode', clearHover) + network.off('blurEdge', clearHover) + network.off('click', handleClick) + network.off('dragStart', handleDragStart) + network.off('dragEnd', handleDragEnd) + network.off('stabilizationProgress', handleProgress) + network.off('stabilizationIterationsDone', handleStabilized) + if (dragSettleTimer) clearTimeout(dragSettleTimer) + networkRef.current?.destroy() + networkRef.current = null + nodesRef.current = null + edgesRef.current = null + } + }, [concepts, relations, conceptByLabel, relationById]) // intentionally exclude color modes — handled by effect 2 + + // ── Effect 2: update colors in-place without destroying the network ─────── + useEffect(() => { + if (!nodesRef.current || !edgesRef.current) return + const total = concepts.length + // Only push color/size updates here — omit x/y so we don't yank nodes back + // to their seed positions on every color-mode change. + nodesRef.current.update(concepts.map((c, i) => { + const n = buildNode(c, i, total) + return { id: n.id, label: n.label, title: n.title, color: n.color, size: n.size } + })) + edgesRef.current.update(relations.map(buildEdge)) + }, [nodeColorMode, edgeColorMode, reviewCounts, buildNode, buildEdge, concepts, relations]) + + useEffect(() => { + setHoverCard(current => { + if (!current) return current + if (current.item.kind === 'node' && conceptByLabel[current.item.concept.label]) return current + if (current.item.kind === 'edge' && Object.values(relationById).includes(current.item.relation)) return current + return null + }) + setSelectedItem(current => { + if (!current) return current + if (current.kind === 'node') return conceptByLabel[current.concept.label] ? current : null + return relations.includes(current.relation) ? current : null + }) + }, [conceptByLabel, relationById, relations]) + + // ── Effect 3: redraw on container resize ────────────────────────────────── + useEffect(() => { + if (!containerRef.current) return + const ro = new ResizeObserver(() => { + if (networkRef.current) { + networkRef.current.setSize('100%', '100%') + networkRef.current.redraw() + } + }) + ro.observe(containerRef.current) + return () => ro.disconnect() + }, []) + + const renderHoverSummary = () => { + if (!hoverCard) return null + const { item, x, y } = hoverCard + const content = item.kind === 'node' + ? [ + { label: 'Concept', value: item.concept.label }, + { label: 'Aliases', value: (item.concept.aliases ?? []).join(', ') || 'None' }, + { label: 'Connections', value: String(degree[item.concept.label] ?? 0) }, + ] + : [ + { label: 'Relation', value: item.relation.relation }, + { label: 'Path', value: `${item.relation.source} -> ${item.relation.target}` }, + { label: 'Evidence', value: EV_LABELS[item.relation.evidence_level ?? 3] ?? String(item.relation.evidence_level ?? 3) }, + ] + + return ( +
+

+ {item.kind === 'node' ? 'Node' : 'Edge'} +

+
+ {content.map(entry => ( +
+ {entry.label}: {entry.value} +
+ ))} +
+
+ ) + } + + const renderDetails = () => { + if (!selectedItem) return null + if (selectedItem.kind === 'node') { + const concept = selectedItem.concept + return ( +
+
+
+

Node details

+

{concept.label}

+
+ +
+
+
+

Aliases

+

{(concept.aliases ?? []).join(', ') || 'None'}

+
+
+
+

Connections

+

{degree[concept.label] ?? 0}

+
+
+

Modified

+

{concept.modification_count ?? 0}

+
+
+

Projects

+

{concept.source_project_ids?.length ?? 0}

+
+
+

Frames

+

{concept.source_frame_ids?.length ?? 0}

+
+
+
+

Source project IDs

+

{concept.source_project_ids?.join(', ') || 'None'}

+
+
+

Source frame IDs

+

{concept.source_frame_ids?.join(', ') || 'None'}

+
+
+
+ ) + } + + const relation = selectedItem.relation + return ( +
+
+
+

Edge details

+

{relation.relation}

+
+ +
+
+
+

Direction

+

{`${relation.source} -> ${relation.target}`}

+
+
+
+

Evidence

+

{EV_LABELS[relation.evidence_level ?? 3] ?? String(relation.evidence_level ?? 3)}

+
+
+

Modified

+

{relation.modification_count ?? 0}

+
+
+

Project

+

{relation.source_project_id ?? 'Unknown'}

+
+
+

Frame

+

{relation.source_frame_id ?? 'Unknown'}

+
+
+
+

Knowledge reference

+ 
+              {relation.knowledge_ref ? JSON.stringify(relation.knowledge_ref, null, 2) : 'None'}
+
+
+
+
+ ) + } + + return ( +
+
+ {stabilizing && ( +
+
+ + Computing layout… {Math.round(stabilizeProgress * 100)}% +
+
+
+
+
+ )} + {renderHoverSummary()} + {renderDetails()} +
+ ) +} + +// ─── Color mode controls ────────────────────────────────────────────────────── + +const NODE_MODES = [ + { value: 'default', label: 'Default (teal)' }, + { value: 'review_coverage', label: 'Review coverage' }, + { value: 'modification_heat', label: 'Modification heat' }, + { value: 'connectivity', label: 'Connectivity' }, +] + +const EDGE_MODES = [ + { value: 'evidence_level', label: 'Evidence level' }, + { value: 'review_coverage', label: 'Review coverage' }, + { value: 'modification_heat', label: 'Modification heat' }, + { value: 'default', label: 'Default (gray)' }, +] + +// ─── Review panel ───────────────────────────────────────────────────────────── + +const REVIEW_MODES = [ + { value: 'auto', label: 'Auto' }, + { value: 'global', label: 'Global' }, + { value: 'local', label: 'Local' }, +] + +const ACTION_ICONS: Record = { + merge: '🔀', standardize: '🏷️', delete: '🗑️', +} +const TOOL_ICONS: Record = { + get_concept_details: '🔍', + get_concept_neighbors: '🌐', + get_relation_type_distribution: '📊', + search_graph_elements: '🔎', + merge_concepts: '🔀', + standardize_relation_name: '🏷️', + delete_concept: '🗑️', + delete_relation: '🗑️', +} + +interface ReviewPanelProps { + onReviewComplete: () => void +} + +function ReviewPanel({ onReviewComplete }: ReviewPanelProps) { + const [mode, setMode] = useState('auto') + const [seedCount, setSeedCount] = useState(10) + const [reviewJobId, setReviewJobId] = useState(null) + const reviewJob = useJobsStore(state => reviewJobId ? state.jobs[reviewJobId] ?? null : null) + const isRunning = !!reviewJobId && (!reviewJob || isJobActive(reviewJob)) + const [expanded, setExpanded] = useState(false) + + const startReview = async () => { + try { + setExpanded(true) + const { job_id } = await reviewGraph({ mode, seed_count: seedCount }) + setReviewJobId(job_id) + } catch { + setReviewJobId(null) + } + } + + useEffect(() => { + const finished = (event: Event) => { + const job = (event as CustomEvent).detail + if (job.job_id === reviewJobId && job.status === 'COMPLETED') onReviewComplete() + } + window.addEventListener(JOB_FINISHED_EVENT, finished) + return () => window.removeEventListener(JOB_FINISHED_EVENT, finished) + }, [onReviewComplete, reviewJobId]) + + return ( +
+ + + {expanded && ( +
+
+
+ + +
+
+ + setSeedCount(Number(e.target.value))} + disabled={isRunning} + className="w-full bg-slate-700 border border-slate-600 rounded px-2 py-1.5 text-sm text-slate-200 focus:outline-none focus:border-teal-500 disabled:opacity-50" + /> +
+
+ +
+
+ + {reviewJob && ( +
+ {isRunning && ( +
+ + Review in progress… +
+ )} + {/* Event log */} + {reviewJob.events && reviewJob.events.length > 0 && ( +
+ {reviewJob.events.slice(-10).reverse().map((ev, i) => { + const action = (ev as Record).action ?? '' + const tool = (ev as Record).tool ?? '' + const label = (ev as Record).label ?? '' + const etype = (ev as Record).element_type ?? '' + const icon = ACTION_ICONS[action] ?? TOOL_ICONS[tool] ?? '·' + return ( +

+ {icon} {tool || action} + {etype && ` — ${etype}`}{label && `: ${label}`} +

+ ) + })} +
+ )} + {/* Result */} + {reviewJob.status === 'COMPLETED' && reviewJob.result && ( +
+ {(() => { + const r = reviewJob.result as Record + const stats = (r.reviewed_elements ?? {}) as Record + return `Review complete — mode: ${r.mode} · examined: ${stats.examined ?? 0} · modified: ${stats.modified ?? 0}` + })()} +
+ )} + {reviewJob.status === 'FAILED' && ( +

Review failed: {reviewJob.error}

+ )} +
+ )} +
+ )} +
+ ) +} + +// ─── Page ───────────────────────────────────────────────────────────────────── + +export default function GraphPage() { + const [payload, setPayload] = useState(null) + const [reviewCounts, setReviewCounts] = useState | null>(null) + const [loading, setLoading] = useState(true) + const [nodeColorMode, setNodeColorMode] = useState('default') + const [edgeColorMode, setEdgeColorMode] = useState('evidence_level') + const [showVizOptions, setShowVizOptions] = useState(false) + + const REVIEW_MODES_SET = new Set(['review_coverage', 'modification_heat']) + + const load = useCallback(async (showLoading = false) => { + if (showLoading) setLoading(true) + try { + const data = await getKnowledgeGraph() + setPayload(data) + } catch { setPayload(null) } + setLoading(false) + }, []) + + useEffect(() => { load(true) }, [load]) + + useEffect(() => { + const refreshOnFinishedJob = (event: Event) => { + const job = (event as CustomEvent).detail + if (job.status === 'COMPLETED' && ['knowledge_graph', 'graph_review'].includes(job.kind)) { + load() + } + } + window.addEventListener(JOB_FINISHED_EVENT, refreshOnFinishedJob) + return () => window.removeEventListener(JOB_FINISHED_EVENT, refreshOnFinishedJob) + }, [load]) + + // Load review counts when a review-based mode is selected + useEffect(() => { + if (REVIEW_MODES_SET.has(nodeColorMode) || REVIEW_MODES_SET.has(edgeColorMode)) { + getReviewCounts() + .then(data => setReviewCounts(data as Record)) + .catch(() => {}) + } + }, [nodeColorMode, edgeColorMode]) + + const graph = payload?.graph + const concepts = graph?.concepts ?? [] + const relations = graph?.relations ?? [] + + if (loading) { + return ( +
+

Dataset Graph

+

Loading…

+
+ ) + } + + return ( +
+ {/* Header */} +
+
+
+

Dataset Graph

+

Merged knowledge graph across all datasets.

+
+
+
+
{concepts.length}
+
Concepts
+
+
+
{relations.length}
+
Relations
+
+
+
{payload?.projection_count ?? 0}
+
Projections
+
+
+
+ + {/* Viz options */} +
+ + {showVizOptions && ( +
+
+ + +
+
+ + +
+
+ )} +
+
+ + {/* Graph canvas or empty state */} + {concepts.length === 0 ? ( +
+

No merged graph data yet. Run graph extraction first.

+ +
+ ) : ( +
+
+
+ +
+
+
+ +
+
+ )} +
+ ) +} diff --git a/frontend/src/features/graph/visualModel.ts b/frontend/src/features/graph/visualModel.ts new file mode 100644 index 0000000..6ac45b0 --- /dev/null +++ b/frontend/src/features/graph/visualModel.ts @@ -0,0 +1,31 @@ +import type { Options } from 'vis-network' + +export const EDGE_COLORS: Record = { 1: '#34d399', 2: '#60a5fa', 3: '#fbbf24', 4: '#f472b6' } +export type NodeColor = { background: string; border: string; highlight: { background: string; border: string }; hover: { background: string; border: string } } + +export const lerpColor = (left: string, right: string, ratio: number) => { + const rgb = (value: string) => [parseInt(value.slice(1, 3), 16), parseInt(value.slice(3, 5), 16), parseInt(value.slice(5, 7), 16)] + const a = rgb(left); const b = rgb(right) + return `#${a.map((value, index) => Math.round(value + (b[index] - value) * ratio).toString(16).padStart(2, '0')).join('')}` +} +export const coverageColor = (count: number, maximum: number) => { + if (maximum <= 0 || count <= 0) return '#6b7280' + const ratio = Math.min(count / maximum, 1) + return ratio < .5 ? lerpColor('#6b7280', '#34d399', ratio * 2) : lerpColor('#34d399', '#f59e0b', (ratio - .5) * 2) +} +export const glowNode = (background: string, border: string): NodeColor => ({ background, border, highlight: { background, border: '#f8fafc' }, hover: { background, border } }) +export const degreeGlow = (ratio: number) => glowNode( + ratio < .5 ? lerpColor('#0c1a2e', '#0a1f18', ratio * 2) : lerpColor('#0a1f18', '#1f1005', (ratio - .5) * 2), + ratio < .5 ? lerpColor('#3b82f6', '#10b981', ratio * 2) : lerpColor('#10b981', '#f97316', (ratio - .5) * 2), +) +export const seedPosition = (index: number, total: number) => { + const radius = Math.max(400, Math.sqrt(total) * 80) + const x = Math.sin(index * 12.9898) * 43758.5453; const y = Math.sin(index * 78.233) * 43758.5453 + return { x: ((x - Math.floor(x)) * 2 - 1) * radius, y: ((y - Math.floor(y)) * 2 - 1) * radius } +} +export function buildVisOptions(nodeCount: number, edgeCount: number): Options { + const large = nodeCount > 400 || edgeCount > 1000 + const iterations = Math.min(800, Math.max(150, Math.round(nodeCount * 1.5))) + const physics = large ? { solver: 'forceAtlas2Based' as const, forceAtlas2Based: { gravitationalConstant: -55, centralGravity: .008, springLength: 120, springConstant: .05, damping: .5, avoidOverlap: .6 }, maxVelocity: 35, minVelocity: .75, timestep: .5, stabilization: { enabled: true, iterations, updateInterval: 25, fit: true } } : { solver: 'barnesHut' as const, barnesHut: { gravitationalConstant: -6000, centralGravity: .3, springLength: 110, springConstant: .05, damping: .12, avoidOverlap: .5 }, stabilization: { enabled: true, iterations, updateInterval: 25, fit: true } } + return { layout: { improvedLayout: !large, randomSeed: 42 }, physics, nodes: { font: { size: 0 }, borderWidth: 2, borderWidthSelected: 3, shape: 'dot', shadow: { enabled: true, color: 'rgba(0, 0, 0, 0.45)', size: 12, x: 0, y: 2 }, scaling: { min: 10, max: 40 }, chosen: { node: ((values: { borderWidth: number; shadowSize: number; shadowColor: string }) => { values.borderWidth = 3; values.shadowSize = 22; values.shadowColor = 'rgba(45, 212, 191, 0.55)' }) as unknown as boolean, label: false } }, edges: { font: { size: 0 }, smooth: { enabled: !large, type: 'continuous', roundness: .25, forceDirection: 'none' }, width: 1.2, selectionWidth: 1.5, arrows: { to: { enabled: true, scaleFactor: .45, type: 'arrow' } }, arrowStrikethrough: false, hoverWidth: .6 }, interaction: { hover: true, tooltipDelay: 80, hideEdgesOnDrag: false, hideNodesOnDrag: false, navigationButtons: false, multiselect: false, dragView: true, zoomView: true } } +} diff --git a/frontend/src/features/projections/ProjectionsFeature.tsx b/frontend/src/features/projections/ProjectionsFeature.tsx new file mode 100644 index 0000000..933bd6e --- /dev/null +++ b/frontend/src/features/projections/ProjectionsFeature.tsx @@ -0,0 +1,641 @@ +import type { JSX } from 'react' +import { useCallback, useEffect, useMemo, useState } from 'react' + +import { JOB_FINISHED_EVENT, isJobTerminal } from '../../api/jobPolling' +import { listProjectGroups } from '../../api/projectGroups' +import { deleteProjection, exportProjections, listProjections, reviewProjections } from '../../api/projections' +import { listProjects } from '../../api/projects' +import { getSpace, listSpaces } from '../../api/spaces' +import ProjectionRow from '../../components/projections/ProjectionRow' +import SectionTable from '../../components/projections/SectionTable' +import { + GLOBAL_KG_SPACE, + buildSectionRows, + paperName, + slugifyExportName, +} from '../../components/projections/helpers' +import StatusBadge from '../../components/StatusBadge' +import type { Job, Project, ProjectGroup, Projection, Space } from '../../types' +import { useJobsStore } from '../../store/jobsStore' +import { useShallow } from 'zustand/react/shallow' + + +export default function ProjectionsPage() { + const [spaces, setSpaces] = useState([]) + const [selectedSpaceId, setSelectedSpaceId] = useState('') + const [spaceDetail, setSpaceDetail] = useState(null) + const [projections, setProjections] = useState([]) + const [allProjects, setAllProjects] = useState([]) + const [groups, setGroups] = useState([]) + const [paperLookup, setPaperLookup] = useState>({}) + const [loading, setLoading] = useState(false) + const [newestOnly, setNewestOnly] = useState(true) + const [showHistory, setShowHistory] = useState(false) + const [reviewJobIds, setReviewJobIds] = useState([]) + const reviewJobs = useJobsStore(useShallow(state => reviewJobIds + .map(jobId => state.jobs[jobId]) + .filter((job): job is Job => !!job))) + const [isReviewing, setIsReviewing] = useState(false) + const [selectedReviewerId, setSelectedReviewerId] = useState('') + const [showSpaceDetail, setShowSpaceDetail] = useState(false) + const [selectedProjectionIds, setSelectedProjectionIds] = useState>(new Set()) + const [batchDeleting, setBatchDeleting] = useState(false) + const [groupBy, setGroupBy] = useState(false) + const [isExporting, setIsExporting] = useState(false) + + const toggleProjectionSelected = useCallback((id: string) => { + setSelectedProjectionIds(prev => { + const next = new Set(prev) + if (next.has(id)) next.delete(id); else next.add(id) + return next + }) + }, []) + + const selectAllProjections = useCallback(() => { + setSelectedProjectionIds(new Set(projections.map(p => p.projection_id))) + }, [projections]) + + const clearProjectionSelection = useCallback(() => setSelectedProjectionIds(new Set()), []) + + const batchDeleteIds = useCallback(async (ids: string[]) => { + if (ids.length === 0) return + if (!confirm(`Delete ${ids.length} projection(s)? This cannot be undone.`)) return + setBatchDeleting(true) + try { + const results = await Promise.allSettled(ids.map(id => deleteProjection(id))) + const deletedOk = ids.filter((_, i) => results[i].status === 'fulfilled') + const failed = results.length - deletedOk.length + const okSet = new Set(deletedOk) + setProjections(prev => prev.filter(p => !okSet.has(p.projection_id))) + setSelectedProjectionIds(prev => { + const next = new Set(prev) + deletedOk.forEach(id => next.delete(id)) + return next + }) + if (failed > 0) alert(`${failed} projection(s) failed to delete.`) + } finally { + setBatchDeleting(false) + } + }, []) + + useEffect(() => { + listSpaces().then(sps => { + const visible = sps.filter(s => s.name !== GLOBAL_KG_SPACE) + setSpaces(visible) + if (visible.length > 0) setSelectedSpaceId(visible[0].space_id) + }).catch(() => {}) + listProjectGroups().then(setGroups).catch(() => {}) + }, []) + + const loadProjections = useCallback(async () => { + if (!selectedSpaceId) return + setLoading(true) + try { + const [projs, projects] = await Promise.all([ + listProjections({ space_id: selectedSpaceId, include_data: true, newest_only: newestOnly, include_history: showHistory, limit: 500 }), + listProjects(500), + ]) + const lookup: Record = {} + projects.forEach(p => { + const name = paperName(p) + if (name) lookup[p.project_id] = name + }) + setPaperLookup(lookup) + setAllProjects(projects) + setProjections(projs) + getSpace(selectedSpaceId).then(setSpaceDetail).catch(() => {}) + } finally { + setLoading(false) + } + }, [selectedSpaceId, newestOnly, showHistory]) + + useEffect(() => { loadProjections() }, [loadProjections]) + + useEffect(() => { + const refreshOnFinishedJob = (event: Event) => { + const job = (event as CustomEvent).detail + if ( + job.status === 'COMPLETED' && + ['project', 'projection_review', 'upload'].includes(job.kind) + ) { + loadProjections() + } + } + window.addEventListener(JOB_FINISHED_EVENT, refreshOnFinishedJob) + return () => window.removeEventListener(JOB_FINISHED_EVENT, refreshOnFinishedJob) + }, [loadProjections]) + + useEffect(() => { + const processors = spaceDetail?.post_processors ?? [] + const enabled = processors.filter(processor => processor.enabled !== false) + if (enabled.length === 0) { + setSelectedReviewerId('') + return + } + if (!enabled.some(processor => processor.id === selectedReviewerId)) { + setSelectedReviewerId(enabled[0].id) + } + }, [spaceDetail?.space_id, spaceDetail?.post_processors, selectedReviewerId]) + + const sectionRows = useMemo(() => buildSectionRows(projections, paperLookup), [projections, paperLookup]) + + const [reviewMode, setReviewMode] = useState<'per_project' | 'session'>('per_project') + + const selectedProjectIds = useMemo(() => { + if (selectedProjectionIds.size === 0) return [] as string[] + const pids = new Set() + for (const proj of projections) { + if (selectedProjectionIds.has(proj.projection_id) && proj.project_id) pids.add(proj.project_id) + } + return Array.from(pids) + }, [selectedProjectionIds, projections]) + + useEffect(() => { + if (!isReviewing || reviewJobs.length !== reviewJobIds.length || reviewJobs.some(job => !isJobTerminal(job.status))) return + setIsReviewing(false) + loadProjections() + }, [isReviewing, loadProjections, reviewJobIds.length, reviewJobs]) + + const startReview = async (overrideProjectionIds?: string[]) => { + try { + setIsReviewing(true) + const projectionIds = overrideProjectionIds && overrideProjectionIds.length > 0 + ? overrideProjectionIds + : Array.from(selectedProjectionIds) + const projectIds = (() => { + if (projectionIds.length === 0) return [] as string[] + const pids = new Set() + for (const proj of projections) { + if (projectionIds.includes(proj.projection_id) && proj.project_id) pids.add(proj.project_id) + } + return Array.from(pids) + })() + const params: { space_id: string; project_ids?: string[]; mode: 'per_project' | 'session'; reviewer_id?: string } = { + space_id: selectedSpaceId, mode: reviewMode, + } + if (projectIds.length > 0) params.project_ids = projectIds + if (selectedReviewerId) params.reviewer_id = selectedReviewerId + const response = await reviewProjections(params) + const jobIds = response.job_ids?.length ? response.job_ids : [response.job_id] + const queuedJobs = jobIds.map(jobId => ({ + job_id: jobId, + kind: 'projection_review', + label: 'Projection Review', + status: 'QUEUED', + project_id: null, + result: null, + error: null, + current_message: 'Queued', + events: [], + created_at: new Date().toISOString(), + updated_at: new Date().toISOString(), + } as Job)) + queuedJobs.forEach(job => useJobsStore.getState().upsertJob(job)) + setReviewJobIds(jobIds) + } catch { setIsReviewing(false) } + } + + const exportSelected = useCallback(async (format: 'yaml' | 'json') => { + const ids = Array.from(selectedProjectionIds) + if (ids.length === 0) return + setIsExporting(true) + try { + await exportProjections(ids, format) + } catch (e) { + alert(`Export failed: ${e instanceof Error ? e.message : String(e)}`) + } finally { + setIsExporting(false) + } + }, [selectedProjectionIds]) + + // Build section rows filtered to a specific set of project IDs (for grouped view) + const groupedSectionRows = useMemo((): Array<{ label: string; color: string | null; rows: Record>> }> => { + if (!groupBy || groups.length === 0) return [] + const projectGroupMap: Record = {} + allProjects.forEach(p => { projectGroupMap[p.project_id] = p.group_id ?? null }) + + const buckets: Array<{ group: ProjectGroup | null; projectIds: Set }> = [ + ...groups.map(g => ({ group: g, projectIds: new Set() })), + { group: null, projectIds: new Set() }, // Ungrouped + ] + allProjects.forEach(p => { + const gid = p.group_id ?? null + const bucket = gid ? buckets.find(b => b.group?.group_id === gid) : buckets[buckets.length - 1] + if (bucket) bucket.projectIds.add(p.project_id) + }) + + return buckets + .filter(b => b.projectIds.size > 0) + .map(b => { + const filteredProjections = projections.filter(p => b.projectIds.has(p.project_id)) + return { + label: b.group?.name ?? 'Ungrouped', + color: b.group?.color ?? null, + rows: buildSectionRows(filteredProjections, paperLookup), + } + }) + .filter(b => Object.keys(b.rows).length > 0) + }, [groupBy, groups, allProjects, projections, paperLookup]) + + const [collapsedGroups, setCollapsedGroups] = useState>(new Set()) + const toggleGroupCollapsed = useCallback((label: string) => { + setCollapsedGroups(prev => { + const next = new Set(prev) + if (next.has(label)) next.delete(label); else next.add(label) + return next + }) + }, []) + + const userSpaces = spaces + const selectedSpaceName = useMemo( + () => userSpaces.find(space => space.space_id === selectedSpaceId)?.name ?? selectedSpaceId ?? 'space', + [selectedSpaceId, userSpaces], + ) + + const reviewSummary = useMemo(() => { + if (reviewJobs.length === 0) return null + const completed = reviewJobs.filter(job => job.status === 'COMPLETED').length + const failed = reviewJobs.filter(job => job.status === 'FAILED').length + const cancelled = reviewJobs.filter(job => job.status === 'CANCELLED').length + const active = reviewJobs.filter(job => !isJobTerminal(job.status)).length + const latestActive = [...reviewJobs] + .reverse() + .find(job => !isJobTerminal(job.status)) + const latestTerminal = [...reviewJobs] + .reverse() + .find(job => isJobTerminal(job.status)) + const visibleJob = latestActive ?? latestTerminal ?? reviewJobs[reviewJobs.length - 1] + const status = failed > 0 + ? 'FAILED' + : cancelled > 0 && active === 0 + ? 'CANCELLED' + : completed === reviewJobs.length + ? 'COMPLETED' + : 'RUNNING' + const parts = [ + `${completed}/${reviewJobs.length} completed`, + active > 0 ? `${active} running or queued` : null, + failed > 0 ? `${failed} failed` : null, + cancelled > 0 ? `${cancelled} cancelled` : null, + ].filter(Boolean) + return { + status: status as Job['status'], + message: reviewJobs.length === 1 + ? (visibleJob.current_message || visibleJob.status) + : parts.join(' · '), + } + }, [reviewJobs]) + + return ( +
+
+
+

Projections

+

Aggregated extraction results per space.

+
+
+ + + + + + {selectedProjectIds.length > 0 + ? `${selectedProjectIds.length} project(s) from selection` + : reviewMode === 'session' + ? 'select projections to enable' + : 'all projects'} + + +
+
+ + {userSpaces.length === 0 ? ( +
+ No spaces defined. Create a space first using the CLI. +
+ ) : ( +
+ {/* Row 1 — space selector + view toggles + utility actions */} +
+
+ Space + +
+
+ {/* View filter pills */} +
+ {([ + { key: 'newest', label: 'Newest only', active: newestOnly, toggle: () => setNewestOnly(v => !v) }, + { key: 'history', label: 'History', active: showHistory, toggle: () => setShowHistory(v => !v) }, + ...(groups.length > 0 ? [{ key: 'group', label: 'Group by', active: groupBy, toggle: () => setGroupBy(v => !v) }] : []), + ] as { key: string; label: string; active: boolean; toggle: () => void }[]).map(({ key, label, active, toggle }) => ( + + ))} +
+
+ + {spaceDetail && ( + + )} +
+ {/* Row 2 — export / selection actions */} +
+ {selectedProjectionIds.size > 0 ? ( + <> + + {selectedProjectionIds.size} selected + + + + + + ) : ( + <> + Export space: + { if (!selectedSpaceId) e.preventDefault() }} + className="inline-flex items-center gap-1 px-2.5 py-0.5 rounded-full text-xs font-medium border border-amber-600/40 text-amber-400 hover:bg-amber-900/20 transition-colors" + title="Download all projections for this space as a ZIP of YAML files" + >⬇ YAML (all) + { if (!selectedSpaceId) e.preventDefault() }} + className="inline-flex items-center gap-1 px-2.5 py-0.5 rounded-full text-xs font-medium border border-amber-600/40 text-amber-400 hover:bg-amber-900/20 transition-colors" + title="Download all projections for this space as a ZIP of JSON files" + >⬇ JSON (all) + + )} +
+
+ )} + + {showSpaceDetail && spaceDetail && ( +
+

Domain: {spaceDetail.domain}

+ {spaceDetail.description &&

{spaceDetail.description}

} +
+ Schema + 
{JSON.stringify(spaceDetail.extraction_schema, null, 2)}
+
+
+ )} + + {reviewSummary && ( +
+ {isReviewing && } + + {reviewSummary.message} +
+ )} + + {loading ? ( +

Loading…

+ ) : projections.length === 0 ? ( +

No projections for this space yet.

+ ) : ( +
+
+

+ All Extracted Data — {projections.length} projection(s) +

+ {Object.keys(sectionRows).length === 0 ? ( +

No completed projection data available yet.

+ ) : groupBy && groupedSectionRows.length > 0 ? ( +
+ {groupedSectionRows.map(({ label, color, rows }) => { + const isCollapsed = collapsedGroups.has(label) + const totalRows = Object.values(rows).reduce((s, r) => s + r.length, 0) + return ( +
+ + {!isCollapsed && ( +
+ {Object.entries(rows).map(([section, sectionRowData]) => { + const sectionSchema = spaceDetail?.extraction_schema?.[section] as Record | undefined + const itemSchema = sectionSchema?.item_schema as Record | undefined + const schemaOrder = itemSchema ? Object.keys(itemSchema) : undefined + return ( + startReview(ids)} + reviewDisabled={isReviewing || !selectedSpaceId} + selectedProjectionIds={selectedProjectionIds} + onToggleProjection={toggleProjectionSelected} + onClearSelection={clearProjectionSelection} + /> + ) + })} +
+ )} +
+ ) + })} +
+ ) : ( +
+ {Object.entries(sectionRows).map(([section, rows]) => { + const sectionSchema = spaceDetail?.extraction_schema?.[section] as Record | undefined + const itemSchema = sectionSchema?.item_schema as Record | undefined + const schemaOrder = itemSchema ? Object.keys(itemSchema) : undefined + return ( + startReview(ids)} + reviewDisabled={isReviewing || !selectedSpaceId} + selectedProjectionIds={selectedProjectionIds} + onToggleProjection={toggleProjectionSelected} + onClearSelection={clearProjectionSelection} + /> + ) + })} +
+ )} +
+ +
+
+

Individual Projections

+ + {selectedProjectionIds.size > 0 && ( + <> + {selectedProjectionIds.size} selected + + + + )} +
+
+ {showHistory + ? (() => { + const byId = new Map(projections.map(p => [p.projection_id, p])) + const live = projections.filter(p => !p.superseded_by_id) + const renderChain = (root: Projection): JSX.Element => { + const ancestors: Projection[] = [] + const walk = (ids: string[] | null | undefined) => { + if (!ids) return + for (const id of ids) { + const anc = byId.get(id) + if (anc) { ancestors.push(anc); walk(anc.supersedes_ids) } + } + } + walk(root.supersedes_ids) + const onDeleted = (id: string) => { + setProjections(prev => prev.filter(x => x.projection_id !== id)) + setSelectedProjectionIds(prev => { const n = new Set(prev); n.delete(id); return n }) + } + return ( +
+ + {ancestors.map((anc, i) => ( +
+
+ + {i === 0 ? '↳ supersedes' : ' ↳'} + +
+ +
+ ))} +
+ ) + } + return live.map(renderChain) + })() + : projections.map(p => ( + { + setProjections(prev => prev.filter(x => x.projection_id !== id)) + setSelectedProjectionIds(prev => { + const next = new Set(prev) + next.delete(id) + return next + }) + }} + /> + )) + } +
+
+
+ )} +
+ ) +} diff --git a/frontend/src/features/projections/tableModel.ts b/frontend/src/features/projections/tableModel.ts new file mode 100644 index 0000000..cd7f573 --- /dev/null +++ b/frontend/src/features/projections/tableModel.ts @@ -0,0 +1,49 @@ +export type ProjectionTableRow = Record +export type ColumnDataType = 'boolean' | 'number' | 'date' | 'text' + +const PAGE_STORAGE_PREFIX = 'mkb:projection-table-page:' + +export const isBlank = (value: unknown) => String(value ?? '').trim() === '' +export const parseBoolean = (value: unknown): number | null => { + const normalized = String(value ?? '').trim().toLowerCase() + if (['true', 'yes', 'y', '1'].includes(normalized)) return 1 + if (['false', 'no', 'n', '0'].includes(normalized)) return 0 + return null +} +export const parseNumber = (value: unknown): number | null => { + const normalized = String(value ?? '').trim().replace(/,/g, '') + if (!normalized) return null + const parsed = Number(normalized) + return Number.isFinite(parsed) ? parsed : null +} +export const parseDate = (value: unknown): number | null => { + const parsed = Date.parse(String(value ?? '').trim()) + return Number.isNaN(parsed) ? null : parsed +} +export function inferColumnDataType(rows: ProjectionTableRow[], column: string): ColumnDataType { + const values = rows.map(row => row[column]).filter(value => !isBlank(value)) + if (!values.length) return 'text' + if (values.every(value => parseBoolean(value) !== null)) return 'boolean' + if (values.every(value => parseNumber(value) !== null)) return 'number' + if (values.every(value => parseDate(value) !== null)) return 'date' + return 'text' +} +export const compareText = (left: unknown, right: unknown) => String(left ?? '').localeCompare( + String(right ?? ''), undefined, { numeric: true, sensitivity: 'base' }, +) +export function sortLabel(type: ColumnDataType, sorted: false | 'asc' | 'desc'): string { + if (!sorted) return 'Click to sort' + if (type === 'boolean') return sorted === 'asc' ? 'False → True (click for True → False)' : 'True → False (click to clear)' + if (type === 'number') return sorted === 'asc' ? '0 → 9 (click for 9 → 0)' : '9 → 0 (click to clear)' + if (type === 'date') return sorted === 'asc' ? 'Old → New (click for New → Old)' : 'New → Old (click to clear)' + return sorted === 'asc' ? 'A → Z (click for Z → A)' : 'Z → A (click to clear)' +} +export const sortArrow = (sorted: false | 'asc' | 'desc') => !sorted ? '⇅' : sorted === 'asc' ? '↑' : '↓' +export function loadSavedPage(key: string): number { + if (typeof window === 'undefined') return 1 + const page = Number(window.sessionStorage.getItem(`${PAGE_STORAGE_PREFIX}${key}`) || 1) + return Number.isFinite(page) && page > 0 ? Math.floor(page) : 1 +} +export function savePage(key: string, page: number): void { + if (typeof window !== 'undefined') window.sessionStorage.setItem(`${PAGE_STORAGE_PREFIX}${key}`, String(page)) +} diff --git a/frontend/src/features/projects/useDragAutoScroll.ts b/frontend/src/features/projects/useDragAutoScroll.ts new file mode 100644 index 0000000..fc79499 --- /dev/null +++ b/frontend/src/features/projects/useDragAutoScroll.ts @@ -0,0 +1,60 @@ +import { useEffect, useRef } from 'react' + +export function useDragAutoScroll({ edgePx = 120, maxSpeed = 18 } = {}) { + const position = useRef(null) + const scrollElement = useRef(null) + const animation = useRef(null) + const dragging = useRef(false) + + useEffect(() => { + const findScrollParent = (element: HTMLElement | null): HTMLElement => { + if (!element || element === document.documentElement) return document.documentElement + const { overflowY } = window.getComputedStyle(element) + return (overflowY === 'auto' || overflowY === 'scroll') && element.scrollHeight > element.clientHeight + ? element + : findScrollParent(element.parentElement) + } + const tick = () => { + const element = scrollElement.current + if (position.current !== null && element) { + const rect = element.getBoundingClientRect() + const relativeY = position.current - rect.top + let speed = 0 + if (relativeY < edgePx) speed = -maxSpeed * (1 - relativeY / edgePx) + else if (relativeY > rect.height - edgePx) speed = maxSpeed * ((relativeY - (rect.height - edgePx)) / edgePx) + element.scrollTop += speed + } + animation.current = requestAnimationFrame(tick) + } + const start = (event: DragEvent) => { + dragging.current = true + scrollElement.current = findScrollParent(event.target as HTMLElement) + animation.current = requestAnimationFrame(tick) + } + const stop = () => { + dragging.current = false + if (animation.current) cancelAnimationFrame(animation.current) + animation.current = null + position.current = null + } + const drag = (event: DragEvent) => { position.current = event.clientY } + const wheel = (event: WheelEvent) => { + if (!dragging.current || !scrollElement.current) return + scrollElement.current.scrollTop += event.deltaY + event.preventDefault() + } + document.addEventListener('dragstart', start, true) + document.addEventListener('dragover', drag) + document.addEventListener('dragend', stop) + document.addEventListener('drop', stop) + document.addEventListener('wheel', wheel, { passive: false, capture: true }) + return () => { + document.removeEventListener('dragstart', start, true) + document.removeEventListener('dragover', drag) + document.removeEventListener('dragend', stop) + document.removeEventListener('drop', stop) + document.removeEventListener('wheel', wheel, { capture: true }) + if (animation.current) cancelAnimationFrame(animation.current) + } + }, [edgePx, maxSpeed]) +} diff --git a/frontend/src/features/spaces/SpacesFeature.tsx b/frontend/src/features/spaces/SpacesFeature.tsx new file mode 100644 index 0000000..4f228e1 --- /dev/null +++ b/frontend/src/features/spaces/SpacesFeature.tsx @@ -0,0 +1,1272 @@ +import { useEffect, useRef, useState } from 'react' +import { + listSpaces, + getSpace, + createSpace, + updateSpace, + deleteSpace, + getDefaultReviewPrompt, +} from '../../api/spaces' +import { listSkills } from '../../api/skills' +import { listPostProcessorScripts, uploadPostProcessorScript } from '../../api/postProcessorScripts' +import { getSettings } from '../../api/settings' +import type { CustomSkill, PostProcessorProfile, PostProcessorScript, Space } from '../../types' +import { + draftFromObject, draftFromSpace, draftToPayload, EMPTY_DRAFT, FIELD_TYPE_OPTIONS, + normalizeOutputColumns, normalizeSchema, outputColumnNames, parseOutputColumns, POST_PROCESSOR_TOOL_OPTIONS, + PURPOSE_COLORS, PURPOSE_OPTIONS, REVIEW_SEARCH_TOOL_OPTIONS, slugifyProcessorId, toRecord, + type EditorState, type SchemaField, type SchemaSection, type SpaceDraft, +} from './model' + +export default function SpacesPage() { + const [spaces, setSpaces] = useState([]) + const [selected, setSelected] = useState(null) + const [editor, setEditor] = useState(null) + const [busy, setBusy] = useState(false) + const [skills, setSkills] = useState([]) + const [scripts, setScripts] = useState([]) + const [uploadedPythonEnabled, setUploadedPythonEnabled] = useState(false) + const [error, setError] = useState(null) + const [info, setInfo] = useState(null) + const importFileRef = useRef(null) + + const refresh = async () => { + try { + const [list, skillList, scriptList, runtimeSettings] = await Promise.all([ + listSpaces(), + listSkills(), + listPostProcessorScripts(), + getSettings(), + ]) + setSpaces(list) + setSkills(skillList) + setScripts(scriptList) + setUploadedPythonEnabled(runtimeSettings.allow_uploaded_python) + if (selected) { + const fresh = list.find(s => s.space_id === selected.space_id) ?? null + if (fresh) { + // pull full detail + const full = await getSpace(fresh.space_id) + setSelected(full) + } else { + setSelected(null) + } + } + } catch (e) { + setError(e instanceof Error ? e.message : String(e)) + } + } + + useEffect(() => { refresh() /* eslint-disable-next-line react-hooks/exhaustive-deps */ }, []) + + const openSpace = async (s: Space) => { + setError(null); setInfo(null) + try { + const full = await getSpace(s.space_id) + setSelected(full) + setEditor(null) + } catch (e) { + setError(e instanceof Error ? e.message : String(e)) + } + } + + const openEditor = (state: EditorState) => { + setEditor(state); setError(null); setInfo(null) + } + + const handleSave = async () => { + if (!editor) return + setBusy(true); setError(null); setInfo(null) + try { + const payload = draftToPayload(editor.draft) + if (!payload.name || !payload.extraction_schema) { + throw new Error('Space must include at least a name and extraction schema.') + } + if (editor.mode === 'edit' && editor.space) { + const res = await updateSpace(editor.space.space_id, payload) + setInfo(`Updated. New version: ${res.version}`) + } else { + const res = await createSpace(payload) + setInfo(`Created space ${res.name} (${res.space_id.slice(0, 8)}…)`) + } + setEditor(null) + await refresh() + } catch (e) { + setError(e instanceof Error ? e.message : String(e)) + } finally { + setBusy(false) + } + } + + const handleDelete = async (s: Space) => { + if (!confirm(`Delete space "${s.name}"? This cannot be undone.`)) return + setBusy(true); setError(null); setInfo(null) + try { + await deleteSpace(s.space_id) + setInfo(`Deleted "${s.name}".`) + if (selected?.space_id === s.space_id) setSelected(null) + await refresh() + } catch (e) { + setError(e instanceof Error ? e.message : String(e)) + } finally { + setBusy(false) + } + } + + return ( +
+
+
+

Spaces

+

+ Projection schemas for tabular DB, QA benchmarks, skill cards, or freeform extraction. + Create one with the Assistant, import JSON, or tune it with structured controls. +

+
+
+ + + { + const file = e.target.files?.[0] + if (!file) return + const reader = new FileReader() + reader.onload = evt => { + const text = evt.target?.result as string + try { + openEditor({ mode: 'import', draft: draftFromObject(JSON.parse(text)) }) + } catch (err) { + setError(err instanceof Error ? err.message : String(err)) + } + } + reader.readAsText(file) + // reset so the same file can be re-selected + e.target.value = '' + }} + /> +
+
+ + {(error || info) && ( +
+ {error && ( +
+ {error} +
+ )} + {info && !error && ( +
+ {info} +
+ )} +
+ )} + +
+ {/* List */} +
+ {spaces.length === 0 && ( +

+ No spaces yet. Ask the Assistant to design one with you, or click "+ New space". +

+ )} + {spaces.map(s => { + const purpose = s.purpose ?? 'tabular_database' + const isActive = selected?.space_id === s.space_id + return ( + + ) + })} +
+ + {/* Detail / editor */} +
+ {editor ? ( +
+
+

+ {editor.mode === 'edit' + ? `Edit ${editor.space?.name}` + : editor.mode === 'import' + ? 'Import space JSON' + : 'New space'} +

+ + Edit with normal fields; long prompts use real line breaks. + +
+ + +
+
+ { + const script = await uploadPostProcessorScript(file) + setScripts(current => [...current, script].sort((a, b) => a.name.localeCompare(b.name))) + return script + }} + onChange={draft => setEditor({ ...editor, draft })} + /> +
+ ) : selected ? ( + + openEditor({ + mode: 'edit', + space: selected, + draft: draftFromSpace(selected), + }) + } + onDelete={() => handleDelete(selected)} + onCustomizeReview={defaultPrompt => + openEditor({ + mode: 'edit', + space: selected, + draft: draftFromSpace(selected, defaultPrompt), + }) + } + /> + ) : ( +

+ Select a space to view its full definition. +

+ )} +
+
+
+ ) +} + +function SpaceForm({ + draft, + skills, + scripts, + uploadedPythonEnabled, + onUploadScript, + onChange, +}: { + draft: SpaceDraft + skills: CustomSkill[] + scripts: PostProcessorScript[] + uploadedPythonEnabled: boolean + onUploadScript: (file: File) => Promise + onChange: (draft: SpaceDraft) => void +}) { + const setDraft = (patch: Partial) => onChange({ ...draft, ...patch }) + const schemaEntries = Object.entries(draft.extraction_schema) + + const updateSection = (sectionKey: string, patch: Partial) => { + setDraft({ + extraction_schema: { + ...draft.extraction_schema, + [sectionKey]: { ...draft.extraction_schema[sectionKey], ...patch }, + }, + }) + } + + const renameSection = (oldKey: string, newKey: string) => { + const clean = newKey.trim() + if (!clean || clean === oldKey || draft.extraction_schema[clean]) return + const next = Object.fromEntries( + Object.entries(draft.extraction_schema).map(([key, value]) => + key === oldKey ? [clean, value] : [key, value], + ), + ) + setDraft({ extraction_schema: next }) + } + + const addSection = () => { + const base = 'new_section' + let key = base + let index = 2 + while (draft.extraction_schema[key]) { + key = `${base}_${index}` + index += 1 + } + setDraft({ + extraction_schema: { + ...draft.extraction_schema, + [key]: { type: 'list', description: '', filter: {}, item_schema: {} }, + }, + }) + } + + const removeSection = (sectionKey: string) => { + const { [sectionKey]: _removed, ...nextSchema } = draft.extraction_schema + setDraft({ extraction_schema: nextSchema }) + } + + const updateField = (sectionKey: string, fieldKey: string, patch: Partial) => { + const section = draft.extraction_schema[sectionKey] + updateSection(sectionKey, { + item_schema: { + ...(section.item_schema ?? {}), + [fieldKey]: { ...(section.item_schema?.[fieldKey] ?? {}), ...patch }, + }, + }) + } + + const renameField = (sectionKey: string, oldKey: string, newKey: string) => { + const clean = newKey.trim() + const section = draft.extraction_schema[sectionKey] + const itemSchema = section.item_schema ?? {} + if (!clean || clean === oldKey || itemSchema[clean]) return + updateSection(sectionKey, { + item_schema: Object.fromEntries( + Object.entries(itemSchema).map(([key, value]) => + key === oldKey ? [clean, value] : [key, value], + ), + ), + }) + } + + const addField = (sectionKey: string) => { + const section = draft.extraction_schema[sectionKey] + const itemSchema = section.item_schema ?? {} + const base = 'new_field' + let key = base + let index = 2 + while (itemSchema[key]) { + key = `${base}_${index}` + index += 1 + } + updateSection(sectionKey, { + item_schema: { + ...itemSchema, + [key]: { type: 'string', item_type: 'string', required: false, description: '' }, + }, + }) + } + + const removeField = (sectionKey: string, fieldKey: string) => { + const section = draft.extraction_schema[sectionKey] + const { [fieldKey]: _removed, ...nextFields } = section.item_schema ?? {} + updateSection(sectionKey, { item_schema: nextFields }) + } + + return ( +
+
+ setDraft({ name })} /> + setDraft({ domain })} /> + + setDraft({ description })} + /> +
+ +
+
+ {schemaEntries.length === 0 && ( +

No schema sections yet.

+ )} + {schemaEntries.map(([sectionKey, section]) => ( +
+
+ renameSection(sectionKey, value)} + /> + + +
+