Dev

.env.example

@@ -1,3 +1,18 @@
+# ── Deployment boundary ─────────────────────────────────────
+# development/local are localhost-only. Production intentionally refuses to
+# start until authentication is implemented and explicitly enabled.
+MKB_DEPLOYMENT_MODE=development
+MKB_API_HOST=127.0.0.1
+MKB_API_PORT=8503
+# JSON list because pydantic-settings parses complex environment values.
+MKB_CORS_ORIGINS=["http://127.0.0.1:5173","http://localhost:5173"]
+MKB_AUTHENTICATION_ENABLED=false
+# Required when authentication is enabled. JSON maps random tokens (32+ chars)
+# to reader, editor, or admin roles. Generate tokens with `openssl rand -hex 32`.
+# MKB_AUTH_TOKENS={"replace-with-a-random-token-at-least-32-chars":"admin"}
+# DANGEROUS: uploaded Python runs with this process's host access.
+MKB_ALLOW_UPLOADED_PYTHON=false
+
 # ── PostgreSQL ───────────────────────────────────────────────
 MKB_PG_HOST=localhost
 MKB_PG_PORT=5432
@@ -10,6 +25,7 @@ MKB_S3_ENDPOINT=http://localhost:9000
 MKB_S3_ACCESS_KEY=minioadmin
 MKB_S3_SECRET_KEY=minioadmin
 MKB_S3_BUCKET_RAW=raw
+MKB_S3_BUCKET_PROCESSED=processed
 MKB_S3_BUCKET_ARCHIVE=archive
 MKB_S3_BUCKET_TEMP=temp
 
@@ -27,10 +43,25 @@ OPENAI_API_BASE=
 # MKB_OPENAI_API_BASE=
 
 # ── Logging ─────────────────────────────────────────────────
-# DEBUG (default) — verbose: agent dialogs, tool calls, full MinerU
+# DEBUG           — verbose: agent dialogs, tool calls, full MinerU
 #                   output, and third-party traces.
-# INFO            — concise app-level messages only.
-MKB_LOG_LEVEL=DEBUG
+# INFO (default)  — concise app-level messages only.
+MKB_LOG_LEVEL=INFO
 MKB_LOG_DIR=logs
 # MKB_LOG_FILE_MAX_MB=20
 # MKB_LOG_FILE_BACKUP_COUNT=5
+
+# ── Upload/archive resource budgets ─────────────────────────
+# MKB_UPLOAD_MAX_FILE_MB=100
+# MKB_UPLOAD_MAX_TOTAL_MB=500
+# MKB_UPLOAD_MAX_FILES=1000
+# MKB_UPLOAD_MAX_PATH_DEPTH=12
+# MKB_ARCHIVE_MAX_EXPANDED_MB=500
+# MKB_ARCHIVE_MAX_MEMBER_MB=100
+# MKB_ARCHIVE_MAX_MEMBERS=2000
+# MKB_ARCHIVE_MAX_COMPRESSION_RATIO=100
+# MKB_ARCHIVE_MAX_NESTING=3
+# MKB_RATE_LIMIT_UPLOAD_PER_MINUTE=30
+# MKB_RATE_LIMIT_ASSISTANT_PER_MINUTE=20
+# MKB_RATE_LIMIT_JOB_START_PER_MINUTE=30
+# MKB_RATE_LIMIT_AUTH_FAILURES_PER_MINUTE=10

.github/CODEOWNERS

@@ -0,0 +1,14 @@
+# Default review owner.
+* @theAfish
+
+# Database access requires database review.
+/src/mkb/db/ @theAfish
+
+# Authentication, uploads, executable content, and deployment boundaries require security review.
+/src/mkb/web/security.py @theAfish
+/src/mkb/web/uploads.py @theAfish
+/src/mkb/config.py @theAfish
+/src/mkb/post_processors/ @theAfish
+/docker-compose.yaml @theAfish
+/SECURITY.md @theAfish
+/docs/security.md @theAfish

.github/pull_request_template.md

@@ -0,0 +1,21 @@
+## Summary
+
+Describe the user-visible change and why it is needed.
+
+## Validation
+
+- [ ] `make lint`
+- [ ] `make test`
+- [ ] `make test-frontend` (or not applicable, explained below)
+- [ ] Documentation updated for observable API/operator behavior
+
+## Risk and ownership
+
+- [ ] No database migration, or migration owner requested
+- [ ] No security-sensitive change, or security owner requested with threat/risk notes
+- [ ] No new secret, generated data, logs, or local environment files committed
+- [ ] Current React path used; legacy canonical compatibility impact noted
+
+## Migration / rollback
+
+Describe data migration, compatibility, and rollback/restore steps, or write “none”.

.github/workflows/frontend.yml

@@ -0,0 +1,23 @@
+name: Frontend
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+
+jobs:
+  build-and-budget:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: frontend
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 22
+          cache: npm
+          cache-dependency-path: frontend/package-lock.json
+      - run: npm ci
+      - run: npm run lint
+      - run: npm run build

.github/workflows/python.yml

@@ -0,0 +1,26 @@
+name: Python distributions
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+
+jobs:
+  test-and-install:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.12"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: pip
+      - run: python -m pip install --upgrade pip build
+      - run: python -m pip install ".[all,dev]"
+      - run: python -m pytest
+      - run: python -m build
+      - run: python -m venv /tmp/mkb-consumer
+      - run: /tmp/mkb-consumer/bin/python -m pip install dist/*.whl
+      - run: cd /tmp && /tmp/mkb-consumer/bin/python "$GITHUB_WORKSPACE/examples/portable_quickstart.py"

.gitignore

@@ -17,16 +17,14 @@ build/
 *.swo
 *~
 
+.vscode/
+
 # Environment
 .env
 
 # Data
-data/inbox/*
-!data/inbox/.gitkeep
-
-data/processed/*
-data/papers/*
-data/uploads/*
+data/
+migration-snapshots/
 
 # Docker volumes
 docker_volumes/
@@ -45,4 +43,4 @@ Thumbs.db
 data/runtime_settings.json
 
 # Logs
-logs/
+logs/

CHANGELOG.md

@@ -0,0 +1,17 @@
+# Changelog
+
+MKB follows [Semantic Versioning](https://semver.org/). Until 1.0, minor releases
+may refine the SDK while documented public imports and persisted-data compatibility
+remain protected by tests and migration gates.
+
+## 0.1.0 - Unreleased
+
+- Introduce the configured `KnowledgeBase` SDK and typed grouped services.
+- Add portable SQLite/filesystem repositories, pipelines, jobs, graph operations,
+  evidence, schemas, projections, and public extension registries.
+- Preserve the existing PostgreSQL/MinIO materials application through injected
+  compatibility adapters.
+- Add inventory, reconciliation, preflight comparison, snapshot validation, and
+  checksum-gated missing-object repair tooling.
+- Split PostgreSQL, S3, PDF, server, materials, Neo4j, and full application support
+  into optional installation extras.

CONTRIBUTING.md

@@ -0,0 +1,20 @@
+# Contributing
+
+Start with `make bootstrap`, review `.env`, then run `make up`, `make doctor`, and
+`make check`. Keep adapters thin: domain behavior belongs in `src/mkb/services/`, HTTP
+mapping in `src/mkb/web/`, CLI parsing in `src/mkb/cli.py`, and current UI work in
+`frontend/`. Canonical-workflow compatibility paths are legacy.
+
+Use a focused branch and include tests for behavior changes. Before opening a pull
+request, run `make lint`, `make test`, and `make test-frontend`. Update the Python API,
+HTTP contract, operator, or security docs whenever their observable behavior changes.
+
+Database migrations require review from the database/migrations owner. Never edit a
+published migration; add a new one and test upgrade plus restore. Changes involving
+authentication, authorization, secrets, uploads/archives, executable processors,
+CORS, filesystem paths, network access, or destructive endpoints require security
+owner review and a short threat/risk note in the pull request.
+
+Do not commit `.env`, credentials, research data, generated exports, logs, or local
+database/object-store state. Report vulnerabilities privately according to
+[SECURITY.md](SECURITY.md), not in a public issue.

Makefile

@@ -1,44 +1,63 @@
-.PHONY: up down logs migrate ingest list batches info purge install test
+.PHONY: setup bootstrap install install-python install-frontend up down logs doctor ingest list batches info purge test lint test-python test-frontend build dev server check ci cleanup reconcile pack unpack restore-drill
+
+PYTHON ?= .venv/bin/python
+BOOTSTRAP_PYTHON ?= python3
+NPM ?= npm
+export PYTHONPATH := $(CURDIR)/src$(if $(PYTHONPATH),:$(PYTHONPATH))
+
+# ── Clean-clone bootstrap ──────────────────────────────────────
+setup: bootstrap
+
+bootstrap:
+	$(BOOTSTRAP_PYTHON) -m venv .venv
+	$(PYTHON) -m pip install --upgrade pip
+	$(MAKE) install
+	@test -f .env || cp .env.example .env
+	@echo "Bootstrap complete. Review .env, then run 'make up' and 'make dev'."
+
+install: install-python install-frontend
+
+install-python:
+	$(PYTHON) -m pip install -e ".[all,dev]"
+
+install-frontend:
+	cd frontend && $(NPM) ci
 
 # ── Infrastructure ──────────────────────────────────────────────
 up:
-	docker compose up -d
-	@echo "Waiting for services…"
-	@docker compose exec postgres pg_isready -U mkb -q && echo "PostgreSQL ready" || true
-	@echo "MinIO console: http://localhost:9001  (minioadmin / minioadmin)"
+	docker compose up -d --wait
+	@echo "MKB data services are healthy."
 
 down:
 	docker compose down
 
 logs:
 	docker compose logs -f
 
-# ── Database ────────────────────────────────────────────────────
-migrate:
-	alembic upgrade head
-
-migration:  ## usage: make migration msg="add foo table"
-	alembic revision --autogenerate -m "$(msg)"
-
-# ── Python ──────────────────────────────────────────────────────
-install:
-	pip install -e ".[dev]"
+doctor:
+	$(PYTHON) -m mkb.doctor
 
 # ── CLI shortcuts ───────────────────────────────────────────────
 ingest:  ## usage: make ingest dir=./data/inbox
-	python -m mkb.cli ingest $(dir)
+	$(PYTHON) -m mkb.cli ingest $(dir)
 
 list:
-	python -m mkb.cli list
+	$(PYTHON) -m mkb.cli list
 
 batches:
-	python -m mkb.cli batches
+	$(PYTHON) -m mkb.cli batches
 
 info:  ## usage: make info id=<asset_id or sha256_prefix>
-	python -m mkb.cli info $(id)
+	$(PYTHON) -m mkb.cli info $(id)
 
 purge:
-	python -m mkb.cli purge
+	$(PYTHON) -m mkb.cli purge
+
+cleanup:
+	$(PYTHON) -m mkb.cli cleanup
+
+reconcile:
+	$(PYTHON) -m mkb.cli reconcile
 
 # ── Data sharing ────────────────────────────────────────────────
 pack:  ## Create a portable snapshot: make pack [out=my_snapshot.tar.gz]
@@ -48,10 +67,36 @@ unpack:  ## Restore from snapshot: make unpack file=mkb_data_YYYYMMDD.tar.gz
 	@[ -n "$(file)" ] || (echo "Usage: make unpack file=<archive.tar.gz>"; exit 1)
 	bash scripts/unpack_data.sh $(file)
 
+restore-drill:
+	bash scripts/restore_drill.sh $(if $(file),$(file),)
+
 # ── Server ──────────────────────────────────────────────────────
 server:
-	python -m mkb.cli api --host 127.0.0.1 --port 8503
+	$(PYTHON) -m mkb.cli api --host 127.0.0.1 --port 8503
+
+dev:
+	PYTHON="$(PYTHON)" NPM="$(NPM)" bash scripts/dev.sh
+
+build:
+	$(PYTHON) -m pip wheel --no-deps --wheel-dir build/wheels .
+	cd frontend && $(NPM) run build
 
 # ── Tests ───────────────────────────────────────────────────────
 test:
-	pytest tests/ -v
+	$(PYTHON) -m pytest tests/ -v
+
+lint:
+	$(PYTHON) -m ruff check src tests
+	cd frontend && $(NPM) run lint
+
+test-python:
+	$(PYTHON) -m pytest
+
+test-frontend:
+	cd frontend && $(NPM) run build
+
+check: lint test-python test-frontend
+
+ci:
+	$(PYTHON) -m ruff check src tests
+	$(PYTHON) -m pytest --collect-only -q