From 1363fe4065524fb6aaa6d4d56ca2159482820f10 Mon Sep 17 00:00:00 2001 From: theAfish Date: Tue, 30 Jun 2026 15:19:33 +0800 Subject: [PATCH 01/26] feat: custom skills --- .gitignore | 7 +- alembic/versions/0020_custom_skills.py | 48 +++ frontend/src/App.tsx | 2 + frontend/src/api/jobPolling.ts | 12 + frontend/src/api/projections.ts | 2 +- frontend/src/api/skills.ts | 25 ++ frontend/src/components/Layout.tsx | 1 + frontend/src/components/frames/AssetsTab.tsx | 25 +- frontend/src/components/frames/GraphTab.tsx | 25 +- .../components/frames/KnowledgeFrameTab.tsx | 25 +- .../src/components/frames/ProjectionsTab.tsx | 25 +- .../components/projections/SectionTable.tsx | 27 +- .../src/components/projects/BrowseTab.tsx | 27 +- .../projects/ProjectAssetsPanel.tsx | 18 +- frontend/src/hooks/useGlobalJobsPoller.ts | 14 +- frontend/src/pages/FramesPage.tsx | 16 +- frontend/src/pages/GraphPage.tsx | 19 +- frontend/src/pages/ProjectionsPage.tsx | 114 ++++++- frontend/src/pages/SkillsPage.tsx | 245 +++++++++++++++ frontend/src/pages/SpacesPage.tsx | 60 +++- frontend/src/types/index.ts | 20 +- src/mkb/agents/projection_reviewer.py | 26 +- src/mkb/db/models.py | 27 ++ src/mkb/skills/__init__.py | 2 + src/mkb/skills/registry.py | 289 ++++++++++++++++++ src/mkb/spaces/registry.py | 19 ++ src/mkb/web/api_server.py | 2 + src/mkb/web/routers/projections.py | 17 ++ src/mkb/web/routers/skills.py | 48 +++ tests/test_projection_review_space_search.py | 29 ++ tests/test_space_post_processors.py | 15 + 31 files changed, 1178 insertions(+), 53 deletions(-) create mode 100644 alembic/versions/0020_custom_skills.py create mode 100644 frontend/src/api/skills.ts create mode 100644 frontend/src/pages/SkillsPage.tsx create mode 100644 src/mkb/skills/__init__.py create mode 100644 src/mkb/skills/registry.py create mode 100644 src/mkb/web/routers/skills.py diff --git a/.gitignore b/.gitignore index a5417a2..0ecc25f 100644 --- a/.gitignore +++ b/.gitignore @@ -21,12 +21,7 @@ build/ .env # Data -data/inbox/* -!data/inbox/.gitkeep - -data/processed/* -data/papers/* -data/uploads/* +data/ # Docker volumes docker_volumes/ diff --git a/alembic/versions/0020_custom_skills.py b/alembic/versions/0020_custom_skills.py new file mode 100644 index 0000000..f79151c --- /dev/null +++ b/alembic/versions/0020_custom_skills.py @@ -0,0 +1,48 @@ +"""Add custom skills. + +Revision ID: 0020_custom_skills +Revises: 0019_space_post_processors +Create Date: 2026-06-30 +""" + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + + +revision = "0020_custom_skills" +down_revision = "0019_space_post_processors" +branch_labels = None +depends_on = None + + +def _has_table(table_name: str) -> bool: + bind = op.get_bind() + inspector = sa.inspect(bind) + return inspector.has_table(table_name) + + +def upgrade() -> None: + if _has_table("custom_skills"): + return + op.create_table( + "custom_skills", + sa.Column("skill_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("name", sa.String(length=255), nullable=False), + sa.Column("slug", sa.String(length=255), nullable=False), + sa.Column("description", sa.Text(), nullable=True), + sa.Column("source_type", sa.String(length=32), nullable=False), + sa.Column("storage_path", sa.Text(), nullable=False), + sa.Column("skill_md", sa.Text(), nullable=False), + sa.Column("file_count", sa.Integer(), nullable=False, server_default="1"), + sa.Column("metadata", postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=True), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=True), + sa.PrimaryKeyConstraint("skill_id"), + sa.UniqueConstraint("slug"), + ) + + +def downgrade() -> None: + if _has_table("custom_skills"): + op.drop_table("custom_skills") diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 4bb17e0..779651b 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -6,6 +6,7 @@ import FramesPage from './pages/FramesPage' import GraphPage from './pages/GraphPage' import ProjectionsPage from './pages/ProjectionsPage' import SpacesPage from './pages/SpacesPage' +import SkillsPage from './pages/SkillsPage' import FeedbackPage from './pages/FeedbackPage' import SettingsPage from './pages/SettingsPage' @@ -20,6 +21,7 @@ function App() { case 'graph': return case 'projections': return case 'spaces': return + case 'skills': return case 'feedback': return case 'settings': return default: return diff --git a/frontend/src/api/jobPolling.ts b/frontend/src/api/jobPolling.ts index fe47256..b582951 100644 --- a/frontend/src/api/jobPolling.ts +++ b/frontend/src/api/jobPolling.ts @@ -7,12 +7,23 @@ const MAX_CONSECUTIVE_ERRORS = 6 /** Statuses that mean the job is still running and we should keep polling. */ export const ACTIVE_JOB_STATUSES = new Set(['QUEUED', 'PENDING', 'RUNNING']) +export const JOB_FINISHED_EVENT = 'mkb:job-finished' /** Returns true when the job has reached a terminal state and polling should stop. */ export function isJobTerminal(status: string): boolean { return !ACTIVE_JOB_STATUSES.has(status) } +const announcedFinishedJobs = new Set() + +export function announceJobFinished(job: Job): void { + if (!isJobTerminal(job.status) || announcedFinishedJobs.has(job.job_id)) return + announcedFinishedJobs.add(job.job_id) + if (typeof window !== 'undefined') { + window.dispatchEvent(new CustomEvent(JOB_FINISHED_EVENT, { detail: job })) + } +} + /** * Returns true when polling should stop for this error. * - 404 means the job is gone (server restart or stale id). @@ -109,6 +120,7 @@ export function startJobPolling(opts: StartJobPollingOptions): JobPollHandle { useJobsStore.getState().upsertJob(job) onUpdate?.(job) if (isJobTerminal(job.status)) { + announceJobFinished(job) if (job.status === 'COMPLETED') { onComplete?.(job) } else { diff --git a/frontend/src/api/projections.ts b/frontend/src/api/projections.ts index 27fcf8c..8cf2555 100644 --- a/frontend/src/api/projections.ts +++ b/frontend/src/api/projections.ts @@ -22,7 +22,7 @@ export const reviewProjections = (params: { mode?: ReviewMode reviewer_id?: string }) => - client.post<{ job_id: string }>('/projections/review', params).then(r => r.data) + client.post<{ job_id: string; job_ids?: string[] }>('/projections/review', params).then(r => r.data) export const deleteProjection = (id: string) => client.delete(`/projections/${id}`) diff --git a/frontend/src/api/skills.ts b/frontend/src/api/skills.ts new file mode 100644 index 0000000..3718475 --- /dev/null +++ b/frontend/src/api/skills.ts @@ -0,0 +1,25 @@ +import client from './client' +import type { CustomSkill } from '../types' + +export const listSkills = () => + client.get('/skills').then(r => r.data) + +export const getSkill = (idOrSlug: string) => + client.get(`/skills/${idOrSlug}`).then(r => r.data) + +export const uploadSkill = (files: File[]) => { + const form = new FormData() + files.forEach(file => { + const relPath = (file as File & { webkitRelativePath?: string }).webkitRelativePath + form.append('files', file, relPath || file.name) + }) + return client + .post('/skills/upload', form, { + headers: { 'Content-Type': 'multipart/form-data' }, + timeout: 120_000, + }) + .then(r => r.data) +} + +export const deleteSkill = (skillId: string) => + client.delete<{ ok: boolean; deleted?: string }>(`/skills/${skillId}`).then(r => r.data) diff --git a/frontend/src/components/Layout.tsx b/frontend/src/components/Layout.tsx index 9a0b986..f93cd0b 100644 --- a/frontend/src/components/Layout.tsx +++ b/frontend/src/components/Layout.tsx @@ -11,6 +11,7 @@ const NAV_ITEMS: { page: Page; label: string; icon: string }[] = [ { page: 'graph', label: 'Dataset Graph', icon: '🕸️' }, { page: 'projections', label: 'Projections', icon: '📊' }, { page: 'spaces', label: 'Spaces', icon: '🗂️' }, + { page: 'skills', label: 'Skills', icon: '🧠' }, { page: 'feedback', label: 'Feedback', icon: '💬' }, { page: 'settings', label: 'Settings', icon: '⚙️' }, ] diff --git a/frontend/src/components/frames/AssetsTab.tsx b/frontend/src/components/frames/AssetsTab.tsx index 3d85a23..158b04e 100644 --- a/frontend/src/components/frames/AssetsTab.tsx +++ b/frontend/src/components/frames/AssetsTab.tsx @@ -1,7 +1,8 @@ -import { useEffect, useState } from 'react' +import { useCallback, useEffect, useState } from 'react' +import { JOB_FINISHED_EVENT } from '../../api/jobPolling' import { listAssets, listProcessedAssets } from '../../api/projects' -import type { Asset, ProcessedAsset } from '../../types' +import type { Asset, Job, ProcessedAsset } from '../../types' import AssetPreviewModal from './AssetPreviewModal' type Preview = { @@ -16,12 +17,30 @@ export default function AssetsTab({ projectId }: { projectId: string }) { const [loading, setLoading] = useState(true) const [preview, setPreview] = useState(null) - useEffect(() => { + const load = useCallback((showLoading = false) => { + if (showLoading) setLoading(true) Promise.all([listAssets(projectId), listProcessedAssets(projectId)]) .then(([a, p]) => { setAssets(a); setProcessed(p) }) .finally(() => setLoading(false)) }, [projectId]) + useEffect(() => { load(true) }, [load]) + + useEffect(() => { + const refreshOnFinishedJob = (event: Event) => { + const job = (event as CustomEvent).detail + if ( + job.status === 'COMPLETED' && + job.project_id === projectId && + ['process', 'upload'].includes(job.kind) + ) { + load() + } + } + window.addEventListener(JOB_FINISHED_EVENT, refreshOnFinishedJob) + return () => window.removeEventListener(JOB_FINISHED_EVENT, refreshOnFinishedJob) + }, [load, projectId]) + if (loading) return

Loading…

if (assets.length === 0) return

No assets ingested yet.

diff --git a/frontend/src/components/frames/GraphTab.tsx b/frontend/src/components/frames/GraphTab.tsx index b14ea78..85b8b53 100644 --- a/frontend/src/components/frames/GraphTab.tsx +++ b/frontend/src/components/frames/GraphTab.tsx @@ -1,7 +1,8 @@ -import { useEffect, useState } from 'react' +import { useCallback, useEffect, useState } from 'react' import { getKnowledgeGraph } from '../../api/graph' -import type { GraphConcept, GraphRelation } from '../../types' +import { JOB_FINISHED_EVENT } from '../../api/jobPolling' +import type { GraphConcept, GraphRelation, Job } from '../../types' import MiniGraph from './MiniGraph' @@ -11,13 +12,31 @@ export default function GraphTab({ projectId }: { projectId: string }) { const [loading, setLoading] = useState(true) const [showList, setShowList] = useState(false) - useEffect(() => { + const load = useCallback((showLoading = false) => { + if (showLoading) setLoading(true) getKnowledgeGraph({ project_id: projectId }) .then(d => { setConcepts(d.graph?.concepts ?? []); setRelations(d.graph?.relations ?? []) }) .catch(() => {}) .finally(() => setLoading(false)) }, [projectId]) + useEffect(() => { load(true) }, [load]) + + useEffect(() => { + const refreshOnFinishedJob = (event: Event) => { + const job = (event as CustomEvent).detail + if ( + job.status === 'COMPLETED' && + job.project_id === projectId && + ['knowledge_graph', 'graph_review'].includes(job.kind) + ) { + load() + } + } + window.addEventListener(JOB_FINISHED_EVENT, refreshOnFinishedJob) + return () => window.removeEventListener(JOB_FINISHED_EVENT, refreshOnFinishedJob) + }, [load, projectId]) + if (loading) return

Loading…

if (concepts.length === 0) return

No graph elements for this project yet. Run Extract Graph.

diff --git a/frontend/src/components/frames/KnowledgeFrameTab.tsx b/frontend/src/components/frames/KnowledgeFrameTab.tsx index eb70fa6..afefeb4 100644 --- a/frontend/src/components/frames/KnowledgeFrameTab.tsx +++ b/frontend/src/components/frames/KnowledgeFrameTab.tsx @@ -1,8 +1,9 @@ -import { useEffect, useState } from 'react' +import { useCallback, useEffect, useState } from 'react' import { getFrame, getFrameHistory } from '../../api/frames' +import { JOB_FINISHED_EVENT } from '../../api/jobPolling' import { getProject } from '../../api/projects' -import type { ExtractionPass, Frame, Project } from '../../types' +import type { ExtractionPass, Frame, Job, Project } from '../../types' import StatusBadge from '../StatusBadge' import { FrameHeader, FrameSection } from './frameRender' @@ -14,7 +15,8 @@ export default function KnowledgeFrameTab({ projectId }: { projectId: string }) const [loading, setLoading] = useState(true) const [showRaw, setShowRaw] = useState(false) - useEffect(() => { + const load = useCallback((showLoading = false) => { + if (showLoading) setLoading(true) Promise.all([getFrame(projectId), getFrameHistory(projectId), getProject(projectId)]) .then(([f, h, p]) => { setFrame(f) @@ -25,6 +27,23 @@ export default function KnowledgeFrameTab({ projectId }: { projectId: string }) .finally(() => setLoading(false)) }, [projectId]) + useEffect(() => { load(true) }, [load]) + + useEffect(() => { + const refreshOnFinishedJob = (event: Event) => { + const job = (event as CustomEvent).detail + if ( + job.status === 'COMPLETED' && + job.project_id === projectId && + ['extract', 'raw_workflow', 'canonical_workflow', 'workflow_maintenance'].includes(job.kind) + ) { + load() + } + } + window.addEventListener(JOB_FINISHED_EVENT, refreshOnFinishedJob) + return () => window.removeEventListener(JOB_FINISHED_EVENT, refreshOnFinishedJob) + }, [load, projectId]) + if (loading) return

Loading…

if (!frame) return

No knowledge frame yet. Run Extract to generate one.

diff --git a/frontend/src/components/frames/ProjectionsTab.tsx b/frontend/src/components/frames/ProjectionsTab.tsx index 759bfcc..ba99f72 100644 --- a/frontend/src/components/frames/ProjectionsTab.tsx +++ b/frontend/src/components/frames/ProjectionsTab.tsx @@ -1,8 +1,9 @@ -import { useEffect, useState } from 'react' +import { useCallback, useEffect, useState } from 'react' +import { JOB_FINISHED_EVENT } from '../../api/jobPolling' import { listProjections } from '../../api/projections' import { listSpaces } from '../../api/spaces' -import type { Projection, Space } from '../../types' +import type { Job, Projection, Space } from '../../types' import StatusBadge from '../StatusBadge' import { FrameSection } from './frameRender' @@ -12,7 +13,8 @@ export default function ProjectionsTab({ projectId }: { projectId: string }) { const [spaces, setSpaces] = useState([]) const [loading, setLoading] = useState(true) - useEffect(() => { + const load = useCallback((showLoading = false) => { + if (showLoading) setLoading(true) Promise.all([ listProjections({ project_id: projectId, include_data: true, limit: 50 }), listSpaces(), @@ -22,6 +24,23 @@ export default function ProjectionsTab({ projectId }: { projectId: string }) { }).finally(() => setLoading(false)) }, [projectId]) + useEffect(() => { load(true) }, [load]) + + useEffect(() => { + const refreshOnFinishedJob = (event: Event) => { + const job = (event as CustomEvent).detail + if ( + job.status === 'COMPLETED' && + job.project_id === projectId && + ['project', 'projection_review'].includes(job.kind) + ) { + load() + } + } + window.addEventListener(JOB_FINISHED_EVENT, refreshOnFinishedJob) + return () => window.removeEventListener(JOB_FINISHED_EVENT, refreshOnFinishedJob) + }, [load, projectId]) + if (loading) return

Loading…

if (projections.length === 0) return

No projections yet. Select a space and run Project.

diff --git a/frontend/src/components/projections/SectionTable.tsx b/frontend/src/components/projections/SectionTable.tsx index 5fe94c6..beda788 100644 --- a/frontend/src/components/projections/SectionTable.tsx +++ b/frontend/src/components/projections/SectionTable.tsx @@ -24,6 +24,7 @@ type ProjectionTableRow = Record type ColumnDataType = 'boolean' | 'number' | 'date' | 'text' const SELECTION_COLUMN_ID = '__projection_selection__' +const PAGE_STORAGE_PREFIX = 'mkb:projection-table-page:' function isBlank(value: unknown): boolean { return String(value ?? '').trim() === '' @@ -79,6 +80,18 @@ function sortArrow(sorted: false | 'asc' | 'desc'): string { return sorted === 'asc' ? '↑' : '↓' } +function loadSavedPage(key: string): number { + if (typeof window === 'undefined') return 1 + const saved = window.sessionStorage.getItem(`${PAGE_STORAGE_PREFIX}${key}`) + const page = saved ? Number(saved) : 1 + return Number.isFinite(page) && page > 0 ? Math.floor(page) : 1 +} + +function savePage(key: string, page: number): void { + if (typeof window === 'undefined') return + window.sessionStorage.setItem(`${PAGE_STORAGE_PREFIX}${key}`, String(page)) +} + export default function SectionTable({ name, rows, @@ -102,7 +115,8 @@ export default function SectionTable({ onClearSelection: () => void reviewDisabled?: boolean }) { - const [page, setPage] = useState(1) + const pageStorageKey = exportBasename ?? name + const [page, setPage] = useState(() => loadSavedPage(pageStorageKey)) const [sorting, setSorting] = useState([]) const [exportingFormat, setExportingFormat] = useState<'csv' | 'excel' | null>(null) const totalPages = Math.max(1, Math.ceil(rows.length / PAGE_SIZE)) @@ -112,6 +126,14 @@ export default function SectionTable({ if (page > totalPages) setPage(totalPages) }, [page, totalPages]) + useEffect(() => { + setPage(loadSavedPage(pageStorageKey)) + }, [pageStorageKey]) + + useEffect(() => { + savePage(pageStorageKey, Math.min(page, totalPages)) + }, [page, pageStorageKey, totalPages]) + const allCols = useMemo(() => { const seen = new Set() for (const r of rows) for (const k of Object.keys(r)) seen.add(k) @@ -209,6 +231,7 @@ export default function SectionTable({ const cycleSort = (columnId: string) => { const current = sorting.find(sort => sort.id === columnId) setPage(1) + savePage(pageStorageKey, 1) if (!current) setSorting([{ id: columnId, desc: false }]) else if (!current.desc) setSorting([{ id: columnId, desc: true }]) else setSorting([]) @@ -289,9 +312,11 @@ export default function SectionTable({ onPaginationChange: updater => { const next = functionalUpdate(updater, pagination) setPage(next.pageIndex + 1) + savePage(pageStorageKey, next.pageIndex + 1) }, onSortingChange: updater => { setPage(1) + savePage(pageStorageKey, 1) setSorting(functionalUpdate(updater, sorting)) }, getCoreRowModel: getCoreRowModel(), diff --git a/frontend/src/components/projects/BrowseTab.tsx b/frontend/src/components/projects/BrowseTab.tsx index 8fb5fef..a238140 100644 --- a/frontend/src/components/projects/BrowseTab.tsx +++ b/frontend/src/components/projects/BrowseTab.tsx @@ -1,8 +1,9 @@ import { useCallback, useEffect, useState } from 'react' +import { JOB_FINISHED_EVENT } from '../../api/jobPolling' import { listProjects } from '../../api/projects' import ProjectGroupedList from '../ProjectGroupedList' -import type { Project, Space } from '../../types' +import type { Job, Project, Space } from '../../types' import ProjectDetail from './ProjectDetail' import StatusLights from './StatusLights' @@ -27,6 +28,30 @@ export default function BrowseTab({ spaces }: { spaces: Space[] }) { useEffect(() => { load() }, [load]) + useEffect(() => { + const refreshOnFinishedJob = (event: Event) => { + const job = (event as CustomEvent).detail + if ( + job.status === 'COMPLETED' && + [ + 'process', + 'extract', + 'project', + 'knowledge_graph', + 'raw_workflow', + 'canonical_workflow', + 'workflow_maintenance', + 'workflow_maintenance_batch', + 'upload', + ].includes(job.kind) + ) { + load() + } + } + window.addEventListener(JOB_FINISHED_EVENT, refreshOnFinishedJob) + return () => window.removeEventListener(JOB_FINISHED_EVENT, refreshOnFinishedJob) + }, [load]) + const getStatus = useCallback( (id: string) => projects.find(p => p.project_id === id)?.frame_status ?? 'NO_FRAME', [projects], diff --git a/frontend/src/components/projects/ProjectAssetsPanel.tsx b/frontend/src/components/projects/ProjectAssetsPanel.tsx index f0914cc..b5dab36 100644 --- a/frontend/src/components/projects/ProjectAssetsPanel.tsx +++ b/frontend/src/components/projects/ProjectAssetsPanel.tsx @@ -1,8 +1,9 @@ import { useCallback, useEffect, useState } from 'react' +import { JOB_FINISHED_EVENT } from '../../api/jobPolling' import { listAssets, listProcessedAssets } from '../../api/projects' import { uploadProcessedAsset } from '../../api/upload' -import type { Asset, ProcessedAsset } from '../../types' +import type { Asset, Job, ProcessedAsset } from '../../types' function UploadProcessedModal({ @@ -144,6 +145,21 @@ export default function ProjectAssetsPanel({ projectId }: { projectId: string }) useEffect(() => { load() }, [load]) + useEffect(() => { + const refreshOnFinishedJob = (event: Event) => { + const job = (event as CustomEvent).detail + if ( + job.status === 'COMPLETED' && + job.project_id === projectId && + ['process', 'upload'].includes(job.kind) + ) { + load() + } + } + window.addEventListener(JOB_FINISHED_EVENT, refreshOnFinishedJob) + return () => window.removeEventListener(JOB_FINISHED_EVENT, refreshOnFinishedJob) + }, [load, projectId]) + const processedByAsset = new Map(processed.map(p => [p.asset_id, p])) return ( diff --git a/frontend/src/hooks/useGlobalJobsPoller.ts b/frontend/src/hooks/useGlobalJobsPoller.ts index 5953709..d24fd28 100644 --- a/frontend/src/hooks/useGlobalJobsPoller.ts +++ b/frontend/src/hooks/useGlobalJobsPoller.ts @@ -1,8 +1,9 @@ import { useEffect, useRef } from 'react' import { listJobs } from '../api/jobs' -import { nextJobPollDelayMs } from '../api/jobPolling' +import { announceJobFinished, isJobTerminal, nextJobPollDelayMs } from '../api/jobPolling' import { JOB_STARTED_EVENT } from '../api/client' import { useJobsStore, isJobActive } from '../store/jobsStore' +import type { Job } from '../types' /** * Single global poll of /api/jobs. Mount exactly once near the app root. @@ -17,6 +18,7 @@ export function useGlobalJobsPoller() { const timerRef = useRef | null>(null) const errorsRef = useRef(0) const fetchingRef = useRef(false) + const previousJobsRef = useRef | null>(null) useEffect(() => { let cancelled = false @@ -25,6 +27,16 @@ export function useGlobalJobsPoller() { try { const data = await listJobs({ limit: 200 }) errorsRef.current = 0 + const previousJobs = previousJobsRef.current + if (previousJobs) { + for (const job of data) { + const previous = previousJobs.get(job.job_id) + if (previous && isJobActive(previous) && isJobTerminal(job.status)) { + announceJobFinished(job) + } + } + } + previousJobsRef.current = new Map(data.map(job => [job.job_id, job])) useJobsStore.getState().setJobs(data) return data } catch { diff --git a/frontend/src/pages/FramesPage.tsx b/frontend/src/pages/FramesPage.tsx index 5409832..cbd8d05 100644 --- a/frontend/src/pages/FramesPage.tsx +++ b/frontend/src/pages/FramesPage.tsx @@ -1,12 +1,13 @@ import { useCallback, useEffect, useState } from 'react' import { listFrames } from '../api/frames' +import { JOB_FINISHED_EVENT } from '../api/jobPolling' import { listProjects } from '../api/projects' import { listSpaces } from '../api/spaces' import ProjectDetail from '../components/frames/ProjectDetail' import ProjectGroupedList from '../components/ProjectGroupedList' import StatusBadge from '../components/StatusBadge' -import type { Project, Space } from '../types' +import type { Job, Project, Space } from '../types' export default function FramesPage() { @@ -34,6 +35,19 @@ export default function FramesPage() { }, []) useEffect(() => { load() }, [load]) + useEffect(() => { + const refreshOnFinishedJob = (event: Event) => { + const job = (event as CustomEvent).detail + if ( + job.status === 'COMPLETED' && + ['process', 'extract', 'raw_workflow', 'canonical_workflow', 'workflow_maintenance', 'upload'].includes(job.kind) + ) { + load() + } + } + window.addEventListener(JOB_FINISHED_EVENT, refreshOnFinishedJob) + return () => window.removeEventListener(JOB_FINISHED_EVENT, refreshOnFinishedJob) + }, [load]) useEffect(() => { listSpaces().then(setSpaces).catch(() => {}) }, []) const getStatus = useCallback( diff --git a/frontend/src/pages/GraphPage.tsx b/frontend/src/pages/GraphPage.tsx index f702d2e..ab90727 100644 --- a/frontend/src/pages/GraphPage.tsx +++ b/frontend/src/pages/GraphPage.tsx @@ -2,7 +2,7 @@ import { useState, useCallback, useEffect, useRef, useMemo } from 'react' import { Network, type Options } from 'vis-network' import { DataSet } from 'vis-data' import { getKnowledgeGraph, getReviewCounts, reviewGraph, clearGraph } from '../api/graph' -import { startJobPolling } from '../api/jobPolling' +import { JOB_FINISHED_EVENT, startJobPolling } from '../api/jobPolling' import JobProgress from '../components/JobProgress' import type { GraphConcept, GraphRelation, GraphPayload, Job } from '../types' @@ -783,8 +783,8 @@ export default function GraphPage() { const REVIEW_MODES_SET = new Set(['review_coverage', 'modification_heat']) - const load = useCallback(async () => { - setLoading(true) + const load = useCallback(async (showLoading = false) => { + if (showLoading) setLoading(true) try { const data = await getKnowledgeGraph() setPayload(data) @@ -792,7 +792,18 @@ export default function GraphPage() { setLoading(false) }, []) - useEffect(() => { load() }, [load]) + useEffect(() => { load(true) }, [load]) + + useEffect(() => { + const refreshOnFinishedJob = (event: Event) => { + const job = (event as CustomEvent).detail + if (job.status === 'COMPLETED' && ['knowledge_graph', 'graph_review'].includes(job.kind)) { + load() + } + } + window.addEventListener(JOB_FINISHED_EVENT, refreshOnFinishedJob) + return () => window.removeEventListener(JOB_FINISHED_EVENT, refreshOnFinishedJob) + }, [load]) // Load review counts when a review-based mode is selected useEffect(() => { diff --git a/frontend/src/pages/ProjectionsPage.tsx b/frontend/src/pages/ProjectionsPage.tsx index 3d3e086..9d62b9b 100644 --- a/frontend/src/pages/ProjectionsPage.tsx +++ b/frontend/src/pages/ProjectionsPage.tsx @@ -1,7 +1,7 @@ import type { JSX } from 'react' import { useCallback, useEffect, useMemo, useState } from 'react' -import { startJobPolling } from '../api/jobPolling' +import { JOB_FINISHED_EVENT, isJobTerminal, startJobPolling } from '../api/jobPolling' import { listProjectGroups } from '../api/projectGroups' import { deleteProjection, exportProjections, listProjections, reviewProjections } from '../api/projections' import { listProjects } from '../api/projects' @@ -29,7 +29,7 @@ export default function ProjectionsPage() { const [loading, setLoading] = useState(false) const [newestOnly, setNewestOnly] = useState(true) const [showHistory, setShowHistory] = useState(false) - const [reviewJob, setReviewJob] = useState(null) + const [reviewJobs, setReviewJobs] = useState([]) const [isReviewing, setIsReviewing] = useState(false) const [selectedReviewerId, setSelectedReviewerId] = useState('') const [showSpaceDetail, setShowSpaceDetail] = useState(false) @@ -106,6 +106,20 @@ export default function ProjectionsPage() { useEffect(() => { loadProjections() }, [loadProjections]) + useEffect(() => { + const refreshOnFinishedJob = (event: Event) => { + const job = (event as CustomEvent).detail + if ( + job.status === 'COMPLETED' && + ['project', 'projection_review', 'upload'].includes(job.kind) + ) { + loadProjections() + } + } + window.addEventListener(JOB_FINISHED_EVENT, refreshOnFinishedJob) + return () => window.removeEventListener(JOB_FINISHED_EVENT, refreshOnFinishedJob) + }, [loadProjections]) + useEffect(() => { const processors = spaceDetail?.post_processors ?? [] const enabled = processors.filter(processor => processor.enabled !== false) @@ -131,14 +145,34 @@ export default function ProjectionsPage() { return Array.from(pids) }, [selectedProjectionIds, projections]) - const pollJob = useCallback((jobId: string) => { - startJobPolling({ - jobId, - onUpdate: setReviewJob, - onComplete: () => { setIsReviewing(false); loadProjections() }, - onFailed: () => setIsReviewing(false), + const updateReviewJob = useCallback((updatedJob: Job) => { + setReviewJobs(prev => { + const index = prev.findIndex(job => job.job_id === updatedJob.job_id) + if (index === -1) return [...prev, updatedJob] + const next = [...prev] + next[index] = updatedJob + return next }) - }, [loadProjections]) + }, []) + + const pollJobs = useCallback((jobIds: string[]) => { + const remaining = new Set(jobIds) + const finishJob = (jobId: string) => { + remaining.delete(jobId) + if (remaining.size === 0) { + setIsReviewing(false) + loadProjections() + } + } + jobIds.forEach(jobId => { + startJobPolling({ + jobId, + onUpdate: updateReviewJob, + onComplete: job => finishJob(job.job_id), + onFailed: job => finishJob(job?.job_id ?? jobId), + }) + }) + }, [loadProjections, updateReviewJob]) const startReview = async (overrideProjectionIds?: string[]) => { try { @@ -159,8 +193,22 @@ export default function ProjectionsPage() { } if (projectIds.length > 0) params.project_ids = projectIds if (selectedReviewerId) params.reviewer_id = selectedReviewerId - const { job_id } = await reviewProjections(params) - pollJob(job_id) + const response = await reviewProjections(params) + const jobIds = response.job_ids?.length ? response.job_ids : [response.job_id] + setReviewJobs(jobIds.map(jobId => ({ + job_id: jobId, + kind: 'projection_review', + label: 'Projection Review', + status: 'QUEUED', + project_id: null, + result: null, + error: null, + current_message: 'Queued', + events: [], + created_at: new Date().toISOString(), + updated_at: new Date().toISOString(), + }))) + pollJobs(jobIds) } catch { setIsReviewing(false) } } @@ -221,6 +269,40 @@ export default function ProjectionsPage() { [selectedSpaceId, userSpaces], ) + const reviewSummary = useMemo(() => { + if (reviewJobs.length === 0) return null + const completed = reviewJobs.filter(job => job.status === 'COMPLETED').length + const failed = reviewJobs.filter(job => job.status === 'FAILED').length + const cancelled = reviewJobs.filter(job => job.status === 'CANCELLED').length + const active = reviewJobs.filter(job => !isJobTerminal(job.status)).length + const latestActive = [...reviewJobs] + .reverse() + .find(job => !isJobTerminal(job.status)) + const latestTerminal = [...reviewJobs] + .reverse() + .find(job => isJobTerminal(job.status)) + const visibleJob = latestActive ?? latestTerminal ?? reviewJobs[reviewJobs.length - 1] + const status = failed > 0 + ? 'FAILED' + : cancelled > 0 && active === 0 + ? 'CANCELLED' + : completed === reviewJobs.length + ? 'COMPLETED' + : 'RUNNING' + const parts = [ + `${completed}/${reviewJobs.length} completed`, + active > 0 ? `${active} running or queued` : null, + failed > 0 ? `${failed} failed` : null, + cancelled > 0 ? `${cancelled} cancelled` : null, + ].filter(Boolean) + return { + status: status as Job['status'], + message: reviewJobs.length === 1 + ? (visibleJob.current_message || visibleJob.status) + : parts.join(' · '), + } + }, [reviewJobs]) + return (
@@ -387,15 +469,15 @@ export default function ProjectionsPage() {
)} - {reviewJob && ( + {reviewSummary && (
{isReviewing && } - - {reviewJob.current_message || reviewJob.status} + + {reviewSummary.message}
)} diff --git a/frontend/src/pages/SkillsPage.tsx b/frontend/src/pages/SkillsPage.tsx new file mode 100644 index 0000000..30e3738 --- /dev/null +++ b/frontend/src/pages/SkillsPage.tsx @@ -0,0 +1,245 @@ +import type { ReactNode } from 'react' +import { useEffect, useRef, useState } from 'react' +import { deleteSkill, getSkill, listSkills, uploadSkill } from '../api/skills' +import type { CustomSkill } from '../types' + +export default function SkillsPage() { + const [skills, setSkills] = useState([]) + const [selected, setSelected] = useState(null) + const [busy, setBusy] = useState(false) + const [error, setError] = useState(null) + const [info, setInfo] = useState(null) + const fileRef = useRef(null) + const folderRef = useRef(null) + + const refresh = async () => { + try { + const next = await listSkills() + setSkills(next) + if (selected) { + const fresh = next.find(skill => skill.skill_id === selected.skill_id) + setSelected(fresh ? await getSkill(fresh.skill_id) : null) + } + } catch (e) { + setError(e instanceof Error ? e.message : String(e)) + } + } + + useEffect(() => { refresh() /* eslint-disable-next-line react-hooks/exhaustive-deps */ }, []) + + const handleUpload = async (files: File[]) => { + if (files.length === 0) return + setBusy(true); setError(null); setInfo(null) + try { + const created = await uploadSkill(files) + setInfo(`Uploaded ${created.name}.`) + await refresh() + setSelected(await getSkill(created.skill_id)) + } catch (e) { + setError(e instanceof Error ? e.message : String(e)) + } finally { + setBusy(false) + } + } + + const openSkill = async (skill: CustomSkill) => { + setError(null); setInfo(null) + try { + setSelected(await getSkill(skill.skill_id)) + } catch (e) { + setError(e instanceof Error ? e.message : String(e)) + } + } + + const removeSkill = async (skill: CustomSkill) => { + if (!confirm(`Delete skill "${skill.name}"?`)) return + setBusy(true); setError(null); setInfo(null) + try { + await deleteSkill(skill.skill_id) + setInfo(`Deleted ${skill.name}.`) + setSelected(null) + await refresh() + } catch (e) { + setError(e instanceof Error ? e.message : String(e)) + } finally { + setBusy(false) + } + } + + return ( +
+
+
+

Skills

+

+ Custom SKILL.md bundles that can be attached to space post processors. +

+
+
+ + + { + handleUpload(Array.from(e.target.files ?? [])) + e.target.value = '' + }} + /> + { + handleUpload(Array.from(e.target.files ?? [])) + e.target.value = '' + }} + {...({ webkitdirectory: '', directory: '' } as Record)} + /> +
+
+ + {(error || info) && ( +
+ {error && ( +
+ {error} +
+ )} + {info && !error && ( +
+ {info} +
+ )} +
+ )} + +
+
+ {skills.length === 0 && ( +

+ No custom skills uploaded yet. +

+ )} + {skills.map(skill => { + const active = selected?.skill_id === skill.skill_id + return ( + + ) + })} +
+ +
+ {selected ? ( +
+
+
+
+
+ + {selected.source_type} + + + {selected.file_count} file{selected.file_count === 1 ? '' : 's'} + +
+

+ {selected.name} +

+
{selected.slug}
+
+ +
+ {selected.description && ( +

{selected.description}

+ )} + {selected.storage_path && ( +
+ {selected.storage_path} +
+ )} +
+ +
+
+ {(selected.metadata.files ?? ['SKILL.md']).map(file => ( + + {file} + + ))} +
+
+ +
+ 
+                  {selected.skill_md ?? ''}
+
+
+
+ ) : ( +

+ Select a skill to view its bundle. +

+ )} +
+
+
+ ) +} + +function Section({ + title, + children, +}: { + title: string + children: ReactNode +}) { + return ( +
+

+ {title} +

+ {children} +
+ ) +} diff --git a/frontend/src/pages/SpacesPage.tsx b/frontend/src/pages/SpacesPage.tsx index 39d3075..a34347f 100644 --- a/frontend/src/pages/SpacesPage.tsx +++ b/frontend/src/pages/SpacesPage.tsx @@ -7,7 +7,8 @@ import { deleteSpace, getDefaultReviewPrompt, } from '../api/spaces' -import type { PostProcessorProfile, Space, SpaceCreatePayload } from '../types' +import { listSkills } from '../api/skills' +import type { CustomSkill, PostProcessorProfile, Space, SpaceCreatePayload } from '../types' const PURPOSE_OPTIONS = ['tabular_database', 'qa_benchmark', 'skill_cards', 'freeform'] as const const REVIEW_SEARCH_TOOL_OPTIONS = ['web', 'uniprot', 'ncbi', 'crossref'] as const @@ -76,6 +77,7 @@ const EMPTY_DRAFT: SpaceDraft = { description: 'General projection review and correction.', prompt: null, tool_groups: ['reading'], + skill_ids: [], enabled: true, }, ], @@ -154,6 +156,9 @@ const normalizePostProcessors = (value: unknown): PostProcessorProfile[] => { description: typeof obj.description === 'string' ? obj.description : '', prompt: typeof obj.prompt === 'string' && obj.prompt.trim() ? obj.prompt : null, tool_groups: toolGroups.length > 0 ? Array.from(new Set(toolGroups)) : ['reading'], + skill_ids: Array.isArray(obj.skill_ids) + ? Array.from(new Set(obj.skill_ids.map(String).filter(Boolean))) + : [], enabled: typeof obj.enabled === 'boolean' ? obj.enabled : true, } }) @@ -218,14 +223,16 @@ export default function SpacesPage() { const [selected, setSelected] = useState(null) const [editor, setEditor] = useState(null) const [busy, setBusy] = useState(false) + const [skills, setSkills] = useState([]) const [error, setError] = useState(null) const [info, setInfo] = useState(null) const importFileRef = useRef(null) const refresh = async () => { try { - const list = await listSpaces() + const [list, skillList] = await Promise.all([listSpaces(), listSkills()]) setSpaces(list) + setSkills(skillList) if (selected) { const fresh = list.find(s => s.space_id === selected.space_id) ?? null if (fresh) { @@ -442,12 +449,14 @@ export default function SpacesPage() {
setEditor({ ...editor, draft })} /> ) : selected ? ( openEditor({ mode: 'edit', @@ -477,9 +486,11 @@ export default function SpacesPage() { function SpaceForm({ draft, + skills, onChange, }: { draft: SpaceDraft + skills: CustomSkill[] onChange: (draft: SpaceDraft) => void }) { const setDraft = (patch: Partial) => onChange({ ...draft, ...patch }) @@ -792,6 +803,7 @@ function SpaceForm({
setDraft({ post_processors })} />
@@ -801,9 +813,11 @@ function SpaceForm({ function PostProcessorEditor({ processors, + skills, onChange, }: { processors: PostProcessorProfile[] + skills: CustomSkill[] onChange: (processors: PostProcessorProfile[]) => void }) { const update = (index: number, patch: Partial) => { @@ -833,6 +847,7 @@ function PostProcessorEditor({ description: '', prompt: null, tool_groups: ['reading'], + skill_ids: [], enabled: true, }, ]) @@ -895,6 +910,35 @@ function PostProcessorEditor({ ) })} +
+
Attached skills
+ {skills.length === 0 ? ( +
No custom skills uploaded.
+ ) : ( +
+ {skills.map(skill => { + const selected = (processor.skill_ids ?? []).includes(skill.skill_id) + return ( + + ) + })} +
+ )} +
diff --git a/frontend/src/types/index.ts b/frontend/src/types/index.ts index 2359d6d..ab0cd3d 100644 --- a/frontend/src/types/index.ts +++ b/frontend/src/types/index.ts @@ -131,9 +131,27 @@ export interface PostProcessorProfile { description?: string prompt?: string | null tool_groups: string[] + skill_ids?: string[] enabled?: boolean } +export interface CustomSkill { + skill_id: string + name: string + slug: string + description: string | null + source_type: string + file_count: number + metadata: { + files?: string[] + [key: string]: unknown + } + skill_md?: string + storage_path?: string + created_at?: string | null + updated_at?: string | null +} + export interface SpaceCreatePayload { name: string domain: string @@ -445,4 +463,4 @@ export interface UploadExpandResponse { // ─── UI ─────────────────────────────────────────────────────────────────────── -export type Page = 'assistant' | 'projects' | 'frames' | 'graph' | 'projections' | 'feedback' | 'spaces' | 'settings' +export type Page = 'assistant' | 'projects' | 'frames' | 'graph' | 'projections' | 'feedback' | 'spaces' | 'skills' | 'settings' diff --git a/src/mkb/agents/projection_reviewer.py b/src/mkb/agents/projection_reviewer.py index 18edb95..52e3b02 100644 --- a/src/mkb/agents/projection_reviewer.py +++ b/src/mkb/agents/projection_reviewer.py @@ -36,6 +36,7 @@ Space, ) from mkb.spaces.registry import resolve_post_processor +from mkb.skills.registry import skill_instruction_block logger = logging.getLogger(__name__) @@ -54,6 +55,7 @@ def build_projection_reviewer_agent( purpose: str | None = None, custom_prompt: str | None = None, tool_groups: list[str] | None = None, + skill_ids: list[str] | None = None, ) -> Agent: """Create a projection reviewer agent. @@ -76,6 +78,9 @@ def build_projection_reviewer_agent( if purpose_key in {"skill_cards", "freeform"} else "projection_reviewer" ) + skill_block = skill_instruction_block(skill_ids) + if skill_block: + instruction = f"{instruction.rstrip()}\n\n{skill_block}" groups = [str(group).strip().lower() for group in (tool_groups or ["reading"]) if str(group).strip()] optional_tools = [] if "reading" in groups: @@ -91,14 +96,16 @@ def build_projection_reviewer_agent( ) -def _processor_runtime(space: Space, reviewer_id: str | None = None) -> tuple[str | None, str | None, list[str], dict]: +def _processor_runtime(space: Space, reviewer_id: str | None = None) -> tuple[str | None, str | None, list[str], list[str], dict]: processor = resolve_post_processor(space, reviewer_id) prompt = processor.get("prompt") or getattr(space, "review_prompt", None) groups = processor.get("tool_groups") or ["reading"] + skill_ids = [str(skill_id).strip() for skill_id in (processor.get("skill_ids") or []) if str(skill_id).strip()] return ( str(processor.get("id") or "default"), prompt, [str(group).strip().lower() for group in groups], + skill_ids, processor, ) @@ -150,13 +157,14 @@ async def _run_review_async( ] space_name = space.name space_purpose = getattr(space, "purpose", None) - selected_reviewer_id, processor_prompt, tool_groups, processor = _processor_runtime(space, reviewer_id) + selected_reviewer_id, processor_prompt, tool_groups, skill_ids, processor = _processor_runtime(space, reviewer_id) agent = build_projection_reviewer_agent( model, purpose=space_purpose, custom_prompt=processor_prompt, tool_groups=tool_groups, + skill_ids=skill_ids, ) runner = AgentRunner(agent=agent, app_name=APP_NAME) @@ -193,6 +201,8 @@ async def _run_review_async( f"(id={selected_reviewer_id}) with these tool groups: " f"{', '.join(tool_groups)}." ) + if skill_ids: + message += f" Apply attached skill IDs: {', '.join(skill_ids)}." result = await runner.run( session_id=session_id, @@ -305,13 +315,14 @@ async def run_projection_review_followup( space_name = space.name space_purpose = getattr(space, "purpose", None) - selected_reviewer_id, processor_prompt, tool_groups, processor = _processor_runtime(space, reviewer_id) + selected_reviewer_id, processor_prompt, tool_groups, skill_ids, processor = _processor_runtime(space, reviewer_id) agent = build_projection_reviewer_agent( model, purpose=space_purpose, custom_prompt=processor_prompt, tool_groups=tool_groups, + skill_ids=skill_ids, ) runner = AgentRunner(agent=agent, app_name=APP_NAME) session_id = f"review_followup_{space_id}_{project_id}_{uuid.uuid4().hex[:8]}" @@ -333,7 +344,8 @@ async def run_projection_review_followup( f"You are continuing a completed projection review for space {space_id} " f"('{space_name}') and project {project_id}.\n\n" f"Selected post-processor: {processor.get('name')} " - f"(id={selected_reviewer_id}); tool groups: {', '.join(tool_groups)}.\n\n" + f"(id={selected_reviewer_id}); tool groups: {', '.join(tool_groups)}; " + f"skill IDs: {', '.join(skill_ids) if skill_ids else 'none'}.\n\n" f"User follow-up request:\n{cleaned_message}\n\n" f"Previous review job result, if available:\n" f"{_compact_followup_context(prior_result)}\n\n" @@ -541,13 +553,14 @@ async def run_projection_review_session( space_name = space.name space_purpose = getattr(space, "purpose", None) - selected_reviewer_id, processor_prompt, tool_groups, processor = _processor_runtime(space, reviewer_id) + selected_reviewer_id, processor_prompt, tool_groups, skill_ids, processor = _processor_runtime(space, reviewer_id) agent = build_projection_reviewer_agent( model, purpose=space_purpose, custom_prompt=processor_prompt, tool_groups=tool_groups, + skill_ids=skill_ids, ) runner = AgentRunner(agent=agent, app_name=APP_NAME) @@ -562,7 +575,8 @@ async def run_projection_review_session( f"You are running a SINGLE consolidated review session over " f"{len(per_project_counts)} project(s) in space {sid} ('{space_name}').\n\n" f"Selected post-processor: {processor.get('name')} " - f"(id={selected_reviewer_id}); tool groups: {', '.join(tool_groups)}.\n\n" + f"(id={selected_reviewer_id}); tool groups: {', '.join(tool_groups)}; " + f"skill IDs: {', '.join(skill_ids) if skill_ids else 'none'}.\n\n" f"Projects to review (one at a time, in order):\n{project_lines}\n\n" f"For EACH project, in order:\n" f" 1. Call get_all_projections_for_review(space_id, project_id) to " diff --git a/src/mkb/db/models.py b/src/mkb/db/models.py index 977db1e..1f78dfa 100644 --- a/src/mkb/db/models.py +++ b/src/mkb/db/models.py @@ -550,6 +550,33 @@ class Space(Base): ) +# ── Custom agent skills ───────────────────────────────────────── + + +class CustomSkill(Base): + __tablename__ = "custom_skills" + + skill_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), primary_key=True, default=uuid.uuid4 + ) + name: Mapped[str] = mapped_column(String(255), nullable=False) + slug: Mapped[str] = mapped_column(String(255), unique=True, nullable=False) + description: Mapped[str | None] = mapped_column(Text, nullable=True) + source_type: Mapped[str] = mapped_column(String(32), nullable=False) + storage_path: Mapped[str] = mapped_column(Text, nullable=False) + skill_md: Mapped[str] = mapped_column(Text, nullable=False) + file_count: Mapped[int] = mapped_column(Integer, default=1, nullable=False) + metadata_: Mapped[dict | None] = mapped_column("metadata", JSONB, default=dict) + + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now() + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now(), + onupdate=lambda: datetime.now(timezone.utc), + ) + + # ── Projections ────────────────────────────────────────────────── diff --git a/src/mkb/skills/__init__.py b/src/mkb/skills/__init__.py new file mode 100644 index 0000000..935998f --- /dev/null +++ b/src/mkb/skills/__init__.py @@ -0,0 +1,2 @@ +"""Custom user-uploaded skills.""" + diff --git a/src/mkb/skills/registry.py b/src/mkb/skills/registry.py new file mode 100644 index 0000000..10fce1d --- /dev/null +++ b/src/mkb/skills/registry.py @@ -0,0 +1,289 @@ +"""Registry and storage helpers for user-uploaded skills.""" + +from __future__ import annotations + +import re +import shutil +import uuid +import zipfile +from pathlib import Path +from typing import BinaryIO + +from mkb.db.engine import SyncSessionLocal +from mkb.db.models import CustomSkill + +SKILLS_ROOT = Path("data/skills") +MAX_SKILL_MD_CHARS = 80_000 + + +def _slugify(value: str, fallback: str = "skill") -> str: + text = (value or fallback).strip().lower() + slug = re.sub(r"[^a-z0-9]+", "_", text).strip("_") + return slug or fallback + + +def _safe_relpath(value: str) -> Path | None: + raw = (value or "").replace("\\", "/").lstrip("/") + if not raw or raw.endswith("/"): + return None + path = Path(raw) + if any(part in {"", ".", ".."} for part in path.parts): + return None + return path + + +def _extract_title(skill_md: str, fallback: str) -> str: + for line in skill_md.splitlines(): + stripped = line.strip() + if stripped.startswith("# "): + title = stripped[2:].strip() + if title: + return title[:255] + return fallback[:255] + + +def _extract_description(skill_md: str) -> str | None: + for line in skill_md.splitlines(): + stripped = line.strip() + if not stripped or stripped.startswith("#"): + continue + return stripped[:1000] + return None + + +def _unique_slug(session, base: str, existing_id: uuid.UUID | None = None) -> str: + slug = base + suffix = 2 + while True: + existing = session.query(CustomSkill).filter_by(slug=slug).first() + if not existing or (existing_id and existing.skill_id == existing_id): + return slug + slug = f"{base}_{suffix}" + suffix += 1 + + +def _root_with_skill_md(root: Path) -> Path: + direct = root / "SKILL.md" + if direct.is_file(): + return root + + children = [path for path in root.iterdir() if path.is_dir()] + files = [path for path in root.iterdir() if path.is_file()] + if len(children) == 1 and not files and (children[0] / "SKILL.md").is_file(): + return children[0] + + raise ValueError("Skill upload must contain a SKILL.md file at the skill root.") + + +def _write_stream(target: Path, stream: BinaryIO) -> int: + target.parent.mkdir(parents=True, exist_ok=True) + size = 0 + with target.open("wb") as out: + while chunk := stream.read(1024 * 1024): + out.write(chunk) + size += len(chunk) + return size + + +def _safe_extract_zip(zip_path: Path, dest_dir: Path) -> int: + dest_root = dest_dir.resolve() + count = 0 + with zipfile.ZipFile(zip_path) as zf: + for member in zf.infolist(): + if member.is_dir(): + continue + rel = _safe_relpath(member.filename) + if rel is None: + continue + if rel.parts[0] == "__MACOSX" or rel.name == ".DS_Store": + continue + info_mode = member.external_attr >> 16 + if info_mode and (info_mode & 0o170000) == 0o120000: + continue + target = (dest_dir / rel).resolve() + try: + target.relative_to(dest_root) + except ValueError: + continue + target.parent.mkdir(parents=True, exist_ok=True) + with zf.open(member) as src, target.open("wb") as out: + shutil.copyfileobj(src, out) + count += 1 + return count + + +def _finalize_skill(staging_root: Path, *, source_type: str, fallback_name: str) -> dict: + skill_root = _root_with_skill_md(staging_root) + skill_md = (skill_root / "SKILL.md").read_text(encoding="utf-8", errors="replace") + if not skill_md.strip(): + raise ValueError("SKILL.md must not be empty.") + if len(skill_md) > MAX_SKILL_MD_CHARS: + raise ValueError("SKILL.md is too large.") + + name = _extract_title(skill_md, fallback_name) + description = _extract_description(skill_md) + files = sorted(path for path in skill_root.rglob("*") if path.is_file()) + skill_id = uuid.uuid4() + + SKILLS_ROOT.mkdir(parents=True, exist_ok=True) + with SyncSessionLocal() as session: + slug = _unique_slug(session, _slugify(name)) + final_root = SKILLS_ROOT / f"{slug}_{skill_id.hex[:8]}" + shutil.move(str(skill_root), final_root) + + skill = CustomSkill( + skill_id=skill_id, + name=name, + slug=slug, + description=description, + source_type=source_type, + storage_path=str(final_root), + skill_md=skill_md, + file_count=len(files), + metadata_={ + "files": [ + path.relative_to(skill_root).as_posix() + for path in files + ], + }, + ) + session.add(skill) + session.commit() + return _skill_to_dict(skill) + + +def create_skill_from_single_file(filename: str, stream: BinaryIO) -> dict: + if Path(filename or "").name.lower() != "skill.md": + raise ValueError("Single-file skill uploads must be named SKILL.md.") + staging = SKILLS_ROOT / "_staging" / uuid.uuid4().hex + if staging.exists(): + shutil.rmtree(staging) + staging.mkdir(parents=True, exist_ok=True) + try: + _write_stream(staging / "SKILL.md", stream) + return _finalize_skill(staging, source_type="file", fallback_name="Uploaded skill") + finally: + if staging.exists(): + shutil.rmtree(staging, ignore_errors=True) + + +def create_skill_from_zip(filename: str, stream: BinaryIO) -> dict: + if not (filename or "").lower().endswith(".zip"): + raise ValueError("Archive skill uploads must be .zip files.") + staging = SKILLS_ROOT / "_staging" / uuid.uuid4().hex + archive = staging / "upload.zip" + if staging.exists(): + shutil.rmtree(staging) + staging.mkdir(parents=True, exist_ok=True) + try: + _write_stream(archive, stream) + extract_root = staging / "extract" + extracted = _safe_extract_zip(archive, extract_root) + if extracted == 0: + raise ValueError("Zip did not contain any regular files.") + return _finalize_skill( + extract_root, + source_type="zip", + fallback_name=Path(filename).stem or "Uploaded skill", + ) + finally: + if staging.exists(): + shutil.rmtree(staging, ignore_errors=True) + + +def create_skill_from_files(files: list[tuple[str, BinaryIO]]) -> dict: + if not files: + raise ValueError("Folder skill upload must include at least one file.") + staging = SKILLS_ROOT / "_staging" / uuid.uuid4().hex + if staging.exists(): + shutil.rmtree(staging) + staging.mkdir(parents=True, exist_ok=True) + try: + top_name = "Uploaded skill" + for relname, stream in files: + rel = _safe_relpath(relname) + if rel is None: + continue + if top_name == "Uploaded skill" and rel.parts: + top_name = rel.parts[0] + _write_stream(staging / rel, stream) + return _finalize_skill(staging, source_type="folder", fallback_name=top_name) + finally: + if staging.exists(): + shutil.rmtree(staging, ignore_errors=True) + + +def _skill_to_dict(skill: CustomSkill, *, include_content: bool = False) -> dict: + payload = { + "skill_id": str(skill.skill_id), + "name": skill.name, + "slug": skill.slug, + "description": skill.description, + "source_type": skill.source_type, + "file_count": skill.file_count, + "metadata": skill.metadata_ or {}, + "created_at": skill.created_at.isoformat() if skill.created_at else None, + "updated_at": skill.updated_at.isoformat() if skill.updated_at else None, + } + if include_content: + payload["skill_md"] = skill.skill_md + payload["storage_path"] = skill.storage_path + return payload + + +def list_skills() -> list[dict]: + with SyncSessionLocal() as session: + skills = session.query(CustomSkill).order_by(CustomSkill.name).all() + return [_skill_to_dict(skill) for skill in skills] + + +def get_skill(skill_id_or_slug: str, *, include_content: bool = False) -> dict | None: + with SyncSessionLocal() as session: + try: + sid = uuid.UUID(str(skill_id_or_slug)) + skill = session.query(CustomSkill).filter_by(skill_id=sid).first() + except ValueError: + skill = session.query(CustomSkill).filter_by(slug=skill_id_or_slug).first() + return _skill_to_dict(skill, include_content=include_content) if skill else None + + +def delete_skill(skill_id: str | uuid.UUID) -> dict: + sid = uuid.UUID(str(skill_id)) + with SyncSessionLocal() as session: + skill = session.query(CustomSkill).filter_by(skill_id=sid).first() + if not skill: + return {"error": f"Skill {skill_id} not found."} + root = Path(skill.storage_path) + name = skill.name + session.delete(skill) + session.commit() + if root.exists() and root.is_dir(): + shutil.rmtree(root, ignore_errors=True) + return {"ok": True, "deleted": name} + + +def skill_instruction_block(skill_ids: list[str] | None) -> str: + if not skill_ids: + return "" + blocks: list[str] = [] + with SyncSessionLocal() as session: + for raw_id in skill_ids: + try: + sid = uuid.UUID(str(raw_id)) + skill = session.query(CustomSkill).filter_by(skill_id=sid).first() + except ValueError: + skill = session.query(CustomSkill).filter_by(slug=str(raw_id)).first() + if not skill: + continue + blocks.append( + f"## {skill.name} ({skill.slug})\n" + f"Source folder: {skill.storage_path}\n\n" + f"{skill.skill_md.strip()}" + ) + if not blocks: + return "" + return ( + "\n\nAttached user skills. Follow these SKILL.md instructions when relevant. " + "If a skill references files in its source folder, inspect them only through available reading tools.\n\n" + + "\n\n---\n\n".join(blocks) + ) diff --git a/src/mkb/spaces/registry.py b/src/mkb/spaces/registry.py index 6c2afb7..fe23f69 100644 --- a/src/mkb/spaces/registry.py +++ b/src/mkb/spaces/registry.py @@ -93,6 +93,7 @@ def _default_post_processor_from_legacy( "description": "General projection review and correction.", "prompt": review_prompt or None, "tool_groups": tool_groups, + "skill_ids": [], "enabled": True, } @@ -111,6 +112,7 @@ def _normalize_post_processors(value, *, legacy_defaults: dict | None = None) -> "description": str(raw.get("description") or ""), "prompt": raw.get("prompt") if isinstance(raw.get("prompt"), str) and raw.get("prompt").strip() else None, "tool_groups": _normalize_tool_groups(raw.get("tool_groups") or raw.get("tools")), + "skill_ids": _normalize_skill_ids(raw.get("skill_ids") or raw.get("skills")), "enabled": bool(raw.get("enabled", True)), }) @@ -137,6 +139,23 @@ def _normalize_post_processors(value, *, legacy_defaults: dict | None = None) -> return unique +def _normalize_skill_ids(value) -> list[str]: + if value is None: + return [] + if isinstance(value, str): + candidates = [value] + elif isinstance(value, list): + candidates = value + else: + candidates = [] + skill_ids: list[str] = [] + for item in candidates: + key = str(item).strip() + if key and key not in skill_ids: + skill_ids.append(key) + return skill_ids + + def resolve_post_processor(space: Space, processor_id: str | None = None) -> dict: processors = _normalize_post_processors( getattr(space, "post_processors", None), diff --git a/src/mkb/web/api_server.py b/src/mkb/web/api_server.py index 4cbc2f5..662544d 100644 --- a/src/mkb/web/api_server.py +++ b/src/mkb/web/api_server.py @@ -345,6 +345,7 @@ def emit(msg: str) -> None: projections, projects, settings as settings_router, + skills, spaces, ) @@ -353,6 +354,7 @@ def emit(msg: str) -> None: projects, frames, spaces, + skills, projections, feedback, graph, diff --git a/src/mkb/web/routers/projections.py b/src/mkb/web/routers/projections.py index 7131162..b8ae3c2 100644 --- a/src/mkb/web/routers/projections.py +++ b/src/mkb/web/routers/projections.py @@ -216,6 +216,23 @@ def review_projections(body: ProjectionReviewRequest): target=api.review_projections, kwargs={"space_id": body.space_id, "project_id": project_ids[0], "reviewer_id": body.reviewer_id}, ) + elif project_ids: + job_ids = [] + for project_id in project_ids: + job_ids.append( + jobs.start_job( + kind="projection_review", + label="Projection Review", + project_id=project_id, + target=api.review_projections, + kwargs={ + "space_id": body.space_id, + "project_id": project_id, + "reviewer_id": body.reviewer_id, + }, + ) + ) + return {"job_id": job_ids[0], "job_ids": job_ids} else: job_id = jobs.start_job( kind="projection_review", diff --git a/src/mkb/web/routers/skills.py b/src/mkb/web/routers/skills.py new file mode 100644 index 0000000..4a7de2e --- /dev/null +++ b/src/mkb/web/routers/skills.py @@ -0,0 +1,48 @@ +from fastapi import APIRouter, File, HTTPException, UploadFile + +from mkb.skills import registry +from mkb.web._helpers import _parse_uuid + +router = APIRouter() + + +@router.get("/api/skills") +def list_skills(): + return registry.list_skills() + + +@router.get("/api/skills/{skill_id_or_slug}") +def get_skill(skill_id_or_slug: str): + skill = registry.get_skill(skill_id_or_slug, include_content=True) + if not skill: + raise HTTPException(status_code=404, detail="Skill not found") + return skill + + +@router.post("/api/skills/upload") +async def upload_skill(files: list[UploadFile] = File(...)): + if not files: + raise HTTPException(status_code=400, detail="No files uploaded") + try: + if len(files) == 1: + upload = files[0] + filename = upload.filename or "" + if filename.lower().endswith(".zip"): + return registry.create_skill_from_zip(filename, upload.file) + return registry.create_skill_from_single_file(filename, upload.file) + + payload = [] + for upload in files: + payload.append((upload.filename or "", upload.file)) + return registry.create_skill_from_files(payload) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + +@router.delete("/api/skills/{skill_id}") +def delete_skill(skill_id: str): + _parse_uuid(skill_id, "skill_id") + result = registry.delete_skill(skill_id) + if isinstance(result, dict) and result.get("error"): + raise HTTPException(status_code=404, detail=result["error"]) + return result diff --git a/tests/test_projection_review_space_search.py b/tests/test_projection_review_space_search.py index 2bd20d7..01b461c 100644 --- a/tests/test_projection_review_space_search.py +++ b/tests/test_projection_review_space_search.py @@ -75,6 +75,35 @@ def fake_start_job(**kwargs): assert "allow_search" not in calls[0]["kwargs"] +def test_projection_review_selected_projects_starts_isolated_jobs( + monkeypatch, + projections_router, +): + calls = [] + + def fake_start_job(**kwargs): + calls.append(kwargs) + return f"job-{len(calls)}" + + monkeypatch.setattr(projections_router.jobs, "start_job", fake_start_job) + + result = projections_router.review_projections( + ProjectionReviewRequest( + space_id=SPACE_ID, + project_ids=[PROJECT_ID, OTHER_PROJECT_ID], + ) + ) + + assert result == {"job_id": "job-1", "job_ids": ["job-1", "job-2"]} + assert [call["target"] for call in calls] == [ + projections_router.api.review_projections, + projections_router.api.review_projections, + ] + assert [call["project_id"] for call in calls] == [PROJECT_ID, OTHER_PROJECT_ID] + assert [call["kwargs"]["project_id"] for call in calls] == [PROJECT_ID, OTHER_PROJECT_ID] + assert all("allow_search" not in call["kwargs"] for call in calls) + + def test_projection_review_session_uses_space_search_settings( monkeypatch, projections_router, diff --git a/tests/test_space_post_processors.py b/tests/test_space_post_processors.py index 9794628..562910a 100644 --- a/tests/test_space_post_processors.py +++ b/tests/test_space_post_processors.py @@ -20,6 +20,7 @@ def test_normalize_post_processors_builds_default_from_legacy_settings(): "description": "General projection review and correction.", "prompt": "legacy prompt", "tool_groups": ["reading", "uniprot"], + "skill_ids": [], "enabled": True, } ] @@ -52,3 +53,17 @@ def test_resolve_post_processor_selects_named_profile_without_forcing_reading(): assert selected["prompt"] == "sequence prompt" assert selected["tool_groups"] == ["uniprot"] + +def test_normalize_post_processors_preserves_attached_skill_ids(): + processors = _normalize_post_processors( + [ + { + "id": "skilled", + "name": "Skilled reviewer", + "tool_groups": ["reading"], + "skill_ids": ["abc", "abc", "def"], + } + ] + ) + + assert processors[0]["skill_ids"] == ["abc", "def"] From 36ec353a43112f6ad2448498dbfc540e8fd9b1a4 Mon Sep 17 00:00:00 2001 From: theAfish Date: Wed, 1 Jul 2026 13:07:03 +0800 Subject: [PATCH 02/26] feat: refactor v1 --- Makefile | 18 +- TODO.md | 619 ++-- docs/architecture-map.md | 52 + docs/development.md | 30 + docs/workflow-card-architecture.md | 3 + docs/workflow-lifecycle-policy.md | 38 + frontend/package.json | 1 + frontend/src/App.tsx | 28 +- .../src/components/frames/ProjectDetail.tsx | 7 +- .../src/components/projects/ProjectDetail.tsx | 6 +- .../components/projects/WorkflowCanvas.tsx | 631 +++- .../components/projects/WorkflowGraphTab.tsx | 10 +- frontend/src/hooks/useProjectRefresh.ts | 15 + frontend/src/types/index.ts | 4 +- pyproject.toml | 1 + src/mkb/agents/__init__.py | 9 + src/mkb/agents/projection.py | 1 - src/mkb/agents/projection_reviewer.py | 6 +- src/mkb/agents/prompts/workflow_extraction.py | 42 +- src/mkb/agents/review.py | 1 - src/mkb/agents/tools/frames.py | 2 +- src/mkb/agents/tools/graph_review.py | 5 +- src/mkb/agents/tools/orchestrator_tools.py | 7 +- src/mkb/agents/tools/reading.py | 7 +- src/mkb/agents/tools/schema_curator.py | 17 +- .../agents/tools/workflow_canonicalization.py | 84 +- src/mkb/agents/tools/workflows.py | 219 +- src/mkb/api.py | 3277 ++--------------- src/mkb/processors/coordinator.py | 35 +- src/mkb/processors/image_processor.py | 2 - src/mkb/services/__init__.py | 7 + src/mkb/services/_api_common.py | 144 + src/mkb/services/assets.py | 434 +++ src/mkb/services/feedback.py | 171 + src/mkb/services/frames.py | 122 + src/mkb/services/graphs.py | 193 + src/mkb/services/ids.py | 21 + src/mkb/services/ingest.py | 57 + src/mkb/services/processing.py | 2 + src/mkb/services/projections.py | 392 ++ src/mkb/services/projects.py | 504 +++ src/mkb/services/result.py | 52 + src/mkb/services/runtime.py | 26 + src/mkb/services/spaces.py | 72 + src/mkb/services/workflows.py | 1000 +++++ src/mkb/ui/README.md | 9 + src/mkb/ui/background_jobs.py | 26 +- src/mkb/ui/pages/assistant.py | 63 +- src/mkb/ui/pages/projections.py | 20 + src/mkb/ui/pages/projects.py | 64 +- src/mkb/web/_helpers.py | 43 +- src/mkb/web/_state.py | 62 +- src/mkb/web/api_server.py | 308 +- src/mkb/web/content.py | 27 + src/mkb/web/job_actions.py | 386 ++ src/mkb/web/routers/assistant.py | 23 +- src/mkb/web/routers/feedback.py | 33 +- src/mkb/web/routers/graph.py | 8 +- src/mkb/web/routers/jobs.py | 21 +- src/mkb/web/routers/projections.py | 60 +- src/mkb/web/routers/projects.py | 176 +- src/mkb/web/routers/skills.py | 6 +- src/mkb/web/routers/spaces.py | 14 +- src/mkb/web/uploads.py | 287 ++ src/mkb/workflows/contract.py | 10 +- src/mkb/workflows/review.py | 11 +- src/mkb/workflows/validation.py | 23 + tests/test_job_action_registry.py | 47 + tests/test_projection_review_patch.py | 3 +- tests/test_projection_review_space_search.py | 2 + tests/test_service_result_convention.py | 31 + tests/test_workflow_contract.py | 34 + tests/test_workflow_resume.py | 4 - tests/test_workflow_review_curator.py | 25 + 74 files changed, 6060 insertions(+), 4140 deletions(-) create mode 100644 docs/architecture-map.md create mode 100644 docs/development.md create mode 100644 docs/workflow-lifecycle-policy.md create mode 100644 frontend/src/hooks/useProjectRefresh.ts create mode 100644 src/mkb/services/__init__.py create mode 100644 src/mkb/services/_api_common.py create mode 100644 src/mkb/services/assets.py create mode 100644 src/mkb/services/feedback.py create mode 100644 src/mkb/services/frames.py create mode 100644 src/mkb/services/graphs.py create mode 100644 src/mkb/services/ids.py create mode 100644 src/mkb/services/ingest.py create mode 100644 src/mkb/services/processing.py create mode 100644 src/mkb/services/projections.py create mode 100644 src/mkb/services/projects.py create mode 100644 src/mkb/services/result.py create mode 100644 src/mkb/services/runtime.py create mode 100644 src/mkb/services/spaces.py create mode 100644 src/mkb/services/workflows.py create mode 100644 src/mkb/ui/README.md create mode 100644 src/mkb/web/content.py create mode 100644 src/mkb/web/job_actions.py create mode 100644 src/mkb/web/uploads.py create mode 100644 tests/test_job_action_registry.py create mode 100644 tests/test_service_result_convention.py diff --git a/Makefile b/Makefile index 4c2180b..8cbb08a 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: up down logs migrate ingest list batches info purge install test +.PHONY: up down logs migrate ingest list batches info purge install test lint test-python test-frontend check ci # ── Infrastructure ────────────────────────────────────────────── up: @@ -55,3 +55,19 @@ server: # ── Tests ─────────────────────────────────────────────────────── test: pytest tests/ -v + +lint: + .venv/bin/python -m ruff check src tests + cd frontend && npm run lint + +test-python: + .venv/bin/python -m pytest + +test-frontend: + cd frontend && npm run build + +check: lint test-python test-frontend + +ci: + .venv/bin/python -m ruff check src tests + .venv/bin/python -m pytest --collect-only -q diff --git a/TODO.md b/TODO.md index 5b21656..3496cbd 100644 --- a/TODO.md +++ b/TODO.md @@ -1,288 +1,331 @@ -## 0. Define Scope (Do Not Skip) - -* [ ] Audit current Streamlit app - - * [ ] List all UI modules - * [ ] Categorize: - - * [ ] Graph visualization (core) - * [ ] Tables / text panels - * [ ] Controls (filters, buttons, inputs) - -* [ ] Define MVP (keep it minimal) - - * [ ] Render graph using React Flow - * [ ] Node click interaction - * [ ] Call backend APIs - * [ ] Basic dynamic graph updates - ---- - -## 1. Initialize Frontend Project - -* [ ] Create project - -```bash -npm create vite@latest graph-ui -- --template react-ts -cd graph-ui -npm install -``` - -* [ ] Install dependencies - -```bash -npm install reactflow zustand axios -``` - -* [ ] Optional utilities - -```bash -npm install classnames -``` - ---- - -## 2. Project Structure - -* [ ] Set up directories - -``` -src/ - components/ - pages/ - store/ - api/ - types/ - utils/ -``` - ---- - -## 3. Define Core Data Model - -* [ ] Normalize graph schema (critical step) - -```ts -// src/types/graph.ts - -export interface GraphNode { - id: string; - type?: string; - data: { - label: string; - [key: string]: any; - }; -} - -export interface GraphEdge { - id: string; - source: string; - target: string; - type?: string; -} - -export interface GraphData { - nodes: GraphNode[]; - edges: GraphEdge[]; -} -``` - ---- - -## 4. API Layer (Replace Streamlit Backend Calls) - -* [ ] Create API wrapper - -```ts -// src/api/graph.ts -import axios from "axios"; - -export const fetchGraph = async () => { - const res = await axios.get("/api/graph"); - return res.data; -}; -``` - ---- - -## 5. State Management (Replace Streamlit Session State) - -* [ ] Create Zustand store - -```ts -// src/store/graphStore.ts -import { create } from "zustand"; -import { GraphData } from "../types/graph"; - -interface GraphState { - graph: GraphData | null; - setGraph: (g: GraphData) => void; -} - -export const useGraphStore = create((set) => ({ - graph: null, - setGraph: (g) => set({ graph: g }), -})); -``` - ---- - -## 6. Integrate React Flow - -* [ ] Create graph component - -```tsx -// src/components/GraphView.tsx -import ReactFlow from "reactflow"; -import "reactflow/dist/style.css"; -import { useGraphStore } from "../store/graphStore"; - -export default function GraphView() { - const graph = useGraphStore((s) => s.graph); - - if (!graph) return
Loading...
; - - return ( -
- -
- ); -} -``` - ---- - -## 7. Page Integration (Replace Streamlit Pages) - -* [ ] Create main page - -```tsx -// src/pages/MainPage.tsx -import { useEffect } from "react"; -import GraphView from "../components/GraphView"; -import { fetchGraph } from "../api/graph"; -import { useGraphStore } from "../store/graphStore"; - -export default function MainPage() { - const setGraph = useGraphStore((s) => s.setGraph); - - useEffect(() => { - fetchGraph().then(setGraph); - }, []); - - return ( -
- -
- ); -} -``` - ---- - -## 8. Migrate Interactions - -### Node Click - -```tsx - { - console.log("clicked:", node); - }} -/> -``` - ---- - -### Backend Interaction (e.g. expand node) - -```ts -export const fetchNeighbors = async (nodeId: string) => { - const res = await axios.get(`/api/neighbors?id=${nodeId}`); - return res.data; -}; -``` - ---- - -### Update Graph State - -```ts -onNodeClick={async (_, node) => { - const subgraph = await fetchNeighbors(node.id); - setGraph(mergeGraph(graph, subgraph)); -}} -``` - ---- - -## 9. Layout Refactor (Replace Streamlit Layout) - -* [ ] Introduce layout structure - -``` -[ Sidebar ] [ Graph Canvas ] -``` - -* [ ] Sidebar responsibilities: - - * [ ] Search - * [ ] Filters - * [ ] Action controls - ---- - -## 10. Styling Strategy - -* [ ] Choose one: - - * [ ] CSS Modules (simple) - * [ ] Tailwind CSS (recommended for scalability) - ---- - -## 11. Backend Integration - -* [ ] Ensure backend endpoints exist: - - * [ ] `/api/graph` - * [ ] `/api/neighbors` - * [ ] `/api/search` - -* [ ] Enable CORS in backend (e.g. FastAPI) - ---- - -## 12. Decommission Streamlit - -* [ ] Mark Streamlit UI as deprecated -* [ ] Keep backend logic -* [ ] Fully switch UI to React frontend - ---- - -## 13. Acceptance Criteria - -Migration is complete when: - -* [ ] Graph renders correctly -* [ ] Node click works -* [ ] API calls succeed -* [ ] Graph updates dynamically -* [ ] No major UI blocking issues (small graphs) - ---- - -## 🚧 Future Work (Out of Scope for Now) - -* [ ] Large graph performance (WebGL, virtualization) -* [ ] Advanced layout algorithms -* [ ] Graph caching -* [ ] Undo / redo -* [ ] Multi-user collaboration \ No newline at end of file +# mat_know_base Refactor and Enhancement TODO + +Repo review date: 2026-06-30. + +This TODO is based on a read-through of the backend API, web routers, agent tools, +workflow modules, React frontend, tests, and dev scripts. It focuses on reducing +duplicate logic, clarifying ownership boundaries, and making the project easier to +operate and develop. + +## P0 - Fix Current Dev/Test Breakages + +- [x] Restore pytest collection. + - Current venv command: `.venv/bin/python -m pytest --collect-only -q`. + - Current failure: `tests/test_projection_utils.py` imports + `_filter_latest_projections` from `src/mkb/ui/pages/projections.py`, but that + helper is no longer present. + - Decide whether to reintroduce the helper, move the test to the React/API + projection path, or delete the legacy Streamlit-specific assertion. + +- [x] Fix Ruff baseline so CI can be meaningful. + - Current command: `.venv/bin/python -m ruff check src tests`. + - Current issues include unused imports, ambiguous loop variable `l`, and real + undefined test variables in `tests/test_workflow_resume.py`. + - Add a CI command that runs Ruff and pytest collection at minimum. + +- [x] Make test commands work without relying on an editable install in a hidden + local venv. + - `python3 -m pytest --collect-only -q` fails with `ModuleNotFoundError: mkb` + on the system interpreter. + - Options: document `.venv/bin/python -m pytest`, add `pythonpath = ["src"]` + to pytest config, or standardize on `pip install -e ".[dev]"`. + - Added pytest `pythonpath = ["src"]`; the system interpreter now reaches repo + modules but still needs project dependencies installed. + +## P1 - Split the Backend API Monolith + +- [x] Break up `src/mkb/api.py`. + - It is currently about 3,000 lines and owns ingestion, processing, frames, + project groups, deletion, workflow extraction, workflow schema review, + projection, export, graph, and feedback. + - Suggested service modules: + - `mkb.services.projects` + - `mkb.services.assets` + - `mkb.services.processing` + - `mkb.services.frames` + - `mkb.services.workflows` + - `mkb.services.projections` + - `mkb.services.graphs` + - `mkb.services.feedback` + - Keep `mkb.api` as a compatibility facade that imports and delegates public + functions until callers migrate. + - `mkb.api` is now a compatibility facade over domain modules: + `mkb.services.runtime`, `ingest`, `assets`, `frames`, `projects`, + `workflows`, `spaces`, `projections`, `graphs`, and `feedback`. + +- [x] Move database serialization helpers next to their domain services. + - Examples in `src/mkb/api.py`: `_serialize_group`, + `_serialize_raw_workflow`, `_serialize_canonical_workflow`, + `_serialize_projection_payload`. + - This will make router, CLI, API, and agent tool behavior easier to keep + aligned. + +- [x] Create a shared `Result`/error convention. + - Many API functions return `{"error": ...}` while routers translate those to + HTTP exceptions manually. + - Pick a single internal exception/result style and let web, CLI, and agent + adapters map it to their own surfaces. + - Added `mkb.services.result.ServiceError`, `error_result`, + `is_error_result`, and `result_status_code`. + - Web adapters now use `require_service_result` / + `require_service_result_or_not_found` instead of hand-unpacking + `{"error": ...}` dictionaries. + - Legacy `{"error": ...}` returns remain supported so service modules can + migrate incrementally. + +## P1 - Consolidate Duplicate Web, CLI, and Agent Adapters + +- [x] Introduce a single job action registry. + - Duplicate job-starting logic exists in: + - `src/mkb/web/routers/projects.py` + - `src/mkb/web/_state.py` + - `src/mkb/agents/tools/orchestrator_tools.py` + - Streamlit background job paths under `src/mkb/ui` + - Define one table of job kinds, labels, target functions, validation, and + active-job conflict policy. + - Use that registry for REST routes, assistant-triggered workflows, batch + actions, and any remaining Streamlit actions. + - Done: `mkb.web.job_actions` now defines action metadata, target lookup, + argument validation, and active-job conflict policy for REST, assistant + workflow dispatch, batch actions, upload ingest, and legacy Streamlit + project actions. + +- [x] Replace forced thread cancellation in `JobManager`. + - `src/mkb/web/_state.py` uses `ctypes.pythonapi.PyThreadState_SetAsyncExc`. + - This can interrupt database sessions, file writes, S3 operations, or agent + tool calls at unsafe points. + - Prefer cooperative cancellation through `progress_callback`, cancellation + tokens, and explicit checks in long-running loops. + +- [x] Extract upload/archive handling from `src/mkb/web/api_server.py`. + - The file notes that upload logic is inline for test monkeypatch compatibility. + - Move implementation to `mkb.web.uploads` and re-export wrapper functions in + `api_server.py` so tests and callers retain the same patch points. + +- [x] Centralize preview/content response logic. + - `src/mkb/web/routers/projects.py` owns `_inline_headers` and + `_asset_media_type`. + - Move content negotiation, safe filename headers, and S3 download response + creation into a shared web helper before adding more preview types. + +## P1 - Clarify the Workflow Migration Boundary + +- [x] Decide whether canonical workflows are legacy, active, or compatibility-only. + - `docs/workflow-card-architecture.md` says extraction to canonicalization is + retired. + - The code still exposes canonical workflow contracts, API functions, router + endpoints, frontend tabs, maintenance tasks, and tests. + - Document the current policy and mark each public endpoint as active, + deprecated, or internal compatibility. + - Documented in `docs/workflow-lifecycle-policy.md`. + +- [ ] Group workflow code by lifecycle. + - Current workflow behavior spans: + - `src/mkb/workflows/*` + - `src/mkb/agents/workflow_extraction.py` + - `src/mkb/agents/workflow_canonicalization.py` + - `src/mkb/agents/schema_curator.py` + - `src/mkb/agents/tools/workflows.py` + - `src/mkb/agents/tools/workflow_canonicalization.py` + - `src/mkb/agents/tools/schema_curator.py` + - many sections of `src/mkb/api.py` + - Create a workflow service package with explicit submodules for extraction, + validation, schema review, indexing, and legacy canonicalization. + - Started: workflow serialization now lives under `mkb.services.workflows`; + full lifecycle package split remains. + +- [ ] Remove duplicated schema/card operations between + `src/mkb/agents/tools/workflows.py` and + `src/mkb/agents/tools/workflow_canonicalization.py`. + - Both modules normalize payloads, expose card/template operations, and + manipulate draft graphs or schema libraries. + - Keep one low-level workflow editing library and make agent tools thin + adapters. + +## P1 - Reduce Frontend Duplication + +- [ ] Merge the two project detail experiences. + - `frontend/src/components/projects/ProjectDetail.tsx` + - `frontend/src/components/frames/ProjectDetail.tsx` + - Both own project actions, job polling, space selection, graph/workflow + display, and project refresh logic. + - Extract shared hooks/components: + - `useProjectActions` + - `useProjectRefresh` + - `ProjectActionBar` + - `ProjectStatusHeader` + - `ProjectTabs` + - Started: extracted `useProjectRefresh`; action bar/status/tabs are still + duplicated. + +- [ ] Split large React pages into feature modules. + - Biggest current files: + - `frontend/src/pages/SpacesPage.tsx` at about 1,400 lines + - `frontend/src/components/projects/WorkflowCanvas.tsx` at about 1,100 lines + - `frontend/src/pages/GraphPage.tsx` at about 900 lines + - `frontend/src/components/projections/SectionTable.tsx` at about 700 lines + - Prioritize extracting pure transformation helpers first, then reusable + controls, then page-level containers. + +- [ ] Finish job polling consolidation. + - `frontend/src/api/jobPolling.ts` is a good shared primitive. + - Continue removing local `pollJob`, `refreshOnFinishedJob`, and manual job + list merging patterns from page components. + - Route all job updates through `frontend/src/store/jobsStore.ts` unless a + component truly needs isolated state. + +- [x] Add route-level code splitting. + - `npm run build` succeeds, but Vite reports a large JS chunk around 1.6 MB. + - Lazy-load heavy pages/components such as graph visualization, PDF preview, + projections table, workflow canvas, and skills/settings pages. + +## P2 - Remove Legacy Streamlit Surface or Fence It Off + +- [x] Decide the long-term owner for `src/mkb/ui`. + - README says React replaces the legacy Streamlit UI, but tests and modules + still import Streamlit page helpers. + - Either: + - remove Streamlit pages after migrating tests and any missing behavior, or + - move them under `mkb.legacy_ui` and mark as compatibility-only. + - Marked `src/mkb/ui` compatibility-only in `src/mkb/ui/README.md`. + +- [ ] Stop testing new behavior through Streamlit helper functions. + - `tests/test_projection_utils.py` and upload grouping tests still target + `src/mkb/ui/pages/*`. + - Prefer tests against pure helper modules, backend service functions, or REST + router behavior. + +- [ ] Deduplicate upload grouping behavior between Streamlit and React/API. + - Similar project naming, collision handling, archive expansion, and grouping + logic appears in: + - `src/mkb/ui/pages/projects.py` + - `src/mkb/web/api_server.py` + - `frontend/src/components/projects/uploadHelpers.ts` + - Put backend-safe path/name logic in one Python module and mirror only the UI + preview heuristics in TypeScript. + +## P2 - Improve Type and Schema Contracts + +- [ ] Generate or validate frontend API types from backend models. + - Backend request models live in `src/mkb/web/_models.py`. + - Frontend domain types live in `frontend/src/types/index.ts`. + - Add an OpenAPI export and a type generation step, or add zod schemas at the + client boundary for high-risk payloads. + +- [ ] Standardize identifiers at boundaries. + - UUID parsing is repeated with `_parse_uuid`, `parse_uuidish`, ad hoc + `uuid.UUID(str(...))`, and frontend string handling. + - Define per-boundary helpers: + - web request validation + - agent tool tolerant parsing + - internal strict UUID conversion + - Started: added `mkb.services.ids` and wired web UUID parsing through it. + +- [ ] Audit JSON/blob fields in `src/mkb/db/models.py`. + - Many important states live in `metadata_`, `content`, `data`, `result`, + `checkpoint`, and `provenance`. + - Add Pydantic contracts for high-value payloads before they enter the DB, + especially workflow checkpoints, projection review results, and job results. + +## P2 - Processor and Storage Cleanup + +- [x] Make processor registration declarative. + - `src/mkb/processors/coordinator.py` owns a hard-coded `PROCESSORS` list and + special cases ambiguous text files. + - Add a registry that can rank processors by MIME type, extension, and content + sniffing confidence. + +- [ ] Reuse bundle hashing and processed-output inspection. + - `src/mkb/api.py` manually inspects handmade processed directories. + - `src/mkb/processors/base.py` and `src/mkb/processors/coordinator.py` compute + processed result hashes and artifact metadata. + - Extract one processed bundle model/helper so manual and automatic processing + share the same hashing, artifact list, and primary file rules. + +- [ ] Add lifecycle cleanup for local processed/upload temp files. + - Upload temp folders, processed local mirrors, and generated exports can grow + quickly during research use. + - Add commands for dry-run cleanup, retention windows, and orphan detection. + +## P2 - Knowledge Graph and Projection Quality + +- [ ] Move graph normalization/dedup logic behind a graph service. + - Logic currently sits in `src/mkb/knowledge_graph.py`, + `src/mkb/agents/tools/knowledge_graph.py`, and graph review tools. + - Keep agent tools thin and centralize concept/relation validation, + deduplication, merge rules, and review counters. + +- [ ] Separate projection extraction, review, patching, and export concerns. + - `src/mkb/agents/tools/projection.py` is over 1,000 lines. + - `src/mkb/agents/tools/projection_review.py` is another large mixed module. + - Suggested split: + - read/query helpers + - projection mutation helpers + - patch/path operations + - review session persistence + - export formatting + +- [ ] Add regression fixtures for duplicate projection and graph merge cases. + - The product depends heavily on deduplication quality. + - Keep small fixtures for same-paper repeated projections, cross-paper concept + aliases, and source-reference preservation. + +## P3 - Dev Experience and Repo Hygiene + +- [x] Add a `make check` target. + - Suggested steps: + - `.venv/bin/python -m ruff check src tests` + - `.venv/bin/python -m pytest` + - `cd frontend && npm run build` + - Add lighter targets for `make lint`, `make test-python`, and + `make test-frontend`. + +- [x] Add frontend linting and formatting. + - `frontend/package.json` has build scripts only. + - Add ESLint/Prettier or a minimal TypeScript-aware lint command so React + cleanup can be enforced incrementally. + +- [x] Add a short architecture map. + - The README is broad and useful, but new developers need a quick owner map: + ingestion, processing, frames, spaces, projections, graph, workflows, jobs, + frontend. + - Put it in `docs/architecture-map.md`. + +- [x] Keep generated/local artifacts out of review noise. + - `.gitignore` covers `data/`, `logs`, `.debug/`, `node_modules/`, and + `__pycache__/`. + - Tracked data exports currently include: + - `data/exports/projections_yaml/*.yaml` + - `data/inbox/.gitkeep` + - Decide whether exported projection YAML files should remain tracked + fixtures or move under examples/fixtures with clear names. + - Documented generated-output policy in `docs/development.md`. + +- [x] Add dependency and environment notes. + - `python` is not available in this environment, but `python3` and `.venv` are. + - Make docs and scripts consistently use `python3` or `.venv/bin/python`. + - Consider pinning high-risk dependencies or adding a constraints file for + reproducible agent and PDF-processing environments. + +## Suggested Refactor Order + +1. Fix pytest collection and Ruff baseline. +2. Introduce service modules behind the existing `mkb.api` facade. +3. Centralize background job action registration and remove unsafe thread + cancellation. +4. Merge frontend project detail/action logic and finish job polling + consolidation. +5. Formalize workflow canonicalization as active or legacy, then prune or fence + matching backend/frontend/tests. +6. Split the largest frontend pages and agent tool modules after the service + boundaries are stable. + +## Verification Notes From This Pass + +- [x] `python3 -m compileall -q src` passed. +- [x] `cd frontend && npm run build` passed. +- [x] `.venv/bin/python -m pytest --collect-only -q` passed with 111 tests + collected. +- [x] `.venv/bin/python -m ruff check src tests` passed. +- [x] `.venv/bin/python -m pytest -q` passed with 111 tests. +- [x] `cd frontend && npm run lint` passed. diff --git a/docs/architecture-map.md b/docs/architecture-map.md new file mode 100644 index 0000000..cebd108 --- /dev/null +++ b/docs/architecture-map.md @@ -0,0 +1,52 @@ +# Architecture Map + +This map names the main ownership areas in the repository so refactors can move +code toward clearer boundaries without changing behavior accidentally. + +## Runtime Surfaces + +- `src/mkb/api.py` is the Python compatibility facade used by scripts, tests, + routers, and legacy UI code. +- `src/mkb/web/` owns the FastAPI app, REST routers, request/response models, + upload handling, and background job state. +- `frontend/src/` owns the React application that replaced the primary + Streamlit workflow. +- `src/mkb/ui/` is the legacy Streamlit surface. It remains compatibility-only + while tests and a few helper paths still import it. +- `src/mkb/agents/` owns agent construction, prompts, tool adapters, and runner + integration. +- `src/mkb/cli.py` owns command-line entry points and should call service/API + functions rather than duplicating behavior. + +## Domain Areas + +- Ingestion: `src/mkb/ingest/`, `src/mkb/api.py`, and upload entry points under + `src/mkb/web/`. +- Processing: `src/mkb/processors/`, processed asset models, and S3 helpers. +- Projects and assets: `ResearchProject`, `Asset`, and `ProjectAsset` models, + plus project routers and frontend project views. +- Frames: `KnowledgeFrame` storage, frame agent code, frame routers, and React + frame/project detail tabs. +- Spaces and projections: `src/mkb/spaces/`, projection agents/tools, projection + routers, and projection table components. +- Workflows: `src/mkb/workflows/`, workflow extraction/canonicalization agents, + schema curator tools, and workflow tabs. +- Knowledge graph: `src/mkb/knowledge_graph.py`, graph agent/tools, graph review + tools, graph router, and graph frontend page. +- Feedback: `src/mkb/feedback/`, feedback agent/tools, feedback router, and + frontend feedback page. +- Jobs: `src/mkb/web/_state.py`, job routers, job polling hooks/stores, and + legacy Streamlit background job helpers. + +## Boundary Direction + +Adapters should stay thin: + +- Web routers validate HTTP input and map service errors to HTTP responses. +- CLI commands parse arguments and print results. +- Agent tools parse tolerant user/agent inputs and call strict domain helpers. +- React components call typed client functions and avoid backend policy logic. + +Shared behavior belongs in domain services or pure helpers before it is reused +by routers, CLI commands, agent tools, and legacy compatibility surfaces. + diff --git a/docs/development.md b/docs/development.md new file mode 100644 index 0000000..3530102 --- /dev/null +++ b/docs/development.md @@ -0,0 +1,30 @@ +# Development Notes + +Use the project virtual environment for Python checks. The repository is +configured with `pythonpath = ["src"]`, so tests can import `mkb` without an +editable install, but the interpreter still needs project dependencies. + +```bash +python3 -m venv .venv +.venv/bin/python -m pip install -e ".[dev]" +cd frontend && npm install +``` + +Common checks: + +```bash +make ci # Ruff plus pytest collection +make lint # Python Ruff plus frontend TypeScript check +make check # Python tests, frontend build, and lint +``` + +The bare `python3` interpreter may collect imports from `src`, but it will fail +unless dependencies such as `pydantic-settings`, `google-adk`, `pgvector`, and +`streamlit` are installed in that interpreter. Prefer `.venv/bin/python` in +scripts and documentation when reproducibility matters. + +Local runtime artifacts live under `data/`, `.debug/`, `logs/`, and +`docker_volumes/`. Projection export YAMLs should be treated as generated output +unless they are deliberately promoted to named fixtures under `examples/` or +`tests/fixtures/`. + diff --git a/docs/workflow-card-architecture.md b/docs/workflow-card-architecture.md index 1c7310f..f125590 100644 --- a/docs/workflow-card-architecture.md +++ b/docs/workflow-card-architecture.md @@ -11,6 +11,9 @@ The workflow subsystem has two agents: The former extraction → canonicalization sequence is retired. Legacy raw and canonical records remain readable during migration. +See `workflow-lifecycle-policy.md` for the current active, deprecated, and +compatibility-only public surfaces. + ## Cards and instances An ontology card describes a reusable concept: diff --git a/docs/workflow-lifecycle-policy.md b/docs/workflow-lifecycle-policy.md new file mode 100644 index 0000000..c9aa14a --- /dev/null +++ b/docs/workflow-lifecycle-policy.md @@ -0,0 +1,38 @@ +# Workflow Lifecycle Policy + +As of June 30, 2026, the active workflow model is the workflow-card extraction +path documented in `workflow-card-architecture.md`. + +## Active + +- Raw workflow extraction records are active. They store the evidence-grounded + workflow graph produced from a project. +- Workflow card/schema validation, review, indexing, and ontology induction are + active. +- Schema proposal review and workflow maintenance tasks are active. + +## Compatibility Only + +- Canonical workflow records and canonicalization endpoints remain available so + older data, tests, and UI tabs continue to work during migration. +- New product behavior should not depend on canonicalization unless it is + explicitly maintaining compatibility with existing records. +- Canonical workflow code should move behind a `legacy` or `compatibility` + service boundary before any larger deletion. + +## Deprecated For New Work + +- The old extraction-to-canonicalization pipeline is retired for new feature + development. +- New workflow features should consume raw/card graphs and schema-library + helpers directly. + +## Public Surface Status + +- Active: raw workflow extraction, raw workflow review/correction, schema + curation, schema proposal review, workflow maintenance, workflow indexing. +- Compatibility: canonical workflow list/get/delete, canonicalization job + routes, canonical workflow frontend tabs, canonical indexes. +- Internal compatibility: checkpoint and draft-edit helpers used to resume or + inspect unfinished legacy canonicalization jobs. + diff --git a/frontend/package.json b/frontend/package.json index d8c26ca..a06017b 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -5,6 +5,7 @@ "type": "module", "scripts": { "dev": "vite", + "lint": "tsc -b --noEmit", "build": "tsc -b && vite build", "preview": "vite preview" }, diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 779651b..9952efe 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -1,14 +1,16 @@ +import { Suspense, lazy } from 'react' import Layout from './components/Layout' import { useUiStore } from './store/uiStore' -import AssistantPage from './pages/AssistantPage' -import ProjectsPage from './pages/ProjectsPage' -import FramesPage from './pages/FramesPage' -import GraphPage from './pages/GraphPage' -import ProjectionsPage from './pages/ProjectionsPage' -import SpacesPage from './pages/SpacesPage' -import SkillsPage from './pages/SkillsPage' -import FeedbackPage from './pages/FeedbackPage' -import SettingsPage from './pages/SettingsPage' + +const AssistantPage = lazy(() => import('./pages/AssistantPage')) +const ProjectsPage = lazy(() => import('./pages/ProjectsPage')) +const FramesPage = lazy(() => import('./pages/FramesPage')) +const GraphPage = lazy(() => import('./pages/GraphPage')) +const ProjectionsPage = lazy(() => import('./pages/ProjectionsPage')) +const SpacesPage = lazy(() => import('./pages/SpacesPage')) +const SkillsPage = lazy(() => import('./pages/SkillsPage')) +const FeedbackPage = lazy(() => import('./pages/FeedbackPage')) +const SettingsPage = lazy(() => import('./pages/SettingsPage')) function App() { const page = useUiStore(s => s.page) @@ -28,7 +30,13 @@ function App() { } } - return {renderPage()} + return ( + + Loading...}> + {renderPage()} + + + ) } export default App diff --git a/frontend/src/components/frames/ProjectDetail.tsx b/frontend/src/components/frames/ProjectDetail.tsx index abcb0de..e588207 100644 --- a/frontend/src/components/frames/ProjectDetail.tsx +++ b/frontend/src/components/frames/ProjectDetail.tsx @@ -4,13 +4,13 @@ import { startJobPolling } from '../../api/jobPolling' import { listFeedback } from '../../api/feedback' import { extractProject, - getProject, kgExtractProject, processProject, projectToSpace, workflowExtractProject, } from '../../api/projects' import { listSpaces } from '../../api/spaces' +import { useProjectRefresh } from '../../hooks/useProjectRefresh' import type { Job, Project, Space } from '../../types' import StatusBadge from '../StatusBadge' import AssetsTab from './AssetsTab' @@ -57,10 +57,7 @@ export default function ProjectDetail({ refreshFeedbackCount() }, [project.project_id, refreshFeedbackCount, selectedSpaceId]) - const refreshProject = useCallback(() => { - if (!onProjectUpdated) return - getProject(project.project_id).then(onProjectUpdated).catch(() => {}) - }, [project.project_id, onProjectUpdated]) + const refreshProject = useProjectRefresh(project.project_id, onProjectUpdated) const pollJob = useCallback((jobId: string, onDone?: () => void) => { setActiveJobId(jobId) diff --git a/frontend/src/components/projects/ProjectDetail.tsx b/frontend/src/components/projects/ProjectDetail.tsx index 4b252bb..34b550d 100644 --- a/frontend/src/components/projects/ProjectDetail.tsx +++ b/frontend/src/components/projects/ProjectDetail.tsx @@ -12,6 +12,7 @@ import { projectToSpace, workflowExtractProject, } from '../../api/projects' +import { useProjectRefresh } from '../../hooks/useProjectRefresh' import type { Job, Project, Space } from '../../types' import JobProgress from '../JobProgress' import StatusBadge from '../StatusBadge' @@ -64,10 +65,7 @@ export default function ProjectDetail({ return () => { pollHandleRef.current?.cancel() } }, [loadJobs]) - const refreshProject = useCallback(() => { - if (!onProjectUpdated) return - getProject(project.project_id).then(onProjectUpdated).catch(() => { /* ignore */ }) - }, [project.project_id, onProjectUpdated]) + const refreshProject = useProjectRefresh(project.project_id, onProjectUpdated) const pollJob = useCallback((jobId: string) => { setActiveJobId(jobId) diff --git a/frontend/src/components/projects/WorkflowCanvas.tsx b/frontend/src/components/projects/WorkflowCanvas.tsx index 2dbe8e3..fb4cb5a 100644 --- a/frontend/src/components/projects/WorkflowCanvas.tsx +++ b/frontend/src/components/projects/WorkflowCanvas.tsx @@ -6,15 +6,17 @@ import ReactFlow, { MarkerType, MiniMap, Position, + applyNodeChanges, type Edge, type Node, + type NodeChange, type NodeProps, useEdgesState, useNodesState, } from 'reactflow' import 'reactflow/dist/style.css' -type WorkflowNodeKind = 'object' | 'operation' +type WorkflowNodeKind = 'object' | 'operation' | 'planning' | 'reasoning' | 'unknown' export interface WorkflowCanvasNode { id: string @@ -43,19 +45,41 @@ interface WorkflowNodeData { const XML_NS = 'http://www.w3.org/2000/svg' const OP_WIDTH = 190 const OBJ_WIDTH = 190 +const CONTEXT_WIDTH = 210 const NODE_HEIGHT = 74 -const X_GAP = 260 +const X_GAP = 300 const Y_GAP = 240 const OBJECT_OFFSET = 145 const MIN_ROW_SPACING = 235 +const COMPONENT_GAP_X = 360 +const COMPONENT_GAP_Y = 150 +const BRANCH_GAP = 320 + +type AnchorSide = 'top' | 'right' | 'bottom' | 'left' +type PositionedNode = WorkflowCanvasNode & { x: number; y: number } function NodeHandles() { + const handles: AnchorSide[] = ['top', 'right', 'bottom', 'left'] return ( <> - - - - + {handles.map(side => ( + + ))} + {handles.map(side => ( + + ))} ) } @@ -109,10 +133,75 @@ function ObjectNode({ data }: NodeProps) { ) } +function PlanningNode({ data }: NodeProps) { + return ( +
+ + +
+ ) +} + +function ReasoningNode({ data }: NodeProps) { + return ( +
+ + +
+ ) +} + +function UnknownNode({ data }: NodeProps) { + return ( +
+ + +
+ ) +} + function average(values: number[]) { return values.length ? values.reduce((sum, value) => sum + value, 0) / values.length : 0 } +function sideToPosition(side: AnchorSide) { + switch (side) { + case 'top': + return Position.Top + case 'right': + return Position.Right + case 'bottom': + return Position.Bottom + case 'left': + return Position.Left + } +} + +function sideVector(side: AnchorSide) { + switch (side) { + case 'top': + return { x: 0, y: -1 } + case 'right': + return { x: 1, y: 0 } + case 'bottom': + return { x: 0, y: 1 } + case 'left': + return { x: -1, y: 0 } + } +} + function escapeXml(value: string) { return value .replace(/&/g, '&') @@ -163,11 +252,14 @@ function labelLines(label: string, maxChars = 18, maxLines = 3) { } function nodeWidth(kind: WorkflowNodeKind) { - return kind === 'operation' ? OP_WIDTH : OBJ_WIDTH + if (kind === 'operation') return OP_WIDTH + if (kind === 'object') return OBJ_WIDTH + return CONTEXT_WIDTH } function nodeBounds(node: Node) { - const width = nodeWidth(node.type === 'operation' ? 'operation' : 'object') + const kind = (node.type ?? node.data.kind) as WorkflowNodeKind + const width = nodeWidth(kind) return { x: node.position.x, y: node.position.y, @@ -178,16 +270,70 @@ function nodeBounds(node: Node) { } } +function sidePoint(box: ReturnType, side: AnchorSide) { + switch (side) { + case 'top': + return { x: box.centerX, y: box.y } + case 'right': + return { x: box.x + box.width, y: box.centerY } + case 'bottom': + return { x: box.centerX, y: box.y + box.height } + case 'left': + return { x: box.x, y: box.centerY } + } +} + +function chooseAnchorSides(source: Node, target: Node) { + const sb = nodeBounds(source) + const tb = nodeBounds(target) + const sourceSides: AnchorSide[] = ['top', 'right', 'bottom', 'left'] + const targetSides: AnchorSide[] = ['top', 'right', 'bottom', 'left'] + let best = { sourceSide: 'bottom' as AnchorSide, targetSide: 'top' as AnchorSide, score: Number.POSITIVE_INFINITY } + + sourceSides.forEach(sourceSide => { + targetSides.forEach(targetSide => { + const start = sidePoint(sb, sourceSide) + const end = sidePoint(tb, targetSide) + const dx = end.x - start.x + const dy = end.y - start.y + let score = Math.abs(dx) + Math.abs(dy) + + const sv = sideVector(sourceSide) + const tv = sideVector(targetSide) + if (Math.sign(dx) !== 0 && Math.sign(dx) !== Math.sign(sv.x)) score += sourceSide === 'left' || sourceSide === 'right' ? 140 : 40 + if (Math.sign(dy) !== 0 && Math.sign(dy) !== Math.sign(sv.y)) score += sourceSide === 'top' || sourceSide === 'bottom' ? 140 : 40 + if (Math.sign(dx) !== 0 && Math.sign(dx) === Math.sign(tv.x)) score += targetSide === 'left' || targetSide === 'right' ? 140 : 40 + if (Math.sign(dy) !== 0 && Math.sign(dy) === Math.sign(tv.y)) score += targetSide === 'top' || targetSide === 'bottom' ? 140 : 40 + + const mostlyVertical = Math.abs(dy) > Math.abs(dx) * 0.9 + const mostlyHorizontal = Math.abs(dx) > Math.abs(dy) * 0.9 + if (mostlyVertical && sourceSide === 'bottom' && targetSide === 'top' && dy > 0) score -= 130 + if (mostlyVertical && sourceSide === 'top' && targetSide === 'bottom' && dy < 0) score -= 130 + if (mostlyHorizontal && sourceSide === 'right' && targetSide === 'left' && dx > 0) score -= 130 + if (mostlyHorizontal && sourceSide === 'left' && targetSide === 'right' && dx < 0) score -= 130 + if (sourceSide === targetSide) score += 80 + + if (score < best.score) { + best = { sourceSide, targetSide, score } + } + }) + }) + + return best +} + function edgePath(source: Node, target: Node) { const sb = nodeBounds(source) const tb = nodeBounds(target) - const sourceBelow = tb.centerY >= sb.centerY - const startX = sb.centerX - const startY = sourceBelow ? sb.y + sb.height : sb.y - const endX = tb.centerX - const endY = sourceBelow ? tb.y : tb.y + tb.height - const midY = startY + (endY - startY) / 2 - return `M ${startX} ${startY} C ${startX} ${midY}, ${endX} ${midY}, ${endX} ${endY}` + const { sourceSide, targetSide } = chooseAnchorSides(source, target) + const start = sidePoint(sb, sourceSide) + const end = sidePoint(tb, targetSide) + const sv = sideVector(sourceSide) + const tv = sideVector(targetSide) + const distance = Math.max(72, Math.min(220, (Math.abs(end.x - start.x) + Math.abs(end.y - start.y)) / 2)) + const c1 = { x: start.x + sv.x * distance, y: start.y + sv.y * distance } + const c2 = { x: end.x + tv.x * distance, y: end.y + tv.y * distance } + return `M ${start.x} ${start.y} C ${c1.x} ${c1.y}, ${c2.x} ${c2.y}, ${end.x} ${end.y}` } function downloadFile(filename: string, mimeType: string, content: string) { @@ -217,10 +363,135 @@ function spreadRow(items: T[], minSpacing: number) { }) } -function buildLayout(nodes: WorkflowCanvasNode[], edges: WorkflowCanvasEdge[]): { nodes: Node[]; edges: Edge[] } { +function connectedComponents(nodes: WorkflowCanvasNode[], edges: WorkflowCanvasEdge[]) { + const nodeIds = new Set(nodes.map(node => node.id)) + const adjacency = new Map>() + nodes.forEach(node => adjacency.set(node.id, new Set())) + edges.forEach(edge => { + if (!nodeIds.has(edge.source) || !nodeIds.has(edge.target)) return + adjacency.get(edge.source)?.add(edge.target) + adjacency.get(edge.target)?.add(edge.source) + }) + + const seen = new Set() + const components: string[][] = [] + nodes.forEach(node => { + if (seen.has(node.id)) return + const queue = [node.id] + const component: string[] = [] + seen.add(node.id) + while (queue.length) { + const current = queue.shift()! + component.push(current) + adjacency.get(current)?.forEach(next => { + if (seen.has(next)) return + seen.add(next) + queue.push(next) + }) + } + components.push(component) + }) + return components +} + +function rowItemsFromAnchors(ids: string[], anchors: Map, fallbackGap: number) { + const rowWidth = (ids.length - 1) * fallbackGap + return ids.map((id, index) => ({ + id, + x: anchors.has(id) ? anchors.get(id)! : index * fallbackGap - rowWidth / 2, + })) +} + +function centeredOffsets(count: number, gap: number) { + const center = (count - 1) / 2 + return Array.from({ length: count }, (_, index) => (index - center) * gap) +} + +function workflowLayoutGroups(nodes: WorkflowCanvasNode[], edges: WorkflowCanvasEdge[]) { + const nodeMap = new Map(nodes.map(node => [node.id, node])) + const operationIds = nodes.filter(node => node.kind === 'operation').map(node => node.id) + if (operationIds.length === 0) return connectedComponents(nodes, edges) + + const producerMap = new Map() + const consumerMap = new Map() + nodes.filter(node => node.kind === 'object').forEach(node => { + producerMap.set(node.id, []) + consumerMap.set(node.id, []) + }) + + edges.forEach(edge => { + const sourceKind = nodeMap.get(edge.source)?.kind + const targetKind = nodeMap.get(edge.target)?.kind + if (sourceKind === 'operation' && targetKind === 'object') { + producerMap.get(edge.target)?.push(edge.source) + } + if (sourceKind === 'object' && targetKind === 'operation') { + consumerMap.get(edge.source)?.push(edge.target) + } + }) + + const opAdjacency = new Map>() + operationIds.forEach(id => opAdjacency.set(id, new Set())) + producerMap.forEach((producers, objectId) => { + const consumers = consumerMap.get(objectId) ?? [] + producers.forEach(producerId => { + consumers.forEach(consumerId => { + if (producerId === consumerId) return + opAdjacency.get(producerId)?.add(consumerId) + opAdjacency.get(consumerId)?.add(producerId) + }) + }) + }) + + const seenOps = new Set() + const groups: string[][] = [] + const groupByOperation = new Map() + operationIds.forEach(operationId => { + if (seenOps.has(operationId)) return + const queue = [operationId] + const group: string[] = [] + seenOps.add(operationId) + while (queue.length) { + const current = queue.shift()! + groupByOperation.set(current, groups.length) + group.push(current) + opAdjacency.get(current)?.forEach(next => { + if (seenOps.has(next)) return + seenOps.add(next) + queue.push(next) + }) + } + groups.push(group) + }) + + const ungroupedObjects: string[] = [] + nodes.filter(node => node.kind === 'object').forEach(node => { + const touchedGroups = new Map() + const touchedOperations = [...(producerMap.get(node.id) ?? []), ...(consumerMap.get(node.id) ?? [])] + touchedOperations.forEach(operationId => { + const groupIndex = groupByOperation.get(operationId) + if (groupIndex === undefined) return + touchedGroups.set(groupIndex, (touchedGroups.get(groupIndex) ?? 0) + 1) + }) + + if (touchedGroups.size === 0) { + ungroupedObjects.push(node.id) + return + } + + const bestGroup = Array.from(touchedGroups.entries()).sort((a, b) => b[1] - a[1] || a[0] - b[0])[0][0] + groups[bestGroup].push(node.id) + }) + + ungroupedObjects.forEach(objectId => groups.push([objectId])) + return groups +} + +function layoutComponent(nodes: WorkflowCanvasNode[], edges: WorkflowCanvasEdge[]) { const nodeMap = new Map(nodes.map(node => [node.id, node])) const operationIds = nodes.filter(node => node.kind === 'operation').map(node => node.id) const objectIds = nodes.filter(node => node.kind === 'object').map(node => node.id) + const contextIds = nodes.filter(node => !['object', 'operation'].includes(node.kind)).map(node => node.id) const producerMap = new Map() const consumerMap = new Map() @@ -279,6 +550,7 @@ function buildLayout(nodes: WorkflowCanvasNode[], edges: WorkflowCanvasEdge[]): queue.push(childId) } }) + queue.sort((a, b) => (nodeMap.get(a)?.label ?? '').localeCompare(nodeMap.get(b)?.label ?? '')) } const levels = new Map() @@ -288,34 +560,107 @@ function buildLayout(nodes: WorkflowCanvasNode[], edges: WorkflowCanvasEdge[]): levels.get(level)!.push(id) }) + const sortedLevels = Array.from(levels.keys()).sort((a, b) => a - b) + const levelOrder = new Map() + sortedLevels.forEach(level => { + levelOrder.set(level, [...(levels.get(level) ?? [])].sort((a, b) => (nodeMap.get(a)?.label ?? '').localeCompare(nodeMap.get(b)?.label ?? ''))) + }) + const opX = new Map() - Array.from(levels.keys()).sort((a, b) => a - b).forEach(level => { - const ids = levels.get(level) ?? [] - ids.sort((a, b) => { - const aParents = Array.from(opParents.get(a) ?? []) - const bParents = Array.from(opParents.get(b) ?? []) - const aAnchor = aParents.length ? average(aParents.map(parentId => opX.get(parentId) ?? 0)) : 0 - const bAnchor = bParents.length ? average(bParents.map(parentId => opX.get(parentId) ?? 0)) : 0 - if (aAnchor !== bAnchor) return aAnchor - bAnchor - return (nodeMap.get(a)?.label ?? '').localeCompare(nodeMap.get(b)?.label ?? '') + for (let pass = 0; pass < 6; pass += 1) { + sortedLevels.forEach(level => { + const ids = levelOrder.get(level) ?? [] + const anchors = new Map() + ids.forEach(id => { + const parents = Array.from(opParents.get(id) ?? []).filter(parent => opX.has(parent)) + if (parents.length) anchors.set(id, average(parents.map(parent => opX.get(parent)!))) + }) + const items = rowItemsFromAnchors(ids, anchors, X_GAP) + spreadRow(items, MIN_ROW_SPACING) + items.forEach(item => opX.set(item.id, item.x)) + levelOrder.set(level, items.sort((a, b) => a.x - b.x).map(item => item.id)) }) - const rowWidth = (ids.length - 1) * X_GAP - ids.forEach((id, index) => { - opX.set(id, index * X_GAP - rowWidth / 2) + sortedLevels.slice().reverse().forEach(level => { + const ids = levelOrder.get(level) ?? [] + const anchors = new Map() + ids.forEach(id => { + const children = Array.from(opChildren.get(id) ?? []).filter(child => opX.has(child)) + if (children.length) anchors.set(id, average(children.map(child => opX.get(child)!))) + }) + const items = rowItemsFromAnchors(ids, anchors, X_GAP) + spreadRow(items, MIN_ROW_SPACING) + items.forEach(item => opX.set(item.id, item.x)) + levelOrder.set(level, items.sort((a, b) => a.x - b.x).map(item => item.id)) + }) + } + + if (operationIds.length === 0) { + objectIds.forEach((id, index) => opX.set(id, index * MIN_ROW_SPACING)) + } + + const branchOffsets = new Map() + const addBranchOffset = (operationId: string, offset: number) => { + const values = branchOffsets.get(operationId) ?? [] + values.push(offset) + branchOffsets.set(operationId, values) + } + + objectIds.forEach(objectId => { + const consumers = [...(consumerMap.get(objectId) ?? [])].sort((a, b) => { + const levelDelta = (opLevel.get(a) ?? 0) - (opLevel.get(b) ?? 0) + if (levelDelta !== 0) return levelDelta + return (nodeMap.get(a)?.label ?? '').localeCompare(nodeMap.get(b)?.label ?? '') }) + if (consumers.length > 1) { + centeredOffsets(consumers.length, BRANCH_GAP).forEach((offset, index) => { + addBranchOffset(consumers[index], offset) + }) + } - const rowItems = ids.map(id => ({ id, x: opX.get(id) ?? 0 })) - spreadRow(rowItems, MIN_ROW_SPACING) - rowItems.forEach(item => { - opX.set(item.id, item.x) + const producers = [...(producerMap.get(objectId) ?? [])].sort((a, b) => { + const levelDelta = (opLevel.get(a) ?? 0) - (opLevel.get(b) ?? 0) + if (levelDelta !== 0) return levelDelta + return (nodeMap.get(a)?.label ?? '').localeCompare(nodeMap.get(b)?.label ?? '') }) + if (producers.length > 1) { + centeredOffsets(producers.length, BRANCH_GAP).forEach((offset, index) => { + addBranchOffset(producers[index], offset) + }) + } + }) + + branchOffsets.forEach((offsets, operationId) => { + opX.set(operationId, (opX.get(operationId) ?? 0) + average(offsets)) + }) + + const levelRows = new Map>() + operationIds.forEach(id => { + const level = opLevel.get(id) ?? 0 + if (!levelRows.has(level)) levelRows.set(level, []) + levelRows.get(level)!.push({ id, x: opX.get(id) ?? 0 }) + }) + levelRows.forEach(items => { + spreadRow(items, MIN_ROW_SPACING) + items.forEach(item => opX.set(item.id, item.x)) }) const objectLayout = objectIds.map(id => { const producers = producerMap.get(id) ?? [] const consumers = consumerMap.get(id) ?? [] + if (producers.length > 0 && consumers.length > 0) { + const latestLevel = Math.max(...producers.map(producerId => opLevel.get(producerId) ?? 0)) + const earliestLevel = Math.min(...consumers.map(consumerId => opLevel.get(consumerId) ?? 0)) + const nearbyProducers = producers.filter(producerId => (opLevel.get(producerId) ?? 0) === latestLevel) + const nearbyConsumers = consumers.filter(consumerId => (opLevel.get(consumerId) ?? 0) === earliestLevel) + return { + id, + x: average([...nearbyProducers, ...nearbyConsumers].map(opId => opX.get(opId) ?? 0)), + y: ((latestLevel + earliestLevel) / 2) * Y_GAP, + } + } + if (consumers.length > 0) { const earliestLevel = Math.min(...consumers.map(consumerId => opLevel.get(consumerId) ?? 0)) const earliestConsumers = consumers.filter(consumerId => (opLevel.get(consumerId) ?? 0) === earliestLevel) @@ -336,7 +681,8 @@ function buildLayout(nodes: WorkflowCanvasNode[], edges: WorkflowCanvasEdge[]): } } - return { id, x: 0, y: -OBJECT_OFFSET } + const isolatedIndex = objectIds.indexOf(id) + return { id, x: isolatedIndex * MIN_ROW_SPACING, y: -OBJECT_OFFSET } }) const objectRows = new Map>() @@ -345,52 +691,188 @@ function buildLayout(nodes: WorkflowCanvasNode[], edges: WorkflowCanvasEdge[]): if (!objectRows.has(rowKey)) objectRows.set(rowKey, []) objectRows.get(rowKey)!.push(item) }) + objectRows.forEach(items => spreadRow(items, MIN_ROW_SPACING)) - objectRows.forEach(items => { - spreadRow(items, MIN_ROW_SPACING) + const basePositions = new Map() + nodes.forEach(node => { + if (node.kind === 'operation') { + basePositions.set(node.id, { ...node, x: opX.get(node.id) ?? 0, y: (opLevel.get(node.id) ?? 0) * Y_GAP }) + return + } + if (node.kind === 'object') { + const objectPosition = objectLayout.find(item => item.id === node.id) ?? { x: 0, y: -OBJECT_OFFSET } + basePositions.set(node.id, { ...node, x: objectPosition.x, y: objectPosition.y }) + } }) - const flowNodes: Node[] = nodes.map(node => { - if (node.kind === 'operation') { + const incoming = new Map() + const outgoing = new Map() + contextIds.forEach(id => { + incoming.set(id, []) + outgoing.set(id, []) + }) + edges.forEach(edge => { + if (contextIds.includes(edge.source)) outgoing.get(edge.source)?.push(edge.target) + if (contextIds.includes(edge.target)) incoming.get(edge.target)?.push(edge.source) + }) + + const contextLayout = contextIds.map((id, index) => { + const downstream = (outgoing.get(id) ?? []).map(targetId => basePositions.get(targetId)).filter(Boolean) as PositionedNode[] + const upstream = (incoming.get(id) ?? []).map(sourceId => basePositions.get(sourceId)).filter(Boolean) as PositionedNode[] + const anchors = downstream.length ? downstream : upstream + if (anchors.length) { return { - id: node.id, - type: 'operation', - position: { x: opX.get(node.id) ?? 0, y: (opLevel.get(node.id) ?? 0) * Y_GAP }, - data: { id: node.id, kind: node.kind, label: node.label, title: node.title, details: node.details }, + id, + x: average(anchors.map(node => node.x)), + y: average(anchors.map(node => node.y)) - OBJECT_OFFSET, } } + return { id, x: index * MIN_ROW_SPACING, y: -OBJECT_OFFSET * 2 } + }) + const contextRows = new Map>() + contextLayout.forEach(item => { + const rowKey = Math.round(item.y) + if (!contextRows.has(rowKey)) contextRows.set(rowKey, []) + contextRows.get(rowKey)!.push(item) + }) + contextRows.forEach(items => spreadRow(items, MIN_ROW_SPACING)) + contextLayout.forEach(item => { + const node = nodeMap.get(item.id) + if (node) basePositions.set(item.id, { ...node, x: item.x, y: item.y }) + }) + + const positioned: PositionedNode[] = nodes.map(node => basePositions.get(node.id) ?? { ...node, x: 0, y: 0 }) + + return positioned +} + +function packComponents(components: PositionedNode[][]) { + const packed: PositionedNode[] = [] + let cursorX = 0 + let cursorY = 0 + let rowHeight = 0 + const maxRowWidth = Math.max(1200, Math.ceil(Math.sqrt(components.length || 1)) * 900) + + const sorted = [...components].sort((a, b) => b.length - a.length) + sorted.forEach(component => { + const bounds = component.reduce( + (acc, node) => { + const width = nodeWidth(node.kind) + return { + minX: Math.min(acc.minX, node.x), + minY: Math.min(acc.minY, node.y), + maxX: Math.max(acc.maxX, node.x + width), + maxY: Math.max(acc.maxY, node.y + NODE_HEIGHT), + } + }, + { minX: Number.POSITIVE_INFINITY, minY: Number.POSITIVE_INFINITY, maxX: Number.NEGATIVE_INFINITY, maxY: Number.NEGATIVE_INFINITY }, + ) + const width = bounds.maxX - bounds.minX + const height = bounds.maxY - bounds.minY + if (cursorX > 0 && cursorX + width > maxRowWidth) { + cursorX = 0 + cursorY += rowHeight + COMPONENT_GAP_Y + rowHeight = 0 + } + + component.forEach(node => { + packed.push({ + ...node, + x: node.x - bounds.minX + cursorX, + y: node.y - bounds.minY + cursorY, + }) + }) - const layout = objectLayout.find(item => item.id === node.id) ?? { x: 0, y: -OBJECT_OFFSET } + cursorX += width + COMPONENT_GAP_X + rowHeight = Math.max(rowHeight, height) + }) + + const bounds = packed.reduce( + (acc, node) => ({ + minX: Math.min(acc.minX, node.x), + maxX: Math.max(acc.maxX, node.x + nodeWidth(node.kind)), + }), + { minX: Number.POSITIVE_INFINITY, maxX: Number.NEGATIVE_INFINITY }, + ) + const centerShift = (bounds.minX + bounds.maxX) / 2 + return packed.map(node => ({ ...node, x: node.x - centerShift })) +} + +function anchorEdges(edgeList: Edge[], nodeList: Node[]) { + const flowNodeMap = new Map(nodeList.map(node => [node.id, node])) + return edgeList.map(edge => { + const source = flowNodeMap.get(String(edge.source)) + const target = flowNodeMap.get(String(edge.target)) + const anchors = source && target ? chooseAnchorSides(source, target) : null return { - id: node.id, - type: 'object', - position: { x: layout.x, y: layout.y }, - data: { id: node.id, kind: node.kind, label: node.label, title: node.title, details: node.details }, + ...edge, + sourceHandle: anchors ? `source-${anchors.sourceSide}` : edge.sourceHandle, + targetHandle: anchors ? `target-${anchors.targetSide}` : edge.targetHandle, } }) +} + +function buildLayout(nodes: WorkflowCanvasNode[], edges: WorkflowCanvasEdge[]): { nodes: Node[]; edges: Edge[] } { + if (nodes.length === 0) return { nodes: [], edges: [] } - const flowEdges: Edge[] = edges.map(edge => ({ - id: edge.id, - source: edge.source, - target: edge.target, - type: 'smoothstep', - label: edge.label, - data: edge.title ? { title: edge.title } : undefined, - animated: false, - markerEnd: { type: MarkerType.ArrowClosed, color: '#64748b' }, - style: { stroke: '#64748b', strokeWidth: 1.5 }, - labelStyle: { fill: '#94a3b8', fontSize: 11 }, - labelBgStyle: { fill: 'rgba(9, 9, 11, 0.92)', fillOpacity: 1 }, - labelBgPadding: [6, 2], - labelBgBorderRadius: 6, + const nodeMap = new Map(nodes.map(node => [node.id, node])) + const validEdges = edges.filter(edge => nodeMap.has(edge.source) && nodeMap.has(edge.target)) + const components = workflowLayoutGroups(nodes, validEdges).map(componentIds => { + const idSet = new Set(componentIds) + const componentNodes = nodes.filter(node => idSet.has(node.id)) + const componentEdges = validEdges.filter(edge => idSet.has(edge.source) && idSet.has(edge.target)) + return layoutComponent(componentNodes, componentEdges) + }) + const packedNodes = packComponents(components) + const flowNodes: Node[] = packedNodes.map(node => ({ + id: node.id, + type: node.kind, + position: { x: node.x, y: node.y }, + data: { id: node.id, kind: node.kind, label: node.label, title: node.title, details: node.details }, })) - return { nodes: flowNodes, edges: flowEdges } + const flowEdges: Edge[] = validEdges.map(edge => { + return { + id: edge.id, + source: edge.source, + target: edge.target, + type: 'smoothstep', + label: edge.label, + data: edge.title ? { title: edge.title } : undefined, + animated: false, + markerEnd: { type: MarkerType.ArrowClosed, color: '#64748b' }, + style: { stroke: '#64748b', strokeWidth: 1.5 }, + labelStyle: { fill: '#94a3b8', fontSize: 11 }, + labelBgStyle: { fill: 'rgba(9, 9, 11, 0.92)', fillOpacity: 1 }, + labelBgPadding: [6, 2], + labelBgBorderRadius: 6, + } + }) + + return { nodes: flowNodes, edges: anchorEdges(flowEdges, flowNodes) } } const nodeTypes = { operation: OperationNode, object: ObjectNode, + planning: PlanningNode, + reasoning: ReasoningNode, + unknown: UnknownNode, +} + +function nodeColor(kind: WorkflowNodeKind) { + switch (kind) { + case 'operation': + return '#8b5cf6' + case 'object': + return '#14b8a6' + case 'planning': + return '#f59e0b' + case 'reasoning': + return '#38bdf8' + case 'unknown': + return '#64748b' + } } export default function WorkflowCanvas({ @@ -402,7 +884,7 @@ export default function WorkflowCanvas({ edges: WorkflowCanvasEdge[] exportBaseName?: string }) { - const [flowNodes, setFlowNodes, onNodesChange] = useNodesState([]) + const [flowNodes, setFlowNodes] = useNodesState([]) const [flowEdges, setFlowEdges, onEdgesChange] = useEdgesState([]) const [selectedNodeId, setSelectedNodeId] = useState(null) @@ -415,6 +897,14 @@ export default function WorkflowCanvas({ const selectedNode = flowNodes.find(node => node.id === selectedNodeId) ?? null + const handleNodesChange = (changes: NodeChange[]) => { + setFlowNodes(currentNodes => { + const nextNodes = applyNodeChanges(changes, currentNodes) + setFlowEdges(currentEdges => anchorEdges(currentEdges, nextNodes)) + return nextNodes + }) + } + const exportSvg = () => { if (flowNodes.length === 0) return @@ -467,11 +957,12 @@ export default function WorkflowCanvas({ for (const node of flowNodes) { const box = nodeBounds(node) const lines = labelLines(node.data.label) - if (node.type === 'operation') { + const kind = node.data.kind + if (kind === 'operation') { svgParts.push( ``, ) - } else { + } else if (kind === 'object') { const slant = 20 const points = [ `${box.x + slant},${box.y}`, @@ -482,6 +973,12 @@ export default function WorkflowCanvas({ svgParts.push( ``, ) + } else { + const fill = kind === 'planning' ? '#78350f' : kind === 'reasoning' ? '#0c4a6e' : '#1e293b' + const stroke = kind === 'planning' ? '#fcd34d' : kind === 'reasoning' ? '#7dd3fc' : '#94a3b8' + svgParts.push( + ``, + ) } const startY = box.centerY - ((lines.length - 1) * 14) / 2 @@ -502,7 +999,7 @@ export default function WorkflowCanvas({ ``, ` `, ...flowNodes.map(node => { - const kind = node.type === 'operation' ? 'operation' : 'object' + const kind = node.data.kind return ` ${node.data.title ? `${escapeXml(node.data.title)}` : ''}` }), ` `, @@ -542,7 +1039,7 @@ export default function WorkflowCanvas({ nodes={flowNodes} edges={flowEdges} nodeTypes={nodeTypes} - onNodesChange={onNodesChange} + onNodesChange={handleNodesChange} onEdgesChange={onEdgesChange} onNodeClick={(_, node) => setSelectedNodeId(node.id)} fitView @@ -556,7 +1053,7 @@ export default function WorkflowCanvas({ node.type === 'operation' ? '#8b5cf6' : '#14b8a6'} + nodeColor={node => nodeColor((node.data as WorkflowNodeData).kind)} maskColor="rgba(9, 9, 11, 0.78)" /> diff --git a/frontend/src/components/projects/WorkflowGraphTab.tsx b/frontend/src/components/projects/WorkflowGraphTab.tsx index 391b809..cdf1b24 100644 --- a/frontend/src/components/projects/WorkflowGraphTab.tsx +++ b/frontend/src/components/projects/WorkflowGraphTab.tsx @@ -4,6 +4,12 @@ import { deleteProjectWorkflowVersion, getProjectWorkflow, listProjectWorkflows import type { RawWorkflowVersion } from '../../types' import WorkflowCanvas, { type WorkflowCanvasEdge, type WorkflowCanvasNode } from './WorkflowCanvas' +function workflowNodeKind(node: { node_kind?: string; node_kind_guess: string }): WorkflowCanvasNode['kind'] { + const kind = node.node_kind ?? node.node_kind_guess + if (kind === 'operation' || kind === 'planning' || kind === 'reasoning' || kind === 'unknown') return kind + return 'object' +} + function RawWorkflowCanvas({ workflow }: { workflow: RawWorkflowVersion }) { const graph = workflow.graph @@ -12,7 +18,7 @@ function RawWorkflowCanvas({ workflow }: { workflow: RawWorkflowVersion }) { const nodes: WorkflowCanvasNode[] = graph.nodes.map(node => ({ id: node.node_id, label: node.canonical_name ?? node.raw_name, - kind: (node.node_kind ?? node.node_kind_guess) === 'operation' ? 'operation' : 'object', + kind: workflowNodeKind(node), title: `${node.canonical_name ?? node.raw_name}\nsource term: ${node.raw_name}\n${node.semantic_type ?? node.node_kind_guess} · confidence ${node.confidence.toFixed(2)}\n\n${node.evidence_text}`, details: { card_id: node.card_id, @@ -133,7 +139,7 @@ export default function WorkflowGraphTab({ )} {selected?.graph ? : null} -

Purple rectangles are operations; teal parallelograms are objects. Drag nodes freely to tidy the canvas and hover nodes for evidence.

+

Purple rectangles are operations; teal parallelograms are objects; amber and blue rectangles are planning and reasoning. Drag nodes freely to tidy the canvas and hover nodes for evidence.

) } diff --git a/frontend/src/hooks/useProjectRefresh.ts b/frontend/src/hooks/useProjectRefresh.ts new file mode 100644 index 0000000..0b12a2c --- /dev/null +++ b/frontend/src/hooks/useProjectRefresh.ts @@ -0,0 +1,15 @@ +import { useCallback } from 'react' + +import { getProject } from '../api/projects' +import type { Project } from '../types' + +export function useProjectRefresh( + projectId: string, + onProjectUpdated?: (project: Project) => void, +) { + return useCallback(() => { + if (!onProjectUpdated) return + getProject(projectId).then(onProjectUpdated).catch(() => { /* ignore refresh failures */ }) + }, [projectId, onProjectUpdated]) +} + diff --git a/frontend/src/types/index.ts b/frontend/src/types/index.ts index ab0cd3d..149e1af 100644 --- a/frontend/src/types/index.ts +++ b/frontend/src/types/index.ts @@ -248,10 +248,10 @@ export interface GraphPayload { export interface RawWorkflowNode { node_id: string raw_name: string - node_kind_guess: 'object' | 'operation' | 'unknown' + node_kind_guess: 'object' | 'operation' | 'planning' | 'reasoning' | 'unknown' canonical_name?: string card_id?: string | null - node_kind?: 'object' | 'operation' | 'unknown' + node_kind?: 'object' | 'operation' | 'planning' | 'reasoning' | 'unknown' semantic_type?: string | null parameters?: Record identity?: Record diff --git a/pyproject.toml b/pyproject.toml index 0ff6267..b144e19 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,4 +56,5 @@ line-length = 100 [tool.pytest.ini_options] asyncio_mode = "auto" +pythonpath = ["src"] testpaths = ["tests"] diff --git a/src/mkb/agents/__init__.py b/src/mkb/agents/__init__.py index b99326e..a6305bd 100644 --- a/src/mkb/agents/__init__.py +++ b/src/mkb/agents/__init__.py @@ -3,3 +3,12 @@ from mkb.agents.extraction import build_extraction_agent, run_extraction, run_extraction_all from mkb.agents.runner import AgentRunner, RunResult from mkb.agents.tools import ALL_TOOLS + +__all__ = [ + "ALL_TOOLS", + "AgentRunner", + "RunResult", + "build_extraction_agent", + "run_extraction", + "run_extraction_all", +] diff --git a/src/mkb/agents/projection.py b/src/mkb/agents/projection.py index f9b4db9..c3d11c3 100644 --- a/src/mkb/agents/projection.py +++ b/src/mkb/agents/projection.py @@ -11,7 +11,6 @@ import logging import uuid -from datetime import datetime, timezone from google.adk.agents import Agent diff --git a/src/mkb/agents/projection_reviewer.py b/src/mkb/agents/projection_reviewer.py index 52e3b02..133504b 100644 --- a/src/mkb/agents/projection_reviewer.py +++ b/src/mkb/agents/projection_reviewer.py @@ -17,11 +17,7 @@ from google.adk.agents import Agent from mkb.agents._utils import create_llm, sync_agent_run -from mkb.agents.prompts.projection_review import ( - PROJECTION_REVIEW_PROMPT, - PROJECTION_REVIEW_QA_PROMPT, - default_review_prompt_for, -) +from mkb.agents.prompts.projection_review import default_review_prompt_for from mkb.agents.runner import AgentRunner from mkb.agents.tools.reading import READING_TOOLS from mkb.agents.tools.projection_review import PROJECTION_REVIEW_TOOLS diff --git a/src/mkb/agents/prompts/workflow_extraction.py b/src/mkb/agents/prompts/workflow_extraction.py index 89ea1bc..47ad386 100644 --- a/src/mkb/agents/prompts/workflow_extraction.py +++ b/src/mkb/agents/prompts/workflow_extraction.py @@ -6,7 +6,8 @@ canonicalization agent, so preserve evidence and reproducibility detail while separating reusable concepts from instance-specific values. -Represent every step as Object -> Operation -> Object: +Represent concrete experimental, computational, and analytical work as +Object -> Operation -> Object: * object -> operation uses `input_to` * operation -> object uses `produces` @@ -15,12 +16,29 @@ * disconnected components and genuinely missing endpoints are allowed; never invent endpoints or routine steps +Also capture explicit planning and reasoning logic when the paper explains why +a step, object, comparison, design choice, hypothesis, or decision is needed: + +* use `planning` for intended strategy, design criteria, experimental plan, + screening strategy, or decision policy +* use `reasoning` for hypothesis, rationale, interpretation, causal argument, + constraint, tradeoff, or conclusion that drives later work +* connect planning/reasoning nodes to downstream nodes with `motivates` when + the text explains why that node is needed, or `leads_to` when the text states + that the plan/reasoning caused the next workflow item +* planning/reasoning nodes may point to objects, operations, or other + planning/reasoning nodes, but do not use `input_to` or `produces` for them +* keep unsupported background claims in `unresolved_information` rather than + adding a planning/reasoning node without direct evidence + Each node is an instantiated card. Fill both the v2 card fields and evidence: * `canonical_name`: short reusable concept, such as `XRD Measurement`, - `Band Structure Calculation`, `Comparison`, `Material`, or `Band Structure` + `Band Structure Calculation`, `Comparison`, `Material`, `Band Structure`, + `Design Rationale`, or `Screening Plan` * `raw_name`: the paper's original phrase (preserves terminology) -* `node_kind` and compatibility field `node_kind_guess` +* `node_kind` and compatibility field `node_kind_guess`; allowed values are + `object`, `operation`, `planning`, `reasoning`, and `unknown` * `semantic_type`: an open, concise scientific type; do not choose from a hand-built closed ontology * `parameters`: run-specific settings, methods, quantities and values @@ -62,19 +80,25 @@ Execution: 1. Call list_project_files and read all relevant assets with paged reads. -2. Before you finalize any node naming or `card_id`, call either +2. Before you finalize object/operation node naming or `card_id`, call either `search_workflow_cards` or `get_active_workflow_card_library` against the newest workflow card base. Reuse an existing card/template when it is a clear semantic match; otherwise keep `card_id` null and mark `ontology_status` as `candidate` or `unmapped`. -3. Repeat card-base lookup whenever you introduce a newly named node family or - revise a node's reusable concept. +3. Repeat card-base lookup whenever you introduce a newly named object or + operation family or revise its reusable concept. Planning/reasoning nodes + usually have no shared card yet; keep their `card_id` null unless the card + base clearly contains a matching planning/reasoning card. 4. On resume, call get_raw_workflow_checkpoint first. 5. Checkpoint after each source or major milestone, stating coverage and work remaining. -6. Use exact request IDs. Node IDs are `raw::n0001`; edge IDs are - `raw::e0001`. +6. Focus on scientific content and evidence. The save/checkpoint tools fill + application-owned envelope fields such as `schema_version`, `paper_id`, + `extraction_id`, sequential node/edge IDs, default empty dict/list fields, + and common edge aliases. Provide stable node/edge references when you have + them, but do not spend turns repairing mechanical schema boilerplate. 7. Save exactly once with save_raw_workflow, including an empty graph when no - supported workflow exists. Use schema_version `workflow-cards/2.0`. + supported workflow exists. The tool will normalize mechanical fields and + return compact validation hints if semantic fixes are still needed. 8. Tool arguments must be strict JSON. """ diff --git a/src/mkb/agents/review.py b/src/mkb/agents/review.py index 9881da6..3c0b6f2 100644 --- a/src/mkb/agents/review.py +++ b/src/mkb/agents/review.py @@ -7,7 +7,6 @@ from __future__ import annotations -import asyncio import logging import uuid diff --git a/src/mkb/agents/tools/frames.py b/src/mkb/agents/tools/frames.py index 611041e..4e0216e 100644 --- a/src/mkb/agents/tools/frames.py +++ b/src/mkb/agents/tools/frames.py @@ -138,7 +138,7 @@ def save_knowledge_frame( return {"error": f"Project {project_id} not found."} links = session.query(ProjectAsset).filter_by(project_id=pid).all() - asset_ids = [str(l.asset_id) for l in links] + asset_ids = [str(link.asset_id) for link in links] source_meta = { "project_label": project.label, "source_path": project.source_path, diff --git a/src/mkb/agents/tools/graph_review.py b/src/mkb/agents/tools/graph_review.py index e66738d..cdfa6e1 100644 --- a/src/mkb/agents/tools/graph_review.py +++ b/src/mkb/agents/tools/graph_review.py @@ -7,7 +7,7 @@ from contextvars import ContextVar from dataclasses import dataclass, field from datetime import datetime, timezone -from typing import Callable, Literal +from typing import Callable from mkb.agents.tools._ids import invalid_identifier_message, parse_uuidish from mkb.agents.tools.knowledge_graph import ( @@ -18,7 +18,6 @@ ) from mkb.db.engine import SyncSessionLocal from mkb.db.models import GraphElementReview, Projection, ProjectionStatus -from mkb.knowledge_graph import ensure_global_kg_space_id MAX_DETAIL_RELATIONS = 120 @@ -303,7 +302,7 @@ def merge_concepts( canonical_label = str(canonical_label).strip() canonical_norm = _normalize_label(canonical_label) - norm_to_merge = {_normalize_label(l) for l in labels_to_merge if str(l).strip()} + norm_to_merge = {_normalize_label(label) for label in labels_to_merge if str(label).strip()} extra_aliases: list[str] = list(aliases or []) _fire_progress({"tool": "merge_concepts", "element_type": "concept", "label": canonical_label, "action": "merge", "merging": labels_to_merge}) diff --git a/src/mkb/agents/tools/orchestrator_tools.py b/src/mkb/agents/tools/orchestrator_tools.py index 4b495b7..6238176 100644 --- a/src/mkb/agents/tools/orchestrator_tools.py +++ b/src/mkb/agents/tools/orchestrator_tools.py @@ -11,10 +11,8 @@ from __future__ import annotations -import json import logging import queue as _queue -import uuid logger = logging.getLogger(__name__) @@ -279,6 +277,7 @@ def trigger_extraction(project_id: str, max_passes: int = 1) -> dict: max_passes: Number of extraction passes (1 = initial only, 2+ includes review). """ _workflow_queue.put({ + "action": "extract_project", "kind": "extraction", "project_id": project_id, "kwargs": {"project_id": project_id, "max_passes": max_passes}, @@ -300,6 +299,7 @@ def trigger_projection(project_id: str, space_id: str) -> dict: space_id: UUID string of the Space to project onto. """ _workflow_queue.put({ + "action": "project_to_space", "kind": "projection", "project_id": project_id, "kwargs": {"project_id": project_id, "space_id": space_id}, @@ -321,6 +321,7 @@ def trigger_knowledge_graph_extraction(project_id: str) -> dict: project_id: UUID string of the project. """ _workflow_queue.put({ + "action": "extract_knowledge_graph", "kind": "kg_extraction", "project_id": project_id, "kwargs": {"project_id": project_id}, @@ -342,6 +343,7 @@ def trigger_feedback_review(project_id: str) -> dict: project_id: UUID string of the project. """ _workflow_queue.put({ + "action": "review_feedback", "kind": "feedback_review", "project_id": project_id, "kwargs": {"project_id": project_id}, @@ -364,6 +366,7 @@ def trigger_projection_review(project_id: str, space_id: str) -> dict: space_id: UUID string of the Space. """ _workflow_queue.put({ + "action": "review_projection", "kind": "projection_review", "project_id": project_id, "kwargs": {"project_id": project_id, "space_id": space_id}, diff --git a/src/mkb/agents/tools/reading.py b/src/mkb/agents/tools/reading.py index a9de0d6..dd68c1e 100644 --- a/src/mkb/agents/tools/reading.py +++ b/src/mkb/agents/tools/reading.py @@ -35,7 +35,8 @@ def _resolve_asset_id(session, asset_ref: str) -> tuple[uuid.UUID | None, str | return asset_id, None candidate = str(asset_ref).strip().strip("\"'") - if candidate: + filename_like = bool(candidate and (Path(candidate).suffix or "/" in candidate or "\\" in candidate)) + if filename_like: asset = ( session.query(Asset) .filter(Asset.filename == candidate) @@ -94,7 +95,7 @@ def list_project_files(project_id: str) -> list[dict]: with SyncSessionLocal() as session: links = session.query(ProjectAsset).filter_by(project_id=pid).all() - asset_ids = [l.asset_id for l in links] + asset_ids = [link.asset_id for link in links] if not asset_ids: return [] assets = session.query(Asset).filter(Asset.asset_id.in_(asset_ids)).all() @@ -354,7 +355,7 @@ def search_in_project(project_id: str, query: str) -> list[dict]: with SyncSessionLocal() as session: links = session.query(ProjectAsset).filter_by(project_id=pid).all() - asset_ids = [l.asset_id for l in links] + asset_ids = [link.asset_id for link in links] if not asset_ids: return [] diff --git a/src/mkb/agents/tools/schema_curator.py b/src/mkb/agents/tools/schema_curator.py index 1e05f5c..4d56754 100644 --- a/src/mkb/agents/tools/schema_curator.py +++ b/src/mkb/agents/tools/schema_curator.py @@ -22,6 +22,7 @@ _CURATOR_AUTHOR: ContextVar[str] = ContextVar( "schema_curator_author", default="schema-curator-agent/unknown-model" ) +SEARCHABLE_NODE_KINDS = {None, "", "object", "operation", "planning", "reasoning", "unknown"} def set_curator_author(author: str): @@ -425,8 +426,8 @@ def search_workflow_cards(query: str, node_kind: str | None = None, limit: int = text = _normalize_text(query) if not text: return {"error": "query is required"} - if node_kind not in {None, "", "object", "operation"}: - return {"error": "node_kind must be 'object', 'operation', or omitted"} + if node_kind not in SEARCHABLE_NODE_KINDS: + return {"error": "node_kind must be one of object, operation, planning, reasoning, unknown, or omitted"} library = get_schema_library_payload() results = [] for card_id, payload in (library.get("cards", {}) or {}).items(): @@ -452,7 +453,7 @@ def search_workflow_cards(query: str, node_kind: str | None = None, limit: int = "status": payload.get("status", "active"), }) for template_id, payload in (library.get("operation_templates", {}) or {}).items(): - if node_kind == "object": + if node_kind and node_kind != "operation": continue haystacks = [ template_id, @@ -490,8 +491,8 @@ def search_similar_workflow_nodes( text = _normalize_text(query) if not text: return {"error": "query is required"} - if node_kind not in {None, "", "object", "operation"}: - return {"error": "node_kind must be 'object', 'operation', or omitted"} + if node_kind not in SEARCHABLE_NODE_KINDS: + return {"error": "node_kind must be one of object, operation, planning, reasoning, unknown, or omitted"} effective_limit = max(1, min(int(limit), 50)) with SyncSessionLocal() as session: rows = _latest_reviewable_workflows(session) @@ -749,7 +750,11 @@ def submit_workflow_review( try: corrected = rebase_graph(corrected, new_id) except Exception as exc: - return {"error": f"corrected graph validation failed: {exc}"} + message = " ".join(str(exc).split()) + return { + "error": "corrected graph validation failed", + "details": message[:1000], + } version = int( session.query(func.coalesce(func.max(RawWorkflowExtraction.version), 0)) .filter_by(project_id=source.project_id) diff --git a/src/mkb/agents/tools/workflow_canonicalization.py b/src/mkb/agents/tools/workflow_canonicalization.py index 6bfe857..2fe5499 100644 --- a/src/mkb/agents/tools/workflow_canonicalization.py +++ b/src/mkb/agents/tools/workflow_canonicalization.py @@ -14,7 +14,7 @@ from mkb.workflows.canonical_contract import CanonicalWorkflowGraph from mkb.workflows.schema_library import get_schema_library_payload from mkb.workflows.indexing import build_index_entries -from mkb.workflows.validation import json_safe_validation_errors +from mkb.workflows.validation import compact_validation_errors def _uuid(value: str) -> uuid.UUID | None: @@ -39,6 +39,83 @@ def _draft_template(row: CanonicalWorkflow, raw: RawWorkflowExtraction) -> dict[ } +def _compact_value(value: Any, *, string_limit: int = 1000, list_limit: int = 30, dict_limit: int = 30) -> Any: + if isinstance(value, str): + return value if len(value) <= string_limit else f"{value[:string_limit]}... [truncated]" + if isinstance(value, list): + items = [_compact_value(item, string_limit=string_limit, list_limit=list_limit, dict_limit=dict_limit) for item in value[:list_limit]] + if len(value) > list_limit: + items.append({"omitted_items": len(value) - list_limit}) + return items + if isinstance(value, dict): + result = {} + for index, (key, item) in enumerate(value.items()): + if index >= dict_limit: + result["omitted_keys"] = len(value) - dict_limit + break + result[key] = _compact_value(item, string_limit=string_limit, list_limit=list_limit, dict_limit=dict_limit) + return result + return value + + +def _raw_graph_context(raw_graph: dict) -> dict: + nodes = raw_graph.get("nodes") if isinstance(raw_graph.get("nodes"), list) else [] + edges = raw_graph.get("edges") if isinstance(raw_graph.get("edges"), list) else [] + return { + "schema_version": raw_graph.get("schema_version"), + "paper_id": raw_graph.get("paper_id"), + "extraction_id": raw_graph.get("extraction_id"), + "nodes": [ + { + "node_id": node.get("node_id"), + "raw_name": node.get("raw_name"), + "canonical_name": node.get("canonical_name"), + "node_kind": node.get("node_kind") or node.get("node_kind_guess"), + "semantic_type": node.get("semantic_type"), + "parameters": _compact_value(node.get("parameters", {}), string_limit=500, list_limit=15, dict_limit=20), + "identity": _compact_value(node.get("identity", {}), string_limit=500, list_limit=15, dict_limit=20), + "state": _compact_value(node.get("state", {}), string_limit=500, list_limit=15, dict_limit=20), + "role": _compact_value(node.get("role", {}), string_limit=500, list_limit=15, dict_limit=20), + "context": _compact_value(node.get("context", {}), string_limit=500, list_limit=15, dict_limit=20), + "evidence_text": _compact_value(node.get("evidence_text", ""), string_limit=800), + "confidence": node.get("confidence"), + } + for node in nodes + if isinstance(node, dict) + ], + "edges": [ + { + "edge_id": edge.get("edge_id"), + "source_node": edge.get("source_node"), + "target_node": edge.get("target_node"), + "relation_type": edge.get("relation_type"), + "evidence_text": _compact_value(edge.get("evidence_text", ""), string_limit=600), + "confidence": edge.get("confidence"), + } + for edge in edges + if isinstance(edge, dict) + ], + "reproducibility": _compact_value(raw_graph.get("reproducibility", {}), string_limit=600, list_limit=20, dict_limit=20), + "unresolved_information": _compact_value(raw_graph.get("unresolved_information", []), string_limit=600, list_limit=30, dict_limit=20), + "note": "This is a compact raw workflow context; final validation still uses the full server-side raw graph.", + } + + +def _normalize_canonical_payload(payload: dict, row: CanonicalWorkflow, raw: RawWorkflowExtraction) -> dict: + normalized = deepcopy(payload if isinstance(payload, dict) else {}) + normalized["schema_version"] = row.schema_version + normalized["canonicalization_id"] = str(row.canonicalization_id) + normalized["paper_id"] = str(row.project_id) + normalized["raw_extraction_id"] = str(raw.extraction_id) + for key in ( + "nodes", "edges", "raw_to_canonical_mappings", "unmatched_raw_information", + "granularity_mappings", "proposed_schema_updates", + ): + if not isinstance(normalized.get(key), list): + normalized[key] = [] + return normalized + + def _load_row_and_raw(session, canonicalization_id: str): cid = _uuid(canonicalization_id) if not cid: @@ -162,7 +239,7 @@ def get_canonicalization_context(canonicalization_id: str) -> dict: "canonicalization_id": str(row.canonicalization_id), "paper_id": str(row.project_id), "raw_extraction_id": str(raw.extraction_id), - "raw_graph": raw.graph, + "raw_graph": _raw_graph_context(raw.graph or {}), "schema_library": get_schema_library_payload(row.schema_version), } @@ -425,13 +502,14 @@ def save_canonical_workflow(canonicalization_id: str, graph: dict | None = None) payload = graph if isinstance(graph, dict) else _ensure_draft(row, raw) if not isinstance(payload, dict): return {"error": "graph must be a JSON object"} + payload = _normalize_canonical_payload(payload, row, raw) try: validated = CanonicalWorkflowGraph.model_validate(payload) except ValidationError as exc: return { "error": "Canonical workflow validation failed", - "details": json_safe_validation_errors(exc), + "details": compact_validation_errors(exc), } cid = row.canonicalization_id diff --git a/src/mkb/agents/tools/workflows.py b/src/mkb/agents/tools/workflows.py index c44ec27..948a6ac 100644 --- a/src/mkb/agents/tools/workflows.py +++ b/src/mkb/agents/tools/workflows.py @@ -3,7 +3,9 @@ from __future__ import annotations import uuid +from copy import deepcopy from datetime import datetime, timezone +from typing import Any from pydantic import ValidationError @@ -12,7 +14,173 @@ from mkb.workflows.contract import RawWorkflowGraph from mkb.workflows.schema_library import get_schema_library_payload from mkb.workflows.review import audit_raw_graph -from mkb.workflows.validation import json_safe_validation_errors +from mkb.workflows.validation import compact_validation_errors + +SEARCHABLE_NODE_KINDS = {None, "", "object", "operation", "planning", "reasoning", "unknown"} +NODE_KINDS = {"object", "operation", "planning", "reasoning", "unknown"} +RELATION_ALIASES = { + "input": "input_to", + "input_to": "input_to", + "produces": "produces", + "output": "produces", + "output_of": "produces", + "same_as": "same_as", + "part_of": "part_of", + "has_part": "has_part", + "expands_to": "expands_to", + "summarized_by": "summarized_by", + "motivates": "motivates", + "leads_to": "leads_to", +} + + +def _dict_or_empty(value: Any) -> dict: + return value if isinstance(value, dict) else {} + + +def _list_or_empty(value: Any) -> list: + return value if isinstance(value, list) else [] + + +def _normalize_kind(value: Any) -> str: + kind = str(value or "unknown").strip().casefold().replace("-", "_") + return kind if kind in NODE_KINDS else "unknown" + + +def _normalize_raw_graph_payload(graph: dict, row: RawWorkflowExtraction, extraction_id: uuid.UUID) -> tuple[dict, dict]: + """Fill app-owned workflow envelope fields and tolerate common LLM aliases.""" + payload = deepcopy(graph if isinstance(graph, dict) else {}) + changes = { + "filled_graph_fields": [], + "assigned_node_ids": 0, + "assigned_edge_ids": 0, + "normalized_nodes": 0, + "normalized_edges": 0, + } + for key, value in { + "schema_version": row.schema_version, + "paper_id": str(row.project_id), + "extraction_id": str(extraction_id), + }.items(): + if payload.get(key) != value: + payload[key] = value + changes["filled_graph_fields"].append(key) + + raw_nodes = payload.get("nodes") + payload["nodes"] = raw_nodes if isinstance(raw_nodes, list) else [] + raw_edges = payload.get("edges") + payload["edges"] = raw_edges if isinstance(raw_edges, list) else [] + payload["unresolved_information"] = [ + item if isinstance(item, dict) else {"description": str(item)} + for item in _list_or_empty(payload.get("unresolved_information")) + ] + if not isinstance(payload.get("reproducibility"), dict): + payload.pop("reproducibility", None) + + old_node_refs: dict[str, str] = {} + for index, node in enumerate(payload["nodes"], 1): + if not isinstance(node, dict): + node = {"raw_name": str(node), "evidence_text": str(node)} + payload["nodes"][index - 1] = node + original_refs = { + str(value).strip() + for value in ( + node.get("node_id"), + node.get("id"), + node.get("name"), + node.get("label"), + node.get("raw_name"), + node.get("canonical_name"), + ) + if value is not None and str(value).strip() + } + expected_id = f"raw:{extraction_id}:n{index:04d}" + node_id = str(node.get("node_id") or node.get("id") or "").strip() + if not node_id or not node_id.startswith(f"raw:{extraction_id}:n"): + node["node_id"] = expected_id + changes["assigned_node_ids"] += 1 + kind = _normalize_kind(node.get("node_kind") or node.get("node_kind_guess") or node.get("kind")) + node["node_kind"] = kind + node["node_kind_guess"] = kind + node["raw_name"] = str(node.get("raw_name") or node.get("canonical_name") or node.get("label") or node["node_id"]) + node.setdefault("canonical_name", node.get("raw_name")) + node.setdefault("semantic_type", kind) + for key in ("parameters", "identity", "state", "role", "context", "attributes_explicitly_mentioned", "paper_location"): + node[key] = _dict_or_empty(node.get(key)) + node["unparsed_modifiers"] = _list_or_empty(node.get("unparsed_modifiers")) + node["aliases_observed"] = _list_or_empty(node.get("aliases_observed")) + status = str(node.get("ontology_status") or "unmapped").strip().casefold() + node["ontology_status"] = "matched" if status == "mapped" else status if status in {"matched", "candidate", "unmapped"} else "unmapped" + node["evidence_text"] = str(node.get("evidence_text") or node.get("evidence") or node.get("raw_name")) + try: + node["confidence"] = float(node.get("confidence", 0.5)) + except (TypeError, ValueError): + node["confidence"] = 0.5 + node["confidence"] = max(0.0, min(1.0, node["confidence"])) + for ref in original_refs: + old_node_refs[ref] = node["node_id"] + changes["normalized_nodes"] += 1 + + for index, edge in enumerate(payload["edges"], 1): + if not isinstance(edge, dict): + edge = {"evidence_text": str(edge)} + payload["edges"][index - 1] = edge + edge_id = str(edge.get("edge_id") or edge.get("id") or "").strip() + if not edge_id or not edge_id.startswith(f"raw:{extraction_id}:e"): + edge["edge_id"] = f"raw:{extraction_id}:e{index:04d}" + changes["assigned_edge_ids"] += 1 + source = edge.get("source_node", edge.get("source")) + target = edge.get("target_node", edge.get("target")) + edge["source_node"] = old_node_refs.get(str(source).strip(), source) + edge["target_node"] = old_node_refs.get(str(target).strip(), target) + relation = str(edge.get("relation_type") or edge.get("kind") or edge.get("relation") or "").strip().casefold().replace("-", "_") + edge["relation_type"] = RELATION_ALIASES.get(relation, relation) + edge["attributes"] = _dict_or_empty(edge.get("attributes")) + edge["evidence_text"] = str(edge.get("evidence_text") or edge.get("evidence") or edge.get("relation_type") or "") + if edge.get("paper_location") is not None: + edge["paper_location"] = _dict_or_empty(edge.get("paper_location")) + try: + edge["confidence"] = float(edge.get("confidence", 0.5)) + except (TypeError, ValueError): + edge["confidence"] = 0.5 + edge["confidence"] = max(0.0, min(1.0, edge["confidence"])) + changes["normalized_edges"] += 1 + + return payload, changes + + +def _compact_raw_checkpoint_manifest(graph: dict | None) -> dict: + if not isinstance(graph, dict): + return {"counts": {"nodes": 0, "edges": 0}, "nodes": [], "edges": []} + nodes = graph.get("nodes") if isinstance(graph.get("nodes"), list) else [] + edges = graph.get("edges") if isinstance(graph.get("edges"), list) else [] + return { + "counts": { + "nodes": len(nodes), + "edges": len(edges), + "unresolved_information": len(graph.get("unresolved_information") or []), + }, + "nodes": [ + { + "node_id": node.get("node_id"), + "raw_name": node.get("raw_name") or node.get("canonical_name") or node.get("label"), + "node_kind": node.get("node_kind") or node.get("node_kind_guess") or node.get("kind"), + } + for node in nodes[:80] + if isinstance(node, dict) + ], + "edges": [ + { + "edge_id": edge.get("edge_id"), + "source_node": edge.get("source_node") or edge.get("source"), + "target_node": edge.get("target_node") or edge.get("target"), + "relation_type": edge.get("relation_type") or edge.get("kind") or edge.get("relation"), + } + for edge in edges[:120] + if isinstance(edge, dict) + ], + "note": "The full checkpoint graph remains server-side. Continue from this manifest and save/checkpoint only changed draft content.", + } def get_active_workflow_card_library( @@ -62,8 +230,8 @@ def search_workflow_cards( text = str(query or "").strip().casefold() if not text: return {"error": "query is required"} - if node_kind not in {None, "", "object", "operation"}: - return {"error": "node_kind must be 'object', 'operation', or omitted"} + if node_kind not in SEARCHABLE_NODE_KINDS: + return {"error": "node_kind must be one of object, operation, planning, reasoning, unknown, or omitted"} library = get_schema_library_payload() results: list[dict] = [] @@ -93,7 +261,7 @@ def search_workflow_cards( }) for template_id, payload in (library.get("operation_templates", {}) or {}).items(): - if node_kind == "object": + if node_kind and node_kind != "operation": continue label = str(payload.get("label") or "") aliases = [str(value) for value in payload.get("aliases", []) if value] @@ -139,12 +307,17 @@ def get_raw_workflow_checkpoint(extraction_id: str) -> dict: row = session.query(RawWorkflowExtraction).filter_by(extraction_id=eid).first() if not row: return {"error": f"Extraction {eid} not found"} + checkpoint = row.checkpoint or {} + draft = checkpoint.get("graph") if isinstance(checkpoint, dict) else None return { "extraction_id": str(eid), "project_id": str(row.project_id), "version": row.version, "status": row.status, - "checkpoint": row.checkpoint, + "checkpoint": { + "summary": checkpoint.get("summary") if isinstance(checkpoint, dict) else None, + "manifest": _compact_raw_checkpoint_manifest(draft), + }, "checkpoint_updated_at": ( row.checkpoint_updated_at.isoformat() if row.checkpoint_updated_at else None ), @@ -203,32 +376,27 @@ def save_raw_workflow(extraction_id: str, graph: dict) -> dict: except (TypeError, ValueError, AttributeError): return {"error": f"Invalid extraction_id: {extraction_id}"} - try: - validated = RawWorkflowGraph.model_validate(graph) - except ValidationError as exc: - return { - "error": "Raw workflow contract validation failed", - "details": json_safe_validation_errors(exc), - } - - if validated.extraction_id != str(eid): - return {"error": "graph.extraction_id does not match extraction_id"} - - expected_node_prefix = f"raw:{eid}:n" - expected_edge_prefix = f"raw:{eid}:e" - if any(not node.node_id.startswith(expected_node_prefix) for node in validated.nodes): - return {"error": f"All node IDs must start with {expected_node_prefix}"} - if any(not edge.edge_id.startswith(expected_edge_prefix) for edge in validated.edges): - return {"error": f"All edge IDs must start with {expected_edge_prefix}"} - with SyncSessionLocal() as session: row = session.query(RawWorkflowExtraction).filter_by(extraction_id=eid).first() if not row: return {"error": f"Extraction {eid} not found"} if row.status != "IN_PROGRESS" or row.graph is not None: return {"error": "This extraction version has already been finalized"} - if validated.paper_id != str(row.project_id): - return {"error": "graph.paper_id does not match the extraction project"} + + payload, normalization = _normalize_raw_graph_payload(graph, row, eid) + try: + validated = RawWorkflowGraph.model_validate(payload) + except ValidationError as exc: + return { + "error": "Raw workflow contract validation failed", + "details": compact_validation_errors(exc), + "normalization": normalization, + "hint": ( + "The tool fills schema_version, paper_id, extraction_id, sequential IDs, " + "default dict/list fields, and common edge aliases. Fix the listed node/edge " + "semantics rather than resending the full source content." + ), + } payload = validated.model_dump(mode="json") asset_ids = sorted({ @@ -267,6 +435,7 @@ def save_raw_workflow(extraction_id: str, graph: dict) -> dict: "version": row.version, "node_count": len(validated.nodes), "edge_count": len(validated.edges), + "normalization": normalization, } diff --git a/src/mkb/api.py b/src/mkb/api.py index eee06be..64a9867 100644 --- a/src/mkb/api.py +++ b/src/mkb/api.py @@ -1,3029 +1,254 @@ -""" -Primary Python API for the Materials Knowledge Base. - -All public functions return plain dicts. This module is the recommended -interface; the CLI is a thin wrapper around these functions. -""" +"""Compatibility facade for the Materials Knowledge Base Python API.""" from __future__ import annotations -import hashlib -import logging -import uuid -from datetime import datetime, timezone -from pathlib import Path -from typing import Any - -from mkb.config import settings -from mkb.db.engine import SyncSessionLocal, init_db - -logger = logging.getLogger(__name__) - - -def _sha256_bytes(data: bytes) -> str: - h = hashlib.sha256() - h.update(data) - return h.hexdigest() - - -def _inspect_manual_processed_dir( - processed_dir: str | Path, - primary_file: str | None = None, -) -> dict: - """Inspect a handmade processed-output directory and describe its bundle.""" - root = Path(processed_dir).resolve() - if not root.is_dir(): - raise FileNotFoundError(f"Processed directory not found: {root}") - - files = sorted(p for p in root.rglob("*") if p.is_file()) - if not files: - raise FileNotFoundError(f"No files found in processed directory: {root}") - - if primary_file: - primary_path = (root / primary_file).resolve() - if not primary_path.is_file(): - raise FileNotFoundError(f"Primary file not found: {primary_path}") - else: - def _priority(path: Path) -> tuple[int, str]: - suffix = path.suffix.lower() - if suffix in {".md", ".markdown"}: - rank = 0 - elif suffix in {".parquet", ".csv", ".tsv"}: - rank = 1 - elif suffix == ".json": - rank = 2 - elif suffix in {".png", ".jpg", ".jpeg", ".gif", ".webp"}: - rank = 4 - else: - rank = 3 - return rank, path.relative_to(root).as_posix() - - primary_path = sorted(files, key=_priority)[0] - - primary_relpath = primary_path.relative_to(root).as_posix() - primary_bytes = primary_path.read_bytes() - - ext = primary_path.suffix.lower() - if ext in {".md", ".markdown", ".txt"}: - processing_type = "MARKDOWN" - elif ext in {".parquet", ".csv", ".tsv", ".json"}: - processing_type = "DATAFRAME" - elif ext in {".png", ".jpg", ".jpeg", ".gif", ".webp"}: - processing_type = "IMAGE" - else: - processing_type = "MARKDOWN" - - artifact_files = sorted( - p.relative_to(root).as_posix() - for p in files - if p != primary_path - ) - - bundle_hash = hashlib.sha256() - bundle_hash.update(primary_bytes) - for relpath in artifact_files: - data = (root / relpath).read_bytes() - bundle_hash.update(relpath.encode("utf-8")) - bundle_hash.update(_sha256_bytes(data).encode("utf-8")) - - from mkb.db.models import ProcessingType - - return { - "local_dir": str(root), - "primary_name": primary_path.name, - "primary_relpath": primary_relpath, - "processing_type": ProcessingType(processing_type), - "output_format": primary_path.suffix.lstrip(".") or "bin", - "artifact_files": artifact_files, - "size_bytes": len(primary_bytes), - "sha256": bundle_hash.hexdigest(), - } - - -def _choose_asset_for_manual_output(assets: list, primary_name: str | None = None): - """Choose the most likely raw asset for a handmade processed bundle.""" - if not assets: - return None - if not primary_name: - return assets[0] - - primary_stem = Path(primary_name).stem.lower() - for asset in assets: - if Path(asset.filename).stem.lower() == primary_stem: - return asset - - for asset in assets: - asset_stem = Path(asset.filename).stem.lower() - if primary_stem in asset_stem or asset_stem in primary_stem: - return asset - - return assets[0] - - -def _normalize_search_query(query: str) -> list[str]: - """Split a free-text query into non-empty keyword tokens.""" - return [token.strip() for token in query.split() if token.strip()] - - -def _matches_search_tokens(*values: Any, tokens: list[str]) -> bool: - """Return True when every token is present in at least one candidate value.""" - haystacks = [str(value).lower() for value in values if value] - if not tokens: - return True - return all(any(token in haystack for haystack in haystacks) for token in tokens) - - -# ── Lifecycle ──────────────────────────────────────────────────── - - -def setup() -> None: - """Ensure database tables exist (idempotent).""" - init_db() - - -def reset_db() -> None: - """Drop all tables and recreate them. Destructive!""" - from mkb.db.engine import sync_engine - from mkb.db.models import Base - - Base.metadata.drop_all(sync_engine) - Base.metadata.create_all(sync_engine) - logger.info("Database reset complete.") - - -# ── Ingestion / Sync ───────────────────────────────────────────── - - -def ingest( - directory: str | Path, - label: str | None = None, - *, - user_named: bool = False, -) -> dict: - """Ingest a single project directory. - - Creates or updates a ResearchProject record keyed on the directory path, - then ingests any new files found inside it. - - ``user_named`` controls whether the provided label should be treated as a - user-given name (in which case the project will be marked as such and - excluded from later automatic renaming during extraction). - - Returns a summary dict with counts (total, ingested, duplicates, errors). - """ - from mkb.ingest.worker import ingest_directory - - return ingest_directory(directory, label=label, user_named=user_named) - - -def rename_project( - project_id: str | uuid.UUID, - label: str, - *, - user_initiated: bool = True, -) -> dict: - """Rename a research project. - - When ``user_initiated`` is True (the default), records - ``metadata_["user_named"] = True`` so that the automatic post-extraction - rename will skip this project. Callers that want to perform an automatic - rename (e.g. from an extracted paper title) should pass - ``user_initiated=False`` to leave that flag alone. - """ - from mkb.db.models import ResearchProject - - pid = uuid.UUID(str(project_id)) - cleaned = (label or "").strip() - if not cleaned: - return {"error": "label must not be empty"} - - with SyncSessionLocal() as session: - project = session.query(ResearchProject).filter_by(project_id=pid).first() - if not project: - return {"error": f"Project {project_id} not found"} - project.label = cleaned - if user_initiated: - meta = dict(project.metadata_ or {}) - meta["user_named"] = True - project.metadata_ = meta - session.commit() - return { - "project_id": str(project.project_id), - "label": project.label, - "user_named": bool((project.metadata_ or {}).get("user_named")), - } - - -def sync(root_dir: str | Path) -> dict: - """Sync all project subfolders under *root_dir*. - - Each immediate subdirectory of *root_dir* is treated as one research - project. New subfolders are registered as new projects; existing projects - are scanned for new files. - - Returns a summary dict with per-project results. - """ - from mkb.ingest.worker import sync_root - - return sync_root(root_dir) - - -def sync_project(project_id: str | uuid.UUID) -> dict: - """Re-scan a single project's source directory for new files. - - Returns a summary dict with counts of newly ingested files. - """ - from mkb.ingest.worker import sync_project as _sync_project - - pid = uuid.UUID(str(project_id)) - return _sync_project(pid) - - -# ── Processing ─────────────────────────────────────────────────── - - -def process(project_id: str | uuid.UUID | None = None, progress_callback=None) -> dict: - """Process assets. If project_id is given, process only that project's assets. - Otherwise process all pending assets. - - Returns a summary dict. - """ - from mkb.processors.coordinator import process_all_pending, process_asset - - if project_id is not None: - pid = uuid.UUID(str(project_id)) - from mkb.db.models import ProjectAsset - with SyncSessionLocal() as session: - links = session.query(ProjectAsset).filter_by(project_id=pid).all() - asset_ids = [l.asset_id for l in links] - - results = [] - for aid in asset_ids: - try: - if progress_callback: - progress_callback({"message": f"Starting asset {len(results) + 1}/{len(asset_ids)}", "asset_id": str(aid)}) - r = process_asset(aid, progress_callback=progress_callback) - results.append(r) - except Exception as exc: - results.append({"asset_id": str(aid), "error": str(exc)}) - return {"project_id": str(pid), "assets_processed": len(results), "results": results} - - return process_all_pending(progress_callback=progress_callback) - - -# ── Extraction ─────────────────────────────────────────────────── - - -def extract( - project_id: str | uuid.UUID | None = None, - model: str | None = None, - verbose: bool = False, - max_passes: int = 1, - progress_callback=None, -) -> dict: - """Run knowledge extraction. If project_id given, extract one project. - Otherwise extract all pending projects. - - Args: - project_id: Optional specific project to extract. - model: LLM model override. - verbose: Enable verbose logging. - max_passes: Number of extraction passes (1=initial only, >1 includes review). - """ - from mkb.agents.extraction import run_extraction, run_extraction_all - - if project_id is not None: - pid = uuid.UUID(str(project_id)) - return run_extraction( - pid, - model=model, - verbose=verbose, - max_passes=max_passes, - progress_callback=progress_callback, - ) - return run_extraction_all(model=model, verbose=verbose, max_passes=max_passes) - - -# ── Knowledge Frames ───────────────────────────────────────────── - - -def get_frame(project_id: str | uuid.UUID) -> dict | None: - """Get the knowledge frame for a project. Returns None if not found.""" - from mkb.db.models import KnowledgeFrame - - init_db() - pid = uuid.UUID(str(project_id)) - with SyncSessionLocal() as session: - frame = session.query(KnowledgeFrame).filter_by(project_id=pid).first() - if not frame: - return None - return { - "frame_id": str(frame.frame_id), - "project_id": str(frame.project_id), - "status": frame.status.value, - "content": frame.content, - "extraction_summary": frame.extraction_summary, - "times_checked": frame.times_checked, - "extraction_version": frame.extraction_version, - "extracted_at": frame.extracted_at.isoformat() if frame.extracted_at else None, - "source_metadata": frame.source_metadata, - "agent_annotations": frame.agent_annotations or {}, - "created_at": frame.created_at.isoformat() if frame.created_at else None, - "updated_at": frame.updated_at.isoformat() if frame.updated_at else None, - } - - -def list_frames(status: str | None = None) -> list[dict]: - """List all knowledge frames, optionally filtered by status.""" - from mkb.db.models import FrameStatus, KnowledgeFrame - - init_db() - with SyncSessionLocal() as session: - q = session.query(KnowledgeFrame).order_by(KnowledgeFrame.created_at.desc()) - if status: - q = q.filter_by(status=FrameStatus(status)) - frames = q.all() - return [ - { - "frame_id": str(f.frame_id), - "project_id": str(f.project_id), - "status": f.status.value, - "times_checked": f.times_checked, - "extraction_version": f.extraction_version, - "extracted_at": f.extracted_at.isoformat() if f.extracted_at else None, - "extraction_summary": f.extraction_summary, - } - for f in frames - ] - - -def get_extraction_history(project_id: str | uuid.UUID) -> list[dict]: - """Get the extraction pass history for a project's frame.""" - from mkb.db.models import ExtractionPass, KnowledgeFrame - - init_db() - pid = uuid.UUID(str(project_id)) - with SyncSessionLocal() as session: - frame = session.query(KnowledgeFrame).filter_by(project_id=pid).first() - if not frame: - return [] - passes = ( - session.query(ExtractionPass) - .filter_by(frame_id=frame.frame_id) - .order_by(ExtractionPass.pass_number) - .all() - ) - return [ - { - "pass_id": str(p.pass_id), - "pass_number": p.pass_number, - "pass_type": p.pass_type, - "changes_made": p.changes_made, - "agent_notes": p.agent_notes, - "created_at": p.created_at.isoformat() if p.created_at else None, - } - for p in passes - ] - - -# ── Projects & Assets ──────────────────────────────────────────── - - -def list_processed_assets( - project_id: str | uuid.UUID | None = None, - limit: int = 100, -) -> list[dict]: - """List processed outputs, optionally filtered by project.""" - from mkb.db.models import Asset, ProcessedAsset, ProjectAsset - - with SyncSessionLocal() as session: - q = session.query(ProcessedAsset).order_by(ProcessedAsset.created_at.desc()) - if project_id is not None: - pid = uuid.UUID(str(project_id)) - links = session.query(ProjectAsset).filter_by(project_id=pid).all() - asset_ids = [l.asset_id for l in links] - if not asset_ids: - return [] - q = q.filter(ProcessedAsset.asset_id.in_(asset_ids)) - - rows = q.limit(limit).all() - result = [] - for row in rows: - asset = session.query(Asset).filter_by(asset_id=row.asset_id).first() - meta = row.conversion_metadata or {} - result.append({ - "processed_asset_id": str(row.processed_asset_id), - "asset_id": str(row.asset_id), - "filename": asset.filename if asset else None, - "processing_type": row.processing_type.value, - "output_format": row.output_format, - "s3_key": row.s3_key, - "local_dir": meta.get("local_dir"), - "primary_relpath": meta.get("primary_relpath"), - "artifact_count": meta.get("artifact_count", 0), - "created_at": row.created_at.isoformat() if row.created_at else None, - }) - return result - - -def link_manual_processed_data( - processed_dir: str | Path, - paper_dir: str | Path | None = None, - project_id: str | uuid.UUID | None = None, - asset_id: str | uuid.UUID | None = None, - primary_file: str | None = None, - processing_type: str | None = None, - output_format: str | None = None, -) -> dict: - """Attach a handmade processed-output folder to an existing project asset. - - This is intended for debugging or backfilling local outputs that were created - outside the normal processing pipeline. - """ - from mkb.db.models import Asset, ProcessedAsset, ProcessingLog, ProcessingType, ProjectAsset, ResearchProject - - bundle = _inspect_manual_processed_dir(processed_dir, primary_file=primary_file) - paper_path = Path(paper_dir).resolve() if paper_dir is not None else None - - if processing_type: - proc_type = ProcessingType(processing_type.upper()) - else: - proc_type = bundle["processing_type"] - out_format = output_format or bundle["output_format"] - - with SyncSessionLocal() as session: - project = None - if project_id is not None: - pid = uuid.UUID(str(project_id)) - project = session.query(ResearchProject).filter_by(project_id=pid).first() - elif paper_path is not None: - project = session.query(ResearchProject).filter_by(source_path=str(paper_path)).first() - if project is None and paper_path.is_dir(): - ingest_result = ingest(paper_path, label=paper_path.name) - pid = uuid.UUID(ingest_result["project_id"]) - project = session.query(ResearchProject).filter_by(project_id=pid).first() - elif asset_id is not None: - # Look up the owning project via ProjectAsset link - aid = uuid.UUID(str(asset_id)) - link = session.query(ProjectAsset).filter_by(asset_id=aid).first() - if link is not None: - project = ( - session.query(ResearchProject) - .filter_by(project_id=link.project_id) - .first() - ) - - if not project: - raise ValueError("Could not find a target project. Provide --paper-dir or --project-id.") - - if asset_id is not None: - target_asset = session.query(Asset).filter_by(asset_id=uuid.UUID(str(asset_id))).first() - else: - links = session.query(ProjectAsset).filter_by(project_id=project.project_id).all() - asset_ids = [l.asset_id for l in links] - assets = session.query(Asset).filter(Asset.asset_id.in_(asset_ids)).all() if asset_ids else [] - target_asset = _choose_asset_for_manual_output(assets, bundle["primary_name"]) - - if not target_asset: - raise ValueError( - "No raw asset found for the target project. Ingest the paper folder first or pass --asset-id." - ) - - link = session.query(ProjectAsset).filter_by( - project_id=project.project_id, - asset_id=target_asset.asset_id, - ).first() - if not link: - session.add(ProjectAsset(project_id=project.project_id, asset_id=target_asset.asset_id)) - - s3_key = f"{project.project_id}/{target_asset.asset_id}/{bundle['primary_relpath']}" - - # Mirror the bundle into the canonical processed-local-root so it survives - # after any caller-supplied temp directory is cleaned up. The local cache - # is used by the idempotency check and by downstream readers. - import shutil - - canonical_root = ( - Path(settings.processed_local_root) - / str(project.project_id) - / str(target_asset.asset_id) - ) - bundle_root = Path(bundle["local_dir"]).resolve() - if bundle_root != canonical_root.resolve(): - canonical_root.mkdir(parents=True, exist_ok=True) - for relpath in [bundle["primary_relpath"], *bundle["artifact_files"]]: - src = bundle_root / relpath - dst = canonical_root / relpath - dst.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(src, dst) - bundle["local_dir"] = str(canonical_root) - bundle_root = canonical_root - - # Upload primary file + artifacts to the processed-assets S3 bucket so that - # downstream consumers (idempotency check, frame extraction, projections, - # etc.) can fetch the bundle the same way as auto-processed outputs. - from mkb.storage.s3 import upload_bytes - - upload_bytes( - (bundle_root / bundle["primary_relpath"]).read_bytes(), - settings.s3_bucket_processed, - s3_key, - ) - for relpath in bundle["artifact_files"]: - artifact_key = f"{project.project_id}/{target_asset.asset_id}/{relpath}" - upload_bytes( - (bundle_root / relpath).read_bytes(), - settings.s3_bucket_processed, - artifact_key, - ) - - metadata = { - "project_id": str(project.project_id), - "local_dir": bundle["local_dir"], - "primary_relpath": bundle["primary_relpath"], - "artifact_files": bundle["artifact_files"], - "artifact_count": len(bundle["artifact_files"]), - "linked_via": "debug_manual_link", - "paper_dir": str(paper_path) if paper_path is not None else None, - } - - existing = ( - session.query(ProcessedAsset) - .filter_by(asset_id=target_asset.asset_id, processing_type=proc_type) - .order_by(ProcessedAsset.created_at.desc()) - .first() - ) - - if existing: - existing.output_format = out_format - existing.s3_bucket = settings.s3_bucket_processed - existing.s3_key = s3_key - existing.sha256 = bundle["sha256"] - existing.size_bytes = bundle["size_bytes"] - existing.conversion_metadata = metadata - existing.raw_asset_hash = target_asset.sha256 - processed_asset = existing - action = "updated" - else: - processed_asset = ProcessedAsset( - processed_asset_id=uuid.uuid4(), - asset_id=target_asset.asset_id, - processing_type=proc_type, - output_format=out_format, - s3_bucket=settings.s3_bucket_processed, - s3_key=s3_key, - sha256=bundle["sha256"], - size_bytes=bundle["size_bytes"], - conversion_metadata=metadata, - raw_asset_hash=target_asset.sha256, - ) - session.add(processed_asset) - action = "created" - - asset_meta = dict(target_asset.metadata_ or {}) - processing_meta = dict(asset_meta.get("processing") or {}) - processing_meta.update( - { - "last_status": "SUCCESS", - "last_processing_type": proc_type.value, - "last_output_format": out_format, - "last_processed_asset_id": str(processed_asset.processed_asset_id), - "last_processed_local_dir": bundle["local_dir"], - "source": "manual_debug_link", - } - ) - asset_meta["processing"] = processing_meta - target_asset.metadata_ = asset_meta - - session.add( - ProcessingLog( - log_id=uuid.uuid4(), - asset_id=target_asset.asset_id, - processing_type=proc_type, - status="SUCCESS", - processed_asset_id=processed_asset.processed_asset_id, - details={ - "debug": True, - "action": action, - "primary_relpath": bundle["primary_relpath"], - "artifact_count": len(bundle["artifact_files"]), - }, - ) - ) - session.commit() - - return { - "status": action, - "project_id": str(project.project_id), - "asset_id": str(target_asset.asset_id), - "processed_asset_id": str(processed_asset.processed_asset_id), - "processing_type": proc_type.value, - "output_format": out_format, - "local_dir": bundle["local_dir"], - "primary_relpath": bundle["primary_relpath"], - "artifact_files": bundle["artifact_files"], - } - - -def list_projects(limit: int = 50) -> list[dict]: - """List research projects.""" - from collections import defaultdict - - from mkb.db.models import CanonicalWorkflow, KnowledgeFrame, ProcessedAsset, ProjectAsset, RawWorkflowExtraction, ResearchProject - - init_db() - with SyncSessionLocal() as session: - projects = ( - session.query(ResearchProject) - .order_by(ResearchProject.created_at.desc()) - .limit(limit) - .all() - ) - if not projects: - return [] - - project_ids = [p.project_id for p in projects] - - # Bulk-fetch asset links for all queried projects - all_links = session.query(ProjectAsset).filter(ProjectAsset.project_id.in_(project_ids)).all() - project_to_asset_ids: dict = defaultdict(list) - for link in all_links: - project_to_asset_ids[link.project_id].append(link.asset_id) - - # Bulk-fetch which assets have at least one ProcessedAsset record - all_asset_ids = [link.asset_id for link in all_links] - if all_asset_ids: - processed_ids = { - row.asset_id - for row in session.query(ProcessedAsset.asset_id) - .filter(ProcessedAsset.asset_id.in_(all_asset_ids)) - .distinct() - .all() - } - else: - processed_ids = set() - - # Bulk-fetch frames - frames = session.query(KnowledgeFrame).filter(KnowledgeFrame.project_id.in_(project_ids)).all() - frame_by_project = {f.project_id: f for f in frames} - workflow_rows = ( - session.query(RawWorkflowExtraction) - .filter(RawWorkflowExtraction.project_id.in_(project_ids)) - .order_by(RawWorkflowExtraction.version.desc()) - .all() - ) - workflow_by_project = {} - for workflow in workflow_rows: - workflow_by_project.setdefault(workflow.project_id, workflow) - current = workflow_by_project[workflow.project_id] - current_is_valid = ( - current.status == "COMPLETED" - and current.record_status in {"active", "needs_review"} - ) - if not current_is_valid and workflow.status == "COMPLETED" and workflow.record_status in {"active", "needs_review"}: - workflow_by_project[workflow.project_id] = workflow - canonical_rows = ( - session.query(CanonicalWorkflow) - .filter(CanonicalWorkflow.project_id.in_(project_ids)) - .order_by(CanonicalWorkflow.version.desc()).all() - ) - canonical_by_project = {} - for canonical in canonical_rows: - canonical_by_project.setdefault(canonical.project_id, canonical) - - result = [] - for p in projects: - asset_ids_for_project = project_to_asset_ids[p.project_id] - total = len(asset_ids_for_project) - processed_count = sum(1 for aid in asset_ids_for_project if aid in processed_ids) - if total == 0: - processing_status = "NO_ASSETS" - elif processed_count == 0: - processing_status = "UNPROCESSED" - elif processed_count < total: - processing_status = "PARTIAL" - else: - processing_status = "PROCESSED" - - frame = frame_by_project.get(p.project_id) - workflow = workflow_by_project.get(p.project_id) - canonical = canonical_by_project.get(p.project_id) - result.append({ - "project_id": str(p.project_id), - "label": p.label, - "source_path": p.source_path, - "file_count": p.file_count, - "asset_count": total, - "processing_status": processing_status, - "frame_status": frame.status.value if frame else "NO_FRAME", - "workflow_status": workflow.status if workflow else "NO_WORKFLOW", - "workflow_version": workflow.version if workflow else None, - "canonical_workflow_status": canonical.status if canonical else "NO_CANONICAL_WORKFLOW", - "canonical_workflow_version": canonical.version if canonical else None, - "created_at": p.created_at.isoformat() if p.created_at else None, - "duplicate_of": (p.metadata_ or {}).get("duplicate_of"), - "group_id": str(p.group_id) if p.group_id else None, - }) - return result - - -# ── Project groups ───────────────────────────────────────────── - - -def _serialize_group(g, project_count: int) -> dict: - return { - "group_id": str(g.group_id), - "name": g.name, - "description": g.description, - "color": g.color, - "display_order": g.display_order, - "project_count": project_count, - "created_at": g.created_at.isoformat() if g.created_at else None, - "updated_at": g.updated_at.isoformat() if g.updated_at else None, - } - - -def list_project_groups() -> list[dict]: - """List all project groups with project counts.""" - from sqlalchemy import func as sa_func - - from mkb.db.models import ProjectGroup, ResearchProject - - init_db() - with SyncSessionLocal() as session: - groups = ( - session.query(ProjectGroup) - .order_by(ProjectGroup.display_order, ProjectGroup.created_at) - .all() - ) - counts = dict( - session.query(ResearchProject.group_id, sa_func.count()) - .filter(ResearchProject.group_id.isnot(None)) - .group_by(ResearchProject.group_id) - .all() - ) - return [_serialize_group(g, counts.get(g.group_id, 0)) for g in groups] - - -def create_project_group( - name: str, - *, - description: str | None = None, - color: str | None = None, - display_order: int | None = None, -) -> dict: - from mkb.db.models import ProjectGroup - - cleaned = (name or "").strip() - if not cleaned: - return {"error": "name must not be empty"} - - init_db() - with SyncSessionLocal() as session: - if display_order is None: - current_max = ( - session.query(ProjectGroup) - .order_by(ProjectGroup.display_order.desc()) - .first() - ) - display_order = (current_max.display_order + 1) if current_max else 0 - group = ProjectGroup( - name=cleaned, - description=(description or None), - color=(color or None), - display_order=int(display_order), - ) - session.add(group) - session.commit() - session.refresh(group) - return _serialize_group(group, 0) - - -def update_project_group( - group_id: str | uuid.UUID, - *, - name: str | None = None, - description: str | None = None, - color: str | None = None, - display_order: int | None = None, -) -> dict: - from sqlalchemy import func as sa_func - - from mkb.db.models import ProjectGroup, ResearchProject - - gid = uuid.UUID(str(group_id)) - init_db() - with SyncSessionLocal() as session: - group = session.query(ProjectGroup).filter_by(group_id=gid).first() - if not group: - return {"error": f"Group {group_id} not found"} - if name is not None: - cleaned = name.strip() - if not cleaned: - return {"error": "name must not be empty"} - group.name = cleaned - if description is not None: - group.description = description.strip() or None - if color is not None: - group.color = color.strip() or None - if display_order is not None: - group.display_order = int(display_order) - session.commit() - session.refresh(group) - count = ( - session.query(sa_func.count()) - .select_from(ResearchProject) - .filter(ResearchProject.group_id == gid) - .scalar() - ) or 0 - return _serialize_group(group, int(count)) - - -def delete_project_group(group_id: str | uuid.UUID) -> dict: - """Delete a group. Projects in it are unassigned (group_id set to NULL).""" - from mkb.db.models import ProjectGroup, ResearchProject - - gid = uuid.UUID(str(group_id)) - init_db() - with SyncSessionLocal() as session: - group = session.query(ProjectGroup).filter_by(group_id=gid).first() - if not group: - return {"error": f"Group {group_id} not found"} - unassigned = ( - session.query(ResearchProject) - .filter(ResearchProject.group_id == gid) - .update({ResearchProject.group_id: None}, synchronize_session=False) - ) - session.delete(group) - session.commit() - return {"group_id": str(gid), "deleted": True, "unassigned_projects": int(unassigned)} - - -def delete_project( - project_id: str | uuid.UUID, - *, - delete_s3_objects: bool = True, -) -> dict: - """Hard-delete a research project and all data exclusively owned by it. - - Cascade: - - ``ProjectAsset`` links for this project are removed. - - ``Asset`` / ``ProcessedAsset`` / ``ProcessingLog`` records are removed - only when the asset is **not** linked to any other project (i.e. not - shared). When ``delete_s3_objects`` is True the corresponding S3 - objects are removed before the DB records. - - ``KnowledgeFrame`` owned by this project is deleted, along with its - ``ExtractionPass``, all ``Projection`` rows (hard delete), and all - ``Feedback`` rows whose ``target_frame_id`` / ``target_project_id`` - match. - - The ``ResearchProject`` record itself is deleted last. - - Returns a summary dict or ``{"error": ...}`` when the project is not found. - """ - from mkb.db.models import ( - Asset, - CanonicalWorkflow, - ExtractionPass, - Feedback, - KnowledgeFrame, - ProcessedAsset, - ProcessingLog, - ProjectAsset, - Projection, - RawWorkflowExtraction, - ResearchProject, - WorkflowIndexEntry, - WorkflowMaintenanceTask, - ) - from mkb.storage.s3 import delete_object - - pid = uuid.UUID(str(project_id)) - init_db() - - with SyncSessionLocal() as session: - project = session.query(ResearchProject).filter_by(project_id=pid).first() - if not project: - return {"error": f"Project {project_id} not found"} - - # ── Collect asset IDs linked to this project ────────────────────── - own_links = session.query(ProjectAsset).filter_by(project_id=pid).all() - own_asset_ids = [lnk.asset_id for lnk in own_links] - - # Determine which of those assets are shared with other projects - shared_asset_ids: set[uuid.UUID] = set() - if own_asset_ids: - other_links = ( - session.query(ProjectAsset.asset_id) - .filter( - ProjectAsset.asset_id.in_(own_asset_ids), - ProjectAsset.project_id != pid, - ) - .distinct() - .all() - ) - shared_asset_ids = {row.asset_id for row in other_links} - - exclusive_asset_ids = [a for a in own_asset_ids if a not in shared_asset_ids] - - # ── Remove S3 objects and DB records for exclusive assets ───────── - deleted_assets = 0 - deleted_processed = 0 - deleted_s3_objects = 0 - - if exclusive_asset_ids: - # ProcessedAsset rows (and their S3 objects) - processed_rows = ( - session.query(ProcessedAsset) - .filter(ProcessedAsset.asset_id.in_(exclusive_asset_ids)) - .all() - ) - for pa in processed_rows: - if delete_s3_objects: - try: - delete_object(pa.s3_bucket, pa.s3_key) - deleted_s3_objects += 1 - except Exception: - logger.warning( - "Failed to delete S3 object %s/%s", pa.s3_bucket, pa.s3_key - ) - session.delete(pa) - deleted_processed = len(processed_rows) - - # ProcessingLog rows - session.query(ProcessingLog).filter( - ProcessingLog.asset_id.in_(exclusive_asset_ids) - ).delete(synchronize_session=False) - - # Raw asset S3 objects + Asset rows - raw_assets = ( - session.query(Asset) - .filter(Asset.asset_id.in_(exclusive_asset_ids)) - .all() - ) - for asset in raw_assets: - if delete_s3_objects: - try: - delete_object(asset.s3_bucket, asset.s3_key) - deleted_s3_objects += 1 - except Exception: - logger.warning( - "Failed to delete S3 object %s/%s", asset.s3_bucket, asset.s3_key - ) - session.delete(asset) - deleted_assets = len(raw_assets) - - # ── Remove ProjectAsset links (including shared ones) ───────────── - for lnk in own_links: - session.delete(lnk) - - # ── Knowledge frame + dependents ────────────────────────────────── - frame = session.query(KnowledgeFrame).filter_by(project_id=pid).first() - deleted_projections = 0 - deleted_passes = 0 - deleted_feedback = 0 - - if frame: - fid = frame.frame_id - - # Projections (hard delete) - deleted_projections = ( - session.query(Projection) - .filter(Projection.frame_id == fid) - .delete(synchronize_session=False) - ) - - # ExtractionPass rows - deleted_passes = ( - session.query(ExtractionPass) - .filter(ExtractionPass.frame_id == fid) - .delete(synchronize_session=False) - ) - - # Feedback rows tied to this frame - deleted_feedback = ( - session.query(Feedback) - .filter(Feedback.target_frame_id == fid) - .delete(synchronize_session=False) - ) - - session.delete(frame) - - # Also remove any feedback rows referencing the project but a different - # (or NULL) frame (defensive clean-up). - extra_feedback = ( - session.query(Feedback) - .filter(Feedback.target_project_id == pid) - .delete(synchronize_session=False) - ) - deleted_feedback += extra_feedback - - deleted_workflows = ( - session.query(RawWorkflowExtraction) - .filter(RawWorkflowExtraction.project_id == pid) - .delete(synchronize_session=False) - ) - deleted_canonical_workflows = ( - session.query(CanonicalWorkflow) - .filter(CanonicalWorkflow.project_id == pid) - .delete(synchronize_session=False) - ) - session.query(WorkflowIndexEntry).filter( - WorkflowIndexEntry.project_id == pid - ).delete(synchronize_session=False) - session.query(WorkflowMaintenanceTask).filter( - WorkflowMaintenanceTask.project_id == pid - ).delete(synchronize_session=False) - - # ── Delete the project itself ───────────────────────────────────── - session.delete(project) - session.commit() - - return { - "project_id": str(pid), - "deleted": True, - "deleted_assets": deleted_assets, - "shared_assets_kept": len(shared_asset_ids), - "deleted_processed_assets": deleted_processed, - "deleted_s3_objects": deleted_s3_objects, - "deleted_projections": deleted_projections, - "deleted_extraction_passes": deleted_passes, - "deleted_feedback": deleted_feedback, - "deleted_workflow_versions": deleted_workflows, - "deleted_canonical_workflow_versions": deleted_canonical_workflows, - } - - -# ── Raw workflow graphs ─────────────────────────────────────── - - -def extract_raw_workflow(project_id: str | uuid.UUID, model: str | None = None, verbose: bool = False, progress_callback=None) -> dict: - """Append a new faithful raw-workflow extraction version for a project.""" - from mkb.agents.workflow_extraction import run_workflow_extraction - - readiness = get_raw_workflow_extraction_readiness(project_id) - if not readiness.get("ready"): - return { - "status": "error", - "message": readiness.get("message") or "Project is not ready for workflow extraction", - } - init_db() - return run_workflow_extraction( - uuid.UUID(str(project_id)), model=model, verbose=verbose, - progress_callback=progress_callback, - ) - - -def get_raw_workflow_extraction_readiness(project_id: str | uuid.UUID) -> dict: - """Check whether a project has readable sources for raw workflow extraction.""" - from mkb.db.models import ( - ProcessedAsset, - ProcessingType, - ProjectAsset, - RawWorkflowExtraction, - ResearchProject, - ) - - pid = uuid.UUID(str(project_id)) - init_db() - with SyncSessionLocal() as session: - project = session.query(ResearchProject).filter_by(project_id=pid).first() - if not project: - return {"ready": False, "message": f"Project {pid} not found"} - - rows = ( - session.query(ProcessedAsset.asset_id) - .join(ProjectAsset, ProjectAsset.asset_id == ProcessedAsset.asset_id) - .filter( - ProjectAsset.project_id == pid, - ProcessedAsset.processing_type == ProcessingType.MARKDOWN, - ) - .distinct() - .all() - ) - asset_ids = [str(row.asset_id) for row in rows] - if not asset_ids: - return { - "ready": False, - "message": ( - "Workflow extraction requires processed Markdown, but this project has no readable " - "processed Markdown files yet. Run Process first and confirm Markdown outputs exist." - ), - } - - unfinished = ( - session.query(RawWorkflowExtraction) - .filter( - RawWorkflowExtraction.project_id == pid, - RawWorkflowExtraction.graph.is_(None), - RawWorkflowExtraction.status.in_(("IN_PROGRESS", "FAILED")), - ) - .order_by(RawWorkflowExtraction.version.desc()) - .first() - ) - return { - "ready": True, - "project_id": str(pid), - "readable_asset_ids": asset_ids, - "resume_extraction_id": str(unfinished.extraction_id) if unfinished else None, - "resume_version": unfinished.version if unfinished else None, - "has_checkpoint": bool(unfinished and unfinished.checkpoint), - } - - -def list_raw_workflows(project_id: str | uuid.UUID, include_graph: bool = False) -> list[dict]: - """List append-only raw workflow versions, newest first.""" - from mkb.db.models import RawWorkflowExtraction - - pid = uuid.UUID(str(project_id)) - init_db() - with SyncSessionLocal() as session: - rows = ( - session.query(RawWorkflowExtraction) - .filter(RawWorkflowExtraction.project_id == pid) - .order_by(RawWorkflowExtraction.version.desc()) - .all() - ) - return [_serialize_raw_workflow(row, include_graph=include_graph) for row in rows] - - -def get_raw_workflow(project_id: str | uuid.UUID, version: int | None = None) -> dict | None: - """Get the latest completed raw workflow, or a specific version.""" - from mkb.db.models import RawWorkflowExtraction - - pid = uuid.UUID(str(project_id)) - init_db() - with SyncSessionLocal() as session: - query = session.query(RawWorkflowExtraction).filter(RawWorkflowExtraction.project_id == pid) - if version is None: - query = query.filter( - RawWorkflowExtraction.status == "COMPLETED", - RawWorkflowExtraction.record_status.in_(("active", "needs_review")), - ).order_by(RawWorkflowExtraction.version.desc()) - else: - query = query.filter(RawWorkflowExtraction.version == version) - row = query.first() - return _serialize_raw_workflow(row, include_graph=True) if row else None - - -def delete_raw_workflow_version(project_id: str | uuid.UUID, version: int) -> dict: - """Delete one raw workflow version when no canonical version depends on it.""" - from mkb.db.models import CanonicalWorkflow, RawWorkflowExtraction - - pid = uuid.UUID(str(project_id)) - init_db() - with SyncSessionLocal() as session: - row = ( - session.query(RawWorkflowExtraction) - .filter( - RawWorkflowExtraction.project_id == pid, - RawWorkflowExtraction.version == version, - ) - .first() - ) - if not row: - return {"error": "Raw workflow version not found"} - dependent_canonical = ( - session.query(CanonicalWorkflow) - .filter(CanonicalWorkflow.raw_extraction_id == row.extraction_id) - .order_by(CanonicalWorkflow.version.desc()) - .first() - ) - if dependent_canonical: - return { - "error": ( - f"Raw workflow v{version} cannot be deleted because canonical workflow " - f"v{dependent_canonical.version} still depends on it" - ) - } - - extraction_id = row.extraction_id - session.delete(row) - session.commit() - return { - "status": "deleted", - "project_id": str(pid), - "version": version, - "extraction_id": str(extraction_id), - } - - -def delete_canonical_workflow_version(project_id: str | uuid.UUID, version: int) -> dict: - """Delete one canonical workflow version and its derived indexes/tasks.""" - from mkb.db.models import CanonicalWorkflow, WorkflowIndexEntry, WorkflowMaintenanceTask - - pid = uuid.UUID(str(project_id)) - init_db() - with SyncSessionLocal() as session: - row = ( - session.query(CanonicalWorkflow) - .filter( - CanonicalWorkflow.project_id == pid, - CanonicalWorkflow.version == version, - ) - .first() - ) - if not row: - return {"error": "Canonical workflow version not found"} - - canonicalization_id = row.canonicalization_id - session.query(WorkflowIndexEntry).filter( - WorkflowIndexEntry.canonicalization_id == canonicalization_id - ).delete(synchronize_session=False) - session.query(WorkflowMaintenanceTask).filter( - WorkflowMaintenanceTask.project_id == pid, - WorkflowMaintenanceTask.source_canonicalization_id == canonicalization_id, - ).delete(synchronize_session=False) - session.delete(row) - session.commit() - return { - "status": "deleted", - "project_id": str(pid), - "version": version, - "canonicalization_id": str(canonicalization_id), - } - - -def _serialize_raw_workflow(row, include_graph: bool) -> dict: - payload = { - "extraction_id": str(row.extraction_id), "project_id": str(row.project_id), - "version": row.version, "schema_version": row.schema_version, - "extractor_version": row.extractor_version, "model": row.model, - "status": row.status, "record_status": row.record_status, - "supersedes_extraction_id": str(row.supersedes_extraction_id) if row.supersedes_extraction_id else None, - "correction_reason": row.correction_reason, - "correction_author": row.correction_author, - "correction_details": row.correction_details or {}, - "review_flags": row.review_flags or [], - "provenance": row.provenance or {}, "error": row.error, - "extracted_at": row.extracted_at.isoformat() if row.extracted_at else None, - "created_at": row.created_at.isoformat() if row.created_at else None, - "has_checkpoint": bool(row.checkpoint), - "checkpoint_summary": (row.checkpoint or {}).get("summary"), - "checkpoint_updated_at": row.checkpoint_updated_at.isoformat() if row.checkpoint_updated_at else None, - "resumable": row.graph is None and row.status in {"IN_PROGRESS", "FAILED"}, - } - if include_graph: - payload["graph"] = row.graph - elif row.graph: - payload["node_count"] = len(row.graph.get("nodes", [])) - payload["edge_count"] = len(row.graph.get("edges", [])) - return payload - - -def review_raw_workflow(extraction_id: str | uuid.UUID, *, status: str | None = None, author: str = "system") -> dict: - """Run automatic checks and optionally set a manual lifecycle status.""" - from mkb.db.models import RawWorkflowExtraction - from mkb.workflows.review import VALID_RECORD_STATUSES, audit_raw_graph - - init_db() - eid = uuid.UUID(str(extraction_id)) - if status is not None and status not in VALID_RECORD_STATUSES: - return {"error": f"Invalid record status: {status}"} - with SyncSessionLocal() as session: - row = session.query(RawWorkflowExtraction).filter_by(extraction_id=eid).first() - if not row or not row.graph: - return {"error": "Completed raw workflow not found"} - later = session.query(RawWorkflowExtraction).filter( - RawWorkflowExtraction.project_id == row.project_id, - RawWorkflowExtraction.version > row.version, - RawWorkflowExtraction.status == "COMPLETED", - ).order_by(RawWorkflowExtraction.version.desc()).first() - flags = audit_raw_graph(row.graph, later_graph=later.graph if later else None) - row.review_flags = flags - if status: - row.record_status = status - elif flags and row.record_status == "active": - row.record_status = "needs_review" - row.provenance = {**(row.provenance or {}), "last_reviewed_by": author} - session.commit() - return _serialize_raw_workflow(row, include_graph=False) - - -def correct_raw_workflow( - extraction_id: str | uuid.UUID, graph: dict, *, reason: str, author: str, - affected_nodes: list[str] | None = None, affected_edges: list[str] | None = None, - evidence: str, -) -> dict: - """Create a corrected immutable version and supersede the source version.""" - from sqlalchemy import func - from mkb.db.models import RawWorkflowExtraction - from mkb.workflows.review import audit_raw_graph, correction_metadata, rebase_graph - - init_db() - eid = uuid.UUID(str(extraction_id)) - new_id = uuid.uuid4() - details = correction_metadata(reason, author, affected_nodes or [], affected_edges or [], evidence) - with SyncSessionLocal() as session: - source = session.query(RawWorkflowExtraction).filter_by(extraction_id=eid).first() - if not source or source.status != "COMPLETED": - return {"error": "Completed source workflow not found"} - corrected = dict(graph) - corrected["paper_id"] = str(source.project_id) - corrected["schema_version"] = source.schema_version - try: - corrected = rebase_graph(corrected, new_id) - except Exception as exc: - return {"error": f"Corrected graph validation failed: {exc}"} - version = (session.query(func.max(RawWorkflowExtraction.version)).filter_by(project_id=source.project_id).scalar() or 0) + 1 - flags = audit_raw_graph(corrected) - row = RawWorkflowExtraction( - extraction_id=new_id, project_id=source.project_id, version=version, - schema_version=source.schema_version, extractor_version=source.extractor_version, - model=source.model, status="COMPLETED", - record_status="needs_review" if flags else "active", - supersedes_extraction_id=source.extraction_id, graph=corrected, - correction_reason=reason, correction_author=author, - correction_details=details, review_flags=flags, - provenance={**(source.provenance or {}), "correction_evidence": evidence}, - extracted_at=datetime.now(timezone.utc), - ) - source.record_status = "superseded" - session.add(row) - session.commit() - return _serialize_raw_workflow(row, include_graph=True) - - -def canonicalize_workflow(project_id: str | uuid.UUID, raw_extraction_id: str | uuid.UUID | None = None, model: str | None = None, verbose: bool = False, progress_callback=None) -> dict: - """Create an append-only canonical view from a valid raw workflow.""" - from mkb.agents.workflow_canonicalization import run_workflow_canonicalization - - init_db() - return run_workflow_canonicalization( - uuid.UUID(str(project_id)), - uuid.UUID(str(raw_extraction_id)) if raw_extraction_id else None, - model=model, verbose=verbose, progress_callback=progress_callback, - ) - - -def list_canonical_workflows(project_id: str | uuid.UUID, include_graph: bool = False) -> list[dict]: - from mkb.db.models import CanonicalWorkflow - - pid = uuid.UUID(str(project_id)) - init_db() - with SyncSessionLocal() as session: - rows = session.query(CanonicalWorkflow).filter_by(project_id=pid).order_by(CanonicalWorkflow.version.desc()).all() - return [_serialize_canonical_workflow(row, include_graph) for row in rows] - - -def get_canonical_workflow(project_id: str | uuid.UUID, version: int | None = None) -> dict | None: - from mkb.db.models import CanonicalWorkflow - - pid = uuid.UUID(str(project_id)) - init_db() - with SyncSessionLocal() as session: - query = session.query(CanonicalWorkflow).filter(CanonicalWorkflow.project_id == pid) - query = ( - query.filter(CanonicalWorkflow.status == "COMPLETED").order_by(CanonicalWorkflow.version.desc()) - if version is None else query.filter(CanonicalWorkflow.version == version) - ) - row = query.first() - return _serialize_canonical_workflow(row, True) if row else None - - -def _serialize_canonical_workflow(row, include_graph: bool) -> dict: - payload = { - "canonicalization_id": str(row.canonicalization_id), "project_id": str(row.project_id), - "raw_extraction_id": str(row.raw_extraction_id), "version": row.version, - "schema_version": row.schema_version, "canonicalizer_version": row.canonicalizer_version, - "model": row.model, "status": row.status, "provenance": row.provenance or {}, - "error": row.error, - "canonicalized_at": row.canonicalized_at.isoformat() if row.canonicalized_at else None, - "created_at": row.created_at.isoformat() if row.created_at else None, - "has_checkpoint": bool(row.checkpoint), - "checkpoint_summary": (row.checkpoint or {}).get("summary"), - "checkpoint_updated_at": row.checkpoint_updated_at.isoformat() if row.checkpoint_updated_at else None, - "resumable": row.graph is None and row.status in {"IN_PROGRESS", "FAILED"}, - } - if include_graph: - payload["graph"] = row.graph - elif row.graph: - payload.update(node_count=len(row.graph.get("nodes", [])), edge_count=len(row.graph.get("edges", []))) - return payload - - -def curate_workflow_schema(*, min_support: int = 2, author: str = "schema-curator/1.0") -> list[dict]: - """Analyze accumulated workflows and persist new evidence-backed proposals.""" - from mkb.db.models import CanonicalWorkflow, RawWorkflowExtraction, SchemaProposal, SchemaProposalRevision, WorkflowSchemaVersion - from mkb.workflows.curator import analyze_canonical_workflows - from mkb.workflows.schema_library import get_schema_library_payload - - init_db() - with SyncSessionLocal() as session: - current = session.query(WorkflowSchemaVersion).filter_by(status="active").order_by(WorkflowSchemaVersion.version.desc()).first() - if not current: - current = WorkflowSchemaVersion(version=1, name="workflow-schema/1.0", payload=get_schema_library_payload(), created_by="seed") - session.add(current) - session.flush() - rows = session.query(CanonicalWorkflow).filter_by(status="COMPLETED").all() - workflows = [] - for row in rows: - raw = session.query(RawWorkflowExtraction).filter_by(extraction_id=row.raw_extraction_id).first() - workflows.append({"canonicalization_id": str(row.canonicalization_id), "graph": row.graph, "raw_graph": raw.graph if raw else {}}) - generated = analyze_canonical_workflows(workflows, min_support=min_support) - results = [] - for item in generated: - duplicate = session.query(SchemaProposal).filter( - SchemaProposal.status.in_(("pending", "revision_requested")), - SchemaProposal.proposal_type == item["proposal_type"], - SchemaProposal.payload == item["payload"], - ).first() - if duplicate: - continue - rationale = ( - f"Deterministic discovery signal: {item.get('analysis', {}).get('signal', 'unknown')} " - f"with support from {len(item.get('evidence_workflow_ids', []))} workflows." - ) - proposal = SchemaProposal( - **item, rationale=rationale, - base_schema_version=current.name, created_by=author, - ) - session.add(proposal) - session.flush() - session.add(SchemaProposalRevision( - proposal_id=proposal.proposal_id, revision_number=1, - payload=proposal.payload, - evidence_workflow_ids=proposal.evidence_workflow_ids, - analysis=proposal.analysis, rationale=proposal.rationale, - author=author, - author_type="agent" if "agent" in author else "system", - change_note="Initial proposal draft", - validation_errors=[], - )) - results.append({**item, "proposal_id": str(proposal.proposal_id), "status": "pending"}) - session.commit() - return results - - -def list_schema_proposals(status: str | None = "pending") -> list[dict]: - from sqlalchemy import func - from mkb.db.models import SchemaProposal, SchemaProposalRevision - - init_db() - with SyncSessionLocal() as session: - query = session.query(SchemaProposal) - if status: - query = query.filter_by(status=status) - rows = query.order_by(SchemaProposal.created_at.desc()).all() - revision_counts = dict( - session.query( - SchemaProposalRevision.proposal_id, - func.count(SchemaProposalRevision.revision_id), - ).group_by(SchemaProposalRevision.proposal_id).all() - ) - return [{ - "proposal_id": str(row.proposal_id), "proposal_type": row.proposal_type, - "status": row.status, "payload": row.payload, - "evidence_workflow_ids": row.evidence_workflow_ids, "analysis": row.analysis, - "base_schema_version": row.base_schema_version, "created_by": row.created_by, - "rationale": row.rationale, - "reviewer_notes": row.reviewer_notes, - "validation_errors": (row.analysis or {}).get("validation_errors", []), - "revision_count": int(revision_counts.get(row.proposal_id, 0)), - "reviewed_by": row.reviewed_by, - "reviewed_at": row.reviewed_at.isoformat() if row.reviewed_at else None, - "created_at": row.created_at.isoformat() if row.created_at else None, - } for row in rows] - - -def get_schema_proposal_revisions(proposal_id: str | uuid.UUID) -> list[dict]: - from mkb.db.models import SchemaProposalRevision - - pid = uuid.UUID(str(proposal_id)) - init_db() - with SyncSessionLocal() as session: - rows = session.query(SchemaProposalRevision).filter_by(proposal_id=pid).order_by( - SchemaProposalRevision.revision_number.desc() - ).all() - return [{ - "revision_id": str(row.revision_id), - "revision_number": row.revision_number, - "payload": row.payload, - "evidence_workflow_ids": row.evidence_workflow_ids, - "analysis": row.analysis, - "rationale": row.rationale, - "author": row.author, - "author_type": row.author_type, - "change_note": row.change_note, - "validation_errors": row.validation_errors, - "created_at": row.created_at.isoformat() if row.created_at else None, - } for row in rows] - - -def edit_schema_proposal( - proposal_id: str | uuid.UUID, *, payload: dict, - evidence_workflow_ids: list[str], rationale: str, - editor: str, change_note: str, -) -> dict: - """Save an attributed proposal draft revision and revalidate it.""" - from sqlalchemy import func - from mkb.db.models import ( - CanonicalWorkflow, RawWorkflowExtraction, SchemaProposal, SchemaProposalRevision, - WorkflowSchemaVersion, - ) - from mkb.workflows.curator import validate_proposal - - pid = uuid.UUID(str(proposal_id)) - if not editor.strip() or not change_note.strip(): - return {"error": "editor and change_note are required"} - try: - evidence_uuids = [uuid.UUID(value) for value in evidence_workflow_ids] - except (TypeError, ValueError, AttributeError): - return {"error": "evidence_workflow_ids must contain canonicalization UUIDs"} - init_db() - with SyncSessionLocal() as session: - row = session.query(SchemaProposal).filter_by(proposal_id=pid).first() - if not row or row.status not in {"pending", "revision_requested"}: - return {"error": "Only pending or revision-requested proposals can be edited"} - current = session.query(WorkflowSchemaVersion).filter_by(status="active").order_by( - WorkflowSchemaVersion.version.desc() - ).first() - if not current: - return {"error": "Active schema library not found"} - known_evidence = { - str(value) for (value,) in session.query(CanonicalWorkflow.canonicalization_id).filter( - CanonicalWorkflow.canonicalization_id.in_(evidence_uuids) - ).all() - } if evidence_workflow_ids else set() - if evidence_workflow_ids: - known_evidence.update({ - str(value) for (value,) in session.query(RawWorkflowExtraction.extraction_id).filter( - RawWorkflowExtraction.extraction_id.in_(evidence_uuids) - ).all() - }) - errors = validate_proposal( - row.proposal_type, payload, evidence_workflow_ids, current.payload, - ) - missing = sorted(set(evidence_workflow_ids) - known_evidence) - if missing: - errors.append(f"unknown evidence workflows: {', '.join(missing)}") - row.payload = payload - row.evidence_workflow_ids = evidence_workflow_ids - row.rationale = rationale.strip() - row.base_schema_version = current.name - row.analysis = {**(row.analysis or {}), "validation_errors": errors} - row.status = "pending" if not errors else "revision_requested" - revision_number = int( - session.query(func.coalesce(func.max(SchemaProposalRevision.revision_number), 0)) - .filter_by(proposal_id=pid).scalar() - ) + 1 - session.add(SchemaProposalRevision( - proposal_id=pid, revision_number=revision_number, - payload=payload, evidence_workflow_ids=evidence_workflow_ids, - analysis=row.analysis, rationale=row.rationale, - author=editor.strip(), author_type="human", - change_note=change_note.strip(), validation_errors=errors, - )) - session.commit() - return { - "proposal_id": str(pid), "status": row.status, - "revision_number": revision_number, "validation_errors": errors, - } - - -def get_workflow_schema_status() -> dict: - """Return global schema and curator queue summary for the frontend.""" - from sqlalchemy import func - from mkb.db.models import SchemaProposal, WorkflowMaintenanceTask, WorkflowSchemaVersion - from mkb.workflows.schema_library import get_schema_library_payload - - init_db() - with SyncSessionLocal() as session: - active = session.query(WorkflowSchemaVersion).filter_by(status="active").order_by( - WorkflowSchemaVersion.version.desc() - ).first() - payload = active.payload if active else get_schema_library_payload() - proposal_counts = dict( - session.query(SchemaProposal.status, func.count(SchemaProposal.proposal_id)) - .group_by(SchemaProposal.status).all() - ) - pending_recanonicalizations = session.query(func.count(WorkflowMaintenanceTask.task_id)).filter( - WorkflowMaintenanceTask.task_type == "recanonicalize", - WorkflowMaintenanceTask.status == "pending", - ).scalar() or 0 - return { - "schema_version": active.name if active else payload["schema_version"], - "version_number": active.version if active else 1, - "status": active.status if active else "seed", - "change_summary": active.change_summary if active else "Built-in seed schema", - "created_by": active.created_by if active else "system", - "created_at": active.created_at.isoformat() if active and active.created_at else None, - "object_schema_count": len(payload.get("object_schemas", {})), - "operation_template_count": len(payload.get("operation_templates", {})), - "card_count": len(payload.get("cards", {})), - "granularity_relation_count": len(payload.get("granularity_relations", [])), - "proposal_counts": proposal_counts, - "pending_recanonicalizations": int(pending_recanonicalizations), - } - - -def review_schema_proposal( - proposal_id: str | uuid.UUID, *, approve: bool | None = None, - reviewer: str, decision: str | None = None, notes: str = "", -) -> dict: - """Validate and approve/reject a proposal; approval creates a schema snapshot.""" - from sqlalchemy import func - from mkb.db.models import ( - CanonicalWorkflow, RawWorkflowExtraction, SchemaProposal, SchemaProposalRevision, - WorkflowMaintenanceTask, WorkflowSchemaVersion, - ) - from mkb.workflows.curator import apply_proposal, validate_proposal - from mkb.workflows.maintenance import recanonicalization_reason_for_proposal - - decision = decision or ("approve" if approve else "reject") - if decision not in {"approve", "reject", "request_revision"}: - return {"error": f"Unsupported review decision: {decision}"} - if not reviewer.strip(): - return {"error": "reviewer is required"} - if decision == "request_revision" and not notes.strip(): - return {"error": "Revision requests require reviewer notes"} - init_db() - with SyncSessionLocal() as session: - row = session.query(SchemaProposal).filter_by(proposal_id=uuid.UUID(str(proposal_id))).first() - if not row or row.status not in {"pending", "revision_requested"}: - return {"error": "Reviewable proposal not found"} - current = session.query(WorkflowSchemaVersion).filter_by(status="active").order_by(WorkflowSchemaVersion.version.desc()).first() - if not current: - return {"error": "Schema library is not initialized; run the curator first"} - errors = validate_proposal(row.proposal_type, row.payload, row.evidence_workflow_ids, current.payload) - known_evidence = { - str(value) for (value,) in session.query(CanonicalWorkflow.canonicalization_id).filter( - CanonicalWorkflow.canonicalization_id.in_([ - uuid.UUID(value) for value in row.evidence_workflow_ids - ]) - ).all() - } if row.evidence_workflow_ids else set() - if row.evidence_workflow_ids: - known_evidence.update({ - str(value) for (value,) in session.query(RawWorkflowExtraction.extraction_id).filter( - RawWorkflowExtraction.extraction_id.in_([ - uuid.UUID(value) for value in row.evidence_workflow_ids - ]) - ).all() - }) - missing = sorted(set(row.evidence_workflow_ids) - known_evidence) - if missing: - errors.append(f"unknown evidence workflows: {', '.join(missing)}") - rebased_from = None - if decision == "approve" and row.base_schema_version != current.name: - rebased_from = row.base_schema_version - row.base_schema_version = current.name - row.analysis = { - **(row.analysis or {}), - "rebased_from_schema": rebased_from, - "rebased_to_schema": current.name, - } - if decision == "approve" and errors: - return {"error": "Schema validation failed", "details": errors} - row.reviewed_by = reviewer.strip() - row.reviewer_notes = notes.strip() or None - row.reviewed_at = datetime.now(timezone.utc) - revision_number = int( - session.query(func.coalesce(func.max(SchemaProposalRevision.revision_number), 0)) - .filter_by(proposal_id=row.proposal_id).scalar() - ) + 1 - session.add(SchemaProposalRevision( - proposal_id=row.proposal_id, revision_number=revision_number, - payload=row.payload, evidence_workflow_ids=row.evidence_workflow_ids, - analysis=row.analysis, rationale=row.rationale, - author=reviewer.strip(), author_type="human", - change_note=( - f"Automatically rebased {rebased_from} to {current.name}. " - if rebased_from else "" - ) + f"Review decision: {decision}. {notes.strip()}".strip(), - validation_errors=errors, - )) - if decision in {"reject", "request_revision"}: - row.status = "rejected" if decision == "reject" else "revision_requested" - session.commit() - return { - "proposal_id": str(row.proposal_id), "status": row.status, - "revision_number": revision_number, - } - next_version = current.version + 1 - next_name = f"workflow-schema/1.{next_version - 1}" - base_payload = {**current.payload, "schema_version": next_name} - payload = apply_proposal(base_payload, row.proposal_type, row.payload) - current.status = "superseded" - session.add(WorkflowSchemaVersion( - version=next_version, name=next_name, payload=payload, - change_summary=f"Applied proposal {row.proposal_id}: {row.proposal_type}", - created_by=reviewer, - )) - row.status = "approved" - affected = 0 - queues_created = 0 - queues_updated = 0 - duplicate_queues_removed = 0 - completed = session.query(CanonicalWorkflow).filter_by(status="COMPLETED").order_by( - CanonicalWorkflow.project_id, CanonicalWorkflow.version.desc() - ).all() - latest_by_project = {} - for canonical in completed: - latest_by_project.setdefault(canonical.project_id, canonical) - # Every immutable schema snapshot has a new version. Even when a - # proposal directly cites only a subset, each latest project view is - # queued so its canonical graph can explicitly target that version. - for canonical in latest_by_project.values(): - canonical.provenance = { - **(canonical.provenance or {}), - "recanonicalization_required": True, - "target_schema_version": next_name, - } - pending_tasks = session.query(WorkflowMaintenanceTask).filter_by( - project_id=canonical.project_id, - task_type="recanonicalize", - status="pending", - ).order_by(WorkflowMaintenanceTask.created_at).all() - proposal_ids = [str(row.proposal_id)] - if pending_tasks: - task = pending_tasks[0] - previous_ids = (task.scope or {}).get("schema_proposal_ids", []) - task.scope = { - **(task.scope or {}), - "schema_proposal_ids": list(dict.fromkeys([ - *previous_ids, *proposal_ids, - ])), - } - task.reason = "schema_version_changed" - task.source_raw_extraction_id = canonical.raw_extraction_id - task.source_canonicalization_id = canonical.canonicalization_id - task.target_schema_version = next_name - task.requested_by = reviewer.strip() - for duplicate in pending_tasks[1:]: - session.delete(duplicate) - duplicate_queues_removed += 1 - queues_updated += 1 - else: - session.add(WorkflowMaintenanceTask( - project_id=canonical.project_id, - task_type="recanonicalize", - reason=recanonicalization_reason_for_proposal(row.proposal_type), - source_raw_extraction_id=canonical.raw_extraction_id, - source_canonicalization_id=canonical.canonicalization_id, - target_schema_version=next_name, - requested_by=reviewer.strip(), - scope={"schema_proposal_ids": proposal_ids}, - )) - queues_created += 1 - affected += 1 - session.commit() - return { - "proposal_id": str(row.proposal_id), "status": "approved", - "schema_version": next_name, - "rebased_from_schema": rebased_from, - "recanonicalization_scheduled": affected, - "queues_created": queues_created, - "queues_updated": queues_updated, - "duplicate_queues_removed": duplicate_queues_removed, - } - - -def schedule_workflow_reextraction(project_id: str | uuid.UUID, *, reason: str, requested_by: str, scope: dict | None = None, raw_extraction_id: str | uuid.UUID | None = None) -> dict: - """Queue an approved full or partial re-extraction request.""" - from mkb.db.models import RawWorkflowExtraction, WorkflowMaintenanceTask - from mkb.workflows.maintenance import validate_reextraction_request - - pid = uuid.UUID(str(project_id)) - validated_scope = validate_reextraction_request(reason, scope) - init_db() - with SyncSessionLocal() as session: - query = session.query(RawWorkflowExtraction).filter( - RawWorkflowExtraction.project_id == pid, - RawWorkflowExtraction.status == "COMPLETED", - RawWorkflowExtraction.record_status.in_(("active", "needs_review")), - ) - raw = query.filter_by(extraction_id=uuid.UUID(str(raw_extraction_id))).first() if raw_extraction_id else query.order_by(RawWorkflowExtraction.version.desc()).first() - if not raw: - return {"error": "No valid raw workflow is available for re-extraction"} - task = WorkflowMaintenanceTask( - project_id=pid, task_type="reextract", reason=reason, - source_raw_extraction_id=raw.extraction_id, scope=validated_scope, - requested_by=requested_by, - ) - session.add(task) - session.commit() - return {"task_id": str(task.task_id), "status": task.status, "task_type": task.task_type, "scope": task.scope} - - -def schedule_workflow_recanonicalization(project_id: str | uuid.UUID, *, reason: str = "manual_request", requested_by: str, raw_extraction_id: str | uuid.UUID | None = None, target_schema_version: str | None = None) -> dict: - from mkb.db.models import RawWorkflowExtraction, WorkflowMaintenanceTask - from mkb.workflows.maintenance import RECANONICALIZATION_REASONS - from mkb.workflows.schema_library import get_schema_library_payload - - if reason not in RECANONICALIZATION_REASONS: - raise ValueError(f"Unsupported recanonicalization reason: {reason}") - pid = uuid.UUID(str(project_id)) - init_db() - with SyncSessionLocal() as session: - query = session.query(RawWorkflowExtraction).filter( - RawWorkflowExtraction.project_id == pid, - RawWorkflowExtraction.status == "COMPLETED", - RawWorkflowExtraction.record_status.in_(("active", "needs_review")), - ) - raw = query.filter_by(extraction_id=uuid.UUID(str(raw_extraction_id))).first() if raw_extraction_id else query.order_by(RawWorkflowExtraction.version.desc()).first() - if not raw: - return {"error": "No valid raw workflow is available for canonicalization"} - task = WorkflowMaintenanceTask( - project_id=pid, task_type="recanonicalize", reason=reason, - source_raw_extraction_id=raw.extraction_id, - target_schema_version=target_schema_version or get_schema_library_payload()["schema_version"], - requested_by=requested_by, - ) - session.add(task) - session.commit() - return {"task_id": str(task.task_id), "status": task.status, "task_type": task.task_type} - - -def list_workflow_maintenance_tasks(*, status: str | None = None, project_id: str | uuid.UUID | None = None) -> list[dict]: - from mkb.db.models import WorkflowMaintenanceTask - - init_db() - with SyncSessionLocal() as session: - query = session.query(WorkflowMaintenanceTask) - if status: - query = query.filter_by(status=status) - if project_id: - query = query.filter_by(project_id=uuid.UUID(str(project_id))) - return [{ - "task_id": str(row.task_id), "project_id": str(row.project_id), - "task_type": row.task_type, "reason": row.reason, "scope": row.scope, - "status": row.status, "target_schema_version": row.target_schema_version, - "result": row.result, "error": row.error, - } for row in query.order_by(WorkflowMaintenanceTask.created_at.desc()).all()] - - -def run_workflow_maintenance_task(task_id: str | uuid.UUID, *, model: str | None = None, verbose: bool = False, progress_callback=None) -> dict: - """Execute one queued task, retaining both raw and canonical history.""" - from mkb.agents.workflow_canonicalization import run_workflow_canonicalization - from mkb.agents.workflow_extraction import run_workflow_extraction - from mkb.db.models import RawWorkflowExtraction, WorkflowMaintenanceTask - - tid = uuid.UUID(str(task_id)) - init_db() - with SyncSessionLocal() as session: - task = session.query(WorkflowMaintenanceTask).filter_by(task_id=tid).first() - if not task or task.status not in {"pending", "failed"}: - return {"error": "Pending or failed maintenance task not found"} - task.status = "running" - task.started_at = datetime.now(timezone.utc) - project_id, task_type, reason = task.project_id, task.task_type, task.reason - source_raw_id, scope = task.source_raw_extraction_id, task.scope - target_schema_version = task.target_schema_version - raw = session.query(RawWorkflowExtraction).filter_by(extraction_id=source_raw_id).first() - baseline = raw.graph if raw else None - session.commit() - try: - if task_type == "reextract": - extraction = run_workflow_extraction( - project_id, model=model, verbose=verbose, progress_callback=progress_callback, - reextraction_request={ - "reason": reason, "scope": scope, - "source_raw_extraction_id": str(source_raw_id), - "baseline_graph": baseline, - }, - ) - if extraction.get("status") != "completed": - raise RuntimeError(extraction.get("message") or "Re-extraction failed") - canonical = run_workflow_canonicalization( - project_id, uuid.UUID(extraction["extraction_id"]), model=model, - verbose=verbose, progress_callback=progress_callback, - recanonicalization_reason="raw_version_changed", - ) - result = {"extraction": extraction, "canonicalization": canonical} - else: - result = run_workflow_canonicalization( - project_id, source_raw_id, model=model, verbose=verbose, - progress_callback=progress_callback, recanonicalization_reason=reason, - target_schema_version=target_schema_version, - ) - successful = result.get("status") == "completed" or result.get("canonicalization", {}).get("status") == "completed" - if not successful: - raise RuntimeError(result.get("message") or "Workflow maintenance failed") - except Exception as exc: - with SyncSessionLocal() as session: - task = session.query(WorkflowMaintenanceTask).filter_by(task_id=tid).first() - task.status, task.error, task.completed_at = "failed", str(exc), datetime.now(timezone.utc) - session.commit() - return {"task_id": str(tid), "status": "failed", "error": str(exc)} - with SyncSessionLocal() as session: - task = session.query(WorkflowMaintenanceTask).filter_by(task_id=tid).first() - task.status, task.result, task.completed_at = "completed", result, datetime.now(timezone.utc) - session.commit() - return {"task_id": str(tid), "status": "completed", "result": result} - - -def run_pending_recanonicalizations( - *, model: str | None = None, verbose: bool = False, progress_callback=None, -) -> dict: - """Run all currently pending recanonicalizations as one global batch job.""" - from mkb.db.models import WorkflowMaintenanceTask - - init_db() - with SyncSessionLocal() as session: - rows = session.query(WorkflowMaintenanceTask).filter_by( - task_type="recanonicalize", status="pending", - ).order_by(WorkflowMaintenanceTask.created_at.desc()).all() - latest_by_project = {} - duplicates = [] - for row in rows: - if row.project_id in latest_by_project: - duplicates.append((row, latest_by_project[row.project_id])) - else: - latest_by_project[row.project_id] = row - for duplicate, retained in duplicates: - duplicate.status = "superseded" - duplicate.result = { - **(duplicate.result or {}), - "superseded_by_task_id": str(retained.task_id), - } - session.commit() - task_ids = [row.task_id for row in latest_by_project.values()] - results = [] - completed = 0 - failed = 0 - for index, task_id in enumerate(task_ids, 1): - if progress_callback: - progress_callback({ - "stage": "recanonicalization_batch", - "message": f"Recanonicalizing project workflow {index}/{len(task_ids)}", - }) - result = run_workflow_maintenance_task( - task_id, model=model, verbose=verbose, - progress_callback=progress_callback, - ) - results.append(result) - if result.get("status") == "completed": - completed += 1 - else: - failed += 1 - return { - "status": "completed" if failed == 0 else "completed_with_errors", - "task_count": len(task_ids), "completed": completed, "failed": failed, - "duplicate_tasks_coalesced": len(duplicates), - "results": results, - } - - -def rebuild_workflow_indexes(project_id: str | uuid.UUID | None = None) -> dict: - from mkb.db.models import CanonicalWorkflow, RawWorkflowExtraction, WorkflowIndexEntry - from mkb.workflows.indexing import build_index_entries - from mkb.workflows.schema_library import get_schema_library_payload - - init_db() - with SyncSessionLocal() as session: - query = session.query(CanonicalWorkflow).filter_by(status="COMPLETED") - if project_id: - query = query.filter_by(project_id=uuid.UUID(str(project_id))) - rows = query.all() - ids = [row.canonicalization_id for row in rows] - if ids: - session.query(WorkflowIndexEntry).filter(WorkflowIndexEntry.canonicalization_id.in_(ids)).delete(synchronize_session=False) - count = 0 - for row in rows: - raw = session.query(RawWorkflowExtraction).filter_by(extraction_id=row.raw_extraction_id).first() - schema = get_schema_library_payload(row.schema_version) - for entry in build_index_entries(row.graph or {}, raw.graph if raw else {}, schema): - session.add(WorkflowIndexEntry(canonicalization_id=row.canonicalization_id, project_id=row.project_id, **entry)) - count += 1 - session.commit() - return {"workflows_indexed": len(rows), "entries_created": count} - - -def search_canonical_workflows(source: str | None = None, operation: str | None = None, target: str | None = None, mode: str = "strict", limit: int = 100) -> list[dict]: - """Search persisted workflow indexes and return evidence-rich explanations.""" - from mkb.db.models import CanonicalWorkflow, WorkflowIndexEntry - from mkb.workflows.indexing import QUERY_MODES, match_index_entry, normalize - - legacy_modes = {"exact": "strict", "relaxed": "alias-expanded", "expanded": "granularity-expanded", "summarized": "granularity-expanded"} - mode = legacy_modes.get(mode, mode) - if mode not in QUERY_MODES: - raise ValueError(f"Unsupported query mode: {mode}") - init_db() - results = [] - seen_paths = set() - with SyncSessionLocal() as session: - completed = session.query(CanonicalWorkflow).filter_by(status="COMPLETED").order_by(CanonicalWorkflow.project_id, CanonicalWorkflow.version.desc()).all() - latest = {} - for row in completed: - latest.setdefault(row.project_id, row) - rows_by_id = {row.canonicalization_id: row for row in latest.values()} - entry_query = session.query(WorkflowIndexEntry).filter( - WorkflowIndexEntry.canonicalization_id.in_(rows_by_id) - ) if rows_by_id else None - if entry_query is not None and mode in {"strict", "alias-expanded", "evidence-required"}: - entry_query = entry_query.filter(WorkflowIndexEntry.index_type == "direct") - if source: - entry_query = entry_query.filter(WorkflowIndexEntry.source_label == normalize(source)) - if target: - entry_query = entry_query.filter(WorkflowIndexEntry.target_label == normalize(target)) - if operation and mode in {"strict", "evidence-required"}: - entry_query = entry_query.filter(WorkflowIndexEntry.operation_label == normalize(operation)) - entries = entry_query.all() if entry_query is not None else [] - for entry in entries: - data = {column.name: getattr(entry, column.name) for column in WorkflowIndexEntry.__table__.columns} - matched, explanation = match_index_entry(data, source=source, operation=operation, target=target, mode=mode) - if not matched: - continue - result_key = (entry.canonicalization_id, tuple(entry.path_node_ids)) - if result_key in seen_paths: - continue - seen_paths.add(result_key) - canonical = rows_by_id[entry.canonicalization_id] - graph_nodes = {node["node_id"]: node for node in (canonical.graph or {}).get("nodes", [])} - results.append({ - "project_id": str(entry.project_id), - "canonicalization_id": str(entry.canonicalization_id), - "version": canonical.version, "mode": mode, - "path": [graph_nodes[node_id] for node_id in entry.path_node_ids if node_id in graph_nodes], - "explanation": explanation, - }) - if len(results) >= limit: - break - return results - - -def assign_projects_to_group( - project_ids: list[str | uuid.UUID], - group_id: str | uuid.UUID | None, -) -> dict: - """Assign multiple projects to a group, or to no group when ``group_id`` is None.""" - from mkb.db.models import ProjectGroup, ResearchProject - - if not project_ids: - return {"updated": 0, "group_id": None} - - pids = [uuid.UUID(str(p)) for p in project_ids] - gid = uuid.UUID(str(group_id)) if group_id else None - - init_db() - with SyncSessionLocal() as session: - if gid is not None: - group = session.query(ProjectGroup).filter_by(group_id=gid).first() - if not group: - return {"error": f"Group {group_id} not found"} - updated = ( - session.query(ResearchProject) - .filter(ResearchProject.project_id.in_(pids)) - .update({ResearchProject.group_id: gid}, synchronize_session=False) - ) - session.commit() - return {"updated": int(updated), "group_id": str(gid) if gid else None} - - -def list_assets(project_id: str | uuid.UUID | None = None, limit: int = 100) -> list[dict]: - """List assets, optionally filtered by project.""" - from mkb.db.models import Asset, ProjectAsset - - with SyncSessionLocal() as session: - if project_id is not None: - pid = uuid.UUID(str(project_id)) - links = session.query(ProjectAsset).filter_by(project_id=pid).all() - asset_ids = [l.asset_id for l in links] - if not asset_ids: - return [] - assets = session.query(Asset).filter(Asset.asset_id.in_(asset_ids)).all() - else: - assets = ( - session.query(Asset) - .order_by(Asset.created_at.desc()) - .limit(limit) - .all() - ) - return [ - { - "asset_id": str(a.asset_id), - "filename": a.filename, - "mime_type": a.mime_type, - "size_bytes": a.size_bytes, - "status": a.status.value, - } - for a in assets - ] - - -def search_library( - query: str, - limit: int = 25, - project_id: str | uuid.UUID | None = None, -) -> dict: - """Search projects and assets by keyword. - - Every keyword token must match somewhere in the target record. Projects are - searched by label and source path. Assets are searched by filename, MIME - type, and selected metadata fields. - """ - from sqlalchemy import and_, or_ - - from mkb.db.models import Asset, ProjectAsset, ResearchProject - - init_db() - tokens = [token.lower() for token in _normalize_search_query(query)] - if not tokens: - return { - "query": query, - "tokens": [], - "project_id": str(project_id) if project_id is not None else None, - "projects": [], - "assets": [], - "total": 0, - } - - pid = uuid.UUID(str(project_id)) if project_id is not None else None - - with SyncSessionLocal() as session: - project_filters = [ - or_( - ResearchProject.label.ilike(f"%{token}%"), - ResearchProject.source_path.ilike(f"%{token}%"), - ) - for token in tokens - ] - project_query = session.query(ResearchProject) - if pid is not None: - project_query = project_query.filter(ResearchProject.project_id == pid) - project_rows = ( - project_query - .filter(and_(*project_filters)) - .order_by(ResearchProject.created_at.desc()) - .limit(limit) - .all() - ) - - asset_filters = [ - or_( - Asset.filename.ilike(f"%{token}%"), - Asset.mime_type.ilike(f"%{token}%"), - Asset.metadata_["title"].astext.ilike(f"%{token}%"), - Asset.metadata_["description"].astext.ilike(f"%{token}%"), - Asset.metadata_["original_path"].astext.ilike(f"%{token}%"), - ) - for token in tokens - ] - asset_query = session.query(Asset, ProjectAsset.project_id).outerjoin( - ProjectAsset, - ProjectAsset.asset_id == Asset.asset_id, - ) - if pid is not None: - asset_query = asset_query.filter(ProjectAsset.project_id == pid) - asset_rows = ( - asset_query - .filter(and_(*asset_filters)) - .order_by(Asset.created_at.desc()) - .limit(limit) - .all() - ) - - projects = [ - { - "project_id": str(row.project_id), - "label": row.label, - "source_path": row.source_path, - "file_count": row.file_count, - "kind": "project", - } - for row in project_rows - ] - - assets = [] - for asset, asset_project_id in asset_rows: - metadata = asset.metadata_ or {} - if not _matches_search_tokens( - asset.filename, - asset.mime_type, - metadata.get("title"), - metadata.get("description"), - metadata.get("original_path"), - tokens=tokens, - ): - continue - - assets.append( - { - "asset_id": str(asset.asset_id), - "project_id": str(asset_project_id) if asset_project_id else None, - "filename": asset.filename, - "mime_type": asset.mime_type, - "size_bytes": asset.size_bytes, - "status": asset.status.value, - "kind": "asset", - } - ) - - return { - "query": query, - "tokens": tokens, - "project_id": str(pid) if pid is not None else None, - "projects": projects, - "assets": assets[:limit], - "total": len(projects) + len(assets[:limit]), - } - - -# ── Spaces ─────────────────────────────────────────────────────── - - -def create_space( - name: str, - domain: str, - extraction_schema: dict, - system_prompt: str, - field_descriptions: dict, - description: str | None = None, - purpose: str = "tabular_database", - review_prompt: str | None = None, - review_trackable: bool = True, - review_allow_search: bool = False, - review_search_tools: list[str] | None = None, - post_processors: list[dict] | None = None, -) -> dict: - """Create a new space (domain-specific extraction configuration).""" - from mkb.spaces.registry import create_space as _create - - return _create( - name=name, - domain=domain, - extraction_schema=extraction_schema, - system_prompt=system_prompt, - field_descriptions=field_descriptions, - description=description, - purpose=purpose, - review_prompt=review_prompt, - review_trackable=review_trackable, - review_allow_search=review_allow_search, - review_search_tools=review_search_tools, - post_processors=post_processors, - ) - - -def update_space(space_id: str | uuid.UUID, **changes) -> dict: - """Update fields on an existing space. Bumps version automatically. - - Accepted keys: name, description, extraction_schema, system_prompt, - field_descriptions, domain, purpose, review_prompt. - """ - from mkb.spaces.registry import update_space as _update - - return _update(space_id, **changes) - - -def delete_space(space_id: str | uuid.UUID) -> dict: - """Delete a space by id.""" - from mkb.spaces.registry import delete_space as _delete - - return _delete(space_id) - - -def list_spaces() -> list[dict]: - """List all spaces.""" - from mkb.spaces.registry import list_spaces as _list - - return _list() - - -def get_space(space_id_or_name: str) -> dict | None: - """Get a space by ID or name.""" - from mkb.spaces.registry import get_space as _get - - return _get(space_id_or_name) - - -# ── Projections ────────────────────────────────────────────────── - - -def project( - space_id: str | uuid.UUID, - frame_id: str | uuid.UUID | None = None, - project_id: str | uuid.UUID | None = None, - model: str | None = None, - verbose: bool = False, - progress_callback=None, - source_type: str = "frame", -) -> dict: - """Run projection on one or more frames using a space definition. - - Args: - source_type: ``"frame"`` (default) to project from the curated knowledge - frame, or ``"markdown"`` to project directly from the processed - Markdown of the project's papers (no extraction step required). - - If frame_id given, project that specific frame. - If project_id given, find or auto-create the frame for that project. - """ - from mkb.agents.projection import run_projection - from mkb.db.models import KnowledgeFrame - - init_db() - sid = uuid.UUID(str(space_id)) - source_kind = (source_type or "frame").strip().lower() - - if frame_id: - fid = uuid.UUID(str(frame_id)) - return run_projection( - sid, fid, model=model, verbose=verbose, - progress_callback=progress_callback, source_type=source_kind, - ) - - if project_id: - pid = uuid.UUID(str(project_id)) - if source_kind == "markdown": - # Frame may not exist yet; the agent runner will create one. - return run_projection( - sid, None, model=model, verbose=verbose, - progress_callback=progress_callback, - source_type=source_kind, project_id=pid, - ) - with SyncSessionLocal() as session: - frame = session.query(KnowledgeFrame).filter_by(project_id=pid).first() - if not frame: - return {"error": f"No frame for project {project_id}"} - fid = frame.frame_id - return run_projection( - sid, fid, model=model, verbose=verbose, - progress_callback=progress_callback, source_type=source_kind, - ) - - return {"error": "Must specify frame_id or project_id"} - - -def project_all( - space_id: str | uuid.UUID, - model: str | None = None, - verbose: bool = False, - source_type: str = "frame", -) -> dict: - """Run projection on all completed frames (or all projects) using a space.""" - from mkb.agents.projection import run_projection_all - - init_db() - sid = uuid.UUID(str(space_id)) - return run_projection_all(sid, model=model, verbose=verbose, source_type=source_type) - - -def get_projection(projection_id: str | uuid.UUID) -> dict | None: - """Get a projection by ID.""" - from mkb.db.models import KnowledgeFrame, Projection, Space - from mkb.spaces.schema_utils import normalize_projection_data - - init_db() - pid = uuid.UUID(str(projection_id)) - with SyncSessionLocal() as session: - proj = session.query(Projection).filter_by(projection_id=pid).first() - if not proj: - return None - frame = session.query(KnowledgeFrame).filter_by(frame_id=proj.frame_id).first() - space = session.query(Space).filter_by(space_id=proj.space_id).first() - normalized_data, normalized_validation = normalize_projection_data( - proj.data or {}, - space.extraction_schema if space else {}, - ) - - validation_result = proj.validation_result or {} - if normalized_validation: - validation_result = { - **normalized_validation, - **validation_result, - } - - return { - "projection_id": str(proj.projection_id), - "space_id": str(proj.space_id), - "frame_id": str(proj.frame_id), - "project_id": str(frame.project_id) if frame else None, - "status": proj.status.value, - "data": normalized_data, - "validation_result": validation_result or None, - "agent_notes": proj.agent_notes, - "extracted_at": proj.extracted_at.isoformat() if proj.extracted_at else None, - "space_version": proj.space_version, - "source_type": getattr(proj, "source_type", "frame"), - "times_reviewed": proj.times_reviewed, - "review_notes": proj.review_notes, - "reviewed_at": proj.reviewed_at.isoformat() if proj.reviewed_at else None, - "created_at": proj.created_at.isoformat() if proj.created_at else None, - } - - -def delete_projection(projection_id: str | uuid.UUID) -> bool: - """Soft-delete a projection by setting deleted_at. Returns True if found.""" - from datetime import datetime, timezone - - from mkb.db.models import Projection - - init_db() - pid = uuid.UUID(str(projection_id)) - with SyncSessionLocal() as session: - proj = session.query(Projection).filter_by(projection_id=pid).first() - if not proj: - return False - proj.deleted_at = datetime.now(timezone.utc) - session.commit() - return True - - -def list_projections( - space_id: str | uuid.UUID | None = None, - frame_id: str | uuid.UUID | None = None, - project_id: str | uuid.UUID | None = None, - include_data: bool = False, - newest_only: bool = False, - include_history: bool = False, -) -> list[dict]: - """List projections, optionally filtered by space, frame, or project. - - Args: - include_history: When False (default), superseded projections (those - replaced by a tracked review) are hidden. When True, the full - history is returned, including ``superseded_by_id`` / - ``supersedes_ids`` pointers so callers can rebuild the chain. - """ - from mkb.db.models import KnowledgeFrame, Projection, Space - from mkb.spaces.schema_utils import normalize_projection_data - - init_db() - with SyncSessionLocal() as session: - q = ( - session.query(Projection, KnowledgeFrame.project_id) - .outerjoin(KnowledgeFrame, Projection.frame_id == KnowledgeFrame.frame_id) - .filter(Projection.deleted_at.is_(None)) - .order_by(Projection.created_at.desc(), Projection.extracted_at.desc()) - ) - if not include_history: - q = q.filter(Projection.superseded_by_id.is_(None)) - if space_id: - q = q.filter(Projection.space_id == uuid.UUID(str(space_id))) - if frame_id: - q = q.filter(Projection.frame_id == uuid.UUID(str(frame_id))) - if project_id: - q = q.filter(KnowledgeFrame.project_id == uuid.UUID(str(project_id))) - projections = q.all() - - results = [] - seen_keys: set[tuple[str, str]] = set() - for projection, projection_project_id in projections: - project_value = str(projection_project_id) if projection_project_id else None - dedupe_key = (str(projection.space_id), project_value or str(projection.frame_id)) - if newest_only and dedupe_key in seen_keys: - continue - seen_keys.add(dedupe_key) - - item = { - "projection_id": str(projection.projection_id), - "space_id": str(projection.space_id), - "frame_id": str(projection.frame_id), - "project_id": project_value, - "status": projection.status.value, - "agent_notes": projection.agent_notes, - "extracted_at": projection.extracted_at.isoformat() if projection.extracted_at else None, - "created_at": projection.created_at.isoformat() if projection.created_at else None, - "space_version": projection.space_version, - "source_type": getattr(projection, "source_type", "frame"), - "times_reviewed": projection.times_reviewed, - "review_notes": projection.review_notes, - "reviewed_at": projection.reviewed_at.isoformat() if projection.reviewed_at else None, - "superseded_by_id": ( - str(projection.superseded_by_id) - if getattr(projection, "superseded_by_id", None) - else None - ), - "supersedes_ids": getattr(projection, "supersedes_ids", None), - } - if include_data: - space = session.query(Space).filter_by(space_id=projection.space_id).first() - normalized_data, _ = normalize_projection_data( - projection.data or {}, - space.extraction_schema if space else {}, - ) - item["data"] = normalized_data - results.append(item) - - return results - - -# ── Projection Exports ────────────────────────────────────────── - - -def _serialize_projection_payload( - projection_id: uuid.UUID, - out_path: Path, - format: str, -) -> Path: - """Generic single-projection dump (used for non-qa_benchmark spaces).""" - import json as _json - - from mkb.db.models import KnowledgeFrame, Projection, Space - - with SyncSessionLocal() as session: - proj = session.query(Projection).filter_by(projection_id=projection_id).first() - if not proj: - raise ValueError(f"Projection {projection_id} not found") - frame = session.query(KnowledgeFrame).filter_by(frame_id=proj.frame_id).first() - space = session.query(Space).filter_by(space_id=proj.space_id).first() - record = { - "projection_id": str(proj.projection_id), - "space": { - "space_id": str(proj.space_id), - "name": space.name if space else None, - "purpose": getattr(space, "purpose", None) if space else None, - "version": proj.space_version, - }, - "frame_id": str(proj.frame_id) if proj.frame_id else None, - "project_id": str(frame.project_id) if frame else None, - "status": proj.status.value, - "source_type": getattr(proj, "source_type", "frame"), - "extracted_at": proj.extracted_at.isoformat() if proj.extracted_at else None, - "agent_notes": proj.agent_notes, - "data": proj.data or {}, - } - - fmt = (format or "yaml").strip().lower() - out_path.parent.mkdir(parents=True, exist_ok=True) - if fmt == "json": - out_path.write_text(_json.dumps(record, indent=2, ensure_ascii=False, default=str)) - else: - import yaml as _yaml - out_path.write_text( - _yaml.safe_dump(record, sort_keys=False, allow_unicode=True) - ) - return out_path - - -def export_projection( - projection_id: str | uuid.UUID, - out_dir: str | Path, - format: str = "yaml", - overwrite: bool = False, -) -> dict: - """Export a single projection to disk. - - For ``qa_benchmark`` spaces, delegates to the mat_agent_bench exporter - (one YAML per question under ``//.yaml``). - For all other purposes, writes a single ``.`` file - containing the projection payload + metadata. - """ - from mkb.db.models import Projection, Space - from mkb.spaces.export_qa_bench import ( - QABenchExportError, - export_projection_to_yaml, - ) - - init_db() - pid = uuid.UUID(str(projection_id)) - out_root = Path(out_dir) - fmt = (format or "yaml").strip().lower() - if fmt not in {"yaml", "json"}: - raise ValueError(f"Unsupported export format: {format}") - - with SyncSessionLocal() as session: - proj = session.query(Projection).filter_by(projection_id=pid).first() - if not proj: - return {"error": f"Projection {projection_id} not found"} - space = session.query(Space).filter_by(space_id=proj.space_id).first() - purpose = getattr(space, "purpose", None) if space else None - - if purpose == "qa_benchmark" and fmt == "yaml": - try: - return export_projection_to_yaml(pid, out_root, overwrite=overwrite) - except QABenchExportError as e: - return {"error": str(e)} - - out_path = out_root / f"{pid}.{fmt}" - if out_path.exists() and not overwrite: - return {"error": f"{out_path} already exists (pass overwrite=True)"} - _serialize_projection_payload(pid, out_path, fmt) - return {"files": [str(out_path)], "skipped": [], "warnings": []} - - -def export_space_projections( - space_id_or_name: str | uuid.UUID, - out_dir: str | Path, - format: str = "yaml", - overwrite: bool = False, - newest_only: bool = True, -) -> dict: - """Export every projection belonging to a space. - - For ``qa_benchmark`` spaces and ``format='yaml'`` this delegates to the - aggregated mat_agent_bench exporter. Otherwise one file is written per - projection: ``/.``. - """ - from mkb.db.models import Projection, ProjectionStatus, Space - from mkb.spaces.export_qa_bench import ( - QABenchExportError, - export_space_to_yaml, - ) - - init_db() - out_root = Path(out_dir) - fmt = (format or "yaml").strip().lower() - if fmt not in {"yaml", "json"}: - raise ValueError(f"Unsupported export format: {format}") - - with SyncSessionLocal() as session: - try: - sid = uuid.UUID(str(space_id_or_name)) - space = session.query(Space).filter_by(space_id=sid).first() - except (ValueError, AttributeError): - space = session.query(Space).filter_by(name=str(space_id_or_name)).first() - if not space: - return {"error": f"Space {space_id_or_name} not found"} - purpose = getattr(space, "purpose", None) - space_id = space.space_id - - if purpose == "qa_benchmark" and fmt == "yaml": - try: - return export_space_to_yaml(space_id, out_root, overwrite=overwrite) - except QABenchExportError as e: - return {"error": str(e)} - - # Generic per-projection dump - with SyncSessionLocal() as session: - q = ( - session.query(Projection) - .filter(Projection.space_id == space_id) - .filter(Projection.deleted_at.is_(None)) - .filter(Projection.status == ProjectionStatus.COMPLETED) - .order_by(Projection.created_at.desc()) - ) - projections = q.all() - if newest_only: - seen: set[str] = set() - unique = [] - for p in projections: - key = str(p.frame_id) - if key in seen: - continue - seen.add(key) - unique.append(p) - projections = unique - ids = [p.projection_id for p in projections] - - written: list[str] = [] - skipped: list[dict] = [] - out_root.mkdir(parents=True, exist_ok=True) - for pid in ids: - out_path = out_root / f"{pid}.{fmt}" - if out_path.exists() and not overwrite: - skipped.append({"id": str(pid), "reason": "exists"}) - continue - _serialize_projection_payload(pid, out_path, fmt) - written.append(str(out_path)) - - return {"files": written, "skipped": skipped, "warnings": []} - - -# ── Knowledge Graphs ──────────────────────────────────────────── - -def clear_knowledge_graphs( - project_id: str | uuid.UUID | None = None, - remove_legacy_frame_sections: bool = True, -) -> dict: - """Delete (soft-delete) old KG projections and optionally purge legacy frame graph sections.""" - from mkb.knowledge_graph import clear_knowledge_graph_projections, purge_legacy_graph_sections - - init_db() - pid = uuid.UUID(str(project_id)) if project_id else None - deleted = clear_knowledge_graph_projections(project_id=pid, include_legacy_spaces=True) - - purged = {"updated_frames": 0, "project_id": str(pid) if pid else None} - if remove_legacy_frame_sections: - purged = purge_legacy_graph_sections(project_id=pid) - - return { - "deleted_projections": deleted, - "purged_legacy_frame_sections": purged, - } - - -def extract_knowledge_graph( - project_id: str | uuid.UUID | None = None, - frame_id: str | uuid.UUID | None = None, - model: str | None = None, - verbose: bool = False, - clear_existing: bool = True, - clear_legacy_frame_sections: bool = True, - progress_callback=None, -) -> dict: - """Run concept-graph extraction using one global cross-domain space. - - If frame_id is provided, extract for that frame. - If project_id is provided, resolve that project's frame and extract. - Otherwise run for all completed frames. - """ - from mkb.agents.knowledge_graph import run_knowledge_graph, run_knowledge_graph_all - from mkb.db.models import KnowledgeFrame - from mkb.knowledge_graph import ensure_global_kg_space_id, purge_legacy_graph_sections - - init_db() - - if clear_legacy_frame_sections: - if project_id: - purge_legacy_graph_sections(project_id=uuid.UUID(str(project_id))) - else: - purge_legacy_graph_sections() - - if frame_id is not None: - fid = uuid.UUID(str(frame_id)) - result = run_knowledge_graph( - fid, - model=model, - verbose=verbose, - clear_existing=clear_existing, - progress_callback=progress_callback, - ) - elif project_id is not None: - pid = uuid.UUID(str(project_id)) - with SyncSessionLocal() as session: - frame = session.query(KnowledgeFrame).filter_by(project_id=pid).first() - if not frame: - return {"error": f"No frame for project {project_id}"} - fid = frame.frame_id - result = run_knowledge_graph( - fid, - model=model, - verbose=verbose, - clear_existing=clear_existing, - progress_callback=progress_callback, - ) - else: - result = run_knowledge_graph_all(model=model, verbose=verbose, clear_existing=clear_existing) - - return { - "global_space_id": str(ensure_global_kg_space_id()), - **result, - } - - -def get_knowledge_graph( - project_id: str | uuid.UUID | None = None, -) -> dict: - """Get the merged concept graph from the singleton global KG space.""" - from mkb.agents.tools.knowledge_graph import normalize_knowledge_graph_payload - from mkb.db.models import KnowledgeFrame, Projection, ProjectionStatus - from mkb.knowledge_graph import ensure_global_kg_space_id - - init_db() - sid = ensure_global_kg_space_id() - - with SyncSessionLocal() as session: - q = ( - session.query(Projection) - .filter(Projection.space_id == sid) - .filter(Projection.deleted_at.is_(None)) - .filter(Projection.status.in_([ProjectionStatus.COMPLETED, ProjectionStatus.REVIEWED])) - ) - if project_id: - pid = uuid.UUID(str(project_id)) - q = q.join(KnowledgeFrame, Projection.frame_id == KnowledgeFrame.frame_id) - q = q.filter(KnowledgeFrame.project_id == pid) - rows = q.all() - - aggregate = {"concepts": [], "relations": []} - for row in rows: - payload, _ = normalize_knowledge_graph_payload(row.data or {}) - aggregate["concepts"].extend(payload["concepts"]) - aggregate["relations"].extend(payload["relations"]) - - merged, validation = normalize_knowledge_graph_payload(aggregate) - return { - "space_id": str(sid), - "projection_count": len(rows), - "graph": merged, - "validation": validation, - "project_id": str(project_id) if project_id else None, - } - - -# ── Feedback ───────────────────────────────────────────────────── - - -def list_feedback( - project_id: str | uuid.UUID | None = None, - status: str | None = None, -) -> list[dict]: - """List feedback items, optionally filtered by project and/or status.""" - from mkb.feedback.manager import list_feedback as _list - - pid = uuid.UUID(str(project_id)) if project_id else None - return _list(project_id=pid, status=status) - - -def review_feedback( - project_id: str | uuid.UUID, - model: str | None = None, - verbose: bool = False, - progress_callback=None, -) -> dict: - """Run feedback review on a project — KB agent reviews and resolves open feedback.""" - from mkb.agents.feedback_reviewer import run_feedback_review - - pid = uuid.UUID(str(project_id)) - if progress_callback: - progress_callback({"message": f"Reviewing feedback for project {str(pid)[:8]}"}) - return run_feedback_review(pid, model=model, verbose=verbose) - - -def review_projections( - space_id: str | uuid.UUID, - project_id: str | uuid.UUID, - model: str | None = None, - verbose: bool = False, - progress_callback=None, - reviewer_id: str | None = None, -) -> dict: - """Run projection review — consolidate and correct all projections for a project. - - A strict reviewer agent compares all projection runs, cross-references - against the knowledge frame and source material, and produces a single - reviewed (consolidated, corrected) projection. - """ - from mkb.agents.projection_reviewer import run_projection_review - - init_db() - sid = uuid.UUID(str(space_id)) - pid = uuid.UUID(str(project_id)) - if progress_callback: - progress_callback({"message": f"Reviewing projections for project {str(pid)[:8]}"}) - return run_projection_review( - sid, - pid, - model=model, - verbose=verbose, - progress_callback=progress_callback, - reviewer_id=reviewer_id, - ) - - -def review_projections_all( - space_id: str | uuid.UUID, - model: str | None = None, - verbose: bool = False, - progress_callback=None, - project_ids: list[str] | None = None, - reviewer_id: str | None = None, -) -> dict: - """Run projection review on projects in a space. - - Args: - project_ids: Restrict to these projects (per-project, separate - sessions). When None, reviews every project that has at least - one completed projection in the space. - """ - from mkb.agents.projection_reviewer import run_projection_review_all - - init_db() - sid = uuid.UUID(str(space_id)) - pids = [uuid.UUID(str(p)) for p in project_ids] if project_ids else None - return run_projection_review_all( - sid, - model=model, - verbose=verbose, - progress_callback=progress_callback, - project_ids=pids, - reviewer_id=reviewer_id, - ) - - -def review_projections_session( - space_id: str | uuid.UUID, - project_ids: list[str], - model: str | None = None, - verbose: bool = False, - progress_callback=None, - reviewer_id: str | None = None, -) -> dict: - """Run a SINGLE reviewer session over multiple selected projects. - - One agent context sees every selected project's projections in turn - and saves each reviewed result before moving on to the next. - """ - from mkb.agents.projection_reviewer import run_projection_review_session - - init_db() - sid = uuid.UUID(str(space_id)) - pids = [uuid.UUID(str(p)) for p in project_ids] - if not pids: - return {"status": "error", "message": "project_ids is required for session mode"} - return run_projection_review_session( - sid, - pids, - model=model, - verbose=verbose, - progress_callback=progress_callback, - reviewer_id=reviewer_id, - ) - - -def review_projection_followup( - space_id: str | uuid.UUID, - project_id: str | uuid.UUID, - message: str, - previous_job: dict | None = None, - model: str | None = None, - verbose: bool = False, - progress_callback=None, - reviewer_id: str | None = None, -) -> dict: - """Run a follow-up turn for a completed projection review job.""" - from mkb.agents.projection_reviewer import run_projection_review_followup - - init_db() - sid = uuid.UUID(str(space_id)) - pid = uuid.UUID(str(project_id)) - if progress_callback: - progress_callback({"message": "Starting review follow-up"}) - return run_projection_review_followup( - sid, - pid, - message, - previous_job=previous_job, - model=model, - verbose=verbose, - progress_callback=progress_callback, - reviewer_id=reviewer_id, - ) - - -def review_knowledge_graph( - mode: str = "auto", - model: str | None = None, - verbose: bool = False, - seed_count: int = 10, - progress_callback=None, -) -> dict: - """Run the graph review agent to deduplicate and clean the knowledge graph. - - Two modes: - - "global": analyzes relation name distributions, standardizes naming, merges - duplicate or synonymous concept nodes across the entire graph. - - "local": selects the least-reviewed concepts as starting points, explores - their neighborhoods, verifies against source frames, and fixes local issues. - - "auto" (default): randomly picks global or local each time. - - After each run, the times_examined and times_modified counters on each visited - graph element are incremented in the graph_element_reviews table. - - Args: - mode: "global", "local", or "auto". - model: LLM model override. - verbose: Enable verbose logging. - seed_count: Number of starting concepts for local mode. - """ - from mkb.agents.graph_review import run_graph_review - - init_db() - return run_graph_review(mode=mode, model=model, verbose=verbose, seed_count=seed_count, progress_callback=progress_callback) - - -def get_graph_review_counts(space_id: str | uuid.UUID | None = None) -> dict: - """Return review counts (times_examined, times_modified) per graph element. - - Returns a dict with two sub-dicts keyed by normalized element key: - - ``concepts``: mapping of normalized concept label → {times_examined, times_modified} - - ``relations``: mapping of "src||rel||tgt" → {times_examined, times_modified} - """ - from mkb.db.models import GraphElementReview - from mkb.knowledge_graph import ensure_global_kg_space_id - - init_db() - sid = uuid.UUID(str(space_id)) if space_id else ensure_global_kg_space_id() - - concepts: dict[str, dict] = {} - relations: dict[str, dict] = {} - - with SyncSessionLocal() as session: - rows = session.query(GraphElementReview).filter_by(space_id=sid).all() - for row in rows: - entry = { - "times_examined": row.times_examined, - "times_modified": row.times_modified, - "last_examined_at": row.last_examined_at.isoformat() if row.last_examined_at else None, - "last_modified_at": row.last_modified_at.isoformat() if row.last_modified_at else None, - } - if row.element_type == "concept": - concepts[row.element_key] = entry - elif row.element_type == "relation": - relations[row.element_key] = entry - - return {"space_id": str(sid), "concepts": concepts, "relations": relations} - - -def get_feedback_summary( - project_id: str | uuid.UUID, -) -> dict: - """Get counts of feedback by category and status for a project.""" - from mkb.feedback.manager import get_feedback_summary as _summary - - pid = uuid.UUID(str(project_id)) - return _summary(pid) - - -def resolve_feedback( - feedback_id: str | uuid.UUID, - status: str, - notes: str, -) -> dict: - """Manually resolve a feedback item.""" - from mkb.feedback.manager import resolve_feedback as _resolve - - fid = uuid.UUID(str(feedback_id)) - return _resolve(fid, status=status, notes=notes, resolved_by="user") +import sys +import types + +from mkb.services import ( + _api_common, + assets as _assets, + feedback as _feedback, + frames as _frames, + graphs as _graphs, + ingest as _ingest, + projects as _projects, + projections as _projections, + runtime as _runtime, + spaces as _spaces, + workflows as _workflows, +) + +from mkb.services.runtime import ( + setup, + reset_db, +) + +from mkb.services.ingest import ( + ingest, + sync, + sync_project, +) + +from mkb.services.frames import ( + extract, + get_frame, + list_frames, + get_extraction_history, +) + +from mkb.services.assets import ( + process, + list_processed_assets, + link_manual_processed_data, + list_assets, + search_library, +) + +from mkb.services.projects import ( + serialize_group, + rename_project, + list_projects, + _serialize_group, + list_project_groups, + create_project_group, + update_project_group, + delete_project_group, + delete_project, + assign_projects_to_group, +) + +from mkb.services.workflows import ( + serialize_raw_workflow, + serialize_canonical_workflow, + extract_raw_workflow, + get_raw_workflow_extraction_readiness, + list_raw_workflows, + get_raw_workflow, + delete_raw_workflow_version, + delete_canonical_workflow_version, + _serialize_raw_workflow, + review_raw_workflow, + correct_raw_workflow, + canonicalize_workflow, + list_canonical_workflows, + get_canonical_workflow, + _serialize_canonical_workflow, + curate_workflow_schema, + list_schema_proposals, + get_schema_proposal_revisions, + edit_schema_proposal, + get_workflow_schema_status, + review_schema_proposal, + schedule_workflow_reextraction, + schedule_workflow_recanonicalization, + list_workflow_maintenance_tasks, + run_workflow_maintenance_task, + run_pending_recanonicalizations, + rebuild_workflow_indexes, + search_canonical_workflows, +) + +from mkb.services.spaces import ( + create_space, + update_space, + delete_space, + list_spaces, + get_space, +) + +from mkb.services.projections import ( + serialize_projection_payload, + project, + project_all, + get_projection, + delete_projection, + list_projections, + _serialize_projection_payload, + export_projection, + export_space_projections, +) + +from mkb.services.graphs import ( + clear_knowledge_graphs, + extract_knowledge_graph, + get_knowledge_graph, + review_knowledge_graph, + get_graph_review_counts, +) + +from mkb.services.feedback import ( + list_feedback, + review_feedback, + review_projections, + review_projections_all, + review_projections_session, + review_projection_followup, + get_feedback_summary, + resolve_feedback, +) + +from mkb.services._api_common import ( + _sha256_bytes, + _inspect_manual_processed_dir, + _choose_asset_for_manual_output, + _normalize_search_query, + _matches_search_tokens, + SyncSessionLocal, + init_db, +) + +__all__ = [ + "SyncSessionLocal", + "_choose_asset_for_manual_output", + "_inspect_manual_processed_dir", + "_matches_search_tokens", + "_normalize_search_query", + "_serialize_canonical_workflow", + "_serialize_group", + "_serialize_projection_payload", + "_serialize_raw_workflow", + "_sha256_bytes", + "assign_projects_to_group", + "canonicalize_workflow", + "clear_knowledge_graphs", + "correct_raw_workflow", + "create_project_group", + "create_space", + "curate_workflow_schema", + "delete_canonical_workflow_version", + "delete_project", + "delete_project_group", + "delete_projection", + "delete_raw_workflow_version", + "delete_space", + "edit_schema_proposal", + "export_projection", + "export_space_projections", + "extract", + "extract_knowledge_graph", + "extract_raw_workflow", + "get_canonical_workflow", + "get_extraction_history", + "get_feedback_summary", + "get_frame", + "get_graph_review_counts", + "get_knowledge_graph", + "get_projection", + "get_raw_workflow", + "get_raw_workflow_extraction_readiness", + "get_schema_proposal_revisions", + "get_space", + "get_workflow_schema_status", + "ingest", + "init_db", + "link_manual_processed_data", + "list_assets", + "list_canonical_workflows", + "list_feedback", + "list_frames", + "list_processed_assets", + "list_project_groups", + "list_projections", + "list_projects", + "list_raw_workflows", + "list_schema_proposals", + "list_spaces", + "list_workflow_maintenance_tasks", + "process", + "project", + "project_all", + "rebuild_workflow_indexes", + "rename_project", + "reset_db", + "resolve_feedback", + "review_feedback", + "review_knowledge_graph", + "review_projection_followup", + "review_projections", + "review_projections_all", + "review_projections_session", + "review_raw_workflow", + "review_schema_proposal", + "run_pending_recanonicalizations", + "run_workflow_maintenance_task", + "schedule_workflow_recanonicalization", + "schedule_workflow_reextraction", + "search_canonical_workflows", + "search_library", + "serialize_canonical_workflow", + "serialize_group", + "serialize_projection_payload", + "serialize_raw_workflow", + "setup", + "sync", + "sync_project", + "update_project_group", + "update_space", +] + +_MIRRORED_MODULES = ( + _api_common, + _assets, + _feedback, + _frames, + _graphs, + _ingest, + _projects, + _projections, + _runtime, + _spaces, + _workflows, +) + +class _ApiFacadeModule(types.ModuleType): + """Mirror monkeypatches on this facade into split service modules.""" + + def __setattr__(self, name: str, value): # type: ignore[override] + super().__setattr__(name, value) + for module in _MIRRORED_MODULES: + if hasattr(module, name): + setattr(module, name, value) + + +sys.modules[__name__].__class__ = _ApiFacadeModule diff --git a/src/mkb/processors/coordinator.py b/src/mkb/processors/coordinator.py index 3823759..748d5d6 100644 --- a/src/mkb/processors/coordinator.py +++ b/src/mkb/processors/coordinator.py @@ -3,10 +3,11 @@ Manages routing to appropriate processors, deduplication, and metadata tracking. """ -import hashlib import logging import uuid +from dataclasses import dataclass from pathlib import Path +from typing import Callable from mkb.db.engine import SyncSessionLocal from mkb.db.models import Asset, ProjectAsset, ProcessedAsset, ProcessingLog, ProcessingType @@ -18,17 +19,29 @@ logger = logging.getLogger(__name__) -# Registry of all available processors -PROCESSORS = [ - PDFProcessor(), - ExcelProcessor(), - CSVProcessor(), - JSONProcessor(), - BasicImageProcessor(), - TextProcessor(), + +@dataclass(frozen=True) +class ProcessorRegistration: + name: str + factory: Callable[[], object] + priority: int = 100 + + +PROCESSOR_REGISTRY = [ + ProcessorRegistration("pdf", PDFProcessor, priority=10), + ProcessorRegistration("excel", ExcelProcessor, priority=20), + ProcessorRegistration("csv", CSVProcessor, priority=30), + ProcessorRegistration("json", JSONProcessor, priority=40), + ProcessorRegistration("image", BasicImageProcessor, priority=50), + ProcessorRegistration("text", TextProcessor, priority=90), ] +def _iter_processors(): + for registration in sorted(PROCESSOR_REGISTRY, key=lambda item: item.priority): + yield registration.factory() + + def _mark_asset_metadata(asset: Asset, update: dict) -> None: """Persist processing markers on the raw asset metadata field.""" metadata = dict(asset.metadata_ or {}) @@ -65,7 +78,7 @@ def _select_processor(asset: Asset, raw_data: bytes): return CSVProcessor() return TextProcessor() - for proc in PROCESSORS: + for proc in _iter_processors(): if proc.can_process(asset.mime_type, asset.filename): return proc return None @@ -590,5 +603,3 @@ def process_all_pending(limit: int | None = None, progress_callback=None) -> dic stats["failed"] += 1 return stats - - diff --git a/src/mkb/processors/image_processor.py b/src/mkb/processors/image_processor.py index fb38356..3fb323c 100644 --- a/src/mkb/processors/image_processor.py +++ b/src/mkb/processors/image_processor.py @@ -6,9 +6,7 @@ import io import json import logging -from pathlib import Path -from mkb.db.models import ProcessingType from mkb.processors.base import ImageProcessor, ProcessingResult logger = logging.getLogger(__name__) diff --git a/src/mkb/services/__init__.py b/src/mkb/services/__init__.py new file mode 100644 index 0000000..dd110ed --- /dev/null +++ b/src/mkb/services/__init__.py @@ -0,0 +1,7 @@ +"""Domain service helpers for MKB. + +The legacy ``mkb.api`` module remains the compatibility facade. New shared +domain logic should live under this package and be called by API, web, CLI, and +agent adapters. +""" + diff --git a/src/mkb/services/_api_common.py b/src/mkb/services/_api_common.py new file mode 100644 index 0000000..e99876e --- /dev/null +++ b/src/mkb/services/_api_common.py @@ -0,0 +1,144 @@ +"""Shared imports and small helpers for API service modules.""" + +from __future__ import annotations + +import hashlib +import logging +import uuid +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from mkb.config import settings +from mkb.db.engine import SyncSessionLocal, init_db + +logger = logging.getLogger(__name__) + +__all__ = [ + "Any", + "Path", + "SyncSessionLocal", + "datetime", + "init_db", + "logger", + "settings", + "timezone", + "uuid", + "_choose_asset_for_manual_output", + "_inspect_manual_processed_dir", + "_matches_search_tokens", + "_normalize_search_query", + "_sha256_bytes", +] + + +def _sha256_bytes(data: bytes) -> str: + h = hashlib.sha256() + h.update(data) + return h.hexdigest() + +def _inspect_manual_processed_dir( + processed_dir: str | Path, + primary_file: str | None = None, +) -> dict: + """Inspect a handmade processed-output directory and describe its bundle.""" + root = Path(processed_dir).resolve() + if not root.is_dir(): + raise FileNotFoundError(f"Processed directory not found: {root}") + + files = sorted(p for p in root.rglob("*") if p.is_file()) + if not files: + raise FileNotFoundError(f"No files found in processed directory: {root}") + + if primary_file: + primary_path = (root / primary_file).resolve() + if not primary_path.is_file(): + raise FileNotFoundError(f"Primary file not found: {primary_path}") + else: + def _priority(path: Path) -> tuple[int, str]: + suffix = path.suffix.lower() + if suffix in {".md", ".markdown"}: + rank = 0 + elif suffix in {".parquet", ".csv", ".tsv"}: + rank = 1 + elif suffix == ".json": + rank = 2 + elif suffix in {".png", ".jpg", ".jpeg", ".gif", ".webp"}: + rank = 4 + else: + rank = 3 + return rank, path.relative_to(root).as_posix() + + primary_path = sorted(files, key=_priority)[0] + + primary_relpath = primary_path.relative_to(root).as_posix() + primary_bytes = primary_path.read_bytes() + + ext = primary_path.suffix.lower() + if ext in {".md", ".markdown", ".txt"}: + processing_type = "MARKDOWN" + elif ext in {".parquet", ".csv", ".tsv", ".json"}: + processing_type = "DATAFRAME" + elif ext in {".png", ".jpg", ".jpeg", ".gif", ".webp"}: + processing_type = "IMAGE" + else: + processing_type = "MARKDOWN" + + artifact_files = sorted( + p.relative_to(root).as_posix() + for p in files + if p != primary_path + ) + + bundle_hash = hashlib.sha256() + bundle_hash.update(primary_bytes) + for relpath in artifact_files: + data = (root / relpath).read_bytes() + bundle_hash.update(relpath.encode("utf-8")) + bundle_hash.update(_sha256_bytes(data).encode("utf-8")) + + from mkb.db.models import ProcessingType + + return { + "local_dir": str(root), + "primary_name": primary_path.name, + "primary_relpath": primary_relpath, + "processing_type": ProcessingType(processing_type), + "output_format": primary_path.suffix.lstrip(".") or "bin", + "artifact_files": artifact_files, + "size_bytes": len(primary_bytes), + "sha256": bundle_hash.hexdigest(), + } + +def _choose_asset_for_manual_output(assets: list, primary_name: str | None = None): + """Choose the most likely raw asset for a handmade processed bundle.""" + if not assets: + return None + if not primary_name: + return assets[0] + + primary_stem = Path(primary_name).stem.lower() + for asset in assets: + if Path(asset.filename).stem.lower() == primary_stem: + return asset + + for asset in assets: + asset_stem = Path(asset.filename).stem.lower() + if primary_stem in asset_stem or asset_stem in primary_stem: + return asset + + return assets[0] + +def _normalize_search_query(query: str) -> list[str]: + """Split a free-text query into non-empty keyword tokens.""" + return [token.strip() for token in query.split() if token.strip()] + +def _matches_search_tokens(*values: Any, tokens: list[str]) -> bool: + """Return True when every token is present in at least one candidate value.""" + haystacks = [str(value).lower() for value in values if value] + if not tokens: + return True + return all(any(token in haystack for haystack in haystacks) for token in tokens) + + +# ── Lifecycle ──────────────────────────────────────────────────── diff --git a/src/mkb/services/assets.py b/src/mkb/services/assets.py new file mode 100644 index 0000000..af949e2 --- /dev/null +++ b/src/mkb/services/assets.py @@ -0,0 +1,434 @@ +"""Assets API service functions.""" + +from __future__ import annotations + +from mkb.services._api_common import ( + Path, + SyncSessionLocal, + init_db, + settings, + uuid, + _choose_asset_for_manual_output, + _inspect_manual_processed_dir, + _matches_search_tokens, + _normalize_search_query, +) + +from mkb.services.ingest import ingest + + +def process(project_id: str | uuid.UUID | None = None, progress_callback=None) -> dict: + """Process assets. If project_id is given, process only that project's assets. + Otherwise process all pending assets. + + Returns a summary dict. + """ + from mkb.processors.coordinator import process_all_pending, process_asset + + if project_id is not None: + pid = uuid.UUID(str(project_id)) + from mkb.db.models import ProjectAsset + with SyncSessionLocal() as session: + links = session.query(ProjectAsset).filter_by(project_id=pid).all() + asset_ids = [link.asset_id for link in links] + + results = [] + for aid in asset_ids: + try: + if progress_callback: + progress_callback({"message": f"Starting asset {len(results) + 1}/{len(asset_ids)}", "asset_id": str(aid)}) + r = process_asset(aid, progress_callback=progress_callback) + results.append(r) + except Exception as exc: + results.append({"asset_id": str(aid), "error": str(exc)}) + return {"project_id": str(pid), "assets_processed": len(results), "results": results} + + return process_all_pending(progress_callback=progress_callback) + + +# ── Extraction ─────────────────────────────────────────────────── + +def list_processed_assets( + project_id: str | uuid.UUID | None = None, + limit: int = 100, +) -> list[dict]: + """List processed outputs, optionally filtered by project.""" + from mkb.db.models import Asset, ProcessedAsset, ProjectAsset + + with SyncSessionLocal() as session: + q = session.query(ProcessedAsset).order_by(ProcessedAsset.created_at.desc()) + if project_id is not None: + pid = uuid.UUID(str(project_id)) + links = session.query(ProjectAsset).filter_by(project_id=pid).all() + asset_ids = [link.asset_id for link in links] + if not asset_ids: + return [] + q = q.filter(ProcessedAsset.asset_id.in_(asset_ids)) + + rows = q.limit(limit).all() + result = [] + for row in rows: + asset = session.query(Asset).filter_by(asset_id=row.asset_id).first() + meta = row.conversion_metadata or {} + result.append({ + "processed_asset_id": str(row.processed_asset_id), + "asset_id": str(row.asset_id), + "filename": asset.filename if asset else None, + "processing_type": row.processing_type.value, + "output_format": row.output_format, + "s3_key": row.s3_key, + "local_dir": meta.get("local_dir"), + "primary_relpath": meta.get("primary_relpath"), + "artifact_count": meta.get("artifact_count", 0), + "created_at": row.created_at.isoformat() if row.created_at else None, + }) + return result + +def link_manual_processed_data( + processed_dir: str | Path, + paper_dir: str | Path | None = None, + project_id: str | uuid.UUID | None = None, + asset_id: str | uuid.UUID | None = None, + primary_file: str | None = None, + processing_type: str | None = None, + output_format: str | None = None, +) -> dict: + """Attach a handmade processed-output folder to an existing project asset. + + This is intended for debugging or backfilling local outputs that were created + outside the normal processing pipeline. + """ + from mkb.db.models import Asset, ProcessedAsset, ProcessingLog, ProcessingType, ProjectAsset, ResearchProject + + bundle = _inspect_manual_processed_dir(processed_dir, primary_file=primary_file) + paper_path = Path(paper_dir).resolve() if paper_dir is not None else None + + if processing_type: + proc_type = ProcessingType(processing_type.upper()) + else: + proc_type = bundle["processing_type"] + out_format = output_format or bundle["output_format"] + + with SyncSessionLocal() as session: + project = None + if project_id is not None: + pid = uuid.UUID(str(project_id)) + project = session.query(ResearchProject).filter_by(project_id=pid).first() + elif paper_path is not None: + project = session.query(ResearchProject).filter_by(source_path=str(paper_path)).first() + if project is None and paper_path.is_dir(): + ingest_result = ingest(paper_path, label=paper_path.name) + pid = uuid.UUID(ingest_result["project_id"]) + project = session.query(ResearchProject).filter_by(project_id=pid).first() + elif asset_id is not None: + # Look up the owning project via ProjectAsset link + aid = uuid.UUID(str(asset_id)) + link = session.query(ProjectAsset).filter_by(asset_id=aid).first() + if link is not None: + project = ( + session.query(ResearchProject) + .filter_by(project_id=link.project_id) + .first() + ) + + if not project: + raise ValueError("Could not find a target project. Provide --paper-dir or --project-id.") + + if asset_id is not None: + target_asset = session.query(Asset).filter_by(asset_id=uuid.UUID(str(asset_id))).first() + else: + links = session.query(ProjectAsset).filter_by(project_id=project.project_id).all() + asset_ids = [link.asset_id for link in links] + assets = session.query(Asset).filter(Asset.asset_id.in_(asset_ids)).all() if asset_ids else [] + target_asset = _choose_asset_for_manual_output(assets, bundle["primary_name"]) + + if not target_asset: + raise ValueError( + "No raw asset found for the target project. Ingest the paper folder first or pass --asset-id." + ) + + link = session.query(ProjectAsset).filter_by( + project_id=project.project_id, + asset_id=target_asset.asset_id, + ).first() + if not link: + session.add(ProjectAsset(project_id=project.project_id, asset_id=target_asset.asset_id)) + + s3_key = f"{project.project_id}/{target_asset.asset_id}/{bundle['primary_relpath']}" + + # Mirror the bundle into the canonical processed-local-root so it survives + # after any caller-supplied temp directory is cleaned up. The local cache + # is used by the idempotency check and by downstream readers. + import shutil + + canonical_root = ( + Path(settings.processed_local_root) + / str(project.project_id) + / str(target_asset.asset_id) + ) + bundle_root = Path(bundle["local_dir"]).resolve() + if bundle_root != canonical_root.resolve(): + canonical_root.mkdir(parents=True, exist_ok=True) + for relpath in [bundle["primary_relpath"], *bundle["artifact_files"]]: + src = bundle_root / relpath + dst = canonical_root / relpath + dst.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src, dst) + bundle["local_dir"] = str(canonical_root) + bundle_root = canonical_root + + # Upload primary file + artifacts to the processed-assets S3 bucket so that + # downstream consumers (idempotency check, frame extraction, projections, + # etc.) can fetch the bundle the same way as auto-processed outputs. + from mkb.storage.s3 import upload_bytes + + upload_bytes( + (bundle_root / bundle["primary_relpath"]).read_bytes(), + settings.s3_bucket_processed, + s3_key, + ) + for relpath in bundle["artifact_files"]: + artifact_key = f"{project.project_id}/{target_asset.asset_id}/{relpath}" + upload_bytes( + (bundle_root / relpath).read_bytes(), + settings.s3_bucket_processed, + artifact_key, + ) + + metadata = { + "project_id": str(project.project_id), + "local_dir": bundle["local_dir"], + "primary_relpath": bundle["primary_relpath"], + "artifact_files": bundle["artifact_files"], + "artifact_count": len(bundle["artifact_files"]), + "linked_via": "debug_manual_link", + "paper_dir": str(paper_path) if paper_path is not None else None, + } + + existing = ( + session.query(ProcessedAsset) + .filter_by(asset_id=target_asset.asset_id, processing_type=proc_type) + .order_by(ProcessedAsset.created_at.desc()) + .first() + ) + + if existing: + existing.output_format = out_format + existing.s3_bucket = settings.s3_bucket_processed + existing.s3_key = s3_key + existing.sha256 = bundle["sha256"] + existing.size_bytes = bundle["size_bytes"] + existing.conversion_metadata = metadata + existing.raw_asset_hash = target_asset.sha256 + processed_asset = existing + action = "updated" + else: + processed_asset = ProcessedAsset( + processed_asset_id=uuid.uuid4(), + asset_id=target_asset.asset_id, + processing_type=proc_type, + output_format=out_format, + s3_bucket=settings.s3_bucket_processed, + s3_key=s3_key, + sha256=bundle["sha256"], + size_bytes=bundle["size_bytes"], + conversion_metadata=metadata, + raw_asset_hash=target_asset.sha256, + ) + session.add(processed_asset) + action = "created" + + asset_meta = dict(target_asset.metadata_ or {}) + processing_meta = dict(asset_meta.get("processing") or {}) + processing_meta.update( + { + "last_status": "SUCCESS", + "last_processing_type": proc_type.value, + "last_output_format": out_format, + "last_processed_asset_id": str(processed_asset.processed_asset_id), + "last_processed_local_dir": bundle["local_dir"], + "source": "manual_debug_link", + } + ) + asset_meta["processing"] = processing_meta + target_asset.metadata_ = asset_meta + + session.add( + ProcessingLog( + log_id=uuid.uuid4(), + asset_id=target_asset.asset_id, + processing_type=proc_type, + status="SUCCESS", + processed_asset_id=processed_asset.processed_asset_id, + details={ + "debug": True, + "action": action, + "primary_relpath": bundle["primary_relpath"], + "artifact_count": len(bundle["artifact_files"]), + }, + ) + ) + session.commit() + + return { + "status": action, + "project_id": str(project.project_id), + "asset_id": str(target_asset.asset_id), + "processed_asset_id": str(processed_asset.processed_asset_id), + "processing_type": proc_type.value, + "output_format": out_format, + "local_dir": bundle["local_dir"], + "primary_relpath": bundle["primary_relpath"], + "artifact_files": bundle["artifact_files"], + } + +def list_assets(project_id: str | uuid.UUID | None = None, limit: int = 100) -> list[dict]: + """List assets, optionally filtered by project.""" + from mkb.db.models import Asset, ProjectAsset + + with SyncSessionLocal() as session: + if project_id is not None: + pid = uuid.UUID(str(project_id)) + links = session.query(ProjectAsset).filter_by(project_id=pid).all() + asset_ids = [link.asset_id for link in links] + if not asset_ids: + return [] + assets = session.query(Asset).filter(Asset.asset_id.in_(asset_ids)).all() + else: + assets = ( + session.query(Asset) + .order_by(Asset.created_at.desc()) + .limit(limit) + .all() + ) + return [ + { + "asset_id": str(a.asset_id), + "filename": a.filename, + "mime_type": a.mime_type, + "size_bytes": a.size_bytes, + "status": a.status.value, + } + for a in assets + ] + +def search_library( + query: str, + limit: int = 25, + project_id: str | uuid.UUID | None = None, +) -> dict: + """Search projects and assets by keyword. + + Every keyword token must match somewhere in the target record. Projects are + searched by label and source path. Assets are searched by filename, MIME + type, and selected metadata fields. + """ + from sqlalchemy import and_, or_ + + from mkb.db.models import Asset, ProjectAsset, ResearchProject + + init_db() + tokens = [token.lower() for token in _normalize_search_query(query)] + if not tokens: + return { + "query": query, + "tokens": [], + "project_id": str(project_id) if project_id is not None else None, + "projects": [], + "assets": [], + "total": 0, + } + + pid = uuid.UUID(str(project_id)) if project_id is not None else None + + with SyncSessionLocal() as session: + project_filters = [ + or_( + ResearchProject.label.ilike(f"%{token}%"), + ResearchProject.source_path.ilike(f"%{token}%"), + ) + for token in tokens + ] + project_query = session.query(ResearchProject) + if pid is not None: + project_query = project_query.filter(ResearchProject.project_id == pid) + project_rows = ( + project_query + .filter(and_(*project_filters)) + .order_by(ResearchProject.created_at.desc()) + .limit(limit) + .all() + ) + + asset_filters = [ + or_( + Asset.filename.ilike(f"%{token}%"), + Asset.mime_type.ilike(f"%{token}%"), + Asset.metadata_["title"].astext.ilike(f"%{token}%"), + Asset.metadata_["description"].astext.ilike(f"%{token}%"), + Asset.metadata_["original_path"].astext.ilike(f"%{token}%"), + ) + for token in tokens + ] + asset_query = session.query(Asset, ProjectAsset.project_id).outerjoin( + ProjectAsset, + ProjectAsset.asset_id == Asset.asset_id, + ) + if pid is not None: + asset_query = asset_query.filter(ProjectAsset.project_id == pid) + asset_rows = ( + asset_query + .filter(and_(*asset_filters)) + .order_by(Asset.created_at.desc()) + .limit(limit) + .all() + ) + + projects = [ + { + "project_id": str(row.project_id), + "label": row.label, + "source_path": row.source_path, + "file_count": row.file_count, + "kind": "project", + } + for row in project_rows + ] + + assets = [] + for asset, asset_project_id in asset_rows: + metadata = asset.metadata_ or {} + if not _matches_search_tokens( + asset.filename, + asset.mime_type, + metadata.get("title"), + metadata.get("description"), + metadata.get("original_path"), + tokens=tokens, + ): + continue + + assets.append( + { + "asset_id": str(asset.asset_id), + "project_id": str(asset_project_id) if asset_project_id else None, + "filename": asset.filename, + "mime_type": asset.mime_type, + "size_bytes": asset.size_bytes, + "status": asset.status.value, + "kind": "asset", + } + ) + + return { + "query": query, + "tokens": tokens, + "project_id": str(pid) if pid is not None else None, + "projects": projects, + "assets": assets[:limit], + "total": len(projects) + len(assets[:limit]), + } + + +# ── Spaces ─────────────────────────────────────────────────────── + diff --git a/src/mkb/services/feedback.py b/src/mkb/services/feedback.py new file mode 100644 index 0000000..7b73c4a --- /dev/null +++ b/src/mkb/services/feedback.py @@ -0,0 +1,171 @@ +"""Feedback API service functions.""" + +from __future__ import annotations + +from mkb.services._api_common import ( + init_db, + uuid, +) + + +def list_feedback( + project_id: str | uuid.UUID | None = None, + status: str | None = None, +) -> list[dict]: + """List feedback items, optionally filtered by project and/or status.""" + from mkb.feedback.manager import list_feedback as _list + + pid = uuid.UUID(str(project_id)) if project_id else None + return _list(project_id=pid, status=status) + +def review_feedback( + project_id: str | uuid.UUID, + model: str | None = None, + verbose: bool = False, + progress_callback=None, +) -> dict: + """Run feedback review on a project — KB agent reviews and resolves open feedback.""" + from mkb.agents.feedback_reviewer import run_feedback_review + + pid = uuid.UUID(str(project_id)) + if progress_callback: + progress_callback({"message": f"Reviewing feedback for project {str(pid)[:8]}"}) + return run_feedback_review(pid, model=model, verbose=verbose) + +def review_projections( + space_id: str | uuid.UUID, + project_id: str | uuid.UUID, + model: str | None = None, + verbose: bool = False, + progress_callback=None, + reviewer_id: str | None = None, +) -> dict: + """Run projection review — consolidate and correct all projections for a project. + + A strict reviewer agent compares all projection runs, cross-references + against the knowledge frame and source material, and produces a single + reviewed (consolidated, corrected) projection. + """ + from mkb.agents.projection_reviewer import run_projection_review + + init_db() + sid = uuid.UUID(str(space_id)) + pid = uuid.UUID(str(project_id)) + if progress_callback: + progress_callback({"message": f"Reviewing projections for project {str(pid)[:8]}"}) + return run_projection_review( + sid, + pid, + model=model, + verbose=verbose, + progress_callback=progress_callback, + reviewer_id=reviewer_id, + ) + +def review_projections_all( + space_id: str | uuid.UUID, + model: str | None = None, + verbose: bool = False, + progress_callback=None, + project_ids: list[str] | None = None, + reviewer_id: str | None = None, +) -> dict: + """Run projection review on projects in a space. + + Args: + project_ids: Restrict to these projects (per-project, separate + sessions). When None, reviews every project that has at least + one completed projection in the space. + """ + from mkb.agents.projection_reviewer import run_projection_review_all + + init_db() + sid = uuid.UUID(str(space_id)) + pids = [uuid.UUID(str(p)) for p in project_ids] if project_ids else None + return run_projection_review_all( + sid, + model=model, + verbose=verbose, + progress_callback=progress_callback, + project_ids=pids, + reviewer_id=reviewer_id, + ) + +def review_projections_session( + space_id: str | uuid.UUID, + project_ids: list[str], + model: str | None = None, + verbose: bool = False, + progress_callback=None, + reviewer_id: str | None = None, +) -> dict: + """Run a SINGLE reviewer session over multiple selected projects. + + One agent context sees every selected project's projections in turn + and saves each reviewed result before moving on to the next. + """ + from mkb.agents.projection_reviewer import run_projection_review_session + + init_db() + sid = uuid.UUID(str(space_id)) + pids = [uuid.UUID(str(p)) for p in project_ids] + if not pids: + return {"status": "error", "message": "project_ids is required for session mode"} + return run_projection_review_session( + sid, + pids, + model=model, + verbose=verbose, + progress_callback=progress_callback, + reviewer_id=reviewer_id, + ) + +def review_projection_followup( + space_id: str | uuid.UUID, + project_id: str | uuid.UUID, + message: str, + previous_job: dict | None = None, + model: str | None = None, + verbose: bool = False, + progress_callback=None, + reviewer_id: str | None = None, +) -> dict: + """Run a follow-up turn for a completed projection review job.""" + from mkb.agents.projection_reviewer import run_projection_review_followup + + init_db() + sid = uuid.UUID(str(space_id)) + pid = uuid.UUID(str(project_id)) + if progress_callback: + progress_callback({"message": "Starting review follow-up"}) + return run_projection_review_followup( + sid, + pid, + message, + previous_job=previous_job, + model=model, + verbose=verbose, + progress_callback=progress_callback, + reviewer_id=reviewer_id, + ) + +def get_feedback_summary( + project_id: str | uuid.UUID, +) -> dict: + """Get counts of feedback by category and status for a project.""" + from mkb.feedback.manager import get_feedback_summary as _summary + + pid = uuid.UUID(str(project_id)) + return _summary(pid) + +def resolve_feedback( + feedback_id: str | uuid.UUID, + status: str, + notes: str, +) -> dict: + """Manually resolve a feedback item.""" + from mkb.feedback.manager import resolve_feedback as _resolve + + fid = uuid.UUID(str(feedback_id)) + return _resolve(fid, status=status, notes=notes, resolved_by="user") + diff --git a/src/mkb/services/frames.py b/src/mkb/services/frames.py new file mode 100644 index 0000000..6154d3d --- /dev/null +++ b/src/mkb/services/frames.py @@ -0,0 +1,122 @@ +"""Frames API service functions.""" + +from __future__ import annotations + +from mkb.services._api_common import ( + SyncSessionLocal, + init_db, + uuid, +) + + +def extract( + project_id: str | uuid.UUID | None = None, + model: str | None = None, + verbose: bool = False, + max_passes: int = 1, + progress_callback=None, +) -> dict: + """Run knowledge extraction. If project_id given, extract one project. + Otherwise extract all pending projects. + + Args: + project_id: Optional specific project to extract. + model: LLM model override. + verbose: Enable verbose logging. + max_passes: Number of extraction passes (1=initial only, >1 includes review). + """ + from mkb.agents.extraction import run_extraction, run_extraction_all + + if project_id is not None: + pid = uuid.UUID(str(project_id)) + return run_extraction( + pid, + model=model, + verbose=verbose, + max_passes=max_passes, + progress_callback=progress_callback, + ) + return run_extraction_all(model=model, verbose=verbose, max_passes=max_passes) + + +# ── Knowledge Frames ───────────────────────────────────────────── + +def get_frame(project_id: str | uuid.UUID) -> dict | None: + """Get the knowledge frame for a project. Returns None if not found.""" + from mkb.db.models import KnowledgeFrame + + init_db() + pid = uuid.UUID(str(project_id)) + with SyncSessionLocal() as session: + frame = session.query(KnowledgeFrame).filter_by(project_id=pid).first() + if not frame: + return None + return { + "frame_id": str(frame.frame_id), + "project_id": str(frame.project_id), + "status": frame.status.value, + "content": frame.content, + "extraction_summary": frame.extraction_summary, + "times_checked": frame.times_checked, + "extraction_version": frame.extraction_version, + "extracted_at": frame.extracted_at.isoformat() if frame.extracted_at else None, + "source_metadata": frame.source_metadata, + "agent_annotations": frame.agent_annotations or {}, + "created_at": frame.created_at.isoformat() if frame.created_at else None, + "updated_at": frame.updated_at.isoformat() if frame.updated_at else None, + } + +def list_frames(status: str | None = None) -> list[dict]: + """List all knowledge frames, optionally filtered by status.""" + from mkb.db.models import FrameStatus, KnowledgeFrame + + init_db() + with SyncSessionLocal() as session: + q = session.query(KnowledgeFrame).order_by(KnowledgeFrame.created_at.desc()) + if status: + q = q.filter_by(status=FrameStatus(status)) + frames = q.all() + return [ + { + "frame_id": str(f.frame_id), + "project_id": str(f.project_id), + "status": f.status.value, + "times_checked": f.times_checked, + "extraction_version": f.extraction_version, + "extracted_at": f.extracted_at.isoformat() if f.extracted_at else None, + "extraction_summary": f.extraction_summary, + } + for f in frames + ] + +def get_extraction_history(project_id: str | uuid.UUID) -> list[dict]: + """Get the extraction pass history for a project's frame.""" + from mkb.db.models import ExtractionPass, KnowledgeFrame + + init_db() + pid = uuid.UUID(str(project_id)) + with SyncSessionLocal() as session: + frame = session.query(KnowledgeFrame).filter_by(project_id=pid).first() + if not frame: + return [] + passes = ( + session.query(ExtractionPass) + .filter_by(frame_id=frame.frame_id) + .order_by(ExtractionPass.pass_number) + .all() + ) + return [ + { + "pass_id": str(p.pass_id), + "pass_number": p.pass_number, + "pass_type": p.pass_type, + "changes_made": p.changes_made, + "agent_notes": p.agent_notes, + "created_at": p.created_at.isoformat() if p.created_at else None, + } + for p in passes + ] + + +# ── Projects & Assets ──────────────────────────────────────────── + diff --git a/src/mkb/services/graphs.py b/src/mkb/services/graphs.py new file mode 100644 index 0000000..462cde3 --- /dev/null +++ b/src/mkb/services/graphs.py @@ -0,0 +1,193 @@ +"""Graphs API service functions.""" + +from __future__ import annotations + +from mkb.services._api_common import ( + SyncSessionLocal, + init_db, + uuid, +) + + +def clear_knowledge_graphs( + project_id: str | uuid.UUID | None = None, + remove_legacy_frame_sections: bool = True, +) -> dict: + """Delete (soft-delete) old KG projections and optionally purge legacy frame graph sections.""" + from mkb.knowledge_graph import clear_knowledge_graph_projections, purge_legacy_graph_sections + + init_db() + pid = uuid.UUID(str(project_id)) if project_id else None + deleted = clear_knowledge_graph_projections(project_id=pid, include_legacy_spaces=True) + + purged = {"updated_frames": 0, "project_id": str(pid) if pid else None} + if remove_legacy_frame_sections: + purged = purge_legacy_graph_sections(project_id=pid) + + return { + "deleted_projections": deleted, + "purged_legacy_frame_sections": purged, + } + +def extract_knowledge_graph( + project_id: str | uuid.UUID | None = None, + frame_id: str | uuid.UUID | None = None, + model: str | None = None, + verbose: bool = False, + clear_existing: bool = True, + clear_legacy_frame_sections: bool = True, + progress_callback=None, +) -> dict: + """Run concept-graph extraction using one global cross-domain space. + + If frame_id is provided, extract for that frame. + If project_id is provided, resolve that project's frame and extract. + Otherwise run for all completed frames. + """ + from mkb.agents.knowledge_graph import run_knowledge_graph, run_knowledge_graph_all + from mkb.db.models import KnowledgeFrame + from mkb.knowledge_graph import ensure_global_kg_space_id, purge_legacy_graph_sections + + init_db() + + if clear_legacy_frame_sections: + if project_id: + purge_legacy_graph_sections(project_id=uuid.UUID(str(project_id))) + else: + purge_legacy_graph_sections() + + if frame_id is not None: + fid = uuid.UUID(str(frame_id)) + result = run_knowledge_graph( + fid, + model=model, + verbose=verbose, + clear_existing=clear_existing, + progress_callback=progress_callback, + ) + elif project_id is not None: + pid = uuid.UUID(str(project_id)) + with SyncSessionLocal() as session: + frame = session.query(KnowledgeFrame).filter_by(project_id=pid).first() + if not frame: + return {"error": f"No frame for project {project_id}"} + fid = frame.frame_id + result = run_knowledge_graph( + fid, + model=model, + verbose=verbose, + clear_existing=clear_existing, + progress_callback=progress_callback, + ) + else: + result = run_knowledge_graph_all(model=model, verbose=verbose, clear_existing=clear_existing) + + return { + "global_space_id": str(ensure_global_kg_space_id()), + **result, + } + +def get_knowledge_graph( + project_id: str | uuid.UUID | None = None, +) -> dict: + """Get the merged concept graph from the singleton global KG space.""" + from mkb.agents.tools.knowledge_graph import normalize_knowledge_graph_payload + from mkb.db.models import KnowledgeFrame, Projection, ProjectionStatus + from mkb.knowledge_graph import ensure_global_kg_space_id + + init_db() + sid = ensure_global_kg_space_id() + + with SyncSessionLocal() as session: + q = ( + session.query(Projection) + .filter(Projection.space_id == sid) + .filter(Projection.deleted_at.is_(None)) + .filter(Projection.status.in_([ProjectionStatus.COMPLETED, ProjectionStatus.REVIEWED])) + ) + if project_id: + pid = uuid.UUID(str(project_id)) + q = q.join(KnowledgeFrame, Projection.frame_id == KnowledgeFrame.frame_id) + q = q.filter(KnowledgeFrame.project_id == pid) + rows = q.all() + + aggregate = {"concepts": [], "relations": []} + for row in rows: + payload, _ = normalize_knowledge_graph_payload(row.data or {}) + aggregate["concepts"].extend(payload["concepts"]) + aggregate["relations"].extend(payload["relations"]) + + merged, validation = normalize_knowledge_graph_payload(aggregate) + return { + "space_id": str(sid), + "projection_count": len(rows), + "graph": merged, + "validation": validation, + "project_id": str(project_id) if project_id else None, + } + + +# ── Feedback ───────────────────────────────────────────────────── + +def review_knowledge_graph( + mode: str = "auto", + model: str | None = None, + verbose: bool = False, + seed_count: int = 10, + progress_callback=None, +) -> dict: + """Run the graph review agent to deduplicate and clean the knowledge graph. + + Two modes: + - "global": analyzes relation name distributions, standardizes naming, merges + duplicate or synonymous concept nodes across the entire graph. + - "local": selects the least-reviewed concepts as starting points, explores + their neighborhoods, verifies against source frames, and fixes local issues. + - "auto" (default): randomly picks global or local each time. + + After each run, the times_examined and times_modified counters on each visited + graph element are incremented in the graph_element_reviews table. + + Args: + mode: "global", "local", or "auto". + model: LLM model override. + verbose: Enable verbose logging. + seed_count: Number of starting concepts for local mode. + """ + from mkb.agents.graph_review import run_graph_review + + init_db() + return run_graph_review(mode=mode, model=model, verbose=verbose, seed_count=seed_count, progress_callback=progress_callback) + +def get_graph_review_counts(space_id: str | uuid.UUID | None = None) -> dict: + """Return review counts (times_examined, times_modified) per graph element. + + Returns a dict with two sub-dicts keyed by normalized element key: + - ``concepts``: mapping of normalized concept label → {times_examined, times_modified} + - ``relations``: mapping of "src||rel||tgt" → {times_examined, times_modified} + """ + from mkb.db.models import GraphElementReview + from mkb.knowledge_graph import ensure_global_kg_space_id + + init_db() + sid = uuid.UUID(str(space_id)) if space_id else ensure_global_kg_space_id() + + concepts: dict[str, dict] = {} + relations: dict[str, dict] = {} + + with SyncSessionLocal() as session: + rows = session.query(GraphElementReview).filter_by(space_id=sid).all() + for row in rows: + entry = { + "times_examined": row.times_examined, + "times_modified": row.times_modified, + "last_examined_at": row.last_examined_at.isoformat() if row.last_examined_at else None, + "last_modified_at": row.last_modified_at.isoformat() if row.last_modified_at else None, + } + if row.element_type == "concept": + concepts[row.element_key] = entry + elif row.element_type == "relation": + relations[row.element_key] = entry + + return {"space_id": str(sid), "concepts": concepts, "relations": relations} + diff --git a/src/mkb/services/ids.py b/src/mkb/services/ids.py new file mode 100644 index 0000000..ad3b82a --- /dev/null +++ b/src/mkb/services/ids.py @@ -0,0 +1,21 @@ +"""Identifier parsing helpers for service boundaries.""" + +from __future__ import annotations + +import uuid +from typing import Any + + +def strict_uuid(value: Any, field: str = "id") -> uuid.UUID: + try: + return uuid.UUID(str(value)) + except (TypeError, ValueError) as exc: + raise ValueError(f"Invalid {field}: {value!r}") from exc + + +def tolerant_uuid(value: Any) -> uuid.UUID | None: + try: + return uuid.UUID(str(value)) + except (TypeError, ValueError): + return None + diff --git a/src/mkb/services/ingest.py b/src/mkb/services/ingest.py new file mode 100644 index 0000000..59d22af --- /dev/null +++ b/src/mkb/services/ingest.py @@ -0,0 +1,57 @@ +"""Ingest API service functions.""" + +from __future__ import annotations + +from mkb.services._api_common import ( + Path, + uuid, +) + + +def ingest( + directory: str | Path, + label: str | None = None, + *, + user_named: bool = False, +) -> dict: + """Ingest a single project directory. + + Creates or updates a ResearchProject record keyed on the directory path, + then ingests any new files found inside it. + + ``user_named`` controls whether the provided label should be treated as a + user-given name (in which case the project will be marked as such and + excluded from later automatic renaming during extraction). + + Returns a summary dict with counts (total, ingested, duplicates, errors). + """ + from mkb.ingest.worker import ingest_directory + + return ingest_directory(directory, label=label, user_named=user_named) + +def sync(root_dir: str | Path) -> dict: + """Sync all project subfolders under *root_dir*. + + Each immediate subdirectory of *root_dir* is treated as one research + project. New subfolders are registered as new projects; existing projects + are scanned for new files. + + Returns a summary dict with per-project results. + """ + from mkb.ingest.worker import sync_root + + return sync_root(root_dir) + +def sync_project(project_id: str | uuid.UUID) -> dict: + """Re-scan a single project's source directory for new files. + + Returns a summary dict with counts of newly ingested files. + """ + from mkb.ingest.worker import sync_project as _sync_project + + pid = uuid.UUID(str(project_id)) + return _sync_project(pid) + + +# ── Processing ─────────────────────────────────────────────────── + diff --git a/src/mkb/services/processing.py b/src/mkb/services/processing.py new file mode 100644 index 0000000..1148e90 --- /dev/null +++ b/src/mkb/services/processing.py @@ -0,0 +1,2 @@ +"""Processing service namespace.""" + diff --git a/src/mkb/services/projections.py b/src/mkb/services/projections.py new file mode 100644 index 0000000..40aafd9 --- /dev/null +++ b/src/mkb/services/projections.py @@ -0,0 +1,392 @@ +"""Projections API service functions.""" + +from __future__ import annotations + +from mkb.services._api_common import ( + Path, + SyncSessionLocal, + init_db, + uuid, +) + + +import json + + +def serialize_projection_payload(projection_id: uuid.UUID, out_path: Path, format: str) -> Path: + from mkb.db.models import KnowledgeFrame, Projection, Space + + with SyncSessionLocal() as session: + projection = session.query(Projection).filter_by(projection_id=projection_id).first() + if not projection: + raise ValueError(f"Projection {projection_id} not found") + frame = session.query(KnowledgeFrame).filter_by(frame_id=projection.frame_id).first() + space = session.query(Space).filter_by(space_id=projection.space_id).first() + record = { + "projection_id": str(projection.projection_id), + "space": { + "space_id": str(projection.space_id), + "name": space.name if space else None, + "purpose": getattr(space, "purpose", None) if space else None, + "version": projection.space_version, + }, + "frame_id": str(projection.frame_id) if projection.frame_id else None, + "project_id": str(frame.project_id) if frame else None, + "status": projection.status.value, + "source_type": getattr(projection, "source_type", "frame"), + "extracted_at": projection.extracted_at.isoformat() if projection.extracted_at else None, + "agent_notes": projection.agent_notes, + "data": projection.data or {}, + } + + fmt = (format or "yaml").strip().lower() + out_path.parent.mkdir(parents=True, exist_ok=True) + if fmt == "json": + out_path.write_text(json.dumps(record, indent=2, ensure_ascii=False, default=str)) + else: + import yaml + + out_path.write_text(yaml.safe_dump(record, sort_keys=False, allow_unicode=True)) + return out_path + + +def project( + space_id: str | uuid.UUID, + frame_id: str | uuid.UUID | None = None, + project_id: str | uuid.UUID | None = None, + model: str | None = None, + verbose: bool = False, + progress_callback=None, + source_type: str = "frame", +) -> dict: + """Run projection on one or more frames using a space definition. + + Args: + source_type: ``"frame"`` (default) to project from the curated knowledge + frame, or ``"markdown"`` to project directly from the processed + Markdown of the project's papers (no extraction step required). + + If frame_id given, project that specific frame. + If project_id given, find or auto-create the frame for that project. + """ + from mkb.agents.projection import run_projection + from mkb.db.models import KnowledgeFrame + + init_db() + sid = uuid.UUID(str(space_id)) + source_kind = (source_type or "frame").strip().lower() + + if frame_id: + fid = uuid.UUID(str(frame_id)) + return run_projection( + sid, fid, model=model, verbose=verbose, + progress_callback=progress_callback, source_type=source_kind, + ) + + if project_id: + pid = uuid.UUID(str(project_id)) + if source_kind == "markdown": + # Frame may not exist yet; the agent runner will create one. + return run_projection( + sid, None, model=model, verbose=verbose, + progress_callback=progress_callback, + source_type=source_kind, project_id=pid, + ) + with SyncSessionLocal() as session: + frame = session.query(KnowledgeFrame).filter_by(project_id=pid).first() + if not frame: + return {"error": f"No frame for project {project_id}"} + fid = frame.frame_id + return run_projection( + sid, fid, model=model, verbose=verbose, + progress_callback=progress_callback, source_type=source_kind, + ) + + return {"error": "Must specify frame_id or project_id"} + +def project_all( + space_id: str | uuid.UUID, + model: str | None = None, + verbose: bool = False, + source_type: str = "frame", +) -> dict: + """Run projection on all completed frames (or all projects) using a space.""" + from mkb.agents.projection import run_projection_all + + init_db() + sid = uuid.UUID(str(space_id)) + return run_projection_all(sid, model=model, verbose=verbose, source_type=source_type) + +def get_projection(projection_id: str | uuid.UUID) -> dict | None: + """Get a projection by ID.""" + from mkb.db.models import KnowledgeFrame, Projection, Space + from mkb.spaces.schema_utils import normalize_projection_data + + init_db() + pid = uuid.UUID(str(projection_id)) + with SyncSessionLocal() as session: + proj = session.query(Projection).filter_by(projection_id=pid).first() + if not proj: + return None + frame = session.query(KnowledgeFrame).filter_by(frame_id=proj.frame_id).first() + space = session.query(Space).filter_by(space_id=proj.space_id).first() + normalized_data, normalized_validation = normalize_projection_data( + proj.data or {}, + space.extraction_schema if space else {}, + ) + + validation_result = proj.validation_result or {} + if normalized_validation: + validation_result = { + **normalized_validation, + **validation_result, + } + + return { + "projection_id": str(proj.projection_id), + "space_id": str(proj.space_id), + "frame_id": str(proj.frame_id), + "project_id": str(frame.project_id) if frame else None, + "status": proj.status.value, + "data": normalized_data, + "validation_result": validation_result or None, + "agent_notes": proj.agent_notes, + "extracted_at": proj.extracted_at.isoformat() if proj.extracted_at else None, + "space_version": proj.space_version, + "source_type": getattr(proj, "source_type", "frame"), + "times_reviewed": proj.times_reviewed, + "review_notes": proj.review_notes, + "reviewed_at": proj.reviewed_at.isoformat() if proj.reviewed_at else None, + "created_at": proj.created_at.isoformat() if proj.created_at else None, + } + +def delete_projection(projection_id: str | uuid.UUID) -> bool: + """Soft-delete a projection by setting deleted_at. Returns True if found.""" + from datetime import datetime, timezone + + from mkb.db.models import Projection + + init_db() + pid = uuid.UUID(str(projection_id)) + with SyncSessionLocal() as session: + proj = session.query(Projection).filter_by(projection_id=pid).first() + if not proj: + return False + proj.deleted_at = datetime.now(timezone.utc) + session.commit() + return True + +def list_projections( + space_id: str | uuid.UUID | None = None, + frame_id: str | uuid.UUID | None = None, + project_id: str | uuid.UUID | None = None, + include_data: bool = False, + newest_only: bool = False, + include_history: bool = False, +) -> list[dict]: + """List projections, optionally filtered by space, frame, or project. + + Args: + include_history: When False (default), superseded projections (those + replaced by a tracked review) are hidden. When True, the full + history is returned, including ``superseded_by_id`` / + ``supersedes_ids`` pointers so callers can rebuild the chain. + """ + from mkb.db.models import KnowledgeFrame, Projection, Space + from mkb.spaces.schema_utils import normalize_projection_data + + init_db() + with SyncSessionLocal() as session: + q = ( + session.query(Projection, KnowledgeFrame.project_id) + .outerjoin(KnowledgeFrame, Projection.frame_id == KnowledgeFrame.frame_id) + .filter(Projection.deleted_at.is_(None)) + .order_by(Projection.created_at.desc(), Projection.extracted_at.desc()) + ) + if not include_history: + q = q.filter(Projection.superseded_by_id.is_(None)) + if space_id: + q = q.filter(Projection.space_id == uuid.UUID(str(space_id))) + if frame_id: + q = q.filter(Projection.frame_id == uuid.UUID(str(frame_id))) + if project_id: + q = q.filter(KnowledgeFrame.project_id == uuid.UUID(str(project_id))) + projections = q.all() + + results = [] + seen_keys: set[tuple[str, str]] = set() + for projection, projection_project_id in projections: + project_value = str(projection_project_id) if projection_project_id else None + dedupe_key = (str(projection.space_id), project_value or str(projection.frame_id)) + if newest_only and dedupe_key in seen_keys: + continue + seen_keys.add(dedupe_key) + + item = { + "projection_id": str(projection.projection_id), + "space_id": str(projection.space_id), + "frame_id": str(projection.frame_id), + "project_id": project_value, + "status": projection.status.value, + "agent_notes": projection.agent_notes, + "extracted_at": projection.extracted_at.isoformat() if projection.extracted_at else None, + "created_at": projection.created_at.isoformat() if projection.created_at else None, + "space_version": projection.space_version, + "source_type": getattr(projection, "source_type", "frame"), + "times_reviewed": projection.times_reviewed, + "review_notes": projection.review_notes, + "reviewed_at": projection.reviewed_at.isoformat() if projection.reviewed_at else None, + "superseded_by_id": ( + str(projection.superseded_by_id) + if getattr(projection, "superseded_by_id", None) + else None + ), + "supersedes_ids": getattr(projection, "supersedes_ids", None), + } + if include_data: + space = session.query(Space).filter_by(space_id=projection.space_id).first() + normalized_data, _ = normalize_projection_data( + projection.data or {}, + space.extraction_schema if space else {}, + ) + item["data"] = normalized_data + results.append(item) + + return results + + +# ── Projection Exports ────────────────────────────────────────── + +def _serialize_projection_payload( + projection_id: uuid.UUID, + out_path: Path, + format: str, +) -> Path: + """Generic single-projection dump (used for non-qa_benchmark spaces).""" + return serialize_projection_payload(projection_id, out_path, format) + +def export_projection( + projection_id: str | uuid.UUID, + out_dir: str | Path, + format: str = "yaml", + overwrite: bool = False, +) -> dict: + """Export a single projection to disk. + + For ``qa_benchmark`` spaces, delegates to the mat_agent_bench exporter + (one YAML per question under ``//.yaml``). + For all other purposes, writes a single ``.`` file + containing the projection payload + metadata. + """ + from mkb.db.models import Projection, Space + from mkb.spaces.export_qa_bench import ( + QABenchExportError, + export_projection_to_yaml, + ) + + init_db() + pid = uuid.UUID(str(projection_id)) + out_root = Path(out_dir) + fmt = (format or "yaml").strip().lower() + if fmt not in {"yaml", "json"}: + raise ValueError(f"Unsupported export format: {format}") + + with SyncSessionLocal() as session: + proj = session.query(Projection).filter_by(projection_id=pid).first() + if not proj: + return {"error": f"Projection {projection_id} not found"} + space = session.query(Space).filter_by(space_id=proj.space_id).first() + purpose = getattr(space, "purpose", None) if space else None + + if purpose == "qa_benchmark" and fmt == "yaml": + try: + return export_projection_to_yaml(pid, out_root, overwrite=overwrite) + except QABenchExportError as e: + return {"error": str(e)} + + out_path = out_root / f"{pid}.{fmt}" + if out_path.exists() and not overwrite: + return {"error": f"{out_path} already exists (pass overwrite=True)"} + _serialize_projection_payload(pid, out_path, fmt) + return {"files": [str(out_path)], "skipped": [], "warnings": []} + +def export_space_projections( + space_id_or_name: str | uuid.UUID, + out_dir: str | Path, + format: str = "yaml", + overwrite: bool = False, + newest_only: bool = True, +) -> dict: + """Export every projection belonging to a space. + + For ``qa_benchmark`` spaces and ``format='yaml'`` this delegates to the + aggregated mat_agent_bench exporter. Otherwise one file is written per + projection: ``/.``. + """ + from mkb.db.models import Projection, ProjectionStatus, Space + from mkb.spaces.export_qa_bench import ( + QABenchExportError, + export_space_to_yaml, + ) + + init_db() + out_root = Path(out_dir) + fmt = (format or "yaml").strip().lower() + if fmt not in {"yaml", "json"}: + raise ValueError(f"Unsupported export format: {format}") + + with SyncSessionLocal() as session: + try: + sid = uuid.UUID(str(space_id_or_name)) + space = session.query(Space).filter_by(space_id=sid).first() + except (ValueError, AttributeError): + space = session.query(Space).filter_by(name=str(space_id_or_name)).first() + if not space: + return {"error": f"Space {space_id_or_name} not found"} + purpose = getattr(space, "purpose", None) + space_id = space.space_id + + if purpose == "qa_benchmark" and fmt == "yaml": + try: + return export_space_to_yaml(space_id, out_root, overwrite=overwrite) + except QABenchExportError as e: + return {"error": str(e)} + + # Generic per-projection dump + with SyncSessionLocal() as session: + q = ( + session.query(Projection) + .filter(Projection.space_id == space_id) + .filter(Projection.deleted_at.is_(None)) + .filter(Projection.status == ProjectionStatus.COMPLETED) + .order_by(Projection.created_at.desc()) + ) + projections = q.all() + if newest_only: + seen: set[str] = set() + unique = [] + for p in projections: + key = str(p.frame_id) + if key in seen: + continue + seen.add(key) + unique.append(p) + projections = unique + ids = [p.projection_id for p in projections] + + written: list[str] = [] + skipped: list[dict] = [] + out_root.mkdir(parents=True, exist_ok=True) + for pid in ids: + out_path = out_root / f"{pid}.{fmt}" + if out_path.exists() and not overwrite: + skipped.append({"id": str(pid), "reason": "exists"}) + continue + _serialize_projection_payload(pid, out_path, fmt) + written.append(str(out_path)) + + return {"files": written, "skipped": skipped, "warnings": []} + + +# ── Knowledge Graphs ──────────────────────────────────────────── + diff --git a/src/mkb/services/projects.py b/src/mkb/services/projects.py new file mode 100644 index 0000000..a3a468e --- /dev/null +++ b/src/mkb/services/projects.py @@ -0,0 +1,504 @@ +"""Projects API service functions.""" + +from __future__ import annotations + +from mkb.services._api_common import ( + SyncSessionLocal, + init_db, + logger, + uuid, +) + + +def serialize_group(group, project_count: int) -> dict: + return { + "group_id": str(group.group_id), + "name": group.name, + "description": group.description, + "color": group.color, + "display_order": group.display_order, + "project_count": project_count, + "created_at": group.created_at.isoformat() if group.created_at else None, + "updated_at": group.updated_at.isoformat() if group.updated_at else None, + } + + +def rename_project( + project_id: str | uuid.UUID, + label: str, + *, + user_initiated: bool = True, +) -> dict: + """Rename a research project. + + When ``user_initiated`` is True (the default), records + ``metadata_["user_named"] = True`` so that the automatic post-extraction + rename will skip this project. Callers that want to perform an automatic + rename (e.g. from an extracted paper title) should pass + ``user_initiated=False`` to leave that flag alone. + """ + from mkb.db.models import ResearchProject + + pid = uuid.UUID(str(project_id)) + cleaned = (label or "").strip() + if not cleaned: + return {"error": "label must not be empty"} + + with SyncSessionLocal() as session: + project = session.query(ResearchProject).filter_by(project_id=pid).first() + if not project: + return {"error": f"Project {project_id} not found"} + project.label = cleaned + if user_initiated: + meta = dict(project.metadata_ or {}) + meta["user_named"] = True + project.metadata_ = meta + session.commit() + return { + "project_id": str(project.project_id), + "label": project.label, + "user_named": bool((project.metadata_ or {}).get("user_named")), + } + +def list_projects(limit: int = 50) -> list[dict]: + """List research projects.""" + from collections import defaultdict + + from mkb.db.models import CanonicalWorkflow, KnowledgeFrame, ProcessedAsset, ProjectAsset, RawWorkflowExtraction, ResearchProject + + init_db() + with SyncSessionLocal() as session: + projects = ( + session.query(ResearchProject) + .order_by(ResearchProject.created_at.desc()) + .limit(limit) + .all() + ) + if not projects: + return [] + + project_ids = [p.project_id for p in projects] + + # Bulk-fetch asset links for all queried projects + all_links = session.query(ProjectAsset).filter(ProjectAsset.project_id.in_(project_ids)).all() + project_to_asset_ids: dict = defaultdict(list) + for link in all_links: + project_to_asset_ids[link.project_id].append(link.asset_id) + + # Bulk-fetch which assets have at least one ProcessedAsset record + all_asset_ids = [link.asset_id for link in all_links] + if all_asset_ids: + processed_ids = { + row.asset_id + for row in session.query(ProcessedAsset.asset_id) + .filter(ProcessedAsset.asset_id.in_(all_asset_ids)) + .distinct() + .all() + } + else: + processed_ids = set() + + # Bulk-fetch frames + frames = session.query(KnowledgeFrame).filter(KnowledgeFrame.project_id.in_(project_ids)).all() + frame_by_project = {f.project_id: f for f in frames} + workflow_rows = ( + session.query(RawWorkflowExtraction) + .filter(RawWorkflowExtraction.project_id.in_(project_ids)) + .order_by(RawWorkflowExtraction.version.desc()) + .all() + ) + workflow_by_project = {} + for workflow in workflow_rows: + workflow_by_project.setdefault(workflow.project_id, workflow) + current = workflow_by_project[workflow.project_id] + current_is_valid = ( + current.status == "COMPLETED" + and current.record_status in {"active", "needs_review"} + ) + if not current_is_valid and workflow.status == "COMPLETED" and workflow.record_status in {"active", "needs_review"}: + workflow_by_project[workflow.project_id] = workflow + canonical_rows = ( + session.query(CanonicalWorkflow) + .filter(CanonicalWorkflow.project_id.in_(project_ids)) + .order_by(CanonicalWorkflow.version.desc()).all() + ) + canonical_by_project = {} + for canonical in canonical_rows: + canonical_by_project.setdefault(canonical.project_id, canonical) + + result = [] + for p in projects: + asset_ids_for_project = project_to_asset_ids[p.project_id] + total = len(asset_ids_for_project) + processed_count = sum(1 for aid in asset_ids_for_project if aid in processed_ids) + if total == 0: + processing_status = "NO_ASSETS" + elif processed_count == 0: + processing_status = "UNPROCESSED" + elif processed_count < total: + processing_status = "PARTIAL" + else: + processing_status = "PROCESSED" + + frame = frame_by_project.get(p.project_id) + workflow = workflow_by_project.get(p.project_id) + canonical = canonical_by_project.get(p.project_id) + result.append({ + "project_id": str(p.project_id), + "label": p.label, + "source_path": p.source_path, + "file_count": p.file_count, + "asset_count": total, + "processing_status": processing_status, + "frame_status": frame.status.value if frame else "NO_FRAME", + "workflow_status": workflow.status if workflow else "NO_WORKFLOW", + "workflow_version": workflow.version if workflow else None, + "canonical_workflow_status": canonical.status if canonical else "NO_CANONICAL_WORKFLOW", + "canonical_workflow_version": canonical.version if canonical else None, + "created_at": p.created_at.isoformat() if p.created_at else None, + "duplicate_of": (p.metadata_ or {}).get("duplicate_of"), + "group_id": str(p.group_id) if p.group_id else None, + }) + return result + + +# ── Project groups ───────────────────────────────────────────── + +def _serialize_group(g, project_count: int) -> dict: + return serialize_group(g, project_count) + +def list_project_groups() -> list[dict]: + """List all project groups with project counts.""" + from sqlalchemy import func as sa_func + + from mkb.db.models import ProjectGroup, ResearchProject + + init_db() + with SyncSessionLocal() as session: + groups = ( + session.query(ProjectGroup) + .order_by(ProjectGroup.display_order, ProjectGroup.created_at) + .all() + ) + counts = dict( + session.query(ResearchProject.group_id, sa_func.count()) + .filter(ResearchProject.group_id.isnot(None)) + .group_by(ResearchProject.group_id) + .all() + ) + return [_serialize_group(g, counts.get(g.group_id, 0)) for g in groups] + +def create_project_group( + name: str, + *, + description: str | None = None, + color: str | None = None, + display_order: int | None = None, +) -> dict: + from mkb.db.models import ProjectGroup + + cleaned = (name or "").strip() + if not cleaned: + return {"error": "name must not be empty"} + + init_db() + with SyncSessionLocal() as session: + if display_order is None: + current_max = ( + session.query(ProjectGroup) + .order_by(ProjectGroup.display_order.desc()) + .first() + ) + display_order = (current_max.display_order + 1) if current_max else 0 + group = ProjectGroup( + name=cleaned, + description=(description or None), + color=(color or None), + display_order=int(display_order), + ) + session.add(group) + session.commit() + session.refresh(group) + return _serialize_group(group, 0) + +def update_project_group( + group_id: str | uuid.UUID, + *, + name: str | None = None, + description: str | None = None, + color: str | None = None, + display_order: int | None = None, +) -> dict: + from sqlalchemy import func as sa_func + + from mkb.db.models import ProjectGroup, ResearchProject + + gid = uuid.UUID(str(group_id)) + init_db() + with SyncSessionLocal() as session: + group = session.query(ProjectGroup).filter_by(group_id=gid).first() + if not group: + return {"error": f"Group {group_id} not found"} + if name is not None: + cleaned = name.strip() + if not cleaned: + return {"error": "name must not be empty"} + group.name = cleaned + if description is not None: + group.description = description.strip() or None + if color is not None: + group.color = color.strip() or None + if display_order is not None: + group.display_order = int(display_order) + session.commit() + session.refresh(group) + count = ( + session.query(sa_func.count()) + .select_from(ResearchProject) + .filter(ResearchProject.group_id == gid) + .scalar() + ) or 0 + return _serialize_group(group, int(count)) + +def delete_project_group(group_id: str | uuid.UUID) -> dict: + """Delete a group. Projects in it are unassigned (group_id set to NULL).""" + from mkb.db.models import ProjectGroup, ResearchProject + + gid = uuid.UUID(str(group_id)) + init_db() + with SyncSessionLocal() as session: + group = session.query(ProjectGroup).filter_by(group_id=gid).first() + if not group: + return {"error": f"Group {group_id} not found"} + unassigned = ( + session.query(ResearchProject) + .filter(ResearchProject.group_id == gid) + .update({ResearchProject.group_id: None}, synchronize_session=False) + ) + session.delete(group) + session.commit() + return {"group_id": str(gid), "deleted": True, "unassigned_projects": int(unassigned)} + +def delete_project( + project_id: str | uuid.UUID, + *, + delete_s3_objects: bool = True, +) -> dict: + """Hard-delete a research project and all data exclusively owned by it. + + Cascade: + - ``ProjectAsset`` links for this project are removed. + - ``Asset`` / ``ProcessedAsset`` / ``ProcessingLog`` records are removed + only when the asset is **not** linked to any other project (i.e. not + shared). When ``delete_s3_objects`` is True the corresponding S3 + objects are removed before the DB records. + - ``KnowledgeFrame`` owned by this project is deleted, along with its + ``ExtractionPass``, all ``Projection`` rows (hard delete), and all + ``Feedback`` rows whose ``target_frame_id`` / ``target_project_id`` + match. + - The ``ResearchProject`` record itself is deleted last. + + Returns a summary dict or ``{"error": ...}`` when the project is not found. + """ + from mkb.db.models import ( + Asset, + CanonicalWorkflow, + ExtractionPass, + Feedback, + KnowledgeFrame, + ProcessedAsset, + ProcessingLog, + ProjectAsset, + Projection, + RawWorkflowExtraction, + ResearchProject, + WorkflowIndexEntry, + WorkflowMaintenanceTask, + ) + from mkb.storage.s3 import delete_object + + pid = uuid.UUID(str(project_id)) + init_db() + + with SyncSessionLocal() as session: + project = session.query(ResearchProject).filter_by(project_id=pid).first() + if not project: + return {"error": f"Project {project_id} not found"} + + # ── Collect asset IDs linked to this project ────────────────────── + own_links = session.query(ProjectAsset).filter_by(project_id=pid).all() + own_asset_ids = [lnk.asset_id for lnk in own_links] + + # Determine which of those assets are shared with other projects + shared_asset_ids: set[uuid.UUID] = set() + if own_asset_ids: + other_links = ( + session.query(ProjectAsset.asset_id) + .filter( + ProjectAsset.asset_id.in_(own_asset_ids), + ProjectAsset.project_id != pid, + ) + .distinct() + .all() + ) + shared_asset_ids = {row.asset_id for row in other_links} + + exclusive_asset_ids = [a for a in own_asset_ids if a not in shared_asset_ids] + + # ── Remove S3 objects and DB records for exclusive assets ───────── + deleted_assets = 0 + deleted_processed = 0 + deleted_s3_objects = 0 + + if exclusive_asset_ids: + # ProcessedAsset rows (and their S3 objects) + processed_rows = ( + session.query(ProcessedAsset) + .filter(ProcessedAsset.asset_id.in_(exclusive_asset_ids)) + .all() + ) + for pa in processed_rows: + if delete_s3_objects: + try: + delete_object(pa.s3_bucket, pa.s3_key) + deleted_s3_objects += 1 + except Exception: + logger.warning( + "Failed to delete S3 object %s/%s", pa.s3_bucket, pa.s3_key + ) + session.delete(pa) + deleted_processed = len(processed_rows) + + # ProcessingLog rows + session.query(ProcessingLog).filter( + ProcessingLog.asset_id.in_(exclusive_asset_ids) + ).delete(synchronize_session=False) + + # Raw asset S3 objects + Asset rows + raw_assets = ( + session.query(Asset) + .filter(Asset.asset_id.in_(exclusive_asset_ids)) + .all() + ) + for asset in raw_assets: + if delete_s3_objects: + try: + delete_object(asset.s3_bucket, asset.s3_key) + deleted_s3_objects += 1 + except Exception: + logger.warning( + "Failed to delete S3 object %s/%s", asset.s3_bucket, asset.s3_key + ) + session.delete(asset) + deleted_assets = len(raw_assets) + + # ── Remove ProjectAsset links (including shared ones) ───────────── + for lnk in own_links: + session.delete(lnk) + + # ── Knowledge frame + dependents ────────────────────────────────── + frame = session.query(KnowledgeFrame).filter_by(project_id=pid).first() + deleted_projections = 0 + deleted_passes = 0 + deleted_feedback = 0 + + if frame: + fid = frame.frame_id + + # Projections (hard delete) + deleted_projections = ( + session.query(Projection) + .filter(Projection.frame_id == fid) + .delete(synchronize_session=False) + ) + + # ExtractionPass rows + deleted_passes = ( + session.query(ExtractionPass) + .filter(ExtractionPass.frame_id == fid) + .delete(synchronize_session=False) + ) + + # Feedback rows tied to this frame + deleted_feedback = ( + session.query(Feedback) + .filter(Feedback.target_frame_id == fid) + .delete(synchronize_session=False) + ) + + session.delete(frame) + + # Also remove any feedback rows referencing the project but a different + # (or NULL) frame (defensive clean-up). + extra_feedback = ( + session.query(Feedback) + .filter(Feedback.target_project_id == pid) + .delete(synchronize_session=False) + ) + deleted_feedback += extra_feedback + + deleted_workflows = ( + session.query(RawWorkflowExtraction) + .filter(RawWorkflowExtraction.project_id == pid) + .delete(synchronize_session=False) + ) + deleted_canonical_workflows = ( + session.query(CanonicalWorkflow) + .filter(CanonicalWorkflow.project_id == pid) + .delete(synchronize_session=False) + ) + session.query(WorkflowIndexEntry).filter( + WorkflowIndexEntry.project_id == pid + ).delete(synchronize_session=False) + session.query(WorkflowMaintenanceTask).filter( + WorkflowMaintenanceTask.project_id == pid + ).delete(synchronize_session=False) + + # ── Delete the project itself ───────────────────────────────────── + session.delete(project) + session.commit() + + return { + "project_id": str(pid), + "deleted": True, + "deleted_assets": deleted_assets, + "shared_assets_kept": len(shared_asset_ids), + "deleted_processed_assets": deleted_processed, + "deleted_s3_objects": deleted_s3_objects, + "deleted_projections": deleted_projections, + "deleted_extraction_passes": deleted_passes, + "deleted_feedback": deleted_feedback, + "deleted_workflow_versions": deleted_workflows, + "deleted_canonical_workflow_versions": deleted_canonical_workflows, + } + + +# ── Raw workflow graphs ─────────────────────────────────────── + +def assign_projects_to_group( + project_ids: list[str | uuid.UUID], + group_id: str | uuid.UUID | None, +) -> dict: + """Assign multiple projects to a group, or to no group when ``group_id`` is None.""" + from mkb.db.models import ProjectGroup, ResearchProject + + if not project_ids: + return {"updated": 0, "group_id": None} + + pids = [uuid.UUID(str(p)) for p in project_ids] + gid = uuid.UUID(str(group_id)) if group_id else None + + init_db() + with SyncSessionLocal() as session: + if gid is not None: + group = session.query(ProjectGroup).filter_by(group_id=gid).first() + if not group: + return {"error": f"Group {group_id} not found"} + updated = ( + session.query(ResearchProject) + .filter(ResearchProject.project_id.in_(pids)) + .update({ResearchProject.group_id: gid}, synchronize_session=False) + ) + session.commit() + return {"updated": int(updated), "group_id": str(gid) if gid else None} + diff --git a/src/mkb/services/result.py b/src/mkb/services/result.py new file mode 100644 index 0000000..68f2104 --- /dev/null +++ b/src/mkb/services/result.py @@ -0,0 +1,52 @@ +"""Shared service error/result helpers.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + + +@dataclass(frozen=True) +class ServiceError(Exception): + message: str + code: str = "service_error" + status_code: int = 400 + details: dict[str, Any] | None = None + + def to_dict(self) -> dict[str, Any]: + payload: dict[str, Any] = { + "error": self.message, + "code": self.code, + "status_code": self.status_code, + } + if self.details: + payload["details"] = self.details + return payload + + +def error_result( + message: str, + *, + code: str = "service_error", + status_code: int = 400, + **details: Any, +) -> dict[str, Any]: + payload: dict[str, Any] = { + "error": message, + "code": code, + "status_code": status_code, + } + if details: + payload["details"] = details + return payload + + +def is_error_result(value: Any) -> bool: + return isinstance(value, dict) and bool(value.get("error")) + + +def result_status_code(value: dict[str, Any], default: int = 400) -> int: + try: + return int(value.get("status_code") or default) + except (TypeError, ValueError): + return default diff --git a/src/mkb/services/runtime.py b/src/mkb/services/runtime.py new file mode 100644 index 0000000..daca6c7 --- /dev/null +++ b/src/mkb/services/runtime.py @@ -0,0 +1,26 @@ +"""Runtime API service functions.""" + +from __future__ import annotations + +from mkb.services._api_common import ( + init_db, + logger, +) + + +def setup() -> None: + """Ensure database tables exist (idempotent).""" + init_db() + +def reset_db() -> None: + """Drop all tables and recreate them. Destructive!""" + from mkb.db.engine import sync_engine + from mkb.db.models import Base + + Base.metadata.drop_all(sync_engine) + Base.metadata.create_all(sync_engine) + logger.info("Database reset complete.") + + +# ── Ingestion / Sync ───────────────────────────────────────────── + diff --git a/src/mkb/services/spaces.py b/src/mkb/services/spaces.py new file mode 100644 index 0000000..8bd6b63 --- /dev/null +++ b/src/mkb/services/spaces.py @@ -0,0 +1,72 @@ +"""Spaces API service functions.""" + +from __future__ import annotations + +from mkb.services._api_common import ( + uuid, +) + + +def create_space( + name: str, + domain: str, + extraction_schema: dict, + system_prompt: str, + field_descriptions: dict, + description: str | None = None, + purpose: str = "tabular_database", + review_prompt: str | None = None, + review_trackable: bool = True, + review_allow_search: bool = False, + review_search_tools: list[str] | None = None, + post_processors: list[dict] | None = None, +) -> dict: + """Create a new space (domain-specific extraction configuration).""" + from mkb.spaces.registry import create_space as _create + + return _create( + name=name, + domain=domain, + extraction_schema=extraction_schema, + system_prompt=system_prompt, + field_descriptions=field_descriptions, + description=description, + purpose=purpose, + review_prompt=review_prompt, + review_trackable=review_trackable, + review_allow_search=review_allow_search, + review_search_tools=review_search_tools, + post_processors=post_processors, + ) + +def update_space(space_id: str | uuid.UUID, **changes) -> dict: + """Update fields on an existing space. Bumps version automatically. + + Accepted keys: name, description, extraction_schema, system_prompt, + field_descriptions, domain, purpose, review_prompt. + """ + from mkb.spaces.registry import update_space as _update + + return _update(space_id, **changes) + +def delete_space(space_id: str | uuid.UUID) -> dict: + """Delete a space by id.""" + from mkb.spaces.registry import delete_space as _delete + + return _delete(space_id) + +def list_spaces() -> list[dict]: + """List all spaces.""" + from mkb.spaces.registry import list_spaces as _list + + return _list() + +def get_space(space_id_or_name: str) -> dict | None: + """Get a space by ID or name.""" + from mkb.spaces.registry import get_space as _get + + return _get(space_id_or_name) + + +# ── Projections ────────────────────────────────────────────────── + diff --git a/src/mkb/services/workflows.py b/src/mkb/services/workflows.py new file mode 100644 index 0000000..b1d09db --- /dev/null +++ b/src/mkb/services/workflows.py @@ -0,0 +1,1000 @@ +"""Workflows API service functions.""" + +from __future__ import annotations + +from mkb.services._api_common import ( + SyncSessionLocal, + datetime, + init_db, + timezone, + uuid, +) + + +def serialize_raw_workflow(row, include_graph: bool) -> dict: + payload = { + "extraction_id": str(row.extraction_id), + "project_id": str(row.project_id), + "version": row.version, + "schema_version": row.schema_version, + "extractor_version": row.extractor_version, + "model": row.model, + "status": row.status, + "record_status": row.record_status, + "supersedes_extraction_id": ( + str(row.supersedes_extraction_id) if row.supersedes_extraction_id else None + ), + "correction_reason": row.correction_reason, + "correction_author": row.correction_author, + "correction_details": row.correction_details or {}, + "review_flags": row.review_flags or [], + "provenance": row.provenance or {}, + "error": row.error, + "extracted_at": row.extracted_at.isoformat() if row.extracted_at else None, + "created_at": row.created_at.isoformat() if row.created_at else None, + "has_checkpoint": bool(row.checkpoint), + "checkpoint_summary": (row.checkpoint or {}).get("summary"), + "checkpoint_updated_at": ( + row.checkpoint_updated_at.isoformat() if row.checkpoint_updated_at else None + ), + "resumable": row.graph is None and row.status in {"IN_PROGRESS", "FAILED"}, + } + if include_graph: + payload["graph"] = row.graph + elif row.graph: + payload["node_count"] = len(row.graph.get("nodes", [])) + payload["edge_count"] = len(row.graph.get("edges", [])) + return payload + + +def serialize_canonical_workflow(row, include_graph: bool) -> dict: + payload = { + "canonicalization_id": str(row.canonicalization_id), + "project_id": str(row.project_id), + "raw_extraction_id": str(row.raw_extraction_id), + "version": row.version, + "schema_version": row.schema_version, + "canonicalizer_version": row.canonicalizer_version, + "model": row.model, + "status": row.status, + "provenance": row.provenance or {}, + "error": row.error, + "canonicalized_at": row.canonicalized_at.isoformat() if row.canonicalized_at else None, + "created_at": row.created_at.isoformat() if row.created_at else None, + "has_checkpoint": bool(row.checkpoint), + "checkpoint_summary": (row.checkpoint or {}).get("summary"), + "checkpoint_updated_at": ( + row.checkpoint_updated_at.isoformat() if row.checkpoint_updated_at else None + ), + "resumable": row.graph is None and row.status in {"IN_PROGRESS", "FAILED"}, + } + if include_graph: + payload["graph"] = row.graph + elif row.graph: + payload.update( + node_count=len(row.graph.get("nodes", [])), + edge_count=len(row.graph.get("edges", [])), + ) + return payload + + +def extract_raw_workflow(project_id: str | uuid.UUID, model: str | None = None, verbose: bool = False, progress_callback=None) -> dict: + """Append a new faithful raw-workflow extraction version for a project.""" + from mkb.agents.workflow_extraction import run_workflow_extraction + + readiness = get_raw_workflow_extraction_readiness(project_id) + if not readiness.get("ready"): + return { + "status": "error", + "message": readiness.get("message") or "Project is not ready for workflow extraction", + } + init_db() + return run_workflow_extraction( + uuid.UUID(str(project_id)), model=model, verbose=verbose, + progress_callback=progress_callback, + ) + +def get_raw_workflow_extraction_readiness(project_id: str | uuid.UUID) -> dict: + """Check whether a project has readable sources for raw workflow extraction.""" + from mkb.db.models import ( + ProcessedAsset, + ProcessingType, + ProjectAsset, + RawWorkflowExtraction, + ResearchProject, + ) + + pid = uuid.UUID(str(project_id)) + init_db() + with SyncSessionLocal() as session: + project = session.query(ResearchProject).filter_by(project_id=pid).first() + if not project: + return {"ready": False, "message": f"Project {pid} not found"} + + rows = ( + session.query(ProcessedAsset.asset_id) + .join(ProjectAsset, ProjectAsset.asset_id == ProcessedAsset.asset_id) + .filter( + ProjectAsset.project_id == pid, + ProcessedAsset.processing_type == ProcessingType.MARKDOWN, + ) + .distinct() + .all() + ) + asset_ids = [str(row.asset_id) for row in rows] + if not asset_ids: + return { + "ready": False, + "message": ( + "Workflow extraction requires processed Markdown, but this project has no readable " + "processed Markdown files yet. Run Process first and confirm Markdown outputs exist." + ), + } + + unfinished = ( + session.query(RawWorkflowExtraction) + .filter( + RawWorkflowExtraction.project_id == pid, + RawWorkflowExtraction.graph.is_(None), + RawWorkflowExtraction.status.in_(("IN_PROGRESS", "FAILED")), + ) + .order_by(RawWorkflowExtraction.version.desc()) + .first() + ) + return { + "ready": True, + "project_id": str(pid), + "readable_asset_ids": asset_ids, + "resume_extraction_id": str(unfinished.extraction_id) if unfinished else None, + "resume_version": unfinished.version if unfinished else None, + "has_checkpoint": bool(unfinished and unfinished.checkpoint), + } + +def list_raw_workflows(project_id: str | uuid.UUID, include_graph: bool = False) -> list[dict]: + """List append-only raw workflow versions, newest first.""" + from mkb.db.models import RawWorkflowExtraction + + pid = uuid.UUID(str(project_id)) + init_db() + with SyncSessionLocal() as session: + rows = ( + session.query(RawWorkflowExtraction) + .filter(RawWorkflowExtraction.project_id == pid) + .order_by(RawWorkflowExtraction.version.desc()) + .all() + ) + return [_serialize_raw_workflow(row, include_graph=include_graph) for row in rows] + +def get_raw_workflow(project_id: str | uuid.UUID, version: int | None = None) -> dict | None: + """Get the latest completed raw workflow, or a specific version.""" + from mkb.db.models import RawWorkflowExtraction + + pid = uuid.UUID(str(project_id)) + init_db() + with SyncSessionLocal() as session: + query = session.query(RawWorkflowExtraction).filter(RawWorkflowExtraction.project_id == pid) + if version is None: + query = query.filter( + RawWorkflowExtraction.status == "COMPLETED", + RawWorkflowExtraction.record_status.in_(("active", "needs_review")), + ).order_by(RawWorkflowExtraction.version.desc()) + else: + query = query.filter(RawWorkflowExtraction.version == version) + row = query.first() + return _serialize_raw_workflow(row, include_graph=True) if row else None + +def delete_raw_workflow_version(project_id: str | uuid.UUID, version: int) -> dict: + """Delete one raw workflow version when no canonical version depends on it.""" + from mkb.db.models import CanonicalWorkflow, RawWorkflowExtraction + + pid = uuid.UUID(str(project_id)) + init_db() + with SyncSessionLocal() as session: + row = ( + session.query(RawWorkflowExtraction) + .filter( + RawWorkflowExtraction.project_id == pid, + RawWorkflowExtraction.version == version, + ) + .first() + ) + if not row: + return {"error": "Raw workflow version not found"} + dependent_canonical = ( + session.query(CanonicalWorkflow) + .filter(CanonicalWorkflow.raw_extraction_id == row.extraction_id) + .order_by(CanonicalWorkflow.version.desc()) + .first() + ) + if dependent_canonical: + return { + "error": ( + f"Raw workflow v{version} cannot be deleted because canonical workflow " + f"v{dependent_canonical.version} still depends on it" + ) + } + + extraction_id = row.extraction_id + session.delete(row) + session.commit() + return { + "status": "deleted", + "project_id": str(pid), + "version": version, + "extraction_id": str(extraction_id), + } + +def delete_canonical_workflow_version(project_id: str | uuid.UUID, version: int) -> dict: + """Delete one canonical workflow version and its derived indexes/tasks.""" + from mkb.db.models import CanonicalWorkflow, WorkflowIndexEntry, WorkflowMaintenanceTask + + pid = uuid.UUID(str(project_id)) + init_db() + with SyncSessionLocal() as session: + row = ( + session.query(CanonicalWorkflow) + .filter( + CanonicalWorkflow.project_id == pid, + CanonicalWorkflow.version == version, + ) + .first() + ) + if not row: + return {"error": "Canonical workflow version not found"} + + canonicalization_id = row.canonicalization_id + session.query(WorkflowIndexEntry).filter( + WorkflowIndexEntry.canonicalization_id == canonicalization_id + ).delete(synchronize_session=False) + session.query(WorkflowMaintenanceTask).filter( + WorkflowMaintenanceTask.project_id == pid, + WorkflowMaintenanceTask.source_canonicalization_id == canonicalization_id, + ).delete(synchronize_session=False) + session.delete(row) + session.commit() + return { + "status": "deleted", + "project_id": str(pid), + "version": version, + "canonicalization_id": str(canonicalization_id), + } + +def _serialize_raw_workflow(row, include_graph: bool) -> dict: + return serialize_raw_workflow(row, include_graph) + +def review_raw_workflow(extraction_id: str | uuid.UUID, *, status: str | None = None, author: str = "system") -> dict: + """Run automatic checks and optionally set a manual lifecycle status.""" + from mkb.db.models import RawWorkflowExtraction + from mkb.workflows.review import VALID_RECORD_STATUSES, audit_raw_graph + + init_db() + eid = uuid.UUID(str(extraction_id)) + if status is not None and status not in VALID_RECORD_STATUSES: + return {"error": f"Invalid record status: {status}"} + with SyncSessionLocal() as session: + row = session.query(RawWorkflowExtraction).filter_by(extraction_id=eid).first() + if not row or not row.graph: + return {"error": "Completed raw workflow not found"} + later = session.query(RawWorkflowExtraction).filter( + RawWorkflowExtraction.project_id == row.project_id, + RawWorkflowExtraction.version > row.version, + RawWorkflowExtraction.status == "COMPLETED", + ).order_by(RawWorkflowExtraction.version.desc()).first() + flags = audit_raw_graph(row.graph, later_graph=later.graph if later else None) + row.review_flags = flags + if status: + row.record_status = status + elif flags and row.record_status == "active": + row.record_status = "needs_review" + row.provenance = {**(row.provenance or {}), "last_reviewed_by": author} + session.commit() + return _serialize_raw_workflow(row, include_graph=False) + +def correct_raw_workflow( + extraction_id: str | uuid.UUID, graph: dict, *, reason: str, author: str, + affected_nodes: list[str] | None = None, affected_edges: list[str] | None = None, + evidence: str, +) -> dict: + """Create a corrected immutable version and supersede the source version.""" + from sqlalchemy import func + from mkb.db.models import RawWorkflowExtraction + from mkb.workflows.review import audit_raw_graph, correction_metadata, rebase_graph + + init_db() + eid = uuid.UUID(str(extraction_id)) + new_id = uuid.uuid4() + details = correction_metadata(reason, author, affected_nodes or [], affected_edges or [], evidence) + with SyncSessionLocal() as session: + source = session.query(RawWorkflowExtraction).filter_by(extraction_id=eid).first() + if not source or source.status != "COMPLETED": + return {"error": "Completed source workflow not found"} + corrected = dict(graph) + corrected["paper_id"] = str(source.project_id) + corrected["schema_version"] = source.schema_version + try: + corrected = rebase_graph(corrected, new_id) + except Exception as exc: + return {"error": f"Corrected graph validation failed: {exc}"} + version = (session.query(func.max(RawWorkflowExtraction.version)).filter_by(project_id=source.project_id).scalar() or 0) + 1 + flags = audit_raw_graph(corrected) + row = RawWorkflowExtraction( + extraction_id=new_id, project_id=source.project_id, version=version, + schema_version=source.schema_version, extractor_version=source.extractor_version, + model=source.model, status="COMPLETED", + record_status="needs_review" if flags else "active", + supersedes_extraction_id=source.extraction_id, graph=corrected, + correction_reason=reason, correction_author=author, + correction_details=details, review_flags=flags, + provenance={**(source.provenance or {}), "correction_evidence": evidence}, + extracted_at=datetime.now(timezone.utc), + ) + source.record_status = "superseded" + session.add(row) + session.commit() + return _serialize_raw_workflow(row, include_graph=True) + +def canonicalize_workflow(project_id: str | uuid.UUID, raw_extraction_id: str | uuid.UUID | None = None, model: str | None = None, verbose: bool = False, progress_callback=None) -> dict: + """Create an append-only canonical view from a valid raw workflow.""" + from mkb.agents.workflow_canonicalization import run_workflow_canonicalization + + init_db() + return run_workflow_canonicalization( + uuid.UUID(str(project_id)), + uuid.UUID(str(raw_extraction_id)) if raw_extraction_id else None, + model=model, verbose=verbose, progress_callback=progress_callback, + ) + +def list_canonical_workflows(project_id: str | uuid.UUID, include_graph: bool = False) -> list[dict]: + from mkb.db.models import CanonicalWorkflow + + pid = uuid.UUID(str(project_id)) + init_db() + with SyncSessionLocal() as session: + rows = session.query(CanonicalWorkflow).filter_by(project_id=pid).order_by(CanonicalWorkflow.version.desc()).all() + return [_serialize_canonical_workflow(row, include_graph) for row in rows] + +def get_canonical_workflow(project_id: str | uuid.UUID, version: int | None = None) -> dict | None: + from mkb.db.models import CanonicalWorkflow + + pid = uuid.UUID(str(project_id)) + init_db() + with SyncSessionLocal() as session: + query = session.query(CanonicalWorkflow).filter(CanonicalWorkflow.project_id == pid) + query = ( + query.filter(CanonicalWorkflow.status == "COMPLETED").order_by(CanonicalWorkflow.version.desc()) + if version is None else query.filter(CanonicalWorkflow.version == version) + ) + row = query.first() + return _serialize_canonical_workflow(row, True) if row else None + +def _serialize_canonical_workflow(row, include_graph: bool) -> dict: + return serialize_canonical_workflow(row, include_graph) + +def curate_workflow_schema(*, min_support: int = 2, author: str = "schema-curator/1.0") -> list[dict]: + """Analyze accumulated workflows and persist new evidence-backed proposals.""" + from mkb.db.models import CanonicalWorkflow, RawWorkflowExtraction, SchemaProposal, SchemaProposalRevision, WorkflowSchemaVersion + from mkb.workflows.curator import analyze_canonical_workflows + from mkb.workflows.schema_library import get_schema_library_payload + + init_db() + with SyncSessionLocal() as session: + current = session.query(WorkflowSchemaVersion).filter_by(status="active").order_by(WorkflowSchemaVersion.version.desc()).first() + if not current: + current = WorkflowSchemaVersion(version=1, name="workflow-schema/1.0", payload=get_schema_library_payload(), created_by="seed") + session.add(current) + session.flush() + rows = session.query(CanonicalWorkflow).filter_by(status="COMPLETED").all() + workflows = [] + for row in rows: + raw = session.query(RawWorkflowExtraction).filter_by(extraction_id=row.raw_extraction_id).first() + workflows.append({"canonicalization_id": str(row.canonicalization_id), "graph": row.graph, "raw_graph": raw.graph if raw else {}}) + generated = analyze_canonical_workflows(workflows, min_support=min_support) + results = [] + for item in generated: + duplicate = session.query(SchemaProposal).filter( + SchemaProposal.status.in_(("pending", "revision_requested")), + SchemaProposal.proposal_type == item["proposal_type"], + SchemaProposal.payload == item["payload"], + ).first() + if duplicate: + continue + rationale = ( + f"Deterministic discovery signal: {item.get('analysis', {}).get('signal', 'unknown')} " + f"with support from {len(item.get('evidence_workflow_ids', []))} workflows." + ) + proposal = SchemaProposal( + **item, rationale=rationale, + base_schema_version=current.name, created_by=author, + ) + session.add(proposal) + session.flush() + session.add(SchemaProposalRevision( + proposal_id=proposal.proposal_id, revision_number=1, + payload=proposal.payload, + evidence_workflow_ids=proposal.evidence_workflow_ids, + analysis=proposal.analysis, rationale=proposal.rationale, + author=author, + author_type="agent" if "agent" in author else "system", + change_note="Initial proposal draft", + validation_errors=[], + )) + results.append({**item, "proposal_id": str(proposal.proposal_id), "status": "pending"}) + session.commit() + return results + +def list_schema_proposals(status: str | None = "pending") -> list[dict]: + from sqlalchemy import func + from mkb.db.models import SchemaProposal, SchemaProposalRevision + + init_db() + with SyncSessionLocal() as session: + query = session.query(SchemaProposal) + if status: + query = query.filter_by(status=status) + rows = query.order_by(SchemaProposal.created_at.desc()).all() + revision_counts = dict( + session.query( + SchemaProposalRevision.proposal_id, + func.count(SchemaProposalRevision.revision_id), + ).group_by(SchemaProposalRevision.proposal_id).all() + ) + return [{ + "proposal_id": str(row.proposal_id), "proposal_type": row.proposal_type, + "status": row.status, "payload": row.payload, + "evidence_workflow_ids": row.evidence_workflow_ids, "analysis": row.analysis, + "base_schema_version": row.base_schema_version, "created_by": row.created_by, + "rationale": row.rationale, + "reviewer_notes": row.reviewer_notes, + "validation_errors": (row.analysis or {}).get("validation_errors", []), + "revision_count": int(revision_counts.get(row.proposal_id, 0)), + "reviewed_by": row.reviewed_by, + "reviewed_at": row.reviewed_at.isoformat() if row.reviewed_at else None, + "created_at": row.created_at.isoformat() if row.created_at else None, + } for row in rows] + +def get_schema_proposal_revisions(proposal_id: str | uuid.UUID) -> list[dict]: + from mkb.db.models import SchemaProposalRevision + + pid = uuid.UUID(str(proposal_id)) + init_db() + with SyncSessionLocal() as session: + rows = session.query(SchemaProposalRevision).filter_by(proposal_id=pid).order_by( + SchemaProposalRevision.revision_number.desc() + ).all() + return [{ + "revision_id": str(row.revision_id), + "revision_number": row.revision_number, + "payload": row.payload, + "evidence_workflow_ids": row.evidence_workflow_ids, + "analysis": row.analysis, + "rationale": row.rationale, + "author": row.author, + "author_type": row.author_type, + "change_note": row.change_note, + "validation_errors": row.validation_errors, + "created_at": row.created_at.isoformat() if row.created_at else None, + } for row in rows] + +def edit_schema_proposal( + proposal_id: str | uuid.UUID, *, payload: dict, + evidence_workflow_ids: list[str], rationale: str, + editor: str, change_note: str, +) -> dict: + """Save an attributed proposal draft revision and revalidate it.""" + from sqlalchemy import func + from mkb.db.models import ( + CanonicalWorkflow, RawWorkflowExtraction, SchemaProposal, SchemaProposalRevision, + WorkflowSchemaVersion, + ) + from mkb.workflows.curator import validate_proposal + + pid = uuid.UUID(str(proposal_id)) + if not editor.strip() or not change_note.strip(): + return {"error": "editor and change_note are required"} + try: + evidence_uuids = [uuid.UUID(value) for value in evidence_workflow_ids] + except (TypeError, ValueError, AttributeError): + return {"error": "evidence_workflow_ids must contain canonicalization UUIDs"} + init_db() + with SyncSessionLocal() as session: + row = session.query(SchemaProposal).filter_by(proposal_id=pid).first() + if not row or row.status not in {"pending", "revision_requested"}: + return {"error": "Only pending or revision-requested proposals can be edited"} + current = session.query(WorkflowSchemaVersion).filter_by(status="active").order_by( + WorkflowSchemaVersion.version.desc() + ).first() + if not current: + return {"error": "Active schema library not found"} + known_evidence = { + str(value) for (value,) in session.query(CanonicalWorkflow.canonicalization_id).filter( + CanonicalWorkflow.canonicalization_id.in_(evidence_uuids) + ).all() + } if evidence_workflow_ids else set() + if evidence_workflow_ids: + known_evidence.update({ + str(value) for (value,) in session.query(RawWorkflowExtraction.extraction_id).filter( + RawWorkflowExtraction.extraction_id.in_(evidence_uuids) + ).all() + }) + errors = validate_proposal( + row.proposal_type, payload, evidence_workflow_ids, current.payload, + ) + missing = sorted(set(evidence_workflow_ids) - known_evidence) + if missing: + errors.append(f"unknown evidence workflows: {', '.join(missing)}") + row.payload = payload + row.evidence_workflow_ids = evidence_workflow_ids + row.rationale = rationale.strip() + row.base_schema_version = current.name + row.analysis = {**(row.analysis or {}), "validation_errors": errors} + row.status = "pending" if not errors else "revision_requested" + revision_number = int( + session.query(func.coalesce(func.max(SchemaProposalRevision.revision_number), 0)) + .filter_by(proposal_id=pid).scalar() + ) + 1 + session.add(SchemaProposalRevision( + proposal_id=pid, revision_number=revision_number, + payload=payload, evidence_workflow_ids=evidence_workflow_ids, + analysis=row.analysis, rationale=row.rationale, + author=editor.strip(), author_type="human", + change_note=change_note.strip(), validation_errors=errors, + )) + session.commit() + return { + "proposal_id": str(pid), "status": row.status, + "revision_number": revision_number, "validation_errors": errors, + } + +def get_workflow_schema_status() -> dict: + """Return global schema and curator queue summary for the frontend.""" + from sqlalchemy import func + from mkb.db.models import SchemaProposal, WorkflowMaintenanceTask, WorkflowSchemaVersion + from mkb.workflows.schema_library import get_schema_library_payload + + init_db() + with SyncSessionLocal() as session: + active = session.query(WorkflowSchemaVersion).filter_by(status="active").order_by( + WorkflowSchemaVersion.version.desc() + ).first() + payload = active.payload if active else get_schema_library_payload() + proposal_counts = dict( + session.query(SchemaProposal.status, func.count(SchemaProposal.proposal_id)) + .group_by(SchemaProposal.status).all() + ) + pending_recanonicalizations = session.query(func.count(WorkflowMaintenanceTask.task_id)).filter( + WorkflowMaintenanceTask.task_type == "recanonicalize", + WorkflowMaintenanceTask.status == "pending", + ).scalar() or 0 + return { + "schema_version": active.name if active else payload["schema_version"], + "version_number": active.version if active else 1, + "status": active.status if active else "seed", + "change_summary": active.change_summary if active else "Built-in seed schema", + "created_by": active.created_by if active else "system", + "created_at": active.created_at.isoformat() if active and active.created_at else None, + "object_schema_count": len(payload.get("object_schemas", {})), + "operation_template_count": len(payload.get("operation_templates", {})), + "card_count": len(payload.get("cards", {})), + "granularity_relation_count": len(payload.get("granularity_relations", [])), + "proposal_counts": proposal_counts, + "pending_recanonicalizations": int(pending_recanonicalizations), + } + +def review_schema_proposal( + proposal_id: str | uuid.UUID, *, approve: bool | None = None, + reviewer: str, decision: str | None = None, notes: str = "", +) -> dict: + """Validate and approve/reject a proposal; approval creates a schema snapshot.""" + from sqlalchemy import func + from mkb.db.models import ( + CanonicalWorkflow, RawWorkflowExtraction, SchemaProposal, SchemaProposalRevision, + WorkflowMaintenanceTask, WorkflowSchemaVersion, + ) + from mkb.workflows.curator import apply_proposal, validate_proposal + from mkb.workflows.maintenance import recanonicalization_reason_for_proposal + + decision = decision or ("approve" if approve else "reject") + if decision not in {"approve", "reject", "request_revision"}: + return {"error": f"Unsupported review decision: {decision}"} + if not reviewer.strip(): + return {"error": "reviewer is required"} + if decision == "request_revision" and not notes.strip(): + return {"error": "Revision requests require reviewer notes"} + init_db() + with SyncSessionLocal() as session: + row = session.query(SchemaProposal).filter_by(proposal_id=uuid.UUID(str(proposal_id))).first() + if not row or row.status not in {"pending", "revision_requested"}: + return {"error": "Reviewable proposal not found"} + current = session.query(WorkflowSchemaVersion).filter_by(status="active").order_by(WorkflowSchemaVersion.version.desc()).first() + if not current: + return {"error": "Schema library is not initialized; run the curator first"} + errors = validate_proposal(row.proposal_type, row.payload, row.evidence_workflow_ids, current.payload) + known_evidence = { + str(value) for (value,) in session.query(CanonicalWorkflow.canonicalization_id).filter( + CanonicalWorkflow.canonicalization_id.in_([ + uuid.UUID(value) for value in row.evidence_workflow_ids + ]) + ).all() + } if row.evidence_workflow_ids else set() + if row.evidence_workflow_ids: + known_evidence.update({ + str(value) for (value,) in session.query(RawWorkflowExtraction.extraction_id).filter( + RawWorkflowExtraction.extraction_id.in_([ + uuid.UUID(value) for value in row.evidence_workflow_ids + ]) + ).all() + }) + missing = sorted(set(row.evidence_workflow_ids) - known_evidence) + if missing: + errors.append(f"unknown evidence workflows: {', '.join(missing)}") + rebased_from = None + if decision == "approve" and row.base_schema_version != current.name: + rebased_from = row.base_schema_version + row.base_schema_version = current.name + row.analysis = { + **(row.analysis or {}), + "rebased_from_schema": rebased_from, + "rebased_to_schema": current.name, + } + if decision == "approve" and errors: + return {"error": "Schema validation failed", "details": errors} + row.reviewed_by = reviewer.strip() + row.reviewer_notes = notes.strip() or None + row.reviewed_at = datetime.now(timezone.utc) + revision_number = int( + session.query(func.coalesce(func.max(SchemaProposalRevision.revision_number), 0)) + .filter_by(proposal_id=row.proposal_id).scalar() + ) + 1 + session.add(SchemaProposalRevision( + proposal_id=row.proposal_id, revision_number=revision_number, + payload=row.payload, evidence_workflow_ids=row.evidence_workflow_ids, + analysis=row.analysis, rationale=row.rationale, + author=reviewer.strip(), author_type="human", + change_note=( + f"Automatically rebased {rebased_from} to {current.name}. " + if rebased_from else "" + ) + f"Review decision: {decision}. {notes.strip()}".strip(), + validation_errors=errors, + )) + if decision in {"reject", "request_revision"}: + row.status = "rejected" if decision == "reject" else "revision_requested" + session.commit() + return { + "proposal_id": str(row.proposal_id), "status": row.status, + "revision_number": revision_number, + } + next_version = current.version + 1 + next_name = f"workflow-schema/1.{next_version - 1}" + base_payload = {**current.payload, "schema_version": next_name} + payload = apply_proposal(base_payload, row.proposal_type, row.payload) + current.status = "superseded" + session.add(WorkflowSchemaVersion( + version=next_version, name=next_name, payload=payload, + change_summary=f"Applied proposal {row.proposal_id}: {row.proposal_type}", + created_by=reviewer, + )) + row.status = "approved" + affected = 0 + queues_created = 0 + queues_updated = 0 + duplicate_queues_removed = 0 + completed = session.query(CanonicalWorkflow).filter_by(status="COMPLETED").order_by( + CanonicalWorkflow.project_id, CanonicalWorkflow.version.desc() + ).all() + latest_by_project = {} + for canonical in completed: + latest_by_project.setdefault(canonical.project_id, canonical) + # Every immutable schema snapshot has a new version. Even when a + # proposal directly cites only a subset, each latest project view is + # queued so its canonical graph can explicitly target that version. + for canonical in latest_by_project.values(): + canonical.provenance = { + **(canonical.provenance or {}), + "recanonicalization_required": True, + "target_schema_version": next_name, + } + pending_tasks = session.query(WorkflowMaintenanceTask).filter_by( + project_id=canonical.project_id, + task_type="recanonicalize", + status="pending", + ).order_by(WorkflowMaintenanceTask.created_at).all() + proposal_ids = [str(row.proposal_id)] + if pending_tasks: + task = pending_tasks[0] + previous_ids = (task.scope or {}).get("schema_proposal_ids", []) + task.scope = { + **(task.scope or {}), + "schema_proposal_ids": list(dict.fromkeys([ + *previous_ids, *proposal_ids, + ])), + } + task.reason = "schema_version_changed" + task.source_raw_extraction_id = canonical.raw_extraction_id + task.source_canonicalization_id = canonical.canonicalization_id + task.target_schema_version = next_name + task.requested_by = reviewer.strip() + for duplicate in pending_tasks[1:]: + session.delete(duplicate) + duplicate_queues_removed += 1 + queues_updated += 1 + else: + session.add(WorkflowMaintenanceTask( + project_id=canonical.project_id, + task_type="recanonicalize", + reason=recanonicalization_reason_for_proposal(row.proposal_type), + source_raw_extraction_id=canonical.raw_extraction_id, + source_canonicalization_id=canonical.canonicalization_id, + target_schema_version=next_name, + requested_by=reviewer.strip(), + scope={"schema_proposal_ids": proposal_ids}, + )) + queues_created += 1 + affected += 1 + session.commit() + return { + "proposal_id": str(row.proposal_id), "status": "approved", + "schema_version": next_name, + "rebased_from_schema": rebased_from, + "recanonicalization_scheduled": affected, + "queues_created": queues_created, + "queues_updated": queues_updated, + "duplicate_queues_removed": duplicate_queues_removed, + } + +def schedule_workflow_reextraction(project_id: str | uuid.UUID, *, reason: str, requested_by: str, scope: dict | None = None, raw_extraction_id: str | uuid.UUID | None = None) -> dict: + """Queue an approved full or partial re-extraction request.""" + from mkb.db.models import RawWorkflowExtraction, WorkflowMaintenanceTask + from mkb.workflows.maintenance import validate_reextraction_request + + pid = uuid.UUID(str(project_id)) + validated_scope = validate_reextraction_request(reason, scope) + init_db() + with SyncSessionLocal() as session: + query = session.query(RawWorkflowExtraction).filter( + RawWorkflowExtraction.project_id == pid, + RawWorkflowExtraction.status == "COMPLETED", + RawWorkflowExtraction.record_status.in_(("active", "needs_review")), + ) + raw = query.filter_by(extraction_id=uuid.UUID(str(raw_extraction_id))).first() if raw_extraction_id else query.order_by(RawWorkflowExtraction.version.desc()).first() + if not raw: + return {"error": "No valid raw workflow is available for re-extraction"} + task = WorkflowMaintenanceTask( + project_id=pid, task_type="reextract", reason=reason, + source_raw_extraction_id=raw.extraction_id, scope=validated_scope, + requested_by=requested_by, + ) + session.add(task) + session.commit() + return {"task_id": str(task.task_id), "status": task.status, "task_type": task.task_type, "scope": task.scope} + +def schedule_workflow_recanonicalization(project_id: str | uuid.UUID, *, reason: str = "manual_request", requested_by: str, raw_extraction_id: str | uuid.UUID | None = None, target_schema_version: str | None = None) -> dict: + from mkb.db.models import RawWorkflowExtraction, WorkflowMaintenanceTask + from mkb.workflows.maintenance import RECANONICALIZATION_REASONS + from mkb.workflows.schema_library import get_schema_library_payload + + if reason not in RECANONICALIZATION_REASONS: + raise ValueError(f"Unsupported recanonicalization reason: {reason}") + pid = uuid.UUID(str(project_id)) + init_db() + with SyncSessionLocal() as session: + query = session.query(RawWorkflowExtraction).filter( + RawWorkflowExtraction.project_id == pid, + RawWorkflowExtraction.status == "COMPLETED", + RawWorkflowExtraction.record_status.in_(("active", "needs_review")), + ) + raw = query.filter_by(extraction_id=uuid.UUID(str(raw_extraction_id))).first() if raw_extraction_id else query.order_by(RawWorkflowExtraction.version.desc()).first() + if not raw: + return {"error": "No valid raw workflow is available for canonicalization"} + task = WorkflowMaintenanceTask( + project_id=pid, task_type="recanonicalize", reason=reason, + source_raw_extraction_id=raw.extraction_id, + target_schema_version=target_schema_version or get_schema_library_payload()["schema_version"], + requested_by=requested_by, + ) + session.add(task) + session.commit() + return {"task_id": str(task.task_id), "status": task.status, "task_type": task.task_type} + +def list_workflow_maintenance_tasks(*, status: str | None = None, project_id: str | uuid.UUID | None = None) -> list[dict]: + from mkb.db.models import WorkflowMaintenanceTask + + init_db() + with SyncSessionLocal() as session: + query = session.query(WorkflowMaintenanceTask) + if status: + query = query.filter_by(status=status) + if project_id: + query = query.filter_by(project_id=uuid.UUID(str(project_id))) + return [{ + "task_id": str(row.task_id), "project_id": str(row.project_id), + "task_type": row.task_type, "reason": row.reason, "scope": row.scope, + "status": row.status, "target_schema_version": row.target_schema_version, + "result": row.result, "error": row.error, + } for row in query.order_by(WorkflowMaintenanceTask.created_at.desc()).all()] + +def run_workflow_maintenance_task(task_id: str | uuid.UUID, *, model: str | None = None, verbose: bool = False, progress_callback=None) -> dict: + """Execute one queued task, retaining both raw and canonical history.""" + from mkb.agents.workflow_canonicalization import run_workflow_canonicalization + from mkb.agents.workflow_extraction import run_workflow_extraction + from mkb.db.models import RawWorkflowExtraction, WorkflowMaintenanceTask + + tid = uuid.UUID(str(task_id)) + init_db() + with SyncSessionLocal() as session: + task = session.query(WorkflowMaintenanceTask).filter_by(task_id=tid).first() + if not task or task.status not in {"pending", "failed"}: + return {"error": "Pending or failed maintenance task not found"} + task.status = "running" + task.started_at = datetime.now(timezone.utc) + project_id, task_type, reason = task.project_id, task.task_type, task.reason + source_raw_id, scope = task.source_raw_extraction_id, task.scope + target_schema_version = task.target_schema_version + raw = session.query(RawWorkflowExtraction).filter_by(extraction_id=source_raw_id).first() + baseline = raw.graph if raw else None + session.commit() + try: + if task_type == "reextract": + extraction = run_workflow_extraction( + project_id, model=model, verbose=verbose, progress_callback=progress_callback, + reextraction_request={ + "reason": reason, "scope": scope, + "source_raw_extraction_id": str(source_raw_id), + "baseline_graph": baseline, + }, + ) + if extraction.get("status") != "completed": + raise RuntimeError(extraction.get("message") or "Re-extraction failed") + canonical = run_workflow_canonicalization( + project_id, uuid.UUID(extraction["extraction_id"]), model=model, + verbose=verbose, progress_callback=progress_callback, + recanonicalization_reason="raw_version_changed", + ) + result = {"extraction": extraction, "canonicalization": canonical} + else: + result = run_workflow_canonicalization( + project_id, source_raw_id, model=model, verbose=verbose, + progress_callback=progress_callback, recanonicalization_reason=reason, + target_schema_version=target_schema_version, + ) + successful = result.get("status") == "completed" or result.get("canonicalization", {}).get("status") == "completed" + if not successful: + raise RuntimeError(result.get("message") or "Workflow maintenance failed") + except Exception as exc: + with SyncSessionLocal() as session: + task = session.query(WorkflowMaintenanceTask).filter_by(task_id=tid).first() + task.status, task.error, task.completed_at = "failed", str(exc), datetime.now(timezone.utc) + session.commit() + return {"task_id": str(tid), "status": "failed", "error": str(exc)} + with SyncSessionLocal() as session: + task = session.query(WorkflowMaintenanceTask).filter_by(task_id=tid).first() + task.status, task.result, task.completed_at = "completed", result, datetime.now(timezone.utc) + session.commit() + return {"task_id": str(tid), "status": "completed", "result": result} + +def run_pending_recanonicalizations( + *, model: str | None = None, verbose: bool = False, progress_callback=None, +) -> dict: + """Run all currently pending recanonicalizations as one global batch job.""" + from mkb.db.models import WorkflowMaintenanceTask + + init_db() + with SyncSessionLocal() as session: + rows = session.query(WorkflowMaintenanceTask).filter_by( + task_type="recanonicalize", status="pending", + ).order_by(WorkflowMaintenanceTask.created_at.desc()).all() + latest_by_project = {} + duplicates = [] + for row in rows: + if row.project_id in latest_by_project: + duplicates.append((row, latest_by_project[row.project_id])) + else: + latest_by_project[row.project_id] = row + for duplicate, retained in duplicates: + duplicate.status = "superseded" + duplicate.result = { + **(duplicate.result or {}), + "superseded_by_task_id": str(retained.task_id), + } + session.commit() + task_ids = [row.task_id for row in latest_by_project.values()] + results = [] + completed = 0 + failed = 0 + for index, task_id in enumerate(task_ids, 1): + if progress_callback: + progress_callback({ + "stage": "recanonicalization_batch", + "message": f"Recanonicalizing project workflow {index}/{len(task_ids)}", + }) + result = run_workflow_maintenance_task( + task_id, model=model, verbose=verbose, + progress_callback=progress_callback, + ) + results.append(result) + if result.get("status") == "completed": + completed += 1 + else: + failed += 1 + return { + "status": "completed" if failed == 0 else "completed_with_errors", + "task_count": len(task_ids), "completed": completed, "failed": failed, + "duplicate_tasks_coalesced": len(duplicates), + "results": results, + } + +def rebuild_workflow_indexes(project_id: str | uuid.UUID | None = None) -> dict: + from mkb.db.models import CanonicalWorkflow, RawWorkflowExtraction, WorkflowIndexEntry + from mkb.workflows.indexing import build_index_entries + from mkb.workflows.schema_library import get_schema_library_payload + + init_db() + with SyncSessionLocal() as session: + query = session.query(CanonicalWorkflow).filter_by(status="COMPLETED") + if project_id: + query = query.filter_by(project_id=uuid.UUID(str(project_id))) + rows = query.all() + ids = [row.canonicalization_id for row in rows] + if ids: + session.query(WorkflowIndexEntry).filter(WorkflowIndexEntry.canonicalization_id.in_(ids)).delete(synchronize_session=False) + count = 0 + for row in rows: + raw = session.query(RawWorkflowExtraction).filter_by(extraction_id=row.raw_extraction_id).first() + schema = get_schema_library_payload(row.schema_version) + for entry in build_index_entries(row.graph or {}, raw.graph if raw else {}, schema): + session.add(WorkflowIndexEntry(canonicalization_id=row.canonicalization_id, project_id=row.project_id, **entry)) + count += 1 + session.commit() + return {"workflows_indexed": len(rows), "entries_created": count} + +def search_canonical_workflows(source: str | None = None, operation: str | None = None, target: str | None = None, mode: str = "strict", limit: int = 100) -> list[dict]: + """Search persisted workflow indexes and return evidence-rich explanations.""" + from mkb.db.models import CanonicalWorkflow, WorkflowIndexEntry + from mkb.workflows.indexing import QUERY_MODES, match_index_entry, normalize + + legacy_modes = {"exact": "strict", "relaxed": "alias-expanded", "expanded": "granularity-expanded", "summarized": "granularity-expanded"} + mode = legacy_modes.get(mode, mode) + if mode not in QUERY_MODES: + raise ValueError(f"Unsupported query mode: {mode}") + init_db() + results = [] + seen_paths = set() + with SyncSessionLocal() as session: + completed = session.query(CanonicalWorkflow).filter_by(status="COMPLETED").order_by(CanonicalWorkflow.project_id, CanonicalWorkflow.version.desc()).all() + latest = {} + for row in completed: + latest.setdefault(row.project_id, row) + rows_by_id = {row.canonicalization_id: row for row in latest.values()} + entry_query = session.query(WorkflowIndexEntry).filter( + WorkflowIndexEntry.canonicalization_id.in_(rows_by_id) + ) if rows_by_id else None + if entry_query is not None and mode in {"strict", "alias-expanded", "evidence-required"}: + entry_query = entry_query.filter(WorkflowIndexEntry.index_type == "direct") + if source: + entry_query = entry_query.filter(WorkflowIndexEntry.source_label == normalize(source)) + if target: + entry_query = entry_query.filter(WorkflowIndexEntry.target_label == normalize(target)) + if operation and mode in {"strict", "evidence-required"}: + entry_query = entry_query.filter(WorkflowIndexEntry.operation_label == normalize(operation)) + entries = entry_query.all() if entry_query is not None else [] + for entry in entries: + data = {column.name: getattr(entry, column.name) for column in WorkflowIndexEntry.__table__.columns} + matched, explanation = match_index_entry(data, source=source, operation=operation, target=target, mode=mode) + if not matched: + continue + result_key = (entry.canonicalization_id, tuple(entry.path_node_ids)) + if result_key in seen_paths: + continue + seen_paths.add(result_key) + canonical = rows_by_id[entry.canonicalization_id] + graph_nodes = {node["node_id"]: node for node in (canonical.graph or {}).get("nodes", [])} + results.append({ + "project_id": str(entry.project_id), + "canonicalization_id": str(entry.canonicalization_id), + "version": canonical.version, "mode": mode, + "path": [graph_nodes[node_id] for node_id in entry.path_node_ids if node_id in graph_nodes], + "explanation": explanation, + }) + if len(results) >= limit: + break + return results + diff --git a/src/mkb/ui/README.md b/src/mkb/ui/README.md new file mode 100644 index 0000000..22cf118 --- /dev/null +++ b/src/mkb/ui/README.md @@ -0,0 +1,9 @@ +# Legacy Streamlit UI + +The React frontend is the active user interface. This package is +compatibility-only while older helpers, tests, and local workflows are migrated. + +Do not add new product behavior here. Shared behavior should move to backend +services, pure helpers, or the React/FastAPI surface, then this package can be +retired once remaining tests stop importing Streamlit page modules. + diff --git a/src/mkb/ui/background_jobs.py b/src/mkb/ui/background_jobs.py index 475c68f..79b8ef1 100644 --- a/src/mkb/ui/background_jobs.py +++ b/src/mkb/ui/background_jobs.py @@ -11,6 +11,8 @@ import streamlit as st +from mkb.web.job_actions import job_action_start_params + _EVENT_LIMIT = 25 _HISTORY_LIMIT = 12 @@ -83,6 +85,7 @@ def start_job( metadata: dict[str, Any] | None = None, args: tuple[Any, ...] | None = None, kwargs: dict[str, Any] | None = None, + on_complete: Callable[[Any], None] | None = None, ) -> str: """Start a daemon-thread background job and register progress state.""" _init_job_state() @@ -118,6 +121,8 @@ def _run() -> None: if "progress_callback" not in worker_kwargs: worker_kwargs["progress_callback"] = _emit_progress result = target(*worker_args, **worker_kwargs) + if on_complete is not None: + on_complete(result) progress_queue.put({"type": "done", "result": result}) except Exception as exc: # noqa: BLE001 progress_queue.put({"type": "error", "error": str(exc)}) @@ -126,6 +131,25 @@ def _run() -> None: return job_id +def start_job_action( + action: str, + *, + job_project_id: str | None = None, + label: str | None = None, + metadata: dict[str, Any] | None = None, + args: tuple[Any, ...] | None = None, + on_complete: Callable[[Any], None] | None = None, + **kwargs: Any, +) -> str: + params = job_action_start_params( + action, + job_project_id=job_project_id, + label=label, + **kwargs, + ) + return start_job(metadata=metadata, args=args, on_complete=on_complete, **params) + + def poll_jobs() -> bool: """Drain all background-job queues. Returns True if any state changed.""" _init_job_state() @@ -246,4 +270,4 @@ def _auto_refresh() -> None: poll_jobs() render_sidebar_monitor() - _auto_refresh() \ No newline at end of file + _auto_refresh() diff --git a/src/mkb/ui/pages/assistant.py b/src/mkb/ui/pages/assistant.py index 1eed9a8..bfbaa25 100644 --- a/src/mkb/ui/pages/assistant.py +++ b/src/mkb/ui/pages/assistant.py @@ -4,8 +4,8 @@ import streamlit as st -from mkb import api -from mkb.ui.background_jobs import get_running_job, start_job +from mkb.ui.background_jobs import start_job_action +from mkb.web.job_actions import action_for_workflow_kind # Session state keys _KEY_RUNNER = "_orch_runner" @@ -35,50 +35,11 @@ def _dispatch_pending_workflows() -> None: pending = get_pending_workflows() for req in pending: kind = req["kind"] + action = req.get("action") or action_for_workflow_kind(kind) project_id = req.get("project_id") kwargs = req.get("kwargs", {}) label = req.get("label", kind) - - if kind == "extraction": - start_job( - kind="extraction", - label=label, - project_id=project_id, - target=api.extract, - kwargs=kwargs, - ) - elif kind == "projection": - start_job( - kind="projection", - label=label, - project_id=project_id, - target=api.project, - kwargs={"space_id": kwargs["space_id"], "project_id": kwargs["project_id"]}, - ) - elif kind == "kg_extraction": - start_job( - kind="kg_extraction", - label=label, - project_id=project_id, - target=api.extract_knowledge_graph, - kwargs={"project_id": kwargs["project_id"]}, - ) - elif kind == "feedback_review": - start_job( - kind="feedback_review", - label=label, - project_id=project_id, - target=api.review_feedback, - kwargs={"project_id": kwargs["project_id"]}, - ) - elif kind == "projection_review": - start_job( - kind="projection_review", - label=label, - project_id=project_id, - target=api.review_projections, - kwargs={"space_id": kwargs["space_id"], "project_id": kwargs["project_id"]}, - ) + start_job_action(action, job_project_id=project_id, label=label, **kwargs) def _collect_reply() -> None: @@ -215,17 +176,11 @@ def render() -> None: runner = st.session_state[_KEY_RUNNER] session_id = st.session_state[_KEY_SESSION] - from mkb.agents.orchestrator import send_message - - job_id = start_job( - kind="orchestrator_chat", - label="Assistant", - target=send_message, - kwargs={ - "runner": runner, - "session_id": session_id, - "message": prompt, - }, + job_id = start_job_action( + "assistant_chat", + runner=runner, + session_id=session_id, + message=prompt, ) st.session_state[_KEY_JOB_ID] = job_id st.session_state[_KEY_WAITING_FINISH] = True diff --git a/src/mkb/ui/pages/projections.py b/src/mkb/ui/pages/projections.py index 61471b7..06dcbfa 100644 --- a/src/mkb/ui/pages/projections.py +++ b/src/mkb/ui/pages/projections.py @@ -110,6 +110,26 @@ def _projection_timestamp(projection: dict) -> str: return projection.get("extracted_at") or projection.get("created_at") or "" +def _filter_latest_projections(projections: list[dict]) -> list[dict]: + """Return only the newest projection for each project/space pair.""" + latest: dict[tuple[str, str], dict] = {} + order: list[tuple[str, str]] = [] + + for projection in projections: + project_id = str(projection.get("project_id") or "") + space_id = str(projection.get("space_id") or projection.get("frame_id") or "") + key = (project_id, space_id) + if key not in latest: + order.append(key) + latest[key] = projection + continue + + if _projection_timestamp(projection) >= _projection_timestamp(latest[key]): + latest[key] = projection + + return [latest[key] for key in order] + + def _projection_to_section_rows( projection: dict, project_paper_lookup: dict[str, str] | None = None, diff --git a/src/mkb/ui/pages/projects.py b/src/mkb/ui/pages/projects.py index f895d23..26d0a78 100644 --- a/src/mkb/ui/pages/projects.py +++ b/src/mkb/ui/pages/projects.py @@ -9,7 +9,12 @@ from mkb import api from mkb.knowledge_graph import GLOBAL_KG_SPACE_NAME from mkb.ui.data_cache import clear_graph_cache, get_knowledge_graph_cached, search_library_cached -from mkb.ui.background_jobs import get_project_jobs, get_running_job, render_project_job_status, start_job +from mkb.ui.background_jobs import ( + get_project_jobs, + get_running_job, + render_project_job_status, + start_job_action, +) from mkb.ui.upload_server import ensure_upload_server, get_upload_url, session_dir # Custom drop-zone component: captures webkitRelativePath so folder structure @@ -118,12 +123,11 @@ def _render_upload(): if not payload: return - start_job( - kind="upload", - label="Upload Ingest", - project_id="__upload__", - target=_run_upload_ingest, - args=(payload,), + start_job_action( + "upload_ingest", + job_project_id="__upload__", + payload=payload, + ingest_func=_run_upload_ingest, ) st.session_state["_upload_gen"] = st.session_state.get("_upload_gen", 0) + 1 st.rerun() @@ -388,13 +392,7 @@ def _render_project_detail(project_id: str): help="Convert raw files to LLM-readable formats", disabled=process_job is not None, ): - start_job( - kind="process", - label="Process Assets", - project_id=project_id, - target=api.process, - kwargs={"project_id": project_id}, - ) + start_job_action("process_project", job_project_id=project_id, project_id=project_id) st.rerun() if process_job: st.caption(process_job.get("current_message") or "Running") @@ -406,13 +404,7 @@ def _render_project_detail(project_id: str): help="Run LLM knowledge extraction", disabled=extract_job is not None, ): - start_job( - kind="extract", - label="Extract Knowledge Frame", - project_id=project_id, - target=api.extract, - kwargs={"project_id": project_id}, - ) + start_job_action("extract_project", job_project_id=project_id, project_id=project_id) st.rerun() if extract_job: st.caption(extract_job.get("current_message") or "Running") @@ -438,12 +430,12 @@ def _render_project_detail(project_id: str): disabled=projection_job is not None, ): sid = space_name_to_id[selected_space_name] - start_job( - kind="project", + start_job_action( + "project_to_space", + job_project_id=project_id, label="Run Projection", + space_id=sid, project_id=project_id, - target=api.project, - kwargs={"space_id": sid, "project_id": project_id}, ) st.rerun() if projection_job: @@ -456,12 +448,12 @@ def _render_project_detail(project_id: str): help="Extract knowledge graph elements", disabled=kg_job is not None, ): - start_job( - kind="knowledge_graph", + start_job_action( + "extract_knowledge_graph", + job_project_id=project_id, label="Extract Knowledge Graph", project_id=project_id, - target=_run_knowledge_graph_job, - kwargs={"project_id": project_id}, + on_complete=lambda _result: clear_graph_cache(), ) st.rerun() if kg_job: @@ -528,12 +520,6 @@ def _render_assets_tab(project_id: str): cols[2].caption(f".{pa['output_format']}") -def _run_knowledge_graph_job(project_id: str, progress_callback=None) -> dict: - result = api.extract_knowledge_graph(project_id=project_id, progress_callback=progress_callback) - clear_graph_cache() - return result - - def _render_frame_tab(project_id: str): frame = api.get_frame(project_id) if not frame: @@ -668,13 +654,7 @@ def _render_workflow_tab(project_id: str): if not readiness.get("ready"): st.error(readiness.get("message") or "Project is not ready for workflow extraction.") return - start_job( - kind="raw_workflow", - label="Extract Workflow", - project_id=project_id, - target=api.extract_raw_workflow, - kwargs={"project_id": project_id}, - ) + start_job_action("extract_raw_workflow", job_project_id=project_id, project_id=project_id) st.rerun() if raw_job: st.caption(raw_job.get("current_message") or "Running") diff --git a/src/mkb/web/_helpers.py b/src/mkb/web/_helpers.py index 6e294b3..d45b34d 100644 --- a/src/mkb/web/_helpers.py +++ b/src/mkb/web/_helpers.py @@ -5,11 +5,14 @@ from pathlib import Path from fastapi import HTTPException +from mkb.services.ids import strict_uuid +from mkb.services.result import ServiceError, is_error_result, result_status_code +from mkb.web.job_actions import JobActionConflict, start_job_action def _parse_uuid(value: str, field: str) -> uuid.UUID: try: - return uuid.UUID(str(value)) + return strict_uuid(value, field) except ValueError as exc: raise HTTPException(status_code=400, detail=f"Invalid {field}: {value!r}") from exc @@ -25,3 +28,41 @@ def _safe_child(base: Path, rel: str) -> Path: except ValueError as exc: raise HTTPException(status_code=400, detail="Path traversal rejected") from exc return full + + +def service_http_exception( + error: ServiceError | dict, + *, + default_status: int = 400, +) -> HTTPException: + if isinstance(error, ServiceError): + return HTTPException(status_code=error.status_code, detail=error.to_dict()) + return HTTPException( + status_code=result_status_code(error, default_status), + detail=error.get("error") or error, + ) + + +def require_service_result(result, *, default_status: int = 400): + """Return successful service results or raise a normalized HTTP error.""" + if isinstance(result, ServiceError): + raise service_http_exception(result, default_status=default_status) + if is_error_result(result): + raise service_http_exception(result, default_status=default_status) + return result + + +def require_service_result_or_not_found(result, *, default_status: int = 400): + """Map service errors containing 'not found' to 404, otherwise default.""" + if is_error_result(result) and "not found" in str(result.get("error", "")).lower(): + default_status = 404 + return require_service_result(result, default_status=default_status) + + +def start_web_job_action(manager, action: str, **kwargs) -> str: + try: + return start_job_action(manager, action, **kwargs) + except JobActionConflict as exc: + raise HTTPException(status_code=409, detail=str(exc)) from exc + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc diff --git a/src/mkb/web/_state.py b/src/mkb/web/_state.py index efc187c..93093a9 100644 --- a/src/mkb/web/_state.py +++ b/src/mkb/web/_state.py @@ -6,7 +6,6 @@ """ from __future__ import annotations -import ctypes import queue import threading import uuid @@ -14,11 +13,11 @@ from datetime import datetime, timezone from typing import Any -from mkb import api from mkb.agents._utils import JobCancelled from mkb.agents.orchestrator import create_orchestrator_runner from mkb.agents.tools.orchestrator_tools import get_pending_workflows from mkb.config import settings +from mkb.web.job_actions import action_for_workflow_kind, start_job_action _EVENT_LIMIT = 60 @@ -58,7 +57,6 @@ def __init__(self, max_concurrent: int | None = None) -> None: self._queues: dict[str, queue.Queue] = {} self._lock = threading.Lock() self._cancelled: set[str] = set() - self._threads: dict[str, int] = {} # job_id -> thread ident limit = max_concurrent if max_concurrent is not None else settings.max_concurrent_jobs self._semaphore = threading.Semaphore(max(1, limit)) @@ -115,8 +113,6 @@ def runner() -> None: if job_id in self._cancelled: q.put({"type": "cancelled"}) return - with self._lock: - self._threads[job_id] = threading.current_thread().ident # type: ignore[assignment] q.put({"type": "running"}) q.put({"type": "progress", "message": f"Started {label.lower()}"}) result = target(*worker_args, **worker_kwargs) @@ -127,8 +123,6 @@ def runner() -> None: q.put({"type": "error", "error": str(exc)}) finally: self._semaphore.release() - with self._lock: - self._threads.pop(job_id, None) threading.Thread(target=runner, daemon=True).start() return job_id @@ -232,8 +226,8 @@ def cancel_job(self, job_id: str) -> bool: """Request cancellation of a QUEUED or RUNNING job. Returns True if the job was found and a cancellation was initiated. - QUEUED jobs are marked CANCELLED immediately; RUNNING jobs receive an - async exception via ctypes so the worker thread can clean up. + QUEUED jobs are marked CANCELLED immediately; RUNNING jobs observe the + request at cooperative progress/cancellation checkpoints. """ self._drain() with self._lock: @@ -255,14 +249,6 @@ def cancel_job(self, job_id: str) -> bool: # Thread is blocked on semaphore — the runner will see the # cancellation flag when it wakes up and exit cleanly. self._queues.pop(job_id, None) - thread_id = self._threads.get(job_id) - - if thread_id is not None: - # Best-effort: raise JobCancelled in the worker thread. - ctypes.pythonapi.PyThreadState_SetAsyncExc( - ctypes.c_ulong(thread_id), - ctypes.py_object(JobCancelled), - ) return True def cancel_all_active(self, *, project_id: str | None = None) -> list[str]: @@ -341,47 +327,9 @@ def _dispatch_pending_workflows() -> None: pending = get_pending_workflows() for req in pending: kind = req.get("kind", "workflow") + action = req.get("action") or action_for_workflow_kind(kind) pid = req.get("project_id") kwargs = req.get("kwargs", {}) label = req.get("label", kind) - if kind == "extraction": - jobs.start_job(kind="extract", label=label, project_id=pid, target=api.extract, kwargs=kwargs) - elif kind == "projection": - proj_kwargs = { - "space_id": kwargs["space_id"], - "project_id": kwargs["project_id"], - } - if "source_type" in kwargs: - proj_kwargs["source_type"] = kwargs["source_type"] - jobs.start_job( - kind="project", - label=label, - project_id=pid, - target=api.project, - kwargs=proj_kwargs, - ) - elif kind == "kg_extraction": - jobs.start_job( - kind="knowledge_graph", - label=label, - project_id=pid, - target=api.extract_knowledge_graph, - kwargs={"project_id": kwargs["project_id"]}, - ) - elif kind == "feedback_review": - jobs.start_job( - kind="feedback_review", - label=label, - project_id=pid, - target=api.review_feedback, - kwargs={"project_id": kwargs["project_id"]}, - ) - elif kind == "projection_review": - jobs.start_job( - kind="projection_review", - label=label, - project_id=pid, - target=api.review_projections, - kwargs={"space_id": kwargs["space_id"], "project_id": kwargs["project_id"]}, - ) + start_job_action(jobs, action, job_project_id=pid, label=label, **kwargs) diff --git a/src/mkb/web/api_server.py b/src/mkb/web/api_server.py index 662544d..9d7e056 100644 --- a/src/mkb/web/api_server.py +++ b/src/mkb/web/api_server.py @@ -1,23 +1,14 @@ -"""FastAPI application entry point for the MKB web layer. - -Most route logic lives in ``mkb.web.routers.*``. The upload cluster -(temp dirs, archive extraction, ``_run_upload_ingest`` and the -``/api/upload/*`` routes) intentionally stays inline because tests in -``tests/test_api_upload_ingest.py`` monkeypatch module attributes here. -""" +"""FastAPI application entry point for the MKB web layer.""" from __future__ import annotations import shutil -import tarfile import tempfile import uuid -import zipfile from pathlib import Path from typing import Any from fastapi import FastAPI, File, Form, HTTPException, UploadFile from fastapi.middleware.cors import CORSMiddleware -from pydantic import BaseModel from mkb import api from mkb.logging_setup import setup_logging @@ -26,7 +17,7 @@ # (e.g. ``uvicorn mkb.web.api_server:app``) without going through ``mkb.cli``. setup_logging() -from mkb.web._helpers import _parse_uuid, _safe_child # noqa: E402 +from mkb.web._helpers import _parse_uuid, _safe_child, require_service_result, start_web_job_action # noqa: E402 from mkb.web._state import ( # noqa: E402 (re-exported for back-compat tests) AssistantSession, JobManager, @@ -37,291 +28,64 @@ assistant_session, jobs, ) +from mkb.web import uploads as upload_impl # noqa: E402 -# ── Upload cluster (kept inline for test monkeypatch compatibility) ───────── +# ── Upload compatibility wrappers ─────────────────────────────────────────── _UPLOAD_TEMP = Path("data/uploads/_temp") _UPLOAD_ROOT = Path("data/uploads") -class UploadInitResponse(BaseModel): - upload_id: str - - -class UploadCompleteRequest(BaseModel): - upload_id: str - - -class UploadExpandRequest(BaseModel): - upload_id: str - - -class UploadExpandFile(BaseModel): - uploadPath: str - size: int - - -class UploadExpandResponse(BaseModel): - files: list[UploadExpandFile] - extracted: list[dict[str, Any]] = [] - failed: list[dict[str, Any]] = [] - - -class UploadFileItem(BaseModel): - name: str - relativePath: str - uploadPath: str - - -class UploadProject(BaseModel): - name: str - upload_id: str - files: list[UploadFileItem] - # True (default) means ``name`` was auto-generated (e.g. folder basename - # from the upload-preview grouping) and may be overwritten later by an - # auto-rename from extraction. False means the user explicitly typed/edited - # the name and it should be preserved. - name_auto: bool = True +UploadCompleteRequest = upload_impl.UploadCompleteRequest +UploadExpandFile = upload_impl.UploadExpandFile +UploadExpandRequest = upload_impl.UploadExpandRequest +UploadExpandResponse = upload_impl.UploadExpandResponse +UploadFileItem = upload_impl.UploadFileItem +UploadInitResponse = upload_impl.UploadInitResponse +UploadProject = upload_impl.UploadProject def _normalize_project_name(name: str, fallback: str = "project") -> str: - import re - - candidate = (name or "").strip() - if not candidate: - candidate = fallback - candidate = re.sub(r"[^A-Za-z0-9._ -]+", "_", candidate) - candidate = candidate.strip(" ._") - return candidate or fallback + return upload_impl.normalize_project_name(name, fallback) def _create_unique_project_dir(project_name: str) -> Path: - _UPLOAD_ROOT.mkdir(parents=True, exist_ok=True) - base_name = _normalize_project_name(project_name) - candidate = _UPLOAD_ROOT / base_name - suffix = 2 - while candidate.exists(): - candidate = _UPLOAD_ROOT / f"{base_name}_{suffix}" - suffix += 1 - candidate.mkdir(parents=True, exist_ok=False) - return candidate + return upload_impl.create_unique_project_dir(project_name, _UPLOAD_ROOT) def _next_available_path(path: Path) -> Path: - if not path.exists(): - return path - stem = path.stem - suffix = path.suffix - idx = 2 - while True: - candidate = path.with_name(f"{stem}_{idx}{suffix}") - if not candidate.exists(): - return candidate - idx += 1 - - -_ARCHIVE_SUFFIXES = ( - ".zip", - ".tar", - ".tar.gz", ".tgz", - ".tar.bz2", ".tbz2", ".tbz", - ".tar.xz", ".txz", -) + return upload_impl.next_available_path(path) def _is_archive(name: str) -> bool: - lower = name.lower() - return any(lower.endswith(ext) for ext in _ARCHIVE_SUFFIXES) + return upload_impl.is_archive(name) def _strip_archive_ext(name: str) -> str: - lower = name.lower() - for ext in _ARCHIVE_SUFFIXES: - if lower.endswith(ext): - return name[: -len(ext)] - return name + return upload_impl.strip_archive_ext(name) def _safe_extract_archive(archive_path: Path, dest_dir: Path) -> int: - """Extract an archive into ``dest_dir`` safely. - - - Rejects entries whose resolved path would escape ``dest_dir`` (zip-slip). - - Skips symlinks/hardlinks and non-regular entries. - - Resolves collisions via ``_next_available_path``. - Returns the number of regular files extracted. - """ - dest_root = dest_dir.resolve() - dest_dir.mkdir(parents=True, exist_ok=True) - count = 0 - name = archive_path.name.lower() - - if name.endswith(".zip"): - with zipfile.ZipFile(archive_path) as zf: - for member in zf.infolist(): - if member.is_dir(): - continue - member_name = member.filename.replace("\\", "/") - if not member_name or member_name.endswith("/"): - continue - if member_name.startswith("__MACOSX/") or "/.DS_Store" in member_name or member_name.endswith("/.DS_Store"): - continue - target = (dest_dir / member_name).resolve() - try: - target.relative_to(dest_root) - except ValueError: - continue - target = _next_available_path(target) - target.parent.mkdir(parents=True, exist_ok=True) - with zf.open(member) as src, target.open("wb") as out: - shutil.copyfileobj(src, out) - count += 1 - return count - - with tarfile.open(archive_path, "r:*") as tf: - for member in tf.getmembers(): - if not member.isfile(): - continue - member_name = member.name.replace("\\", "/").lstrip("/") - if not member_name: - continue - if member_name.startswith("__MACOSX/") or member_name.endswith("/.DS_Store"): - continue - target = (dest_dir / member_name).resolve() - try: - target.relative_to(dest_root) - except ValueError: - continue - src = tf.extractfile(member) - if src is None: - continue - target = _next_available_path(target) - target.parent.mkdir(parents=True, exist_ok=True) - with target.open("wb") as out: - shutil.copyfileobj(src, out) - count += 1 - return count + return upload_impl.safe_extract_archive(archive_path, dest_dir) def _expand_temp_dir(temp_root: Path, emit=None) -> dict[str, Any]: - """Expand archives in ``temp_root`` and return the resulting file tree.""" - def _emit(msg: str) -> None: - if emit: - emit(msg) - - extracted: list[dict[str, Any]] = [] - failed: list[dict[str, Any]] = [] - - if not temp_root.is_dir(): - return {"files": [], "extracted": extracted, "failed": failed} - - # Bounded loop so nested archives (an archive that contains another - # archive) eventually fully unpack. - for _ in range(8): - archives = [ - p for p in temp_root.rglob("*") - if p.is_file() and _is_archive(p.name) - ] - if not archives: - break - for archive in archives: - extract_target = archive.parent / _strip_archive_ext(archive.name) - if extract_target.exists(): - extract_target = _next_available_path(extract_target) - _emit(f"Extracting {archive.relative_to(temp_root)}") - try: - n = _safe_extract_archive(archive, extract_target) - except (zipfile.BadZipFile, tarfile.TarError, OSError) as exc: - failed.append({ - "archive": str(archive.relative_to(temp_root)), - "error": str(exc), - }) - continue - extracted.append({ - "archive": str(archive.relative_to(temp_root)), - "count": n, - }) - try: - archive.unlink() - except OSError: - pass - - files: list[dict[str, Any]] = [] - root_resolved = temp_root.resolve() - for p in sorted(temp_root.rglob("*")): - if not p.is_file(): - continue - try: - rel = p.resolve().relative_to(root_resolved) - except ValueError: - continue - files.append({ - "uploadPath": rel.as_posix(), - "size": p.stat().st_size, - }) - return {"files": files, "extracted": extracted, "failed": failed} + return upload_impl.expand_temp_dir( + temp_root, + emit=emit, + extract_archive=_safe_extract_archive, + ) def _run_upload_ingest(payload: list[UploadProject], progress_callback=None) -> dict[str, Any]: - def emit(msg: str) -> None: - if progress_callback: - progress_callback({"message": msg}) - - if not payload: - return {"status": "completed", "message": "No projects provided."} - - upload_id = str(payload[0].upload_id) if payload else "" - temp_root = _UPLOAD_TEMP / upload_id if upload_id else _UPLOAD_TEMP - - total_ingested = 0 - total_dupes = 0 - created: list[str] = [] - reused = 0 - - try: - emit(f"Preparing {len(payload)} project(s) for ingest") - for idx, proj in enumerate(payload, start=1): - upload_dir = _create_unique_project_dir(proj.name) - emit(f"Moving files for {upload_dir.name} ({idx}/{len(payload)})") - - for file_info in proj.files: - src = _safe_child(temp_root, file_info.uploadPath) - if not src.is_file(): - continue - rel_full = _safe_child(upload_dir, file_info.relativePath) - dest = upload_dir / rel_full.relative_to(upload_dir.resolve()) - dest.parent.mkdir(parents=True, exist_ok=True) - dest = _next_available_path(dest) - shutil.move(str(src), str(dest)) - - emit(f"Ingesting {upload_dir.name}") - result = api.ingest( - upload_dir, - label=proj.name if not proj.name_auto else None, - user_named=not proj.name_auto, - ) - total_ingested += int(result.get("ingested", 0) or 0) - total_dupes += int(result.get("duplicates", 0) or 0) - if result.get("project_reused"): - reused += 1 - shutil.rmtree(upload_dir, ignore_errors=True) - else: - created.append(upload_dir.name) - finally: - if temp_root.is_dir(): - shutil.rmtree(temp_root, ignore_errors=True) - - return { - "status": "completed", - "message": ( - f"Created {len(created)} project(s), reused {reused} existing project(s) · " - f"{total_ingested} file(s) ingested, {total_dupes} duplicate(s) skipped." - ), - "created_projects": created, - "reused_projects": reused, - "ingested": total_ingested, - "duplicates": total_dupes, - } + return upload_impl.run_upload_ingest( + payload, + upload_temp=_UPLOAD_TEMP, + create_project_dir=_create_unique_project_dir, + api_module=api, + progress_callback=progress_callback, + ) # ── App + middleware ──────────────────────────────────────────────────────── @@ -423,12 +187,12 @@ def upload_expand(body: UploadExpandRequest): def upload_ingest(payload: list[UploadProject]): if not payload: raise HTTPException(status_code=400, detail="No projects uploaded") - job_id = jobs.start_job( - kind="upload", - label="Upload Ingest", - project_id="__upload__", - target=_run_upload_ingest, - args=(payload,), + job_id = start_web_job_action( + jobs, + "upload_ingest", + job_project_id="__upload__", + payload=payload, + ingest_func=_run_upload_ingest, ) return {"job_id": job_id} @@ -481,7 +245,7 @@ def upload_processed_for_asset( raise HTTPException(status_code=400, detail=str(exc)) from exc except ValueError as exc: raise HTTPException(status_code=400, detail=str(exc)) from exc - return result + return require_service_result(result) finally: shutil.rmtree(tmp_dir, ignore_errors=True) diff --git a/src/mkb/web/content.py b/src/mkb/web/content.py new file mode 100644 index 0000000..31fb7b1 --- /dev/null +++ b/src/mkb/web/content.py @@ -0,0 +1,27 @@ +"""Shared response helpers for previewable asset content.""" + +from __future__ import annotations + +from pathlib import Path +from urllib.parse import quote + + +def inline_headers(filename: str) -> dict[str, str]: + safe_name = filename.replace('"', "'").replace("\r", "").replace("\n", "") + ascii_name = safe_name.encode("ascii", "ignore").decode("ascii") or "document" + return { + "Content-Disposition": ( + f'inline; filename="{ascii_name}"; filename*=UTF-8\'\'{quote(safe_name)}' + ), + "X-Content-Type-Options": "nosniff", + } + + +def asset_media_type(filename: str, mime_type: str | None = None) -> str | None: + suffix = Path(filename).suffix.lower() + if suffix == ".pdf" or mime_type == "application/pdf": + return "application/pdf" + if suffix in {".md", ".markdown"} or mime_type in {"text/markdown", "text/x-markdown"}: + return "text/markdown" + return None + diff --git a/src/mkb/web/job_actions.py b/src/mkb/web/job_actions.py new file mode 100644 index 0000000..cae59d4 --- /dev/null +++ b/src/mkb/web/job_actions.py @@ -0,0 +1,386 @@ +"""Registry for background job actions shared by web, agents, and legacy UI.""" + +from __future__ import annotations + +import importlib +from dataclasses import dataclass +from typing import Any, Callable + + +KwargsBuilder = Callable[..., dict[str, Any]] +Validator = Callable[[dict[str, Any]], None] + + +@dataclass(frozen=True) +class JobAction: + action: str + kind: str + label: str + target: Callable[..., Any] | str + build_kwargs: KwargsBuilder = lambda **kwargs: dict(kwargs) + project_arg: str | None = "project_id" + conflict_policy: str = "allow" + validate: Validator | None = None + + +def _api_module(): + return importlib.import_module("mkb.api") + + +def _require_keys(*keys: str) -> Validator: + def _validate(kwargs: dict[str, Any]) -> None: + missing = [key for key in keys if kwargs.get(key) is None] + if missing: + joined = ", ".join(missing) + raise ValueError(f"Missing required job action argument(s): {joined}") + + return _validate + + +def _projection_kwargs(**kwargs) -> dict[str, Any]: + return { + "space_id": kwargs["space_id"], + "project_id": kwargs["project_id"], + "source_type": kwargs.get("source_type", "frame"), + } + + +def _feedback_review_all(progress_callback=None) -> dict[str, Any]: + api = _api_module() + projects = api.list_projects(limit=500) + results = [] + for project in projects: + project_id = project["project_id"] + summary = api.get_feedback_summary(project_id) + if int(summary.get("total", 0) or 0) == 0: + continue + if progress_callback: + progress_callback({"message": f"Reviewing feedback for {project_id[:8]}"}) + results.append(api.review_feedback(project_id=project_id)) + return {"reviewed_projects": len(results), "results": results} + + +def _projection_review_kwargs(**kwargs) -> dict[str, Any]: + return { + "space_id": kwargs["space_id"], + "project_id": kwargs["project_id"], + "reviewer_id": kwargs.get("reviewer_id"), + } + + +def _projection_review_all_kwargs(**kwargs) -> dict[str, Any]: + return { + "space_id": kwargs["space_id"], + "project_ids": kwargs.get("project_ids"), + "reviewer_id": kwargs.get("reviewer_id"), + } + + +def _projection_review_session_kwargs(**kwargs) -> dict[str, Any]: + return { + "space_id": kwargs["space_id"], + "project_ids": kwargs["project_ids"], + "reviewer_id": kwargs.get("reviewer_id"), + } + + +def _projection_review_followup_kwargs(**kwargs) -> dict[str, Any]: + return { + "space_id": kwargs["space_id"], + "project_id": kwargs["project_id"], + "message": kwargs["message"], + "previous_job": kwargs["previous_job"], + "reviewer_id": kwargs.get("reviewer_id"), + } + + +def _graph_review_kwargs(**kwargs) -> dict[str, Any]: + return {"mode": kwargs["mode"], "seed_count": kwargs["seed_count"]} + + +def _run_ontology_induction(**kwargs) -> dict[str, Any]: + from mkb.agents.ontology_induction import run_ontology_induction + + return run_ontology_induction(**kwargs) + + +def _ontology_induction_kwargs(**kwargs) -> dict[str, Any]: + return { + "min_support": kwargs["min_support"], + "author": kwargs["author"], + "mode": kwargs["mode"], + "sample_size": kwargs["sample_size"], + "model": kwargs.get("model"), + "verbose": kwargs.get("verbose", False), + } + + +def _run_assistant_chat( + *, + runner, + session_id: str, + message: str, + dispatch_pending_workflows: Callable[[], Any] | None = None, + progress_callback=None, +): + from mkb.agents.orchestrator import send_message + + result = send_message( + runner=runner, + session_id=session_id, + message=message, + progress_callback=progress_callback, + ) + if dispatch_pending_workflows is not None: + dispatch_pending_workflows() + return result + + +def _assistant_chat_kwargs(**kwargs) -> dict[str, Any]: + return { + "runner": kwargs["runner"], + "session_id": kwargs["session_id"], + "message": kwargs["message"], + "dispatch_pending_workflows": kwargs.get("dispatch_pending_workflows"), + } + + +def _run_upload_ingest_action(*, payload, ingest_func, progress_callback=None): + return ingest_func(payload, progress_callback=progress_callback) + + +def _upload_ingest_kwargs(**kwargs) -> dict[str, Any]: + return {"payload": kwargs["payload"], "ingest_func": kwargs["ingest_func"]} + + +JOB_ACTIONS: dict[str, JobAction] = { + "assistant_chat": JobAction( + "assistant_chat", + "orchestrator_chat", + "Assistant", + _run_assistant_chat, + build_kwargs=_assistant_chat_kwargs, + project_arg=None, + conflict_policy="global_kind", + validate=_require_keys("runner", "session_id", "message"), + ), + "upload_ingest": JobAction( + "upload_ingest", + "upload", + "Upload Ingest", + _run_upload_ingest_action, + build_kwargs=_upload_ingest_kwargs, + project_arg="job_project_id", + conflict_policy="global_kind", + validate=_require_keys("payload", "ingest_func"), + ), + "process_project": JobAction( + "process_project", + "process", + "Process", + "process", + conflict_policy="project_kind", + validate=_require_keys("project_id"), + ), + "extract_project": JobAction( + "extract_project", + "extract", + "Extract", + "extract", + conflict_policy="project_kind", + validate=_require_keys("project_id"), + ), + "project_to_space": JobAction( + "project_to_space", + "project", + "Project", + "project", + build_kwargs=_projection_kwargs, + conflict_policy="project_kind", + validate=_require_keys("space_id", "project_id"), + ), + "extract_knowledge_graph": JobAction( + "extract_knowledge_graph", + "knowledge_graph", + "Extract Graph", + "extract_knowledge_graph", + conflict_policy="project_kind", + validate=_require_keys("project_id"), + ), + "extract_raw_workflow": JobAction( + "extract_raw_workflow", + "raw_workflow", + "Extract Workflow", + "extract_raw_workflow", + conflict_policy="project_kind", + validate=_require_keys("project_id"), + ), + "canonicalize_workflow": JobAction( + "canonicalize_workflow", + "canonical_workflow", + "Canonicalize Workflow", + "canonicalize_workflow", + conflict_policy="project_kind", + validate=_require_keys("project_id"), + ), + "review_feedback": JobAction( + "review_feedback", + "feedback_review", + "Review Feedback", + "review_feedback", + conflict_policy="project_kind", + validate=_require_keys("project_id"), + ), + "review_feedback_all": JobAction( + "review_feedback_all", + "feedback_review", + "Review Feedback", + _feedback_review_all, + build_kwargs=lambda **_kwargs: {}, + project_arg=None, + ), + "review_projection": JobAction( + "review_projection", + "projection_review", + "Review Projection", + "review_projections", + build_kwargs=_projection_review_kwargs, + conflict_policy="project_kind", + validate=_require_keys("space_id", "project_id"), + ), + "review_projection_all": JobAction( + "review_projection_all", + "projection_review", + "Review Projection", + "review_projections_all", + build_kwargs=_projection_review_all_kwargs, + project_arg=None, + validate=_require_keys("space_id"), + ), + "review_projection_session": JobAction( + "review_projection_session", + "projection_review", + "Projection Review (session)", + "review_projections_session", + build_kwargs=_projection_review_session_kwargs, + project_arg=None, + conflict_policy="global_kind", + validate=_require_keys("space_id", "project_ids"), + ), + "review_projection_followup": JobAction( + "review_projection_followup", + "projection_review", + "Projection Review Follow-up", + "review_projection_followup", + build_kwargs=_projection_review_followup_kwargs, + conflict_policy="project_kind", + validate=_require_keys("space_id", "project_id", "message", "previous_job"), + ), + "review_graph": JobAction( + "review_graph", + "graph_review", + "Graph Review", + "review_knowledge_graph", + build_kwargs=_graph_review_kwargs, + project_arg=None, + conflict_policy="global_kind", + validate=_require_keys("mode", "seed_count"), + ), + "curate_workflow_schema": JobAction( + "curate_workflow_schema", + "ontology_induction", + "Workflow Review Agent", + _run_ontology_induction, + build_kwargs=_ontology_induction_kwargs, + project_arg=None, + conflict_policy="global_kind", + validate=_require_keys("min_support", "author", "mode", "sample_size"), + ), + "workflow_maintenance": JobAction( + "workflow_maintenance", + "workflow_maintenance", + "Workflow Maintenance", + "run_workflow_maintenance_task", + project_arg=None, + validate=_require_keys("task_id"), + ), + "workflow_recanonicalization_batch": JobAction( + "workflow_recanonicalization_batch", + "workflow_maintenance_batch", + "Recanonicalize Global Workflow Batch", + "run_pending_recanonicalizations", + build_kwargs=lambda **_kwargs: {}, + project_arg=None, + ), +} + + +WORKFLOW_KIND_ACTIONS = { + "extraction": "extract_project", + "projection": "project_to_space", + "kg_extraction": "extract_knowledge_graph", + "feedback_review": "review_feedback", + "projection_review": "review_projection", +} + + +class JobActionConflict(RuntimeError): + def __init__(self, action: JobAction, active_job: dict[str, Any]) -> None: + self.action = action + self.active_job = active_job + status = str(active_job.get("status") or "active").lower() + super().__init__(f"{action.label} is already {status}.") + + +def action_for_workflow_kind(kind: str) -> str: + return WORKFLOW_KIND_ACTIONS[kind] + + +def job_action_start_params( + action: str, + *, + job_project_id: str | None = None, + label: str | None = None, + **kwargs: Any, +) -> dict[str, Any]: + spec = JOB_ACTIONS[action] + if spec.validate is not None: + spec.validate(kwargs) + job_kwargs = spec.build_kwargs(**kwargs) + project_id = job_project_id if spec.project_arg else None + target = getattr(_api_module(), spec.target) if isinstance(spec.target, str) else spec.target + return { + "kind": spec.kind, + "label": label or spec.label, + "project_id": project_id, + "target": target, + "kwargs": job_kwargs, + } + + +def _find_conflict(manager, spec: JobAction, project_id: str | None) -> dict[str, Any] | None: + if spec.conflict_policy == "allow" or not hasattr(manager, "find_active_job"): + return None + if spec.conflict_policy == "project_kind": + if not project_id: + return None + return manager.find_active_job(project_id=project_id, kind=spec.kind) + if spec.conflict_policy == "global_kind": + return manager.find_active_job(kind=spec.kind) + raise ValueError(f"Unknown job action conflict policy: {spec.conflict_policy}") + + +def start_job_action( + manager, + action: str, + *, + job_project_id: str | None = None, + label: str | None = None, + **kwargs: Any, +) -> str: + spec = JOB_ACTIONS[action] + params = job_action_start_params(action, job_project_id=job_project_id, label=label, **kwargs) + active = _find_conflict(manager, spec, params["project_id"]) + if active: + raise JobActionConflict(spec, active) + return manager.start_job(**params) diff --git a/src/mkb/web/routers/assistant.py b/src/mkb/web/routers/assistant.py index 4d75e33..198df4d 100644 --- a/src/mkb/web/routers/assistant.py +++ b/src/mkb/web/routers/assistant.py @@ -1,6 +1,6 @@ from fastapi import APIRouter, HTTPException -from mkb.agents.orchestrator import send_message +from mkb.web._helpers import start_web_job_action from mkb.web._models import AssistantChatRequest from mkb.web._state import _dispatch_pending_workflows, _get_assistant_session, jobs @@ -15,19 +15,12 @@ def assistant_chat(body: AssistantChatRequest): session = _get_assistant_session() - def _run_chat(progress_callback=None): - result = send_message( - runner=session.runner, - session_id=session.session_id, - message=message, - progress_callback=progress_callback, - ) - _dispatch_pending_workflows() - return result - - job_id = jobs.start_job( - kind="orchestrator_chat", - label="Assistant", - target=_run_chat, + job_id = start_web_job_action( + jobs, + "assistant_chat", + runner=session.runner, + session_id=session.session_id, + message=message, + dispatch_pending_workflows=_dispatch_pending_workflows, ) return {"job_id": job_id} diff --git a/src/mkb/web/routers/feedback.py b/src/mkb/web/routers/feedback.py index 22776dd..4cfca1b 100644 --- a/src/mkb/web/routers/feedback.py +++ b/src/mkb/web/routers/feedback.py @@ -1,9 +1,7 @@ -from typing import Any - from fastapi import APIRouter from mkb import api -from mkb.web._helpers import _parse_uuid +from mkb.web._helpers import _parse_uuid, start_web_job_action from mkb.web._models import FeedbackResolveRequest, FeedbackReviewRequest from mkb.web._state import jobs @@ -26,35 +24,16 @@ def resolve_feedback(feedback_id: str, body: FeedbackResolveRequest): return api.resolve_feedback(feedback_id=feedback_id, status=body.status, notes=body.notes) -def _review_feedback_all(progress_callback=None) -> dict[str, Any]: - projects = api.list_projects(limit=500) - results = [] - for p in projects: - pid = p["project_id"] - summary = api.get_feedback_summary(pid) - if int(summary.get("total", 0) or 0) == 0: - continue - if progress_callback: - progress_callback({"message": f"Reviewing feedback for {pid[:8]}"}) - results.append(api.review_feedback(project_id=pid)) - return {"reviewed_projects": len(results), "results": results} - - @router.post("/api/feedback/review") def review_feedback(body: FeedbackReviewRequest): if body.project_id: _parse_uuid(body.project_id, "project_id") - job_id = jobs.start_job( - kind="feedback_review", - label="Feedback Review", + job_id = start_web_job_action( + jobs, + "review_feedback", + job_project_id=body.project_id, project_id=body.project_id, - target=api.review_feedback, - kwargs={"project_id": body.project_id}, ) else: - job_id = jobs.start_job( - kind="feedback_review", - label="Feedback Review", - target=_review_feedback_all, - ) + job_id = start_web_job_action(jobs, "review_feedback_all") return {"job_id": job_id} diff --git a/src/mkb/web/routers/graph.py b/src/mkb/web/routers/graph.py index 557daa1..14da8d4 100644 --- a/src/mkb/web/routers/graph.py +++ b/src/mkb/web/routers/graph.py @@ -1,6 +1,7 @@ from fastapi import APIRouter from mkb import api +from mkb.web._helpers import start_web_job_action from mkb.web._models import GraphReviewRequest from mkb.web._state import jobs @@ -19,12 +20,7 @@ def get_graph_review_counts(): @router.post("/api/graph/review") def review_graph(body: GraphReviewRequest): - job_id = jobs.start_job( - kind="graph_review", - label="Graph Review", - target=api.review_knowledge_graph, - kwargs={"mode": body.mode, "seed_count": body.seed_count}, - ) + job_id = start_web_job_action(jobs, "review_graph", mode=body.mode, seed_count=body.seed_count) return {"job_id": job_id} diff --git a/src/mkb/web/routers/jobs.py b/src/mkb/web/routers/jobs.py index 5a31d53..f4a21d6 100644 --- a/src/mkb/web/routers/jobs.py +++ b/src/mkb/web/routers/jobs.py @@ -1,6 +1,6 @@ from fastapi import APIRouter, HTTPException -from mkb import api +from mkb.web._helpers import start_web_job_action from mkb.web._models import ReviewJobChatRequest from mkb.web._state import jobs @@ -57,17 +57,14 @@ def review_job_chat(job_id: str, body: ReviewJobChatRequest): detail="This review job does not identify a single space/project for follow-up chat", ) - followup_job_id = jobs.start_job( - kind="projection_review", - label="Projection Review Follow-up", + followup_job_id = start_web_job_action( + jobs, + "review_projection_followup", + job_project_id=str(project_id), + space_id=str(space_id), project_id=str(project_id), - target=api.review_projection_followup, - kwargs={ - "space_id": str(space_id), - "project_id": str(project_id), - "message": message, - "previous_job": row, - "reviewer_id": result.get("reviewer_id"), - }, + message=message, + previous_job=row, + reviewer_id=result.get("reviewer_id"), ) return {"job_id": followup_job_id} diff --git a/src/mkb/web/routers/projections.py b/src/mkb/web/routers/projections.py index b8ae3c2..35fcd1a 100644 --- a/src/mkb/web/routers/projections.py +++ b/src/mkb/web/routers/projections.py @@ -8,7 +8,7 @@ from pydantic import BaseModel from mkb import api -from mkb.web._helpers import _parse_uuid +from mkb.web._helpers import _parse_uuid, require_service_result, start_web_job_action from mkb.web._models import ProjectionReviewRequest from mkb.web._state import jobs @@ -69,8 +69,7 @@ def export_projection_endpoint(projection_id: str, format: str = "yaml"): with tempfile.TemporaryDirectory() as tmp: out_dir = Path(tmp) / "export" result = api.export_projection(projection_id, out_dir, format=fmt, overwrite=True) - if "error" in result: - raise HTTPException(status_code=400, detail=result["error"]) + require_service_result(result) files = [Path(p) for p in result.get("files", [])] if not files: raise HTTPException(status_code=404, detail="Nothing to export") @@ -155,8 +154,7 @@ def export_space_endpoint(space_id_or_name: str, format: str = "yaml"): result = api.export_space_projections( space_id_or_name, out_dir, format=fmt, overwrite=True ) - if "error" in result: - raise HTTPException(status_code=400, detail=result["error"]) + require_service_result(result) files = list(out_dir.rglob("*")) if not any(p.is_file() for p in files): raise HTTPException(status_code=404, detail="Nothing to export") @@ -200,48 +198,44 @@ def review_projections(body: ProjectionReviewRequest): status_code=400, detail="Session-mode review requires explicit project_ids (or project_id).", ) - job_id = jobs.start_job( - kind="projection_review", - label="Projection Review (session)", - target=api.review_projections_session, - kwargs={"space_id": body.space_id, "project_ids": project_ids, "reviewer_id": body.reviewer_id}, + job_id = start_web_job_action( + jobs, + "review_projection_session", + space_id=body.space_id, + project_ids=project_ids, + reviewer_id=body.reviewer_id, ) return {"job_id": job_id} if len(project_ids) == 1: - job_id = jobs.start_job( - kind="projection_review", - label="Projection Review", + job_id = start_web_job_action( + jobs, + "review_projection", + job_project_id=project_ids[0], + space_id=body.space_id, project_id=project_ids[0], - target=api.review_projections, - kwargs={"space_id": body.space_id, "project_id": project_ids[0], "reviewer_id": body.reviewer_id}, + reviewer_id=body.reviewer_id, ) elif project_ids: job_ids = [] for project_id in project_ids: job_ids.append( - jobs.start_job( - kind="projection_review", - label="Projection Review", + start_web_job_action( + jobs, + "review_projection", + job_project_id=project_id, + space_id=body.space_id, project_id=project_id, - target=api.review_projections, - kwargs={ - "space_id": body.space_id, - "project_id": project_id, - "reviewer_id": body.reviewer_id, - }, + reviewer_id=body.reviewer_id, ) ) return {"job_id": job_ids[0], "job_ids": job_ids} else: - job_id = jobs.start_job( - kind="projection_review", - label="Projection Review", - target=api.review_projections_all, - kwargs={ - "space_id": body.space_id, - "project_ids": project_ids or None, - "reviewer_id": body.reviewer_id, - }, + job_id = start_web_job_action( + jobs, + "review_projection_all", + space_id=body.space_id, + project_ids=project_ids or None, + reviewer_id=body.reviewer_id, ) return {"job_id": job_id} diff --git a/src/mkb/web/routers/projects.py b/src/mkb/web/routers/projects.py index 0bcbf8e..58edea5 100644 --- a/src/mkb/web/routers/projects.py +++ b/src/mkb/web/routers/projects.py @@ -1,5 +1,4 @@ from pathlib import Path -from urllib.parse import quote from fastapi import APIRouter, HTTPException, Response @@ -7,7 +6,13 @@ from mkb.db.engine import SyncSessionLocal from mkb.db.models import Asset, ProcessedAsset, ProjectAsset from mkb.storage.s3 import download_bytes -from mkb.web._helpers import _parse_uuid +from mkb.web._helpers import ( + _parse_uuid, + require_service_result, + require_service_result_or_not_found, + start_web_job_action, +) +from mkb.web.content import asset_media_type, inline_headers from mkb.web._models import ( ProjectGroupAssign, ProjectGroupCreate, @@ -26,24 +31,8 @@ router = APIRouter() -def _inline_headers(filename: str) -> dict[str, str]: - safe_name = filename.replace('"', "'").replace("\r", "").replace("\n", "") - ascii_name = safe_name.encode("ascii", "ignore").decode("ascii") or "document" - return { - "Content-Disposition": ( - f'inline; filename="{ascii_name}"; filename*=UTF-8\'\'{quote(safe_name)}' - ), - "X-Content-Type-Options": "nosniff", - } - - -def _asset_media_type(filename: str, mime_type: str | None = None) -> str | None: - suffix = Path(filename).suffix.lower() - if suffix == ".pdf" or mime_type == "application/pdf": - return "application/pdf" - if suffix in {".md", ".markdown"} or mime_type in {"text/markdown", "text/x-markdown"}: - return "text/markdown" - return None +_asset_media_type = asset_media_type +_inline_headers = inline_headers @router.get("/api/projects") @@ -64,18 +53,14 @@ def get_project(project_id: str): def update_project(project_id: str, body: ProjectUpdateRequest): _parse_uuid(project_id, "project_id") result = api.rename_project(project_id, body.label, user_initiated=True) - if "error" in result: - raise HTTPException(status_code=404, detail=result["error"]) - return result + return require_service_result(result, default_status=404) @router.delete("/api/projects/{project_id}") def delete_project(project_id: str, delete_s3: bool = True): _parse_uuid(project_id, "project_id") result = api.delete_project(project_id, delete_s3_objects=delete_s3) - if "error" in result: - raise HTTPException(status_code=404, detail=result["error"]) - return result + return require_service_result(result, default_status=404) @router.get("/api/projects/{project_id}/assets") @@ -147,12 +132,11 @@ def get_project_processed_asset_content(project_id: str, processed_asset_id: str @router.post("/api/projects/{project_id}/process") def process_project(project_id: str): _parse_uuid(project_id, "project_id") - job_id = jobs.start_job( - kind="process", - label="Process", + job_id = start_web_job_action( + jobs, + "process_project", + job_project_id=project_id, project_id=project_id, - target=api.process, - kwargs={"project_id": project_id}, ) return {"job_id": job_id} @@ -160,12 +144,11 @@ def process_project(project_id: str): @router.post("/api/projects/{project_id}/extract") def extract_project(project_id: str): _parse_uuid(project_id, "project_id") - job_id = jobs.start_job( - kind="extract", - label="Extract", + job_id = start_web_job_action( + jobs, + "extract_project", + job_project_id=project_id, project_id=project_id, - target=api.extract, - kwargs={"project_id": project_id}, ) return {"job_id": job_id} @@ -178,16 +161,14 @@ def project_project(project_id: str, body: ProjectionRunRequest): if source_type not in {"frame", "markdown"}: raise HTTPException(status_code=400, detail=f"Invalid source_type: {body.source_type}") label = "Project" if source_type == "frame" else "Project (markdown)" - job_id = jobs.start_job( - kind="project", + job_id = start_web_job_action( + jobs, + "project_to_space", + job_project_id=project_id, label=label, + space_id=body.space_id, project_id=project_id, - target=api.project, - kwargs={ - "space_id": body.space_id, - "project_id": project_id, - "source_type": source_type, - }, + source_type=source_type, ) return {"job_id": job_id} @@ -195,12 +176,11 @@ def project_project(project_id: str, body: ProjectionRunRequest): @router.post("/api/projects/{project_id}/kg-extract") def project_kg_extract(project_id: str): _parse_uuid(project_id, "project_id") - job_id = jobs.start_job( - kind="knowledge_graph", - label="Extract Graph", + job_id = start_web_job_action( + jobs, + "extract_knowledge_graph", + job_project_id=project_id, project_id=project_id, - target=api.extract_knowledge_graph, - kwargs={"project_id": project_id}, ) return {"job_id": job_id} @@ -217,12 +197,11 @@ def project_workflow_extract(project_id: str): readiness = api.get_raw_workflow_extraction_readiness(project_id) if not readiness.get("ready"): raise HTTPException(status_code=400, detail=readiness.get("message") or "Project is not ready for workflow extraction") - job_id = jobs.start_job( - kind="raw_workflow", - label="Extract Workflow", + job_id = start_web_job_action( + jobs, + "extract_raw_workflow", + job_project_id=project_id, project_id=project_id, - target=api.extract_raw_workflow, - kwargs={"project_id": project_id}, ) return {"job_id": job_id} @@ -247,10 +226,12 @@ def canonicalize_project_workflow(project_id: str, body: WorkflowCanonicalizeReq _parse_uuid(project_id, "project_id") if body.raw_extraction_id: _parse_uuid(body.raw_extraction_id, "raw_extraction_id") - job_id = jobs.start_job( - kind="canonical_workflow", label="Canonicalize Workflow", project_id=project_id, - target=api.canonicalize_workflow, - kwargs={"project_id": project_id, "raw_extraction_id": body.raw_extraction_id}, + job_id = start_web_job_action( + jobs, + "canonicalize_workflow", + job_project_id=project_id, + project_id=project_id, + raw_extraction_id=body.raw_extraction_id, ) return {"job_id": job_id} @@ -274,11 +255,7 @@ def delete_project_workflow_version(project_id: str, version: int): detail="Workflow extraction is currently running for this project. Cancel or wait for it to finish before deleting a version.", ) result = api.delete_raw_workflow_version(project_id, version) - if result.get("error"): - detail = result["error"] - status_code = 404 if "not found" in detail.lower() else 400 - raise HTTPException(status_code=status_code, detail=detail) - return result + return require_service_result_or_not_found(result) @router.get("/api/projects/{project_id}/canonical-workflows") @@ -315,11 +292,7 @@ def delete_project_canonical_workflow_version(project_id: str, version: int): detail="Workflow canonicalization is currently running for this project. Cancel or wait for it to finish before deleting a version.", ) result = api.delete_canonical_workflow_version(project_id, version) - if result.get("error"): - detail = result["error"] - status_code = 404 if "not found" in detail.lower() else 400 - raise HTTPException(status_code=status_code, detail=detail) - return result + return require_service_result_or_not_found(result) @router.get("/api/workflows/search") @@ -342,9 +315,7 @@ def schedule_reextraction(project_id: str, body: WorkflowReextractionRequest): ) except ValueError as exc: raise HTTPException(status_code=400, detail=str(exc)) from exc - if result.get("error"): - raise HTTPException(status_code=400, detail=result["error"]) - return result + return require_service_result(result) @router.post("/api/projects/{project_id}/workflow-recanonicalize") @@ -357,18 +328,13 @@ def schedule_recanonicalization(project_id: str, body: WorkflowRecanonicalizatio ) except ValueError as exc: raise HTTPException(status_code=400, detail=str(exc)) from exc - if result.get("error"): - raise HTTPException(status_code=400, detail=result["error"]) - return result + return require_service_result(result) @router.post("/api/workflow-maintenance/{task_id}/run") def run_maintenance_task(task_id: str): _parse_uuid(task_id, "task_id") - job_id = jobs.start_job( - kind="workflow_maintenance", label="Workflow Maintenance", - target=api.run_workflow_maintenance_task, kwargs={"task_id": task_id}, - ) + job_id = start_web_job_action(jobs, "workflow_maintenance", task_id=task_id) return {"job_id": job_id, "task_id": task_id} @@ -379,11 +345,7 @@ def workflow_maintenance_tasks(status: str | None = None, project_id: str | None @router.post("/api/workflow-maintenance-batch/recanonicalize") def run_recanonicalization_batch(): - job_id = jobs.start_job( - kind="workflow_maintenance_batch", - label="Recanonicalize Global Workflow Batch", - target=api.run_pending_recanonicalizations, - ) + job_id = start_web_job_action(jobs, "workflow_recanonicalization_batch") return {"job_id": job_id} @@ -400,20 +362,15 @@ def curate_schema(body: SchemaCurateRequest): raise HTTPException(status_code=400, detail="sample_size must be at least 1") if body.mode not in {"global", "local", "auto"}: raise HTTPException(status_code=400, detail="mode must be global, local, or auto") - from mkb.agents.ontology_induction import run_ontology_induction - - job_id = jobs.start_job( - kind="ontology_induction", - label="Workflow Review Agent", - target=run_ontology_induction, - kwargs={ - "min_support": body.min_support, - "author": body.author.strip() or "workflow-review/ui", - "mode": body.mode, - "sample_size": body.sample_size, - "model": body.model, - "verbose": body.verbose, - }, + job_id = start_web_job_action( + jobs, + "curate_workflow_schema", + min_support=body.min_support, + author=body.author.strip() or "workflow-review/ui", + mode=body.mode, + sample_size=body.sample_size, + model=body.model, + verbose=body.verbose, ) return {"job_id": job_id} @@ -432,9 +389,7 @@ def review_schema_proposal_endpoint(proposal_id: str, body: SchemaProposalReview proposal_id, decision=body.decision, reviewer=body.reviewer.strip(), notes=body.notes, ) - if result.get("error"): - raise HTTPException(status_code=400, detail=result) - return result + return require_service_result(result) @router.patch("/api/workflow-schema/proposals/{proposal_id}") @@ -446,9 +401,7 @@ def edit_schema_proposal_endpoint(proposal_id: str, body: SchemaProposalEditRequ rationale=body.rationale, editor=body.editor, change_note=body.change_note, ) - if result.get("error"): - raise HTTPException(status_code=400, detail=result) - return result + return require_service_result(result) @router.get("/api/workflow-schema/proposals/{proposal_id}/revisions") @@ -478,9 +431,7 @@ def create_project_group_endpoint(body: ProjectGroupCreate): color=body.color, display_order=body.display_order, ) - if "error" in result: - raise HTTPException(status_code=400, detail=result["error"]) - return result + return require_service_result(result) @router.patch("/api/project-groups/{group_id}") @@ -493,19 +444,14 @@ def update_project_group_endpoint(group_id: str, body: ProjectGroupUpdate): color=body.color, display_order=body.display_order, ) - if "error" in result: - status = 404 if "not found" in result["error"] else 400 - raise HTTPException(status_code=status, detail=result["error"]) - return result + return require_service_result_or_not_found(result) @router.delete("/api/project-groups/{group_id}") def delete_project_group_endpoint(group_id: str): _parse_uuid(group_id, "group_id") result = api.delete_project_group(group_id) - if "error" in result: - raise HTTPException(status_code=404, detail=result["error"]) - return result + return require_service_result(result, default_status=404) @router.post("/api/project-groups/assign") @@ -515,6 +461,4 @@ def assign_project_group_endpoint(body: ProjectGroupAssign): if body.group_id: _parse_uuid(body.group_id, "group_id") result = api.assign_projects_to_group(body.project_ids, body.group_id) - if "error" in result: - raise HTTPException(status_code=404, detail=result["error"]) - return result + return require_service_result(result, default_status=404) diff --git a/src/mkb/web/routers/skills.py b/src/mkb/web/routers/skills.py index 4a7de2e..832ec10 100644 --- a/src/mkb/web/routers/skills.py +++ b/src/mkb/web/routers/skills.py @@ -1,7 +1,7 @@ from fastapi import APIRouter, File, HTTPException, UploadFile from mkb.skills import registry -from mkb.web._helpers import _parse_uuid +from mkb.web._helpers import _parse_uuid, require_service_result router = APIRouter() @@ -43,6 +43,4 @@ async def upload_skill(files: list[UploadFile] = File(...)): def delete_skill(skill_id: str): _parse_uuid(skill_id, "skill_id") result = registry.delete_skill(skill_id) - if isinstance(result, dict) and result.get("error"): - raise HTTPException(status_code=404, detail=result["error"]) - return result + return require_service_result(result, default_status=404) diff --git a/src/mkb/web/routers/spaces.py b/src/mkb/web/routers/spaces.py index c2becc0..c97db4f 100644 --- a/src/mkb/web/routers/spaces.py +++ b/src/mkb/web/routers/spaces.py @@ -1,7 +1,7 @@ from fastapi import APIRouter, HTTPException from mkb import api -from mkb.web._helpers import _parse_uuid +from mkb.web._helpers import _parse_uuid, require_service_result from mkb.web._models import SpaceCreateRequest, SpaceUpdateRequest router = APIRouter() @@ -48,9 +48,7 @@ def create_space(body: SpaceCreateRequest): review_search_tools=body.review_search_tools, post_processors=body.post_processors, ) - if isinstance(result, dict) and result.get("error"): - raise HTTPException(status_code=400, detail=result["error"]) - return result + return require_service_result(result) @router.put("/api/spaces/{space_id}") @@ -60,15 +58,11 @@ def update_space(space_id: str, body: SpaceUpdateRequest): if not changes: raise HTTPException(status_code=400, detail="No fields to update") result = api.update_space(space_id, **changes) - if isinstance(result, dict) and result.get("error"): - raise HTTPException(status_code=400, detail=result["error"]) - return result + return require_service_result(result) @router.delete("/api/spaces/{space_id}") def delete_space(space_id: str): _parse_uuid(space_id, "space_id") result = api.delete_space(space_id) - if isinstance(result, dict) and result.get("error"): - raise HTTPException(status_code=404, detail=result["error"]) - return result + return require_service_result(result, default_status=404) diff --git a/src/mkb/web/uploads.py b/src/mkb/web/uploads.py new file mode 100644 index 0000000..c0b1abf --- /dev/null +++ b/src/mkb/web/uploads.py @@ -0,0 +1,287 @@ +"""Upload/archive helpers shared by API routes and compatibility wrappers.""" + +from __future__ import annotations + +import shutil +import tarfile +import zipfile +from pathlib import Path +from typing import Any, Callable, Protocol + +from pydantic import BaseModel + +from mkb.web._helpers import _safe_child + + +class UploadInitResponse(BaseModel): + upload_id: str + + +class UploadCompleteRequest(BaseModel): + upload_id: str + + +class UploadExpandRequest(BaseModel): + upload_id: str + + +class UploadExpandFile(BaseModel): + uploadPath: str + size: int + + +class UploadExpandResponse(BaseModel): + files: list[UploadExpandFile] + extracted: list[dict[str, Any]] = [] + failed: list[dict[str, Any]] = [] + + +class UploadFileItem(BaseModel): + name: str + relativePath: str + uploadPath: str + + +class UploadProject(BaseModel): + name: str + upload_id: str + files: list[UploadFileItem] + name_auto: bool = True + + +class IngestApi(Protocol): + def ingest(self, path: Path, **kwargs): ... + + +_ARCHIVE_SUFFIXES = ( + ".zip", + ".tar", + ".tar.gz", + ".tgz", + ".tar.bz2", + ".tbz2", + ".tbz", + ".tar.xz", + ".txz", +) + + +def normalize_project_name(name: str, fallback: str = "project") -> str: + import re + + candidate = (name or "").strip() + if not candidate: + candidate = fallback + candidate = re.sub(r"[^A-Za-z0-9._ -]+", "_", candidate) + candidate = candidate.strip(" ._") + return candidate or fallback + + +def create_unique_project_dir(project_name: str, upload_root: Path) -> Path: + upload_root.mkdir(parents=True, exist_ok=True) + base_name = normalize_project_name(project_name) + candidate = upload_root / base_name + suffix = 2 + while candidate.exists(): + candidate = upload_root / f"{base_name}_{suffix}" + suffix += 1 + candidate.mkdir(parents=True, exist_ok=False) + return candidate + + +def next_available_path(path: Path) -> Path: + if not path.exists(): + return path + stem = path.stem + suffix = path.suffix + idx = 2 + while True: + candidate = path.with_name(f"{stem}_{idx}{suffix}") + if not candidate.exists(): + return candidate + idx += 1 + + +def is_archive(name: str) -> bool: + lower = name.lower() + return any(lower.endswith(ext) for ext in _ARCHIVE_SUFFIXES) + + +def strip_archive_ext(name: str) -> str: + lower = name.lower() + for ext in _ARCHIVE_SUFFIXES: + if lower.endswith(ext): + return name[: -len(ext)] + return name + + +def safe_extract_archive(archive_path: Path, dest_dir: Path) -> int: + dest_root = dest_dir.resolve() + dest_dir.mkdir(parents=True, exist_ok=True) + count = 0 + name = archive_path.name.lower() + + if name.endswith(".zip"): + with zipfile.ZipFile(archive_path) as zf: + for member in zf.infolist(): + if member.is_dir(): + continue + member_name = member.filename.replace("\\", "/") + if not member_name or member_name.endswith("/"): + continue + if ( + member_name.startswith("__MACOSX/") + or "/.DS_Store" in member_name + or member_name.endswith("/.DS_Store") + ): + continue + target = (dest_dir / member_name).resolve() + try: + target.relative_to(dest_root) + except ValueError: + continue + target = next_available_path(target) + target.parent.mkdir(parents=True, exist_ok=True) + with zf.open(member) as src, target.open("wb") as out: + shutil.copyfileobj(src, out) + count += 1 + return count + + with tarfile.open(archive_path, "r:*") as tf: + for member in tf.getmembers(): + if not member.isfile(): + continue + member_name = member.name.replace("\\", "/").lstrip("/") + if not member_name: + continue + if member_name.startswith("__MACOSX/") or member_name.endswith("/.DS_Store"): + continue + target = (dest_dir / member_name).resolve() + try: + target.relative_to(dest_root) + except ValueError: + continue + src = tf.extractfile(member) + if src is None: + continue + target = next_available_path(target) + target.parent.mkdir(parents=True, exist_ok=True) + with target.open("wb") as out: + shutil.copyfileobj(src, out) + count += 1 + return count + + +def expand_temp_dir( + temp_root: Path, + emit: Callable[[str], None] | None = None, + extract_archive: Callable[[Path, Path], int] = safe_extract_archive, +) -> dict[str, Any]: + extracted: list[dict[str, Any]] = [] + failed: list[dict[str, Any]] = [] + + if not temp_root.is_dir(): + return {"files": [], "extracted": extracted, "failed": failed} + + for _ in range(8): + archives = [p for p in temp_root.rglob("*") if p.is_file() and is_archive(p.name)] + if not archives: + break + for archive in archives: + extract_target = archive.parent / strip_archive_ext(archive.name) + if extract_target.exists(): + extract_target = next_available_path(extract_target) + if emit: + emit(f"Extracting {archive.relative_to(temp_root)}") + try: + count = extract_archive(archive, extract_target) + except (zipfile.BadZipFile, tarfile.TarError, OSError) as exc: + failed.append({"archive": str(archive.relative_to(temp_root)), "error": str(exc)}) + continue + extracted.append({"archive": str(archive.relative_to(temp_root)), "count": count}) + try: + archive.unlink() + except OSError: + pass + + files: list[dict[str, Any]] = [] + root_resolved = temp_root.resolve() + for path in sorted(temp_root.rglob("*")): + if not path.is_file(): + continue + try: + rel = path.resolve().relative_to(root_resolved) + except ValueError: + continue + files.append({"uploadPath": rel.as_posix(), "size": path.stat().st_size}) + return {"files": files, "extracted": extracted, "failed": failed} + + +def run_upload_ingest( + payload: list[UploadProject], + *, + upload_temp: Path, + create_project_dir: Callable[[str], Path], + api_module: IngestApi, + progress_callback=None, +) -> dict[str, Any]: + def emit(msg: str) -> None: + if progress_callback: + progress_callback({"message": msg}) + + if not payload: + return {"status": "completed", "message": "No projects provided."} + + upload_id = str(payload[0].upload_id) if payload else "" + temp_root = upload_temp / upload_id if upload_id else upload_temp + + total_ingested = 0 + total_dupes = 0 + created: list[str] = [] + reused = 0 + + try: + emit(f"Preparing {len(payload)} project(s) for ingest") + for idx, project in enumerate(payload, start=1): + upload_dir = create_project_dir(project.name) + emit(f"Moving files for {upload_dir.name} ({idx}/{len(payload)})") + + for file_info in project.files: + src = _safe_child(temp_root, file_info.uploadPath) + if not src.is_file(): + continue + rel_full = _safe_child(upload_dir, file_info.relativePath) + dest = upload_dir / rel_full.relative_to(upload_dir.resolve()) + dest.parent.mkdir(parents=True, exist_ok=True) + dest = next_available_path(dest) + shutil.move(str(src), str(dest)) + + emit(f"Ingesting {upload_dir.name}") + result = api_module.ingest( + upload_dir, + label=project.name if not project.name_auto else None, + user_named=not project.name_auto, + ) + total_ingested += int(result.get("ingested", 0) or 0) + total_dupes += int(result.get("duplicates", 0) or 0) + if result.get("project_reused"): + reused += 1 + shutil.rmtree(upload_dir, ignore_errors=True) + else: + created.append(upload_dir.name) + finally: + if temp_root.is_dir(): + shutil.rmtree(temp_root, ignore_errors=True) + + return { + "status": "completed", + "message": ( + f"Created {len(created)} project(s), reused {reused} existing project(s) · " + f"{total_ingested} file(s) ingested, {total_dupes} duplicate(s) skipped." + ), + "created_projects": created, + "reused_projects": reused, + "ingested": total_ingested, + "duplicates": total_dupes, + } + diff --git a/src/mkb/workflows/contract.py b/src/mkb/workflows/contract.py index 79f551f..dd9f361 100644 --- a/src/mkb/workflows/contract.py +++ b/src/mkb/workflows/contract.py @@ -17,10 +17,10 @@ LEGACY_RAW_WORKFLOW_SCHEMA_VERSION = "raw-workflow/1.0" EXTRACTOR_VERSION = "workflow-extractor/2.0" -NodeKind = Literal["object", "operation", "unknown"] +NodeKind = Literal["object", "operation", "planning", "reasoning", "unknown"] RelationType = Literal[ "input_to", "produces", "same_as", "part_of", "has_part", - "expands_to", "summarized_by", + "expands_to", "summarized_by", "motivates", "leads_to", ] @@ -175,6 +175,10 @@ def validate_graph(self) -> "RawWorkflowGraph": source.node_kind == "operation" and target.node_kind == "object" ): raise ValueError("produces must connect operation -> object") + if edge.relation_type in {"motivates", "leads_to"} and source.node_kind not in { + "planning", "reasoning", + }: + raise ValueError(f"{edge.relation_type} must start from planning or reasoning") return self @@ -190,5 +194,5 @@ def validate_graph(self) -> "RawWorkflowGraph": "extraction": "UUID", "node_instance": "raw::n", "edge_instance": "raw::e", - "ontology_card": "card:::", + "ontology_card": "card:::", } diff --git a/src/mkb/workflows/review.py b/src/mkb/workflows/review.py index 71a0e2a..adaf1e7 100644 --- a/src/mkb/workflows/review.py +++ b/src/mkb/workflows/review.py @@ -37,10 +37,13 @@ def audit_raw_graph(graph: dict, *, low_confidence_threshold: float = 0.5, later source, target = by_id.get(edge.get("source_node")), by_id.get(edge.get("target_node")) relation = edge.get("relation_type") impossible = not source or not target - impossible |= relation == "input_to" and (source or {}).get("node_kind_guess") != "object" - impossible |= relation == "input_to" and (target or {}).get("node_kind_guess") != "operation" - impossible |= relation == "produces" and (source or {}).get("node_kind_guess") != "operation" - impossible |= relation == "produces" and (target or {}).get("node_kind_guess") != "object" + source_kind = (source or {}).get("node_kind") or (source or {}).get("node_kind_guess") + target_kind = (target or {}).get("node_kind") or (target or {}).get("node_kind_guess") + impossible |= relation == "input_to" and source_kind != "object" + impossible |= relation == "input_to" and target_kind != "operation" + impossible |= relation == "produces" and source_kind != "operation" + impossible |= relation == "produces" and target_kind != "object" + impossible |= relation in {"motivates", "leads_to"} and source_kind not in {"planning", "reasoning"} if impossible: flags.append({"type": "impossible_edge", "item_type": "edge", "item_id": edge.get("edge_id")}) granular_pairs = {(e.get("source_node"), e.get("target_node")) for e in edges if e.get("relation_type") in {"part_of", "has_part", "expands_to", "summarized_by"}} diff --git a/src/mkb/workflows/validation.py b/src/mkb/workflows/validation.py index 4bfc560..79c1daa 100644 --- a/src/mkb/workflows/validation.py +++ b/src/mkb/workflows/validation.py @@ -8,3 +8,26 @@ def json_safe_validation_errors(exc: ValidationError) -> list[dict]: """Remove exception objects from Pydantic error context before ADK sees it.""" return exc.errors(include_url=False, include_context=False, include_input=True) + + +def compact_validation_errors(exc: ValidationError, *, limit: int = 20) -> list[dict]: + """Return bounded validation errors for agent-facing tool responses. + + Pydantic's ``input`` field can contain the full invalid graph section. That + is useful for local debugging but expensive and distracting in LLM history. + """ + errors = exc.errors(include_url=False, include_context=False, include_input=False) + compact = [] + for error in errors[: max(1, int(limit))]: + compact.append({ + "loc": list(error.get("loc", [])), + "msg": error.get("msg"), + "type": error.get("type"), + }) + if len(errors) > len(compact): + compact.append({ + "loc": [], + "msg": f"{len(errors) - len(compact)} additional validation errors omitted", + "type": "truncated", + }) + return compact diff --git a/tests/test_job_action_registry.py b/tests/test_job_action_registry.py new file mode 100644 index 0000000..7d8afac --- /dev/null +++ b/tests/test_job_action_registry.py @@ -0,0 +1,47 @@ +import sys +from types import SimpleNamespace + +import pytest + +from mkb.web import job_actions + + +def test_job_action_start_params_resolves_api_target_lazily(monkeypatch): + fake_process = object() + monkeypatch.setitem(sys.modules, "mkb.api", SimpleNamespace(process=fake_process)) + + params = job_actions.job_action_start_params( + "process_project", + job_project_id="project-1", + project_id="project-1", + ) + + assert params["kind"] == "process" + assert params["label"] == "Process" + assert params["project_id"] == "project-1" + assert params["target"] is fake_process + assert params["kwargs"] == {"project_id": "project-1"} + + +def test_workflow_kind_maps_to_registry_action(): + assert job_actions.action_for_workflow_kind("projection_review") == "review_projection" + + +def test_start_job_action_applies_project_conflict_policy(monkeypatch): + def fake_process(**_kwargs): + return {"ok": True} + + active = {"status": "RUNNING"} + manager = SimpleNamespace( + find_active_job=lambda **_kwargs: active, + start_job=lambda **_kwargs: "job-1", + ) + monkeypatch.setitem(sys.modules, "mkb.api", SimpleNamespace(process=fake_process)) + + with pytest.raises(job_actions.JobActionConflict): + job_actions.start_job_action( + manager, + "process_project", + job_project_id="project-1", + project_id="project-1", + ) diff --git a/tests/test_projection_review_patch.py b/tests/test_projection_review_patch.py index 84a62cd..3bf84c9 100644 --- a/tests/test_projection_review_patch.py +++ b/tests/test_projection_review_patch.py @@ -14,7 +14,8 @@ "mkb.agents.runner", SimpleNamespace(AgentRunner=object, RunResult=object), ) -from mkb.agents.tools.projection_review import _set_patch_value, _summarize_data_changes + +from mkb.agents.tools.projection_review import _set_patch_value, _summarize_data_changes # noqa: E402 def test_set_patch_value_updates_nested_list_field(): diff --git a/tests/test_projection_review_space_search.py b/tests/test_projection_review_space_search.py index 01b461c..fccdb31 100644 --- a/tests/test_projection_review_space_search.py +++ b/tests/test_projection_review_space_search.py @@ -1,4 +1,5 @@ import importlib +import mkb import sys from types import SimpleNamespace @@ -24,6 +25,7 @@ def projections_router(monkeypatch): module_name = "mkb.web.routers.projections" sys.modules.pop(module_name, None) monkeypatch.setitem(sys.modules, "mkb.api", fake_api) + monkeypatch.setattr(mkb, "api", fake_api, raising=False) monkeypatch.setitem(sys.modules, "mkb.web._state", SimpleNamespace(jobs=fake_jobs)) module = importlib.import_module(module_name) diff --git a/tests/test_service_result_convention.py b/tests/test_service_result_convention.py new file mode 100644 index 0000000..0dde655 --- /dev/null +++ b/tests/test_service_result_convention.py @@ -0,0 +1,31 @@ +from fastapi import HTTPException +import pytest + +from mkb.services.result import ServiceError, error_result +from mkb.web._helpers import require_service_result, require_service_result_or_not_found + + +def test_require_service_result_maps_service_error_to_http_exception(): + with pytest.raises(HTTPException) as exc: + require_service_result(ServiceError("No such project", code="not_found", status_code=404)) + + assert exc.value.status_code == 404 + assert exc.value.detail["error"] == "No such project" + assert exc.value.detail["code"] == "not_found" + + +def test_require_service_result_maps_error_dict_status_code(): + with pytest.raises(HTTPException) as exc: + require_service_result(error_result("Invalid payload", code="invalid", status_code=422)) + + assert exc.value.status_code == 422 + assert exc.value.detail == "Invalid payload" + + +def test_require_service_result_or_not_found_preserves_legacy_error_dicts(): + with pytest.raises(HTTPException) as exc: + require_service_result_or_not_found({"error": "Project not found"}) + + assert exc.value.status_code == 404 + assert exc.value.detail == "Project not found" + diff --git a/tests/test_workflow_contract.py b/tests/test_workflow_contract.py index 8910c90..56fb581 100644 --- a/tests/test_workflow_contract.py +++ b/tests/test_workflow_contract.py @@ -98,3 +98,37 @@ def test_raw_workflow_contract_rejects_wrong_object_operation_direction(): ) with pytest.raises(ValidationError, match="object -> operation"): RawWorkflowGraph.model_validate(payload) + + +def test_raw_workflow_contract_accepts_reasoning_that_motivates_workflow_node(): + payload = _graph() + eid = payload["extraction_id"] + payload["nodes"].append({ + "node_id": f"raw:{eid}:n0003", + "raw_name": "need higher crystallinity before XRD", + "node_kind_guess": "reasoning", + "attributes_explicitly_mentioned": {}, + "evidence_text": "To improve crystallinity, the powder was annealed before XRD.", + "paper_location": {"section": "Methods"}, + "confidence": 0.91, + }) + payload["edges"].append({ + "edge_id": f"raw:{eid}:e0002", + "source_node": f"raw:{eid}:n0003", + "target_node": f"raw:{eid}:n0002", + "relation_type": "motivates", + "evidence_text": "To improve crystallinity, the powder was annealed before XRD.", + "confidence": 0.9, + }) + + graph = RawWorkflowGraph.model_validate(payload) + + assert graph.nodes[-1].node_kind == "reasoning" + assert graph.edges[-1].relation_type == "motivates" + + +def test_raw_workflow_contract_rejects_motivates_from_non_reasoning_node(): + payload = _graph() + payload["edges"][0]["relation_type"] = "motivates" + with pytest.raises(ValidationError, match="must start from planning or reasoning"): + RawWorkflowGraph.model_validate(payload) diff --git a/tests/test_workflow_resume.py b/tests/test_workflow_resume.py index 572ca7c..37b02ab 100644 --- a/tests/test_workflow_resume.py +++ b/tests/test_workflow_resume.py @@ -245,10 +245,6 @@ def _start_job(**kwargs): assert captured["kwargs"]["mode"] == "local" assert captured["kwargs"]["sample_size"] == 12 assert captured["kwargs"]["min_support"] == 3 - assert fake_row.checkpoint["summary"].startswith("Read methods section") - assert fake_row.checkpoint["graph"] == {"nodes": [{"node_id": "draft-1"}], "edges": []} - assert fake_row.provenance["checkpoint_count"] == 1 - fake_session.commit.assert_called_once() def test_checkpoint_canonical_workflow_updates_unfinished_row(monkeypatch): diff --git a/tests/test_workflow_review_curator.py b/tests/test_workflow_review_curator.py index 5960d94..5b9af95 100644 --- a/tests/test_workflow_review_curator.py +++ b/tests/test_workflow_review_curator.py @@ -35,6 +35,31 @@ def test_audit_and_rebase_correction_graph(): assert corrected["nodes"][0]["node_id"].startswith(f"raw:{new_id}:n") +def test_audit_allows_planning_edge_to_downstream_operation(): + graph = _raw_graph() + eid = graph["extraction_id"] + graph["nodes"].append({ + "node_id": f"raw:{eid}:n0003", + "raw_name": "screen high temperature phase stability", + "node_kind_guess": "planning", + "evidence_text": "We screened high temperature phase stability before annealing.", + "paper_location": {}, + "confidence": 0.9, + }) + graph["edges"].append({ + "edge_id": f"raw:{eid}:e0002", + "source_node": f"raw:{eid}:n0003", + "target_node": f"raw:{eid}:n0002", + "relation_type": "leads_to", + "evidence_text": "We screened high temperature phase stability before annealing.", + "confidence": 0.9, + }) + + flags = audit_raw_graph(graph) + + assert not [flag for flag in flags if flag["type"] == "impossible_edge"] + + def test_validation_errors_are_safe_for_agent_request_serialization(): graph = _raw_graph() graph["edges"][0]["relation_type"] = "produces" From 39830903dda51193758b9217fd7b22dee3e80e2d Mon Sep 17 00:00:00 2001 From: theAfish Date: Wed, 1 Jul 2026 13:07:28 +0800 Subject: [PATCH 03/26] fix: workflow rendering/extracting bugs --- TODO.md | 17 +- docs/workflow-lifecycle-policy.md | 17 +- .../components/projects/WorkflowCanvas.tsx | 78 +- .../components/projects/WorkflowGraphTab.tsx | 11 + frontend/src/types/index.ts | 1 + src/mkb/agents/prompts/workflow_extraction.py | 17 +- .../agents/tools/workflow_canonicalization.py | 48 +- src/mkb/agents/tools/workflows.py | 295 +---- src/mkb/api.py | 14 + src/mkb/services/workflows.py | 1000 ----------------- src/mkb/services/workflows/__init__.py | 73 ++ src/mkb/services/workflows/extraction.py | 230 ++++ src/mkb/services/workflows/indexing.py | 83 ++ .../workflows/legacy_canonicalization.py | 82 ++ src/mkb/services/workflows/maintenance.py | 190 ++++ src/mkb/services/workflows/schema_review.py | 380 +++++++ src/mkb/services/workflows/serialization.py | 81 ++ src/mkb/web/_models.py | 4 - src/mkb/web/job_actions.py | 8 - src/mkb/web/routers/projects.py | 16 - src/mkb/workflows/editing.py | 442 ++++++++ src/mkb/workflows/review.py | 6 + tests/test_workflow_editing.py | 186 +++ tests/test_workflow_review.py | 41 + tests/test_workflow_review_curator.py | 3 +- tests/test_workflow_serialization.py | 59 + 26 files changed, 1926 insertions(+), 1456 deletions(-) delete mode 100644 src/mkb/services/workflows.py create mode 100644 src/mkb/services/workflows/__init__.py create mode 100644 src/mkb/services/workflows/extraction.py create mode 100644 src/mkb/services/workflows/indexing.py create mode 100644 src/mkb/services/workflows/legacy_canonicalization.py create mode 100644 src/mkb/services/workflows/maintenance.py create mode 100644 src/mkb/services/workflows/schema_review.py create mode 100644 src/mkb/services/workflows/serialization.py create mode 100644 src/mkb/workflows/editing.py create mode 100644 tests/test_workflow_editing.py create mode 100644 tests/test_workflow_review.py create mode 100644 tests/test_workflow_serialization.py diff --git a/TODO.md b/TODO.md index 3496cbd..d41bdb7 100644 --- a/TODO.md +++ b/TODO.md @@ -119,7 +119,7 @@ operate and develop. deprecated, or internal compatibility. - Documented in `docs/workflow-lifecycle-policy.md`. -- [ ] Group workflow code by lifecycle. +- [x] Group workflow code by lifecycle. - Current workflow behavior spans: - `src/mkb/workflows/*` - `src/mkb/agents/workflow_extraction.py` @@ -131,16 +131,21 @@ operate and develop. - many sections of `src/mkb/api.py` - Create a workflow service package with explicit submodules for extraction, validation, schema review, indexing, and legacy canonicalization. - - Started: workflow serialization now lives under `mkb.services.workflows`; - full lifecycle package split remains. + - Done: `mkb.services.workflows` is now a lifecycle package with + `extraction`, `serialization`, `schema_review`, `maintenance`, `indexing`, + and `legacy_canonicalization` modules. Active routes/jobs use raw + workflow extraction and review; canonicalization launch is no longer part + of the active REST/job flow. -- [ ] Remove duplicated schema/card operations between +- [x] Remove duplicated schema/card operations between `src/mkb/agents/tools/workflows.py` and `src/mkb/agents/tools/workflow_canonicalization.py`. - Both modules normalize payloads, expose card/template operations, and manipulate draft graphs or schema libraries. - - Keep one low-level workflow editing library and make agent tools thin - adapters. + - Done: shared card search, schema-library views, raw graph normalization, + checkpoint manifests, compacting, and draft replacement helpers live in + `mkb.workflows.editing`. Agent tools now call that library instead of + owning duplicate low-level operations. ## P1 - Reduce Frontend Duplication diff --git a/docs/workflow-lifecycle-policy.md b/docs/workflow-lifecycle-policy.md index c9aa14a..12f596a 100644 --- a/docs/workflow-lifecycle-policy.md +++ b/docs/workflow-lifecycle-policy.md @@ -9,12 +9,15 @@ path documented in `workflow-card-architecture.md`. workflow graph produced from a project. - Workflow card/schema validation, review, indexing, and ontology induction are active. -- Schema proposal review and workflow maintenance tasks are active. +- Schema proposal review and raw-workflow maintenance tasks are active. ## Compatibility Only -- Canonical workflow records and canonicalization endpoints remain available so - older data, tests, and UI tabs continue to work during migration. +- Canonical workflow records remain readable/deletable so older data, tests, + and UI tabs continue to work during migration. +- Canonicalization agent/tool code is internal compatibility for unfinished + legacy jobs and old records. It is not exposed through the active REST/job + action flow. - New product behavior should not depend on canonicalization unless it is explicitly maintaining compatibility with existing records. - Canonical workflow code should move behind a `legacy` or `compatibility` @@ -30,9 +33,9 @@ path documented in `workflow-card-architecture.md`. ## Public Surface Status - Active: raw workflow extraction, raw workflow review/correction, schema - curation, schema proposal review, workflow maintenance, workflow indexing. -- Compatibility: canonical workflow list/get/delete, canonicalization job - routes, canonical workflow frontend tabs, canonical indexes. + curation, schema proposal review, raw workflow maintenance, workflow-card + editing helpers. +- Compatibility: canonical workflow list/get/delete, canonical workflow + frontend tabs, canonical indexes. - Internal compatibility: checkpoint and draft-edit helpers used to resume or inspect unfinished legacy canonicalization jobs. - diff --git a/frontend/src/components/projects/WorkflowCanvas.tsx b/frontend/src/components/projects/WorkflowCanvas.tsx index fb4cb5a..288dd05 100644 --- a/frontend/src/components/projects/WorkflowCanvas.tsx +++ b/frontend/src/components/projects/WorkflowCanvas.tsx @@ -408,83 +408,7 @@ function centeredOffsets(count: number, gap: number) { } function workflowLayoutGroups(nodes: WorkflowCanvasNode[], edges: WorkflowCanvasEdge[]) { - const nodeMap = new Map(nodes.map(node => [node.id, node])) - const operationIds = nodes.filter(node => node.kind === 'operation').map(node => node.id) - if (operationIds.length === 0) return connectedComponents(nodes, edges) - - const producerMap = new Map() - const consumerMap = new Map() - nodes.filter(node => node.kind === 'object').forEach(node => { - producerMap.set(node.id, []) - consumerMap.set(node.id, []) - }) - - edges.forEach(edge => { - const sourceKind = nodeMap.get(edge.source)?.kind - const targetKind = nodeMap.get(edge.target)?.kind - if (sourceKind === 'operation' && targetKind === 'object') { - producerMap.get(edge.target)?.push(edge.source) - } - if (sourceKind === 'object' && targetKind === 'operation') { - consumerMap.get(edge.source)?.push(edge.target) - } - }) - - const opAdjacency = new Map>() - operationIds.forEach(id => opAdjacency.set(id, new Set())) - producerMap.forEach((producers, objectId) => { - const consumers = consumerMap.get(objectId) ?? [] - producers.forEach(producerId => { - consumers.forEach(consumerId => { - if (producerId === consumerId) return - opAdjacency.get(producerId)?.add(consumerId) - opAdjacency.get(consumerId)?.add(producerId) - }) - }) - }) - - const seenOps = new Set() - const groups: string[][] = [] - const groupByOperation = new Map() - operationIds.forEach(operationId => { - if (seenOps.has(operationId)) return - const queue = [operationId] - const group: string[] = [] - seenOps.add(operationId) - while (queue.length) { - const current = queue.shift()! - groupByOperation.set(current, groups.length) - group.push(current) - opAdjacency.get(current)?.forEach(next => { - if (seenOps.has(next)) return - seenOps.add(next) - queue.push(next) - }) - } - groups.push(group) - }) - - const ungroupedObjects: string[] = [] - nodes.filter(node => node.kind === 'object').forEach(node => { - const touchedGroups = new Map() - const touchedOperations = [...(producerMap.get(node.id) ?? []), ...(consumerMap.get(node.id) ?? [])] - touchedOperations.forEach(operationId => { - const groupIndex = groupByOperation.get(operationId) - if (groupIndex === undefined) return - touchedGroups.set(groupIndex, (touchedGroups.get(groupIndex) ?? 0) + 1) - }) - - if (touchedGroups.size === 0) { - ungroupedObjects.push(node.id) - return - } - - const bestGroup = Array.from(touchedGroups.entries()).sort((a, b) => b[1] - a[1] || a[0] - b[0])[0][0] - groups[bestGroup].push(node.id) - }) - - ungroupedObjects.forEach(objectId => groups.push([objectId])) - return groups + return connectedComponents(nodes, edges) } function layoutComponent(nodes: WorkflowCanvasNode[], edges: WorkflowCanvasEdge[]) { diff --git a/frontend/src/components/projects/WorkflowGraphTab.tsx b/frontend/src/components/projects/WorkflowGraphTab.tsx index cdf1b24..b44d5e4 100644 --- a/frontend/src/components/projects/WorkflowGraphTab.tsx +++ b/frontend/src/components/projects/WorkflowGraphTab.tsx @@ -47,6 +47,11 @@ function RawWorkflowCanvas({ workflow }: { workflow: RawWorkflowVersion }) { return } +function formatReviewFlag(flag: { type: string; item_type?: string; item_id?: string }) { + const label = flag.type.replace(/_/g, ' ') + return flag.item_id ? `${label}: ${flag.item_id}` : label +} + export default function WorkflowGraphTab({ projectId, actionsDisabled = false, @@ -130,6 +135,12 @@ export default function WorkflowGraphTab({ {error &&

{error}

} {selected?.error &&

{selected.error}

} + {selected?.review_flags && selected.review_flags.length > 0 && ( +
+ {selected.review_flags.slice(0, 4).map(flag => formatReviewFlag(flag)).join(' · ')} + {selected.review_flags.length > 4 ? ` · ${selected.review_flags.length - 4} more` : ''} +
+ )} {selected && !selected.graph && selected.resumable && (
{selected.has_checkpoint diff --git a/frontend/src/types/index.ts b/frontend/src/types/index.ts index 149e1af..37ecfdb 100644 --- a/frontend/src/types/index.ts +++ b/frontend/src/types/index.ts @@ -304,6 +304,7 @@ export interface RawWorkflowVersion { model: string | null provenance: Record error: string | null + review_flags?: Array<{ type: string; item_type?: string; item_id?: string }> created_at: string | null extracted_at: string | null has_checkpoint?: boolean diff --git a/src/mkb/agents/prompts/workflow_extraction.py b/src/mkb/agents/prompts/workflow_extraction.py index 47ad386..e9c9693 100644 --- a/src/mkb/agents/prompts/workflow_extraction.py +++ b/src/mkb/agents/prompts/workflow_extraction.py @@ -9,8 +9,8 @@ Represent concrete experimental, computational, and analytical work as Object -> Operation -> Object: -* object -> operation uses `input_to` -* operation -> object uses `produces` +* identify object -> operation and operation -> object connections from the + evidence; the save tool assigns `input_to` and `produces` from node kinds * separate runs are separate operation instances, even when they instantiate the same reusable operation card * disconnected components and genuinely missing endpoints are allowed; never @@ -23,11 +23,11 @@ screening strategy, or decision policy * use `reasoning` for hypothesis, rationale, interpretation, causal argument, constraint, tradeoff, or conclusion that drives later work -* connect planning/reasoning nodes to downstream nodes with `motivates` when - the text explains why that node is needed, or `leads_to` when the text states - that the plan/reasoning caused the next workflow item +* identify planning/reasoning -> downstream connections from the evidence; the + save tool assigns `motivates` by default, or preserves explicit `leads_to` + when the text states that the plan/reasoning caused the next workflow item * planning/reasoning nodes may point to objects, operations, or other - planning/reasoning nodes, but do not use `input_to` or `produces` for them + planning/reasoning nodes * keep unsupported background claims in `unresolved_information` rather than adding a planning/reasoning node without direct evidence @@ -95,8 +95,9 @@ 6. Focus on scientific content and evidence. The save/checkpoint tools fill application-owned envelope fields such as `schema_version`, `paper_id`, `extraction_id`, sequential node/edge IDs, default empty dict/list fields, - and common edge aliases. Provide stable node/edge references when you have - them, but do not spend turns repairing mechanical schema boilerplate. + and deterministic edge relations from endpoint node kinds. Provide stable + node/edge references when you have them, but do not spend turns repairing + mechanical schema boilerplate. 7. Save exactly once with save_raw_workflow, including an empty graph when no supported workflow exists. The tool will normalize mechanical fields and return compact validation hints if semantic fixes are still needed. diff --git a/src/mkb/agents/tools/workflow_canonicalization.py b/src/mkb/agents/tools/workflow_canonicalization.py index 2fe5499..26b33a7 100644 --- a/src/mkb/agents/tools/workflow_canonicalization.py +++ b/src/mkb/agents/tools/workflow_canonicalization.py @@ -15,6 +15,11 @@ from mkb.workflows.schema_library import get_schema_library_payload from mkb.workflows.indexing import build_index_entries from mkb.workflows.validation import compact_validation_errors +from mkb.workflows.editing import ( + compact_value as _compact_value, + replace_by_id as _replace_by_id, + replace_by_raw_ids as _replace_by_raw_ids, +) def _uuid(value: str) -> uuid.UUID | None: @@ -39,25 +44,6 @@ def _draft_template(row: CanonicalWorkflow, raw: RawWorkflowExtraction) -> dict[ } -def _compact_value(value: Any, *, string_limit: int = 1000, list_limit: int = 30, dict_limit: int = 30) -> Any: - if isinstance(value, str): - return value if len(value) <= string_limit else f"{value[:string_limit]}... [truncated]" - if isinstance(value, list): - items = [_compact_value(item, string_limit=string_limit, list_limit=list_limit, dict_limit=dict_limit) for item in value[:list_limit]] - if len(value) > list_limit: - items.append({"omitted_items": len(value) - list_limit}) - return items - if isinstance(value, dict): - result = {} - for index, (key, item) in enumerate(value.items()): - if index >= dict_limit: - result["omitted_keys"] = len(value) - dict_limit - break - result[key] = _compact_value(item, string_limit=string_limit, list_limit=list_limit, dict_limit=dict_limit) - return result - return value - - def _raw_graph_context(raw_graph: dict) -> dict: nodes = raw_graph.get("nodes") if isinstance(raw_graph.get("nodes"), list) else [] edges = raw_graph.get("edges") if isinstance(raw_graph.get("edges"), list) else [] @@ -204,30 +190,6 @@ def _save_draft(row: CanonicalWorkflow, draft: dict[str, Any], *, summary: str | } -def _replace_by_id(items: list[dict[str, Any]], id_key: str, item: dict[str, Any]) -> tuple[list[dict[str, Any]], str]: - item_id = item.get(id_key) - if not isinstance(item_id, str) or not item_id.strip(): - raise ValueError(f"{id_key} is required") - for index, existing in enumerate(items): - if existing.get(id_key) == item_id: - items[index] = item - return items, "updated" - items.append(item) - return items, "added" - - -def _replace_by_raw_ids(items: list[dict[str, Any]], item: dict[str, Any]) -> tuple[list[dict[str, Any]], str]: - raw_ids = tuple(item.get("raw_node_ids") or []) - if not raw_ids: - raise ValueError("raw_node_ids is required") - for index, existing in enumerate(items): - if tuple(existing.get("raw_node_ids") or []) == raw_ids: - items[index] = item - return items, "updated" - items.append(item) - return items, "added" - - def get_canonicalization_context(canonicalization_id: str) -> dict: """Load the raw graph and current schema library for a pending run.""" with SyncSessionLocal() as session: diff --git a/src/mkb/agents/tools/workflows.py b/src/mkb/agents/tools/workflows.py index 948a6ac..d394864 100644 --- a/src/mkb/agents/tools/workflows.py +++ b/src/mkb/agents/tools/workflows.py @@ -3,298 +3,20 @@ from __future__ import annotations import uuid -from copy import deepcopy from datetime import datetime, timezone -from typing import Any - from pydantic import ValidationError from mkb.db.engine import SyncSessionLocal from mkb.db.models import RawWorkflowExtraction from mkb.workflows.contract import RawWorkflowGraph -from mkb.workflows.schema_library import get_schema_library_payload from mkb.workflows.review import audit_raw_graph from mkb.workflows.validation import compact_validation_errors - -SEARCHABLE_NODE_KINDS = {None, "", "object", "operation", "planning", "reasoning", "unknown"} -NODE_KINDS = {"object", "operation", "planning", "reasoning", "unknown"} -RELATION_ALIASES = { - "input": "input_to", - "input_to": "input_to", - "produces": "produces", - "output": "produces", - "output_of": "produces", - "same_as": "same_as", - "part_of": "part_of", - "has_part": "has_part", - "expands_to": "expands_to", - "summarized_by": "summarized_by", - "motivates": "motivates", - "leads_to": "leads_to", -} - - -def _dict_or_empty(value: Any) -> dict: - return value if isinstance(value, dict) else {} - - -def _list_or_empty(value: Any) -> list: - return value if isinstance(value, list) else [] - - -def _normalize_kind(value: Any) -> str: - kind = str(value or "unknown").strip().casefold().replace("-", "_") - return kind if kind in NODE_KINDS else "unknown" - - -def _normalize_raw_graph_payload(graph: dict, row: RawWorkflowExtraction, extraction_id: uuid.UUID) -> tuple[dict, dict]: - """Fill app-owned workflow envelope fields and tolerate common LLM aliases.""" - payload = deepcopy(graph if isinstance(graph, dict) else {}) - changes = { - "filled_graph_fields": [], - "assigned_node_ids": 0, - "assigned_edge_ids": 0, - "normalized_nodes": 0, - "normalized_edges": 0, - } - for key, value in { - "schema_version": row.schema_version, - "paper_id": str(row.project_id), - "extraction_id": str(extraction_id), - }.items(): - if payload.get(key) != value: - payload[key] = value - changes["filled_graph_fields"].append(key) - - raw_nodes = payload.get("nodes") - payload["nodes"] = raw_nodes if isinstance(raw_nodes, list) else [] - raw_edges = payload.get("edges") - payload["edges"] = raw_edges if isinstance(raw_edges, list) else [] - payload["unresolved_information"] = [ - item if isinstance(item, dict) else {"description": str(item)} - for item in _list_or_empty(payload.get("unresolved_information")) - ] - if not isinstance(payload.get("reproducibility"), dict): - payload.pop("reproducibility", None) - - old_node_refs: dict[str, str] = {} - for index, node in enumerate(payload["nodes"], 1): - if not isinstance(node, dict): - node = {"raw_name": str(node), "evidence_text": str(node)} - payload["nodes"][index - 1] = node - original_refs = { - str(value).strip() - for value in ( - node.get("node_id"), - node.get("id"), - node.get("name"), - node.get("label"), - node.get("raw_name"), - node.get("canonical_name"), - ) - if value is not None and str(value).strip() - } - expected_id = f"raw:{extraction_id}:n{index:04d}" - node_id = str(node.get("node_id") or node.get("id") or "").strip() - if not node_id or not node_id.startswith(f"raw:{extraction_id}:n"): - node["node_id"] = expected_id - changes["assigned_node_ids"] += 1 - kind = _normalize_kind(node.get("node_kind") or node.get("node_kind_guess") or node.get("kind")) - node["node_kind"] = kind - node["node_kind_guess"] = kind - node["raw_name"] = str(node.get("raw_name") or node.get("canonical_name") or node.get("label") or node["node_id"]) - node.setdefault("canonical_name", node.get("raw_name")) - node.setdefault("semantic_type", kind) - for key in ("parameters", "identity", "state", "role", "context", "attributes_explicitly_mentioned", "paper_location"): - node[key] = _dict_or_empty(node.get(key)) - node["unparsed_modifiers"] = _list_or_empty(node.get("unparsed_modifiers")) - node["aliases_observed"] = _list_or_empty(node.get("aliases_observed")) - status = str(node.get("ontology_status") or "unmapped").strip().casefold() - node["ontology_status"] = "matched" if status == "mapped" else status if status in {"matched", "candidate", "unmapped"} else "unmapped" - node["evidence_text"] = str(node.get("evidence_text") or node.get("evidence") or node.get("raw_name")) - try: - node["confidence"] = float(node.get("confidence", 0.5)) - except (TypeError, ValueError): - node["confidence"] = 0.5 - node["confidence"] = max(0.0, min(1.0, node["confidence"])) - for ref in original_refs: - old_node_refs[ref] = node["node_id"] - changes["normalized_nodes"] += 1 - - for index, edge in enumerate(payload["edges"], 1): - if not isinstance(edge, dict): - edge = {"evidence_text": str(edge)} - payload["edges"][index - 1] = edge - edge_id = str(edge.get("edge_id") or edge.get("id") or "").strip() - if not edge_id or not edge_id.startswith(f"raw:{extraction_id}:e"): - edge["edge_id"] = f"raw:{extraction_id}:e{index:04d}" - changes["assigned_edge_ids"] += 1 - source = edge.get("source_node", edge.get("source")) - target = edge.get("target_node", edge.get("target")) - edge["source_node"] = old_node_refs.get(str(source).strip(), source) - edge["target_node"] = old_node_refs.get(str(target).strip(), target) - relation = str(edge.get("relation_type") or edge.get("kind") or edge.get("relation") or "").strip().casefold().replace("-", "_") - edge["relation_type"] = RELATION_ALIASES.get(relation, relation) - edge["attributes"] = _dict_or_empty(edge.get("attributes")) - edge["evidence_text"] = str(edge.get("evidence_text") or edge.get("evidence") or edge.get("relation_type") or "") - if edge.get("paper_location") is not None: - edge["paper_location"] = _dict_or_empty(edge.get("paper_location")) - try: - edge["confidence"] = float(edge.get("confidence", 0.5)) - except (TypeError, ValueError): - edge["confidence"] = 0.5 - edge["confidence"] = max(0.0, min(1.0, edge["confidence"])) - changes["normalized_edges"] += 1 - - return payload, changes - - -def _compact_raw_checkpoint_manifest(graph: dict | None) -> dict: - if not isinstance(graph, dict): - return {"counts": {"nodes": 0, "edges": 0}, "nodes": [], "edges": []} - nodes = graph.get("nodes") if isinstance(graph.get("nodes"), list) else [] - edges = graph.get("edges") if isinstance(graph.get("edges"), list) else [] - return { - "counts": { - "nodes": len(nodes), - "edges": len(edges), - "unresolved_information": len(graph.get("unresolved_information") or []), - }, - "nodes": [ - { - "node_id": node.get("node_id"), - "raw_name": node.get("raw_name") or node.get("canonical_name") or node.get("label"), - "node_kind": node.get("node_kind") or node.get("node_kind_guess") or node.get("kind"), - } - for node in nodes[:80] - if isinstance(node, dict) - ], - "edges": [ - { - "edge_id": edge.get("edge_id"), - "source_node": edge.get("source_node") or edge.get("source"), - "target_node": edge.get("target_node") or edge.get("target"), - "relation_type": edge.get("relation_type") or edge.get("kind") or edge.get("relation"), - } - for edge in edges[:120] - if isinstance(edge, dict) - ], - "note": "The full checkpoint graph remains server-side. Continue from this manifest and save/checkpoint only changed draft content.", - } - - -def get_active_workflow_card_library( - max_cards: int = 40, - max_templates: int = 40, -) -> dict: - """Return a bounded view of the newest workflow card/schema library.""" - library = get_schema_library_payload() - cards = library.get("cards", {}) - templates = library.get("operation_templates", {}) - card_items = list(cards.items())[: max(1, min(int(max_cards), 200))] - template_items = list(templates.items())[: max(1, min(int(max_templates), 200))] - return { - "schema_version": library.get("schema_version"), - "cards": [ - { - "card_id": card_id, - "canonical_name": payload.get("canonical_name"), - "kind": payload.get("kind"), - "aliases": payload.get("aliases", []), - "parameter_slots": payload.get("parameter_slots", []), - "status": payload.get("status", "active"), - "replaced_by": payload.get("replaced_by"), - } - for card_id, payload in card_items - ], - "operation_templates": [ - { - "template_id": template_id, - "label": payload.get("label"), - "aliases": payload.get("aliases", []), - "slots": payload.get("slots", []), - "parameters": payload.get("parameters", {}), - "deprecated": bool(payload.get("deprecated")), - } - for template_id, payload in template_items - ], - } - - -def search_workflow_cards( - query: str, - node_kind: str | None = None, - limit: int = 10, -) -> dict: - """Search the newest card base/templates before instantiating workflow nodes.""" - text = str(query or "").strip().casefold() - if not text: - return {"error": "query is required"} - if node_kind not in SEARCHABLE_NODE_KINDS: - return {"error": "node_kind must be one of object, operation, planning, reasoning, unknown, or omitted"} - - library = get_schema_library_payload() - results: list[dict] = [] - effective_limit = max(1, min(int(limit), 50)) - - for card_id, payload in (library.get("cards", {}) or {}).items(): - kind = payload.get("kind") - if node_kind and kind != node_kind: - continue - name = str(payload.get("canonical_name") or "") - aliases = [str(value) for value in payload.get("aliases", []) if value] - haystacks = [card_id, name, *aliases] - score = sum(3 for item in haystacks if text == item.casefold()) - score += sum(1 for item in haystacks if text in item.casefold()) - if score <= 0: - continue - results.append({ - "match_type": "card", - "score": score, - "card_id": card_id, - "canonical_name": name, - "kind": kind, - "aliases": aliases, - "parameter_slots": payload.get("parameter_slots", []), - "status": payload.get("status", "active"), - "replaced_by": payload.get("replaced_by"), - }) - - for template_id, payload in (library.get("operation_templates", {}) or {}).items(): - if node_kind and node_kind != "operation": - continue - label = str(payload.get("label") or "") - aliases = [str(value) for value in payload.get("aliases", []) if value] - haystacks = [template_id, label, *aliases] - score = sum(3 for item in haystacks if text == item.casefold()) - score += sum(1 for item in haystacks if text in item.casefold()) - if score <= 0: - continue - results.append({ - "match_type": "operation_template", - "score": score, - "template_id": template_id, - "label": label, - "kind": "operation", - "aliases": aliases, - "slots": payload.get("slots", []), - "parameters": payload.get("parameters", {}), - "deprecated": bool(payload.get("deprecated")), - }) - - results.sort( - key=lambda item: ( - -int(item.get("score", 0)), - str(item.get("canonical_name") or item.get("label") or item.get("card_id") or item.get("template_id")), - ) - ) - return { - "schema_version": library.get("schema_version"), - "query": query, - "node_kind": node_kind or "any", - "results": results[:effective_limit], - } - +from mkb.workflows.editing import ( + compact_raw_checkpoint_manifest as _compact_raw_checkpoint_manifest, + get_active_workflow_card_library, + normalize_raw_graph_payload as _normalize_raw_graph_payload, + search_workflow_cards, +) def get_raw_workflow_checkpoint(extraction_id: str) -> dict: """Read the latest saved checkpoint for an unfinished raw workflow.""" @@ -393,8 +115,9 @@ def save_raw_workflow(extraction_id: str, graph: dict) -> dict: "normalization": normalization, "hint": ( "The tool fills schema_version, paper_id, extraction_id, sequential IDs, " - "default dict/list fields, and common edge aliases. Fix the listed node/edge " - "semantics rather than resending the full source content." + "default dict/list fields, and deterministic edge relations from endpoint " + "node kinds. If validation still fails, fix the listed endpoints or node " + "kinds rather than resending the full source content." ), } diff --git a/src/mkb/api.py b/src/mkb/api.py index 64a9867..93fd57b 100644 --- a/src/mkb/api.py +++ b/src/mkb/api.py @@ -18,6 +18,14 @@ spaces as _spaces, workflows as _workflows, ) +from mkb.services.workflows import ( + extraction as _workflow_extraction, + indexing as _workflow_indexing, + legacy_canonicalization as _workflow_legacy_canonicalization, + maintenance as _workflow_maintenance, + schema_review as _workflow_schema_review, + serialization as _workflow_serialization, +) from mkb.services.runtime import ( setup, @@ -239,6 +247,12 @@ _runtime, _spaces, _workflows, + _workflow_extraction, + _workflow_indexing, + _workflow_legacy_canonicalization, + _workflow_maintenance, + _workflow_schema_review, + _workflow_serialization, ) class _ApiFacadeModule(types.ModuleType): diff --git a/src/mkb/services/workflows.py b/src/mkb/services/workflows.py deleted file mode 100644 index b1d09db..0000000 --- a/src/mkb/services/workflows.py +++ /dev/null @@ -1,1000 +0,0 @@ -"""Workflows API service functions.""" - -from __future__ import annotations - -from mkb.services._api_common import ( - SyncSessionLocal, - datetime, - init_db, - timezone, - uuid, -) - - -def serialize_raw_workflow(row, include_graph: bool) -> dict: - payload = { - "extraction_id": str(row.extraction_id), - "project_id": str(row.project_id), - "version": row.version, - "schema_version": row.schema_version, - "extractor_version": row.extractor_version, - "model": row.model, - "status": row.status, - "record_status": row.record_status, - "supersedes_extraction_id": ( - str(row.supersedes_extraction_id) if row.supersedes_extraction_id else None - ), - "correction_reason": row.correction_reason, - "correction_author": row.correction_author, - "correction_details": row.correction_details or {}, - "review_flags": row.review_flags or [], - "provenance": row.provenance or {}, - "error": row.error, - "extracted_at": row.extracted_at.isoformat() if row.extracted_at else None, - "created_at": row.created_at.isoformat() if row.created_at else None, - "has_checkpoint": bool(row.checkpoint), - "checkpoint_summary": (row.checkpoint or {}).get("summary"), - "checkpoint_updated_at": ( - row.checkpoint_updated_at.isoformat() if row.checkpoint_updated_at else None - ), - "resumable": row.graph is None and row.status in {"IN_PROGRESS", "FAILED"}, - } - if include_graph: - payload["graph"] = row.graph - elif row.graph: - payload["node_count"] = len(row.graph.get("nodes", [])) - payload["edge_count"] = len(row.graph.get("edges", [])) - return payload - - -def serialize_canonical_workflow(row, include_graph: bool) -> dict: - payload = { - "canonicalization_id": str(row.canonicalization_id), - "project_id": str(row.project_id), - "raw_extraction_id": str(row.raw_extraction_id), - "version": row.version, - "schema_version": row.schema_version, - "canonicalizer_version": row.canonicalizer_version, - "model": row.model, - "status": row.status, - "provenance": row.provenance or {}, - "error": row.error, - "canonicalized_at": row.canonicalized_at.isoformat() if row.canonicalized_at else None, - "created_at": row.created_at.isoformat() if row.created_at else None, - "has_checkpoint": bool(row.checkpoint), - "checkpoint_summary": (row.checkpoint or {}).get("summary"), - "checkpoint_updated_at": ( - row.checkpoint_updated_at.isoformat() if row.checkpoint_updated_at else None - ), - "resumable": row.graph is None and row.status in {"IN_PROGRESS", "FAILED"}, - } - if include_graph: - payload["graph"] = row.graph - elif row.graph: - payload.update( - node_count=len(row.graph.get("nodes", [])), - edge_count=len(row.graph.get("edges", [])), - ) - return payload - - -def extract_raw_workflow(project_id: str | uuid.UUID, model: str | None = None, verbose: bool = False, progress_callback=None) -> dict: - """Append a new faithful raw-workflow extraction version for a project.""" - from mkb.agents.workflow_extraction import run_workflow_extraction - - readiness = get_raw_workflow_extraction_readiness(project_id) - if not readiness.get("ready"): - return { - "status": "error", - "message": readiness.get("message") or "Project is not ready for workflow extraction", - } - init_db() - return run_workflow_extraction( - uuid.UUID(str(project_id)), model=model, verbose=verbose, - progress_callback=progress_callback, - ) - -def get_raw_workflow_extraction_readiness(project_id: str | uuid.UUID) -> dict: - """Check whether a project has readable sources for raw workflow extraction.""" - from mkb.db.models import ( - ProcessedAsset, - ProcessingType, - ProjectAsset, - RawWorkflowExtraction, - ResearchProject, - ) - - pid = uuid.UUID(str(project_id)) - init_db() - with SyncSessionLocal() as session: - project = session.query(ResearchProject).filter_by(project_id=pid).first() - if not project: - return {"ready": False, "message": f"Project {pid} not found"} - - rows = ( - session.query(ProcessedAsset.asset_id) - .join(ProjectAsset, ProjectAsset.asset_id == ProcessedAsset.asset_id) - .filter( - ProjectAsset.project_id == pid, - ProcessedAsset.processing_type == ProcessingType.MARKDOWN, - ) - .distinct() - .all() - ) - asset_ids = [str(row.asset_id) for row in rows] - if not asset_ids: - return { - "ready": False, - "message": ( - "Workflow extraction requires processed Markdown, but this project has no readable " - "processed Markdown files yet. Run Process first and confirm Markdown outputs exist." - ), - } - - unfinished = ( - session.query(RawWorkflowExtraction) - .filter( - RawWorkflowExtraction.project_id == pid, - RawWorkflowExtraction.graph.is_(None), - RawWorkflowExtraction.status.in_(("IN_PROGRESS", "FAILED")), - ) - .order_by(RawWorkflowExtraction.version.desc()) - .first() - ) - return { - "ready": True, - "project_id": str(pid), - "readable_asset_ids": asset_ids, - "resume_extraction_id": str(unfinished.extraction_id) if unfinished else None, - "resume_version": unfinished.version if unfinished else None, - "has_checkpoint": bool(unfinished and unfinished.checkpoint), - } - -def list_raw_workflows(project_id: str | uuid.UUID, include_graph: bool = False) -> list[dict]: - """List append-only raw workflow versions, newest first.""" - from mkb.db.models import RawWorkflowExtraction - - pid = uuid.UUID(str(project_id)) - init_db() - with SyncSessionLocal() as session: - rows = ( - session.query(RawWorkflowExtraction) - .filter(RawWorkflowExtraction.project_id == pid) - .order_by(RawWorkflowExtraction.version.desc()) - .all() - ) - return [_serialize_raw_workflow(row, include_graph=include_graph) for row in rows] - -def get_raw_workflow(project_id: str | uuid.UUID, version: int | None = None) -> dict | None: - """Get the latest completed raw workflow, or a specific version.""" - from mkb.db.models import RawWorkflowExtraction - - pid = uuid.UUID(str(project_id)) - init_db() - with SyncSessionLocal() as session: - query = session.query(RawWorkflowExtraction).filter(RawWorkflowExtraction.project_id == pid) - if version is None: - query = query.filter( - RawWorkflowExtraction.status == "COMPLETED", - RawWorkflowExtraction.record_status.in_(("active", "needs_review")), - ).order_by(RawWorkflowExtraction.version.desc()) - else: - query = query.filter(RawWorkflowExtraction.version == version) - row = query.first() - return _serialize_raw_workflow(row, include_graph=True) if row else None - -def delete_raw_workflow_version(project_id: str | uuid.UUID, version: int) -> dict: - """Delete one raw workflow version when no canonical version depends on it.""" - from mkb.db.models import CanonicalWorkflow, RawWorkflowExtraction - - pid = uuid.UUID(str(project_id)) - init_db() - with SyncSessionLocal() as session: - row = ( - session.query(RawWorkflowExtraction) - .filter( - RawWorkflowExtraction.project_id == pid, - RawWorkflowExtraction.version == version, - ) - .first() - ) - if not row: - return {"error": "Raw workflow version not found"} - dependent_canonical = ( - session.query(CanonicalWorkflow) - .filter(CanonicalWorkflow.raw_extraction_id == row.extraction_id) - .order_by(CanonicalWorkflow.version.desc()) - .first() - ) - if dependent_canonical: - return { - "error": ( - f"Raw workflow v{version} cannot be deleted because canonical workflow " - f"v{dependent_canonical.version} still depends on it" - ) - } - - extraction_id = row.extraction_id - session.delete(row) - session.commit() - return { - "status": "deleted", - "project_id": str(pid), - "version": version, - "extraction_id": str(extraction_id), - } - -def delete_canonical_workflow_version(project_id: str | uuid.UUID, version: int) -> dict: - """Delete one canonical workflow version and its derived indexes/tasks.""" - from mkb.db.models import CanonicalWorkflow, WorkflowIndexEntry, WorkflowMaintenanceTask - - pid = uuid.UUID(str(project_id)) - init_db() - with SyncSessionLocal() as session: - row = ( - session.query(CanonicalWorkflow) - .filter( - CanonicalWorkflow.project_id == pid, - CanonicalWorkflow.version == version, - ) - .first() - ) - if not row: - return {"error": "Canonical workflow version not found"} - - canonicalization_id = row.canonicalization_id - session.query(WorkflowIndexEntry).filter( - WorkflowIndexEntry.canonicalization_id == canonicalization_id - ).delete(synchronize_session=False) - session.query(WorkflowMaintenanceTask).filter( - WorkflowMaintenanceTask.project_id == pid, - WorkflowMaintenanceTask.source_canonicalization_id == canonicalization_id, - ).delete(synchronize_session=False) - session.delete(row) - session.commit() - return { - "status": "deleted", - "project_id": str(pid), - "version": version, - "canonicalization_id": str(canonicalization_id), - } - -def _serialize_raw_workflow(row, include_graph: bool) -> dict: - return serialize_raw_workflow(row, include_graph) - -def review_raw_workflow(extraction_id: str | uuid.UUID, *, status: str | None = None, author: str = "system") -> dict: - """Run automatic checks and optionally set a manual lifecycle status.""" - from mkb.db.models import RawWorkflowExtraction - from mkb.workflows.review import VALID_RECORD_STATUSES, audit_raw_graph - - init_db() - eid = uuid.UUID(str(extraction_id)) - if status is not None and status not in VALID_RECORD_STATUSES: - return {"error": f"Invalid record status: {status}"} - with SyncSessionLocal() as session: - row = session.query(RawWorkflowExtraction).filter_by(extraction_id=eid).first() - if not row or not row.graph: - return {"error": "Completed raw workflow not found"} - later = session.query(RawWorkflowExtraction).filter( - RawWorkflowExtraction.project_id == row.project_id, - RawWorkflowExtraction.version > row.version, - RawWorkflowExtraction.status == "COMPLETED", - ).order_by(RawWorkflowExtraction.version.desc()).first() - flags = audit_raw_graph(row.graph, later_graph=later.graph if later else None) - row.review_flags = flags - if status: - row.record_status = status - elif flags and row.record_status == "active": - row.record_status = "needs_review" - row.provenance = {**(row.provenance or {}), "last_reviewed_by": author} - session.commit() - return _serialize_raw_workflow(row, include_graph=False) - -def correct_raw_workflow( - extraction_id: str | uuid.UUID, graph: dict, *, reason: str, author: str, - affected_nodes: list[str] | None = None, affected_edges: list[str] | None = None, - evidence: str, -) -> dict: - """Create a corrected immutable version and supersede the source version.""" - from sqlalchemy import func - from mkb.db.models import RawWorkflowExtraction - from mkb.workflows.review import audit_raw_graph, correction_metadata, rebase_graph - - init_db() - eid = uuid.UUID(str(extraction_id)) - new_id = uuid.uuid4() - details = correction_metadata(reason, author, affected_nodes or [], affected_edges or [], evidence) - with SyncSessionLocal() as session: - source = session.query(RawWorkflowExtraction).filter_by(extraction_id=eid).first() - if not source or source.status != "COMPLETED": - return {"error": "Completed source workflow not found"} - corrected = dict(graph) - corrected["paper_id"] = str(source.project_id) - corrected["schema_version"] = source.schema_version - try: - corrected = rebase_graph(corrected, new_id) - except Exception as exc: - return {"error": f"Corrected graph validation failed: {exc}"} - version = (session.query(func.max(RawWorkflowExtraction.version)).filter_by(project_id=source.project_id).scalar() or 0) + 1 - flags = audit_raw_graph(corrected) - row = RawWorkflowExtraction( - extraction_id=new_id, project_id=source.project_id, version=version, - schema_version=source.schema_version, extractor_version=source.extractor_version, - model=source.model, status="COMPLETED", - record_status="needs_review" if flags else "active", - supersedes_extraction_id=source.extraction_id, graph=corrected, - correction_reason=reason, correction_author=author, - correction_details=details, review_flags=flags, - provenance={**(source.provenance or {}), "correction_evidence": evidence}, - extracted_at=datetime.now(timezone.utc), - ) - source.record_status = "superseded" - session.add(row) - session.commit() - return _serialize_raw_workflow(row, include_graph=True) - -def canonicalize_workflow(project_id: str | uuid.UUID, raw_extraction_id: str | uuid.UUID | None = None, model: str | None = None, verbose: bool = False, progress_callback=None) -> dict: - """Create an append-only canonical view from a valid raw workflow.""" - from mkb.agents.workflow_canonicalization import run_workflow_canonicalization - - init_db() - return run_workflow_canonicalization( - uuid.UUID(str(project_id)), - uuid.UUID(str(raw_extraction_id)) if raw_extraction_id else None, - model=model, verbose=verbose, progress_callback=progress_callback, - ) - -def list_canonical_workflows(project_id: str | uuid.UUID, include_graph: bool = False) -> list[dict]: - from mkb.db.models import CanonicalWorkflow - - pid = uuid.UUID(str(project_id)) - init_db() - with SyncSessionLocal() as session: - rows = session.query(CanonicalWorkflow).filter_by(project_id=pid).order_by(CanonicalWorkflow.version.desc()).all() - return [_serialize_canonical_workflow(row, include_graph) for row in rows] - -def get_canonical_workflow(project_id: str | uuid.UUID, version: int | None = None) -> dict | None: - from mkb.db.models import CanonicalWorkflow - - pid = uuid.UUID(str(project_id)) - init_db() - with SyncSessionLocal() as session: - query = session.query(CanonicalWorkflow).filter(CanonicalWorkflow.project_id == pid) - query = ( - query.filter(CanonicalWorkflow.status == "COMPLETED").order_by(CanonicalWorkflow.version.desc()) - if version is None else query.filter(CanonicalWorkflow.version == version) - ) - row = query.first() - return _serialize_canonical_workflow(row, True) if row else None - -def _serialize_canonical_workflow(row, include_graph: bool) -> dict: - return serialize_canonical_workflow(row, include_graph) - -def curate_workflow_schema(*, min_support: int = 2, author: str = "schema-curator/1.0") -> list[dict]: - """Analyze accumulated workflows and persist new evidence-backed proposals.""" - from mkb.db.models import CanonicalWorkflow, RawWorkflowExtraction, SchemaProposal, SchemaProposalRevision, WorkflowSchemaVersion - from mkb.workflows.curator import analyze_canonical_workflows - from mkb.workflows.schema_library import get_schema_library_payload - - init_db() - with SyncSessionLocal() as session: - current = session.query(WorkflowSchemaVersion).filter_by(status="active").order_by(WorkflowSchemaVersion.version.desc()).first() - if not current: - current = WorkflowSchemaVersion(version=1, name="workflow-schema/1.0", payload=get_schema_library_payload(), created_by="seed") - session.add(current) - session.flush() - rows = session.query(CanonicalWorkflow).filter_by(status="COMPLETED").all() - workflows = [] - for row in rows: - raw = session.query(RawWorkflowExtraction).filter_by(extraction_id=row.raw_extraction_id).first() - workflows.append({"canonicalization_id": str(row.canonicalization_id), "graph": row.graph, "raw_graph": raw.graph if raw else {}}) - generated = analyze_canonical_workflows(workflows, min_support=min_support) - results = [] - for item in generated: - duplicate = session.query(SchemaProposal).filter( - SchemaProposal.status.in_(("pending", "revision_requested")), - SchemaProposal.proposal_type == item["proposal_type"], - SchemaProposal.payload == item["payload"], - ).first() - if duplicate: - continue - rationale = ( - f"Deterministic discovery signal: {item.get('analysis', {}).get('signal', 'unknown')} " - f"with support from {len(item.get('evidence_workflow_ids', []))} workflows." - ) - proposal = SchemaProposal( - **item, rationale=rationale, - base_schema_version=current.name, created_by=author, - ) - session.add(proposal) - session.flush() - session.add(SchemaProposalRevision( - proposal_id=proposal.proposal_id, revision_number=1, - payload=proposal.payload, - evidence_workflow_ids=proposal.evidence_workflow_ids, - analysis=proposal.analysis, rationale=proposal.rationale, - author=author, - author_type="agent" if "agent" in author else "system", - change_note="Initial proposal draft", - validation_errors=[], - )) - results.append({**item, "proposal_id": str(proposal.proposal_id), "status": "pending"}) - session.commit() - return results - -def list_schema_proposals(status: str | None = "pending") -> list[dict]: - from sqlalchemy import func - from mkb.db.models import SchemaProposal, SchemaProposalRevision - - init_db() - with SyncSessionLocal() as session: - query = session.query(SchemaProposal) - if status: - query = query.filter_by(status=status) - rows = query.order_by(SchemaProposal.created_at.desc()).all() - revision_counts = dict( - session.query( - SchemaProposalRevision.proposal_id, - func.count(SchemaProposalRevision.revision_id), - ).group_by(SchemaProposalRevision.proposal_id).all() - ) - return [{ - "proposal_id": str(row.proposal_id), "proposal_type": row.proposal_type, - "status": row.status, "payload": row.payload, - "evidence_workflow_ids": row.evidence_workflow_ids, "analysis": row.analysis, - "base_schema_version": row.base_schema_version, "created_by": row.created_by, - "rationale": row.rationale, - "reviewer_notes": row.reviewer_notes, - "validation_errors": (row.analysis or {}).get("validation_errors", []), - "revision_count": int(revision_counts.get(row.proposal_id, 0)), - "reviewed_by": row.reviewed_by, - "reviewed_at": row.reviewed_at.isoformat() if row.reviewed_at else None, - "created_at": row.created_at.isoformat() if row.created_at else None, - } for row in rows] - -def get_schema_proposal_revisions(proposal_id: str | uuid.UUID) -> list[dict]: - from mkb.db.models import SchemaProposalRevision - - pid = uuid.UUID(str(proposal_id)) - init_db() - with SyncSessionLocal() as session: - rows = session.query(SchemaProposalRevision).filter_by(proposal_id=pid).order_by( - SchemaProposalRevision.revision_number.desc() - ).all() - return [{ - "revision_id": str(row.revision_id), - "revision_number": row.revision_number, - "payload": row.payload, - "evidence_workflow_ids": row.evidence_workflow_ids, - "analysis": row.analysis, - "rationale": row.rationale, - "author": row.author, - "author_type": row.author_type, - "change_note": row.change_note, - "validation_errors": row.validation_errors, - "created_at": row.created_at.isoformat() if row.created_at else None, - } for row in rows] - -def edit_schema_proposal( - proposal_id: str | uuid.UUID, *, payload: dict, - evidence_workflow_ids: list[str], rationale: str, - editor: str, change_note: str, -) -> dict: - """Save an attributed proposal draft revision and revalidate it.""" - from sqlalchemy import func - from mkb.db.models import ( - CanonicalWorkflow, RawWorkflowExtraction, SchemaProposal, SchemaProposalRevision, - WorkflowSchemaVersion, - ) - from mkb.workflows.curator import validate_proposal - - pid = uuid.UUID(str(proposal_id)) - if not editor.strip() or not change_note.strip(): - return {"error": "editor and change_note are required"} - try: - evidence_uuids = [uuid.UUID(value) for value in evidence_workflow_ids] - except (TypeError, ValueError, AttributeError): - return {"error": "evidence_workflow_ids must contain canonicalization UUIDs"} - init_db() - with SyncSessionLocal() as session: - row = session.query(SchemaProposal).filter_by(proposal_id=pid).first() - if not row or row.status not in {"pending", "revision_requested"}: - return {"error": "Only pending or revision-requested proposals can be edited"} - current = session.query(WorkflowSchemaVersion).filter_by(status="active").order_by( - WorkflowSchemaVersion.version.desc() - ).first() - if not current: - return {"error": "Active schema library not found"} - known_evidence = { - str(value) for (value,) in session.query(CanonicalWorkflow.canonicalization_id).filter( - CanonicalWorkflow.canonicalization_id.in_(evidence_uuids) - ).all() - } if evidence_workflow_ids else set() - if evidence_workflow_ids: - known_evidence.update({ - str(value) for (value,) in session.query(RawWorkflowExtraction.extraction_id).filter( - RawWorkflowExtraction.extraction_id.in_(evidence_uuids) - ).all() - }) - errors = validate_proposal( - row.proposal_type, payload, evidence_workflow_ids, current.payload, - ) - missing = sorted(set(evidence_workflow_ids) - known_evidence) - if missing: - errors.append(f"unknown evidence workflows: {', '.join(missing)}") - row.payload = payload - row.evidence_workflow_ids = evidence_workflow_ids - row.rationale = rationale.strip() - row.base_schema_version = current.name - row.analysis = {**(row.analysis or {}), "validation_errors": errors} - row.status = "pending" if not errors else "revision_requested" - revision_number = int( - session.query(func.coalesce(func.max(SchemaProposalRevision.revision_number), 0)) - .filter_by(proposal_id=pid).scalar() - ) + 1 - session.add(SchemaProposalRevision( - proposal_id=pid, revision_number=revision_number, - payload=payload, evidence_workflow_ids=evidence_workflow_ids, - analysis=row.analysis, rationale=row.rationale, - author=editor.strip(), author_type="human", - change_note=change_note.strip(), validation_errors=errors, - )) - session.commit() - return { - "proposal_id": str(pid), "status": row.status, - "revision_number": revision_number, "validation_errors": errors, - } - -def get_workflow_schema_status() -> dict: - """Return global schema and curator queue summary for the frontend.""" - from sqlalchemy import func - from mkb.db.models import SchemaProposal, WorkflowMaintenanceTask, WorkflowSchemaVersion - from mkb.workflows.schema_library import get_schema_library_payload - - init_db() - with SyncSessionLocal() as session: - active = session.query(WorkflowSchemaVersion).filter_by(status="active").order_by( - WorkflowSchemaVersion.version.desc() - ).first() - payload = active.payload if active else get_schema_library_payload() - proposal_counts = dict( - session.query(SchemaProposal.status, func.count(SchemaProposal.proposal_id)) - .group_by(SchemaProposal.status).all() - ) - pending_recanonicalizations = session.query(func.count(WorkflowMaintenanceTask.task_id)).filter( - WorkflowMaintenanceTask.task_type == "recanonicalize", - WorkflowMaintenanceTask.status == "pending", - ).scalar() or 0 - return { - "schema_version": active.name if active else payload["schema_version"], - "version_number": active.version if active else 1, - "status": active.status if active else "seed", - "change_summary": active.change_summary if active else "Built-in seed schema", - "created_by": active.created_by if active else "system", - "created_at": active.created_at.isoformat() if active and active.created_at else None, - "object_schema_count": len(payload.get("object_schemas", {})), - "operation_template_count": len(payload.get("operation_templates", {})), - "card_count": len(payload.get("cards", {})), - "granularity_relation_count": len(payload.get("granularity_relations", [])), - "proposal_counts": proposal_counts, - "pending_recanonicalizations": int(pending_recanonicalizations), - } - -def review_schema_proposal( - proposal_id: str | uuid.UUID, *, approve: bool | None = None, - reviewer: str, decision: str | None = None, notes: str = "", -) -> dict: - """Validate and approve/reject a proposal; approval creates a schema snapshot.""" - from sqlalchemy import func - from mkb.db.models import ( - CanonicalWorkflow, RawWorkflowExtraction, SchemaProposal, SchemaProposalRevision, - WorkflowMaintenanceTask, WorkflowSchemaVersion, - ) - from mkb.workflows.curator import apply_proposal, validate_proposal - from mkb.workflows.maintenance import recanonicalization_reason_for_proposal - - decision = decision or ("approve" if approve else "reject") - if decision not in {"approve", "reject", "request_revision"}: - return {"error": f"Unsupported review decision: {decision}"} - if not reviewer.strip(): - return {"error": "reviewer is required"} - if decision == "request_revision" and not notes.strip(): - return {"error": "Revision requests require reviewer notes"} - init_db() - with SyncSessionLocal() as session: - row = session.query(SchemaProposal).filter_by(proposal_id=uuid.UUID(str(proposal_id))).first() - if not row or row.status not in {"pending", "revision_requested"}: - return {"error": "Reviewable proposal not found"} - current = session.query(WorkflowSchemaVersion).filter_by(status="active").order_by(WorkflowSchemaVersion.version.desc()).first() - if not current: - return {"error": "Schema library is not initialized; run the curator first"} - errors = validate_proposal(row.proposal_type, row.payload, row.evidence_workflow_ids, current.payload) - known_evidence = { - str(value) for (value,) in session.query(CanonicalWorkflow.canonicalization_id).filter( - CanonicalWorkflow.canonicalization_id.in_([ - uuid.UUID(value) for value in row.evidence_workflow_ids - ]) - ).all() - } if row.evidence_workflow_ids else set() - if row.evidence_workflow_ids: - known_evidence.update({ - str(value) for (value,) in session.query(RawWorkflowExtraction.extraction_id).filter( - RawWorkflowExtraction.extraction_id.in_([ - uuid.UUID(value) for value in row.evidence_workflow_ids - ]) - ).all() - }) - missing = sorted(set(row.evidence_workflow_ids) - known_evidence) - if missing: - errors.append(f"unknown evidence workflows: {', '.join(missing)}") - rebased_from = None - if decision == "approve" and row.base_schema_version != current.name: - rebased_from = row.base_schema_version - row.base_schema_version = current.name - row.analysis = { - **(row.analysis or {}), - "rebased_from_schema": rebased_from, - "rebased_to_schema": current.name, - } - if decision == "approve" and errors: - return {"error": "Schema validation failed", "details": errors} - row.reviewed_by = reviewer.strip() - row.reviewer_notes = notes.strip() or None - row.reviewed_at = datetime.now(timezone.utc) - revision_number = int( - session.query(func.coalesce(func.max(SchemaProposalRevision.revision_number), 0)) - .filter_by(proposal_id=row.proposal_id).scalar() - ) + 1 - session.add(SchemaProposalRevision( - proposal_id=row.proposal_id, revision_number=revision_number, - payload=row.payload, evidence_workflow_ids=row.evidence_workflow_ids, - analysis=row.analysis, rationale=row.rationale, - author=reviewer.strip(), author_type="human", - change_note=( - f"Automatically rebased {rebased_from} to {current.name}. " - if rebased_from else "" - ) + f"Review decision: {decision}. {notes.strip()}".strip(), - validation_errors=errors, - )) - if decision in {"reject", "request_revision"}: - row.status = "rejected" if decision == "reject" else "revision_requested" - session.commit() - return { - "proposal_id": str(row.proposal_id), "status": row.status, - "revision_number": revision_number, - } - next_version = current.version + 1 - next_name = f"workflow-schema/1.{next_version - 1}" - base_payload = {**current.payload, "schema_version": next_name} - payload = apply_proposal(base_payload, row.proposal_type, row.payload) - current.status = "superseded" - session.add(WorkflowSchemaVersion( - version=next_version, name=next_name, payload=payload, - change_summary=f"Applied proposal {row.proposal_id}: {row.proposal_type}", - created_by=reviewer, - )) - row.status = "approved" - affected = 0 - queues_created = 0 - queues_updated = 0 - duplicate_queues_removed = 0 - completed = session.query(CanonicalWorkflow).filter_by(status="COMPLETED").order_by( - CanonicalWorkflow.project_id, CanonicalWorkflow.version.desc() - ).all() - latest_by_project = {} - for canonical in completed: - latest_by_project.setdefault(canonical.project_id, canonical) - # Every immutable schema snapshot has a new version. Even when a - # proposal directly cites only a subset, each latest project view is - # queued so its canonical graph can explicitly target that version. - for canonical in latest_by_project.values(): - canonical.provenance = { - **(canonical.provenance or {}), - "recanonicalization_required": True, - "target_schema_version": next_name, - } - pending_tasks = session.query(WorkflowMaintenanceTask).filter_by( - project_id=canonical.project_id, - task_type="recanonicalize", - status="pending", - ).order_by(WorkflowMaintenanceTask.created_at).all() - proposal_ids = [str(row.proposal_id)] - if pending_tasks: - task = pending_tasks[0] - previous_ids = (task.scope or {}).get("schema_proposal_ids", []) - task.scope = { - **(task.scope or {}), - "schema_proposal_ids": list(dict.fromkeys([ - *previous_ids, *proposal_ids, - ])), - } - task.reason = "schema_version_changed" - task.source_raw_extraction_id = canonical.raw_extraction_id - task.source_canonicalization_id = canonical.canonicalization_id - task.target_schema_version = next_name - task.requested_by = reviewer.strip() - for duplicate in pending_tasks[1:]: - session.delete(duplicate) - duplicate_queues_removed += 1 - queues_updated += 1 - else: - session.add(WorkflowMaintenanceTask( - project_id=canonical.project_id, - task_type="recanonicalize", - reason=recanonicalization_reason_for_proposal(row.proposal_type), - source_raw_extraction_id=canonical.raw_extraction_id, - source_canonicalization_id=canonical.canonicalization_id, - target_schema_version=next_name, - requested_by=reviewer.strip(), - scope={"schema_proposal_ids": proposal_ids}, - )) - queues_created += 1 - affected += 1 - session.commit() - return { - "proposal_id": str(row.proposal_id), "status": "approved", - "schema_version": next_name, - "rebased_from_schema": rebased_from, - "recanonicalization_scheduled": affected, - "queues_created": queues_created, - "queues_updated": queues_updated, - "duplicate_queues_removed": duplicate_queues_removed, - } - -def schedule_workflow_reextraction(project_id: str | uuid.UUID, *, reason: str, requested_by: str, scope: dict | None = None, raw_extraction_id: str | uuid.UUID | None = None) -> dict: - """Queue an approved full or partial re-extraction request.""" - from mkb.db.models import RawWorkflowExtraction, WorkflowMaintenanceTask - from mkb.workflows.maintenance import validate_reextraction_request - - pid = uuid.UUID(str(project_id)) - validated_scope = validate_reextraction_request(reason, scope) - init_db() - with SyncSessionLocal() as session: - query = session.query(RawWorkflowExtraction).filter( - RawWorkflowExtraction.project_id == pid, - RawWorkflowExtraction.status == "COMPLETED", - RawWorkflowExtraction.record_status.in_(("active", "needs_review")), - ) - raw = query.filter_by(extraction_id=uuid.UUID(str(raw_extraction_id))).first() if raw_extraction_id else query.order_by(RawWorkflowExtraction.version.desc()).first() - if not raw: - return {"error": "No valid raw workflow is available for re-extraction"} - task = WorkflowMaintenanceTask( - project_id=pid, task_type="reextract", reason=reason, - source_raw_extraction_id=raw.extraction_id, scope=validated_scope, - requested_by=requested_by, - ) - session.add(task) - session.commit() - return {"task_id": str(task.task_id), "status": task.status, "task_type": task.task_type, "scope": task.scope} - -def schedule_workflow_recanonicalization(project_id: str | uuid.UUID, *, reason: str = "manual_request", requested_by: str, raw_extraction_id: str | uuid.UUID | None = None, target_schema_version: str | None = None) -> dict: - from mkb.db.models import RawWorkflowExtraction, WorkflowMaintenanceTask - from mkb.workflows.maintenance import RECANONICALIZATION_REASONS - from mkb.workflows.schema_library import get_schema_library_payload - - if reason not in RECANONICALIZATION_REASONS: - raise ValueError(f"Unsupported recanonicalization reason: {reason}") - pid = uuid.UUID(str(project_id)) - init_db() - with SyncSessionLocal() as session: - query = session.query(RawWorkflowExtraction).filter( - RawWorkflowExtraction.project_id == pid, - RawWorkflowExtraction.status == "COMPLETED", - RawWorkflowExtraction.record_status.in_(("active", "needs_review")), - ) - raw = query.filter_by(extraction_id=uuid.UUID(str(raw_extraction_id))).first() if raw_extraction_id else query.order_by(RawWorkflowExtraction.version.desc()).first() - if not raw: - return {"error": "No valid raw workflow is available for canonicalization"} - task = WorkflowMaintenanceTask( - project_id=pid, task_type="recanonicalize", reason=reason, - source_raw_extraction_id=raw.extraction_id, - target_schema_version=target_schema_version or get_schema_library_payload()["schema_version"], - requested_by=requested_by, - ) - session.add(task) - session.commit() - return {"task_id": str(task.task_id), "status": task.status, "task_type": task.task_type} - -def list_workflow_maintenance_tasks(*, status: str | None = None, project_id: str | uuid.UUID | None = None) -> list[dict]: - from mkb.db.models import WorkflowMaintenanceTask - - init_db() - with SyncSessionLocal() as session: - query = session.query(WorkflowMaintenanceTask) - if status: - query = query.filter_by(status=status) - if project_id: - query = query.filter_by(project_id=uuid.UUID(str(project_id))) - return [{ - "task_id": str(row.task_id), "project_id": str(row.project_id), - "task_type": row.task_type, "reason": row.reason, "scope": row.scope, - "status": row.status, "target_schema_version": row.target_schema_version, - "result": row.result, "error": row.error, - } for row in query.order_by(WorkflowMaintenanceTask.created_at.desc()).all()] - -def run_workflow_maintenance_task(task_id: str | uuid.UUID, *, model: str | None = None, verbose: bool = False, progress_callback=None) -> dict: - """Execute one queued task, retaining both raw and canonical history.""" - from mkb.agents.workflow_canonicalization import run_workflow_canonicalization - from mkb.agents.workflow_extraction import run_workflow_extraction - from mkb.db.models import RawWorkflowExtraction, WorkflowMaintenanceTask - - tid = uuid.UUID(str(task_id)) - init_db() - with SyncSessionLocal() as session: - task = session.query(WorkflowMaintenanceTask).filter_by(task_id=tid).first() - if not task or task.status not in {"pending", "failed"}: - return {"error": "Pending or failed maintenance task not found"} - task.status = "running" - task.started_at = datetime.now(timezone.utc) - project_id, task_type, reason = task.project_id, task.task_type, task.reason - source_raw_id, scope = task.source_raw_extraction_id, task.scope - target_schema_version = task.target_schema_version - raw = session.query(RawWorkflowExtraction).filter_by(extraction_id=source_raw_id).first() - baseline = raw.graph if raw else None - session.commit() - try: - if task_type == "reextract": - extraction = run_workflow_extraction( - project_id, model=model, verbose=verbose, progress_callback=progress_callback, - reextraction_request={ - "reason": reason, "scope": scope, - "source_raw_extraction_id": str(source_raw_id), - "baseline_graph": baseline, - }, - ) - if extraction.get("status") != "completed": - raise RuntimeError(extraction.get("message") or "Re-extraction failed") - canonical = run_workflow_canonicalization( - project_id, uuid.UUID(extraction["extraction_id"]), model=model, - verbose=verbose, progress_callback=progress_callback, - recanonicalization_reason="raw_version_changed", - ) - result = {"extraction": extraction, "canonicalization": canonical} - else: - result = run_workflow_canonicalization( - project_id, source_raw_id, model=model, verbose=verbose, - progress_callback=progress_callback, recanonicalization_reason=reason, - target_schema_version=target_schema_version, - ) - successful = result.get("status") == "completed" or result.get("canonicalization", {}).get("status") == "completed" - if not successful: - raise RuntimeError(result.get("message") or "Workflow maintenance failed") - except Exception as exc: - with SyncSessionLocal() as session: - task = session.query(WorkflowMaintenanceTask).filter_by(task_id=tid).first() - task.status, task.error, task.completed_at = "failed", str(exc), datetime.now(timezone.utc) - session.commit() - return {"task_id": str(tid), "status": "failed", "error": str(exc)} - with SyncSessionLocal() as session: - task = session.query(WorkflowMaintenanceTask).filter_by(task_id=tid).first() - task.status, task.result, task.completed_at = "completed", result, datetime.now(timezone.utc) - session.commit() - return {"task_id": str(tid), "status": "completed", "result": result} - -def run_pending_recanonicalizations( - *, model: str | None = None, verbose: bool = False, progress_callback=None, -) -> dict: - """Run all currently pending recanonicalizations as one global batch job.""" - from mkb.db.models import WorkflowMaintenanceTask - - init_db() - with SyncSessionLocal() as session: - rows = session.query(WorkflowMaintenanceTask).filter_by( - task_type="recanonicalize", status="pending", - ).order_by(WorkflowMaintenanceTask.created_at.desc()).all() - latest_by_project = {} - duplicates = [] - for row in rows: - if row.project_id in latest_by_project: - duplicates.append((row, latest_by_project[row.project_id])) - else: - latest_by_project[row.project_id] = row - for duplicate, retained in duplicates: - duplicate.status = "superseded" - duplicate.result = { - **(duplicate.result or {}), - "superseded_by_task_id": str(retained.task_id), - } - session.commit() - task_ids = [row.task_id for row in latest_by_project.values()] - results = [] - completed = 0 - failed = 0 - for index, task_id in enumerate(task_ids, 1): - if progress_callback: - progress_callback({ - "stage": "recanonicalization_batch", - "message": f"Recanonicalizing project workflow {index}/{len(task_ids)}", - }) - result = run_workflow_maintenance_task( - task_id, model=model, verbose=verbose, - progress_callback=progress_callback, - ) - results.append(result) - if result.get("status") == "completed": - completed += 1 - else: - failed += 1 - return { - "status": "completed" if failed == 0 else "completed_with_errors", - "task_count": len(task_ids), "completed": completed, "failed": failed, - "duplicate_tasks_coalesced": len(duplicates), - "results": results, - } - -def rebuild_workflow_indexes(project_id: str | uuid.UUID | None = None) -> dict: - from mkb.db.models import CanonicalWorkflow, RawWorkflowExtraction, WorkflowIndexEntry - from mkb.workflows.indexing import build_index_entries - from mkb.workflows.schema_library import get_schema_library_payload - - init_db() - with SyncSessionLocal() as session: - query = session.query(CanonicalWorkflow).filter_by(status="COMPLETED") - if project_id: - query = query.filter_by(project_id=uuid.UUID(str(project_id))) - rows = query.all() - ids = [row.canonicalization_id for row in rows] - if ids: - session.query(WorkflowIndexEntry).filter(WorkflowIndexEntry.canonicalization_id.in_(ids)).delete(synchronize_session=False) - count = 0 - for row in rows: - raw = session.query(RawWorkflowExtraction).filter_by(extraction_id=row.raw_extraction_id).first() - schema = get_schema_library_payload(row.schema_version) - for entry in build_index_entries(row.graph or {}, raw.graph if raw else {}, schema): - session.add(WorkflowIndexEntry(canonicalization_id=row.canonicalization_id, project_id=row.project_id, **entry)) - count += 1 - session.commit() - return {"workflows_indexed": len(rows), "entries_created": count} - -def search_canonical_workflows(source: str | None = None, operation: str | None = None, target: str | None = None, mode: str = "strict", limit: int = 100) -> list[dict]: - """Search persisted workflow indexes and return evidence-rich explanations.""" - from mkb.db.models import CanonicalWorkflow, WorkflowIndexEntry - from mkb.workflows.indexing import QUERY_MODES, match_index_entry, normalize - - legacy_modes = {"exact": "strict", "relaxed": "alias-expanded", "expanded": "granularity-expanded", "summarized": "granularity-expanded"} - mode = legacy_modes.get(mode, mode) - if mode not in QUERY_MODES: - raise ValueError(f"Unsupported query mode: {mode}") - init_db() - results = [] - seen_paths = set() - with SyncSessionLocal() as session: - completed = session.query(CanonicalWorkflow).filter_by(status="COMPLETED").order_by(CanonicalWorkflow.project_id, CanonicalWorkflow.version.desc()).all() - latest = {} - for row in completed: - latest.setdefault(row.project_id, row) - rows_by_id = {row.canonicalization_id: row for row in latest.values()} - entry_query = session.query(WorkflowIndexEntry).filter( - WorkflowIndexEntry.canonicalization_id.in_(rows_by_id) - ) if rows_by_id else None - if entry_query is not None and mode in {"strict", "alias-expanded", "evidence-required"}: - entry_query = entry_query.filter(WorkflowIndexEntry.index_type == "direct") - if source: - entry_query = entry_query.filter(WorkflowIndexEntry.source_label == normalize(source)) - if target: - entry_query = entry_query.filter(WorkflowIndexEntry.target_label == normalize(target)) - if operation and mode in {"strict", "evidence-required"}: - entry_query = entry_query.filter(WorkflowIndexEntry.operation_label == normalize(operation)) - entries = entry_query.all() if entry_query is not None else [] - for entry in entries: - data = {column.name: getattr(entry, column.name) for column in WorkflowIndexEntry.__table__.columns} - matched, explanation = match_index_entry(data, source=source, operation=operation, target=target, mode=mode) - if not matched: - continue - result_key = (entry.canonicalization_id, tuple(entry.path_node_ids)) - if result_key in seen_paths: - continue - seen_paths.add(result_key) - canonical = rows_by_id[entry.canonicalization_id] - graph_nodes = {node["node_id"]: node for node in (canonical.graph or {}).get("nodes", [])} - results.append({ - "project_id": str(entry.project_id), - "canonicalization_id": str(entry.canonicalization_id), - "version": canonical.version, "mode": mode, - "path": [graph_nodes[node_id] for node_id in entry.path_node_ids if node_id in graph_nodes], - "explanation": explanation, - }) - if len(results) >= limit: - break - return results - diff --git a/src/mkb/services/workflows/__init__.py b/src/mkb/services/workflows/__init__.py new file mode 100644 index 0000000..8e7666d --- /dev/null +++ b/src/mkb/services/workflows/__init__.py @@ -0,0 +1,73 @@ +"""Workflow lifecycle service package.""" + +from mkb.services.workflows.extraction import ( + _serialize_raw_workflow, + correct_raw_workflow, + delete_raw_workflow_version, + extract_raw_workflow, + get_raw_workflow, + get_raw_workflow_extraction_readiness, + list_raw_workflows, + review_raw_workflow, +) +from mkb.services.workflows.indexing import ( + rebuild_workflow_indexes, + search_canonical_workflows, +) +from mkb.services.workflows.legacy_canonicalization import ( + _serialize_canonical_workflow, + canonicalize_workflow, + delete_canonical_workflow_version, + get_canonical_workflow, + list_canonical_workflows, +) +from mkb.services.workflows.maintenance import ( + list_workflow_maintenance_tasks, + run_pending_recanonicalizations, + run_workflow_maintenance_task, + schedule_workflow_recanonicalization, + schedule_workflow_reextraction, +) +from mkb.services.workflows.schema_review import ( + curate_workflow_schema, + edit_schema_proposal, + get_schema_proposal_revisions, + get_workflow_schema_status, + list_schema_proposals, + review_schema_proposal, +) +from mkb.services.workflows.serialization import ( + serialize_canonical_workflow, + serialize_raw_workflow, +) + +__all__ = [ + "_serialize_canonical_workflow", + "_serialize_raw_workflow", + "canonicalize_workflow", + "correct_raw_workflow", + "curate_workflow_schema", + "delete_canonical_workflow_version", + "delete_raw_workflow_version", + "edit_schema_proposal", + "extract_raw_workflow", + "get_canonical_workflow", + "get_raw_workflow", + "get_raw_workflow_extraction_readiness", + "get_schema_proposal_revisions", + "get_workflow_schema_status", + "list_canonical_workflows", + "list_raw_workflows", + "list_schema_proposals", + "list_workflow_maintenance_tasks", + "rebuild_workflow_indexes", + "review_raw_workflow", + "review_schema_proposal", + "run_pending_recanonicalizations", + "run_workflow_maintenance_task", + "schedule_workflow_recanonicalization", + "schedule_workflow_reextraction", + "search_canonical_workflows", + "serialize_canonical_workflow", + "serialize_raw_workflow", +] diff --git a/src/mkb/services/workflows/extraction.py b/src/mkb/services/workflows/extraction.py new file mode 100644 index 0000000..748d1ce --- /dev/null +++ b/src/mkb/services/workflows/extraction.py @@ -0,0 +1,230 @@ +from __future__ import annotations + +from mkb.services._api_common import ( + SyncSessionLocal, + datetime, + init_db, + timezone, + uuid, +) +from mkb.services.workflows.serialization import serialize_raw_workflow + +def extract_raw_workflow(project_id: str | uuid.UUID, model: str | None = None, verbose: bool = False, progress_callback=None) -> dict: + """Append a new faithful raw-workflow extraction version for a project.""" + from mkb.agents.workflow_extraction import run_workflow_extraction + + readiness = get_raw_workflow_extraction_readiness(project_id) + if not readiness.get("ready"): + return { + "status": "error", + "message": readiness.get("message") or "Project is not ready for workflow extraction", + } + init_db() + return run_workflow_extraction( + uuid.UUID(str(project_id)), model=model, verbose=verbose, + progress_callback=progress_callback, + ) + +def get_raw_workflow_extraction_readiness(project_id: str | uuid.UUID) -> dict: + """Check whether a project has readable sources for raw workflow extraction.""" + from mkb.db.models import ( + ProcessedAsset, + ProcessingType, + ProjectAsset, + RawWorkflowExtraction, + ResearchProject, + ) + + pid = uuid.UUID(str(project_id)) + init_db() + with SyncSessionLocal() as session: + project = session.query(ResearchProject).filter_by(project_id=pid).first() + if not project: + return {"ready": False, "message": f"Project {pid} not found"} + + rows = ( + session.query(ProcessedAsset.asset_id) + .join(ProjectAsset, ProjectAsset.asset_id == ProcessedAsset.asset_id) + .filter( + ProjectAsset.project_id == pid, + ProcessedAsset.processing_type == ProcessingType.MARKDOWN, + ) + .distinct() + .all() + ) + asset_ids = [str(row.asset_id) for row in rows] + if not asset_ids: + return { + "ready": False, + "message": ( + "Workflow extraction requires processed Markdown, but this project has no readable " + "processed Markdown files yet. Run Process first and confirm Markdown outputs exist." + ), + } + + unfinished = ( + session.query(RawWorkflowExtraction) + .filter( + RawWorkflowExtraction.project_id == pid, + RawWorkflowExtraction.graph.is_(None), + RawWorkflowExtraction.status.in_(("IN_PROGRESS", "FAILED")), + ) + .order_by(RawWorkflowExtraction.version.desc()) + .first() + ) + return { + "ready": True, + "project_id": str(pid), + "readable_asset_ids": asset_ids, + "resume_extraction_id": str(unfinished.extraction_id) if unfinished else None, + "resume_version": unfinished.version if unfinished else None, + "has_checkpoint": bool(unfinished and unfinished.checkpoint), + } + +def list_raw_workflows(project_id: str | uuid.UUID, include_graph: bool = False) -> list[dict]: + """List append-only raw workflow versions, newest first.""" + from mkb.db.models import RawWorkflowExtraction + + pid = uuid.UUID(str(project_id)) + init_db() + with SyncSessionLocal() as session: + rows = ( + session.query(RawWorkflowExtraction) + .filter(RawWorkflowExtraction.project_id == pid) + .order_by(RawWorkflowExtraction.version.desc()) + .all() + ) + return [_serialize_raw_workflow(row, include_graph=include_graph) for row in rows] + +def get_raw_workflow(project_id: str | uuid.UUID, version: int | None = None) -> dict | None: + """Get the latest completed raw workflow, or a specific version.""" + from mkb.db.models import RawWorkflowExtraction + + pid = uuid.UUID(str(project_id)) + init_db() + with SyncSessionLocal() as session: + query = session.query(RawWorkflowExtraction).filter(RawWorkflowExtraction.project_id == pid) + if version is None: + query = query.filter( + RawWorkflowExtraction.status == "COMPLETED", + RawWorkflowExtraction.record_status.in_(("active", "needs_review")), + ).order_by(RawWorkflowExtraction.version.desc()) + else: + query = query.filter(RawWorkflowExtraction.version == version) + row = query.first() + return _serialize_raw_workflow(row, include_graph=True) if row else None + +def delete_raw_workflow_version(project_id: str | uuid.UUID, version: int) -> dict: + """Delete one raw workflow version when no canonical version depends on it.""" + from mkb.db.models import CanonicalWorkflow, RawWorkflowExtraction + + pid = uuid.UUID(str(project_id)) + init_db() + with SyncSessionLocal() as session: + row = ( + session.query(RawWorkflowExtraction) + .filter( + RawWorkflowExtraction.project_id == pid, + RawWorkflowExtraction.version == version, + ) + .first() + ) + if not row: + return {"error": "Raw workflow version not found"} + dependent_canonical = ( + session.query(CanonicalWorkflow) + .filter(CanonicalWorkflow.raw_extraction_id == row.extraction_id) + .order_by(CanonicalWorkflow.version.desc()) + .first() + ) + if dependent_canonical: + return { + "error": ( + f"Raw workflow v{version} cannot be deleted because canonical workflow " + f"v{dependent_canonical.version} still depends on it" + ) + } + + extraction_id = row.extraction_id + session.delete(row) + session.commit() + return { + "status": "deleted", + "project_id": str(pid), + "version": version, + "extraction_id": str(extraction_id), + } + +def _serialize_raw_workflow(row, include_graph: bool) -> dict: + return serialize_raw_workflow(row, include_graph) + +def review_raw_workflow(extraction_id: str | uuid.UUID, *, status: str | None = None, author: str = "system") -> dict: + """Run automatic checks and optionally set a manual lifecycle status.""" + from mkb.db.models import RawWorkflowExtraction + from mkb.workflows.review import VALID_RECORD_STATUSES, audit_raw_graph + + init_db() + eid = uuid.UUID(str(extraction_id)) + if status is not None and status not in VALID_RECORD_STATUSES: + return {"error": f"Invalid record status: {status}"} + with SyncSessionLocal() as session: + row = session.query(RawWorkflowExtraction).filter_by(extraction_id=eid).first() + if not row or not row.graph: + return {"error": "Completed raw workflow not found"} + later = session.query(RawWorkflowExtraction).filter( + RawWorkflowExtraction.project_id == row.project_id, + RawWorkflowExtraction.version > row.version, + RawWorkflowExtraction.status == "COMPLETED", + ).order_by(RawWorkflowExtraction.version.desc()).first() + flags = audit_raw_graph(row.graph, later_graph=later.graph if later else None) + row.review_flags = flags + if status: + row.record_status = status + elif flags and row.record_status == "active": + row.record_status = "needs_review" + row.provenance = {**(row.provenance or {}), "last_reviewed_by": author} + session.commit() + return _serialize_raw_workflow(row, include_graph=False) + +def correct_raw_workflow( + extraction_id: str | uuid.UUID, graph: dict, *, reason: str, author: str, + affected_nodes: list[str] | None = None, affected_edges: list[str] | None = None, + evidence: str, +) -> dict: + """Create a corrected immutable version and supersede the source version.""" + from sqlalchemy import func + from mkb.db.models import RawWorkflowExtraction + from mkb.workflows.review import audit_raw_graph, correction_metadata, rebase_graph + + init_db() + eid = uuid.UUID(str(extraction_id)) + new_id = uuid.uuid4() + details = correction_metadata(reason, author, affected_nodes or [], affected_edges or [], evidence) + with SyncSessionLocal() as session: + source = session.query(RawWorkflowExtraction).filter_by(extraction_id=eid).first() + if not source or source.status != "COMPLETED": + return {"error": "Completed source workflow not found"} + corrected = dict(graph) + corrected["paper_id"] = str(source.project_id) + corrected["schema_version"] = source.schema_version + try: + corrected = rebase_graph(corrected, new_id) + except Exception as exc: + return {"error": f"Corrected graph validation failed: {exc}"} + version = (session.query(func.max(RawWorkflowExtraction.version)).filter_by(project_id=source.project_id).scalar() or 0) + 1 + flags = audit_raw_graph(corrected) + row = RawWorkflowExtraction( + extraction_id=new_id, project_id=source.project_id, version=version, + schema_version=source.schema_version, extractor_version=source.extractor_version, + model=source.model, status="COMPLETED", + record_status="needs_review" if flags else "active", + supersedes_extraction_id=source.extraction_id, graph=corrected, + correction_reason=reason, correction_author=author, + correction_details=details, review_flags=flags, + provenance={**(source.provenance or {}), "correction_evidence": evidence}, + extracted_at=datetime.now(timezone.utc), + ) + source.record_status = "superseded" + session.add(row) + session.commit() + return _serialize_raw_workflow(row, include_graph=True) diff --git a/src/mkb/services/workflows/indexing.py b/src/mkb/services/workflows/indexing.py new file mode 100644 index 0000000..b019d87 --- /dev/null +++ b/src/mkb/services/workflows/indexing.py @@ -0,0 +1,83 @@ +from __future__ import annotations + +from mkb.services._api_common import ( + SyncSessionLocal, + init_db, + uuid, +) + +def rebuild_workflow_indexes(project_id: str | uuid.UUID | None = None) -> dict: + from mkb.db.models import CanonicalWorkflow, RawWorkflowExtraction, WorkflowIndexEntry + from mkb.workflows.indexing import build_index_entries + from mkb.workflows.schema_library import get_schema_library_payload + + init_db() + with SyncSessionLocal() as session: + query = session.query(CanonicalWorkflow).filter_by(status="COMPLETED") + if project_id: + query = query.filter_by(project_id=uuid.UUID(str(project_id))) + rows = query.all() + ids = [row.canonicalization_id for row in rows] + if ids: + session.query(WorkflowIndexEntry).filter(WorkflowIndexEntry.canonicalization_id.in_(ids)).delete(synchronize_session=False) + count = 0 + for row in rows: + raw = session.query(RawWorkflowExtraction).filter_by(extraction_id=row.raw_extraction_id).first() + schema = get_schema_library_payload(row.schema_version) + for entry in build_index_entries(row.graph or {}, raw.graph if raw else {}, schema): + session.add(WorkflowIndexEntry(canonicalization_id=row.canonicalization_id, project_id=row.project_id, **entry)) + count += 1 + session.commit() + return {"workflows_indexed": len(rows), "entries_created": count} + +def search_canonical_workflows(source: str | None = None, operation: str | None = None, target: str | None = None, mode: str = "strict", limit: int = 100) -> list[dict]: + """Search persisted workflow indexes and return evidence-rich explanations.""" + from mkb.db.models import CanonicalWorkflow, WorkflowIndexEntry + from mkb.workflows.indexing import QUERY_MODES, match_index_entry, normalize + + legacy_modes = {"exact": "strict", "relaxed": "alias-expanded", "expanded": "granularity-expanded", "summarized": "granularity-expanded"} + mode = legacy_modes.get(mode, mode) + if mode not in QUERY_MODES: + raise ValueError(f"Unsupported query mode: {mode}") + init_db() + results = [] + seen_paths = set() + with SyncSessionLocal() as session: + completed = session.query(CanonicalWorkflow).filter_by(status="COMPLETED").order_by(CanonicalWorkflow.project_id, CanonicalWorkflow.version.desc()).all() + latest = {} + for row in completed: + latest.setdefault(row.project_id, row) + rows_by_id = {row.canonicalization_id: row for row in latest.values()} + entry_query = session.query(WorkflowIndexEntry).filter( + WorkflowIndexEntry.canonicalization_id.in_(rows_by_id) + ) if rows_by_id else None + if entry_query is not None and mode in {"strict", "alias-expanded", "evidence-required"}: + entry_query = entry_query.filter(WorkflowIndexEntry.index_type == "direct") + if source: + entry_query = entry_query.filter(WorkflowIndexEntry.source_label == normalize(source)) + if target: + entry_query = entry_query.filter(WorkflowIndexEntry.target_label == normalize(target)) + if operation and mode in {"strict", "evidence-required"}: + entry_query = entry_query.filter(WorkflowIndexEntry.operation_label == normalize(operation)) + entries = entry_query.all() if entry_query is not None else [] + for entry in entries: + data = {column.name: getattr(entry, column.name) for column in WorkflowIndexEntry.__table__.columns} + matched, explanation = match_index_entry(data, source=source, operation=operation, target=target, mode=mode) + if not matched: + continue + result_key = (entry.canonicalization_id, tuple(entry.path_node_ids)) + if result_key in seen_paths: + continue + seen_paths.add(result_key) + canonical = rows_by_id[entry.canonicalization_id] + graph_nodes = {node["node_id"]: node for node in (canonical.graph or {}).get("nodes", [])} + results.append({ + "project_id": str(entry.project_id), + "canonicalization_id": str(entry.canonicalization_id), + "version": canonical.version, "mode": mode, + "path": [graph_nodes[node_id] for node_id in entry.path_node_ids if node_id in graph_nodes], + "explanation": explanation, + }) + if len(results) >= limit: + break + return results diff --git a/src/mkb/services/workflows/legacy_canonicalization.py b/src/mkb/services/workflows/legacy_canonicalization.py new file mode 100644 index 0000000..30744a0 --- /dev/null +++ b/src/mkb/services/workflows/legacy_canonicalization.py @@ -0,0 +1,82 @@ +"""Compatibility service functions for retired canonical workflow records.""" + +from __future__ import annotations + +from mkb.services._api_common import ( + SyncSessionLocal, + init_db, + uuid, +) +from mkb.services.workflows.serialization import serialize_canonical_workflow + +def delete_canonical_workflow_version(project_id: str | uuid.UUID, version: int) -> dict: + """Delete one canonical workflow version and its derived indexes/tasks.""" + from mkb.db.models import CanonicalWorkflow, WorkflowIndexEntry, WorkflowMaintenanceTask + + pid = uuid.UUID(str(project_id)) + init_db() + with SyncSessionLocal() as session: + row = ( + session.query(CanonicalWorkflow) + .filter( + CanonicalWorkflow.project_id == pid, + CanonicalWorkflow.version == version, + ) + .first() + ) + if not row: + return {"error": "Canonical workflow version not found"} + + canonicalization_id = row.canonicalization_id + session.query(WorkflowIndexEntry).filter( + WorkflowIndexEntry.canonicalization_id == canonicalization_id + ).delete(synchronize_session=False) + session.query(WorkflowMaintenanceTask).filter( + WorkflowMaintenanceTask.project_id == pid, + WorkflowMaintenanceTask.source_canonicalization_id == canonicalization_id, + ).delete(synchronize_session=False) + session.delete(row) + session.commit() + return { + "status": "deleted", + "project_id": str(pid), + "version": version, + "canonicalization_id": str(canonicalization_id), + } + +def canonicalize_workflow(project_id: str | uuid.UUID, raw_extraction_id: str | uuid.UUID | None = None, model: str | None = None, verbose: bool = False, progress_callback=None) -> dict: + """Create an append-only canonical view from a valid raw workflow.""" + from mkb.agents.workflow_canonicalization import run_workflow_canonicalization + + init_db() + return run_workflow_canonicalization( + uuid.UUID(str(project_id)), + uuid.UUID(str(raw_extraction_id)) if raw_extraction_id else None, + model=model, verbose=verbose, progress_callback=progress_callback, + ) + +def list_canonical_workflows(project_id: str | uuid.UUID, include_graph: bool = False) -> list[dict]: + from mkb.db.models import CanonicalWorkflow + + pid = uuid.UUID(str(project_id)) + init_db() + with SyncSessionLocal() as session: + rows = session.query(CanonicalWorkflow).filter_by(project_id=pid).order_by(CanonicalWorkflow.version.desc()).all() + return [_serialize_canonical_workflow(row, include_graph) for row in rows] + +def get_canonical_workflow(project_id: str | uuid.UUID, version: int | None = None) -> dict | None: + from mkb.db.models import CanonicalWorkflow + + pid = uuid.UUID(str(project_id)) + init_db() + with SyncSessionLocal() as session: + query = session.query(CanonicalWorkflow).filter(CanonicalWorkflow.project_id == pid) + query = ( + query.filter(CanonicalWorkflow.status == "COMPLETED").order_by(CanonicalWorkflow.version.desc()) + if version is None else query.filter(CanonicalWorkflow.version == version) + ) + row = query.first() + return _serialize_canonical_workflow(row, True) if row else None + +def _serialize_canonical_workflow(row, include_graph: bool) -> dict: + return serialize_canonical_workflow(row, include_graph) diff --git a/src/mkb/services/workflows/maintenance.py b/src/mkb/services/workflows/maintenance.py new file mode 100644 index 0000000..fa0b2bb --- /dev/null +++ b/src/mkb/services/workflows/maintenance.py @@ -0,0 +1,190 @@ +from __future__ import annotations + +from mkb.services._api_common import ( + SyncSessionLocal, + datetime, + init_db, + timezone, + uuid, +) + +def schedule_workflow_reextraction(project_id: str | uuid.UUID, *, reason: str, requested_by: str, scope: dict | None = None, raw_extraction_id: str | uuid.UUID | None = None) -> dict: + """Queue an approved full or partial re-extraction request.""" + from mkb.db.models import RawWorkflowExtraction, WorkflowMaintenanceTask + from mkb.workflows.maintenance import validate_reextraction_request + + pid = uuid.UUID(str(project_id)) + validated_scope = validate_reextraction_request(reason, scope) + init_db() + with SyncSessionLocal() as session: + query = session.query(RawWorkflowExtraction).filter( + RawWorkflowExtraction.project_id == pid, + RawWorkflowExtraction.status == "COMPLETED", + RawWorkflowExtraction.record_status.in_(("active", "needs_review")), + ) + raw = query.filter_by(extraction_id=uuid.UUID(str(raw_extraction_id))).first() if raw_extraction_id else query.order_by(RawWorkflowExtraction.version.desc()).first() + if not raw: + return {"error": "No valid raw workflow is available for re-extraction"} + task = WorkflowMaintenanceTask( + project_id=pid, task_type="reextract", reason=reason, + source_raw_extraction_id=raw.extraction_id, scope=validated_scope, + requested_by=requested_by, + ) + session.add(task) + session.commit() + return {"task_id": str(task.task_id), "status": task.status, "task_type": task.task_type, "scope": task.scope} + +def schedule_workflow_recanonicalization(project_id: str | uuid.UUID, *, reason: str = "manual_request", requested_by: str, raw_extraction_id: str | uuid.UUID | None = None, target_schema_version: str | None = None) -> dict: + from mkb.db.models import RawWorkflowExtraction, WorkflowMaintenanceTask + from mkb.workflows.maintenance import RECANONICALIZATION_REASONS + from mkb.workflows.schema_library import get_schema_library_payload + + if reason not in RECANONICALIZATION_REASONS: + raise ValueError(f"Unsupported recanonicalization reason: {reason}") + pid = uuid.UUID(str(project_id)) + init_db() + with SyncSessionLocal() as session: + query = session.query(RawWorkflowExtraction).filter( + RawWorkflowExtraction.project_id == pid, + RawWorkflowExtraction.status == "COMPLETED", + RawWorkflowExtraction.record_status.in_(("active", "needs_review")), + ) + raw = query.filter_by(extraction_id=uuid.UUID(str(raw_extraction_id))).first() if raw_extraction_id else query.order_by(RawWorkflowExtraction.version.desc()).first() + if not raw: + return {"error": "No valid raw workflow is available for canonicalization"} + task = WorkflowMaintenanceTask( + project_id=pid, task_type="recanonicalize", reason=reason, + source_raw_extraction_id=raw.extraction_id, + target_schema_version=target_schema_version or get_schema_library_payload()["schema_version"], + requested_by=requested_by, + ) + session.add(task) + session.commit() + return {"task_id": str(task.task_id), "status": task.status, "task_type": task.task_type} + +def list_workflow_maintenance_tasks(*, status: str | None = None, project_id: str | uuid.UUID | None = None) -> list[dict]: + from mkb.db.models import WorkflowMaintenanceTask + + init_db() + with SyncSessionLocal() as session: + query = session.query(WorkflowMaintenanceTask) + if status: + query = query.filter_by(status=status) + if project_id: + query = query.filter_by(project_id=uuid.UUID(str(project_id))) + return [{ + "task_id": str(row.task_id), "project_id": str(row.project_id), + "task_type": row.task_type, "reason": row.reason, "scope": row.scope, + "status": row.status, "target_schema_version": row.target_schema_version, + "result": row.result, "error": row.error, + } for row in query.order_by(WorkflowMaintenanceTask.created_at.desc()).all()] + +def run_workflow_maintenance_task(task_id: str | uuid.UUID, *, model: str | None = None, verbose: bool = False, progress_callback=None) -> dict: + """Execute one queued task, retaining both raw and canonical history.""" + from mkb.agents.workflow_canonicalization import run_workflow_canonicalization + from mkb.agents.workflow_extraction import run_workflow_extraction + from mkb.db.models import RawWorkflowExtraction, WorkflowMaintenanceTask + + tid = uuid.UUID(str(task_id)) + init_db() + with SyncSessionLocal() as session: + task = session.query(WorkflowMaintenanceTask).filter_by(task_id=tid).first() + if not task or task.status not in {"pending", "failed"}: + return {"error": "Pending or failed maintenance task not found"} + task.status = "running" + task.started_at = datetime.now(timezone.utc) + project_id, task_type, reason = task.project_id, task.task_type, task.reason + source_raw_id, scope = task.source_raw_extraction_id, task.scope + target_schema_version = task.target_schema_version + raw = session.query(RawWorkflowExtraction).filter_by(extraction_id=source_raw_id).first() + baseline = raw.graph if raw else None + session.commit() + try: + if task_type == "reextract": + extraction = run_workflow_extraction( + project_id, model=model, verbose=verbose, progress_callback=progress_callback, + reextraction_request={ + "reason": reason, "scope": scope, + "source_raw_extraction_id": str(source_raw_id), + "baseline_graph": baseline, + }, + ) + if extraction.get("status") != "completed": + raise RuntimeError(extraction.get("message") or "Re-extraction failed") + canonical = run_workflow_canonicalization( + project_id, uuid.UUID(extraction["extraction_id"]), model=model, + verbose=verbose, progress_callback=progress_callback, + recanonicalization_reason="raw_version_changed", + ) + result = {"extraction": extraction, "canonicalization": canonical} + else: + result = run_workflow_canonicalization( + project_id, source_raw_id, model=model, verbose=verbose, + progress_callback=progress_callback, recanonicalization_reason=reason, + target_schema_version=target_schema_version, + ) + successful = result.get("status") == "completed" or result.get("canonicalization", {}).get("status") == "completed" + if not successful: + raise RuntimeError(result.get("message") or "Workflow maintenance failed") + except Exception as exc: + with SyncSessionLocal() as session: + task = session.query(WorkflowMaintenanceTask).filter_by(task_id=tid).first() + task.status, task.error, task.completed_at = "failed", str(exc), datetime.now(timezone.utc) + session.commit() + return {"task_id": str(tid), "status": "failed", "error": str(exc)} + with SyncSessionLocal() as session: + task = session.query(WorkflowMaintenanceTask).filter_by(task_id=tid).first() + task.status, task.result, task.completed_at = "completed", result, datetime.now(timezone.utc) + session.commit() + return {"task_id": str(tid), "status": "completed", "result": result} + +def run_pending_recanonicalizations( + *, model: str | None = None, verbose: bool = False, progress_callback=None, +) -> dict: + """Run all currently pending recanonicalizations as one global batch job.""" + from mkb.db.models import WorkflowMaintenanceTask + + init_db() + with SyncSessionLocal() as session: + rows = session.query(WorkflowMaintenanceTask).filter_by( + task_type="recanonicalize", status="pending", + ).order_by(WorkflowMaintenanceTask.created_at.desc()).all() + latest_by_project = {} + duplicates = [] + for row in rows: + if row.project_id in latest_by_project: + duplicates.append((row, latest_by_project[row.project_id])) + else: + latest_by_project[row.project_id] = row + for duplicate, retained in duplicates: + duplicate.status = "superseded" + duplicate.result = { + **(duplicate.result or {}), + "superseded_by_task_id": str(retained.task_id), + } + session.commit() + task_ids = [row.task_id for row in latest_by_project.values()] + results = [] + completed = 0 + failed = 0 + for index, task_id in enumerate(task_ids, 1): + if progress_callback: + progress_callback({ + "stage": "recanonicalization_batch", + "message": f"Recanonicalizing project workflow {index}/{len(task_ids)}", + }) + result = run_workflow_maintenance_task( + task_id, model=model, verbose=verbose, + progress_callback=progress_callback, + ) + results.append(result) + if result.get("status") == "completed": + completed += 1 + else: + failed += 1 + return { + "status": "completed" if failed == 0 else "completed_with_errors", + "task_count": len(task_ids), "completed": completed, "failed": failed, + "duplicate_tasks_coalesced": len(duplicates), + "results": results, + } diff --git a/src/mkb/services/workflows/schema_review.py b/src/mkb/services/workflows/schema_review.py new file mode 100644 index 0000000..2f79f20 --- /dev/null +++ b/src/mkb/services/workflows/schema_review.py @@ -0,0 +1,380 @@ +from __future__ import annotations + +from mkb.services._api_common import ( + SyncSessionLocal, + datetime, + init_db, + timezone, + uuid, +) + +def curate_workflow_schema(*, min_support: int = 2, author: str = "schema-curator/1.0") -> list[dict]: + """Analyze accumulated workflows and persist new evidence-backed proposals.""" + from mkb.db.models import CanonicalWorkflow, RawWorkflowExtraction, SchemaProposal, SchemaProposalRevision, WorkflowSchemaVersion + from mkb.workflows.curator import analyze_canonical_workflows + from mkb.workflows.schema_library import get_schema_library_payload + + init_db() + with SyncSessionLocal() as session: + current = session.query(WorkflowSchemaVersion).filter_by(status="active").order_by(WorkflowSchemaVersion.version.desc()).first() + if not current: + current = WorkflowSchemaVersion(version=1, name="workflow-schema/1.0", payload=get_schema_library_payload(), created_by="seed") + session.add(current) + session.flush() + rows = session.query(CanonicalWorkflow).filter_by(status="COMPLETED").all() + workflows = [] + for row in rows: + raw = session.query(RawWorkflowExtraction).filter_by(extraction_id=row.raw_extraction_id).first() + workflows.append({"canonicalization_id": str(row.canonicalization_id), "graph": row.graph, "raw_graph": raw.graph if raw else {}}) + generated = analyze_canonical_workflows(workflows, min_support=min_support) + results = [] + for item in generated: + duplicate = session.query(SchemaProposal).filter( + SchemaProposal.status.in_(("pending", "revision_requested")), + SchemaProposal.proposal_type == item["proposal_type"], + SchemaProposal.payload == item["payload"], + ).first() + if duplicate: + continue + rationale = ( + f"Deterministic discovery signal: {item.get('analysis', {}).get('signal', 'unknown')} " + f"with support from {len(item.get('evidence_workflow_ids', []))} workflows." + ) + proposal = SchemaProposal( + **item, rationale=rationale, + base_schema_version=current.name, created_by=author, + ) + session.add(proposal) + session.flush() + session.add(SchemaProposalRevision( + proposal_id=proposal.proposal_id, revision_number=1, + payload=proposal.payload, + evidence_workflow_ids=proposal.evidence_workflow_ids, + analysis=proposal.analysis, rationale=proposal.rationale, + author=author, + author_type="agent" if "agent" in author else "system", + change_note="Initial proposal draft", + validation_errors=[], + )) + results.append({**item, "proposal_id": str(proposal.proposal_id), "status": "pending"}) + session.commit() + return results + +def list_schema_proposals(status: str | None = "pending") -> list[dict]: + from sqlalchemy import func + from mkb.db.models import SchemaProposal, SchemaProposalRevision + + init_db() + with SyncSessionLocal() as session: + query = session.query(SchemaProposal) + if status: + query = query.filter_by(status=status) + rows = query.order_by(SchemaProposal.created_at.desc()).all() + revision_counts = dict( + session.query( + SchemaProposalRevision.proposal_id, + func.count(SchemaProposalRevision.revision_id), + ).group_by(SchemaProposalRevision.proposal_id).all() + ) + return [{ + "proposal_id": str(row.proposal_id), "proposal_type": row.proposal_type, + "status": row.status, "payload": row.payload, + "evidence_workflow_ids": row.evidence_workflow_ids, "analysis": row.analysis, + "base_schema_version": row.base_schema_version, "created_by": row.created_by, + "rationale": row.rationale, + "reviewer_notes": row.reviewer_notes, + "validation_errors": (row.analysis or {}).get("validation_errors", []), + "revision_count": int(revision_counts.get(row.proposal_id, 0)), + "reviewed_by": row.reviewed_by, + "reviewed_at": row.reviewed_at.isoformat() if row.reviewed_at else None, + "created_at": row.created_at.isoformat() if row.created_at else None, + } for row in rows] + +def get_schema_proposal_revisions(proposal_id: str | uuid.UUID) -> list[dict]: + from mkb.db.models import SchemaProposalRevision + + pid = uuid.UUID(str(proposal_id)) + init_db() + with SyncSessionLocal() as session: + rows = session.query(SchemaProposalRevision).filter_by(proposal_id=pid).order_by( + SchemaProposalRevision.revision_number.desc() + ).all() + return [{ + "revision_id": str(row.revision_id), + "revision_number": row.revision_number, + "payload": row.payload, + "evidence_workflow_ids": row.evidence_workflow_ids, + "analysis": row.analysis, + "rationale": row.rationale, + "author": row.author, + "author_type": row.author_type, + "change_note": row.change_note, + "validation_errors": row.validation_errors, + "created_at": row.created_at.isoformat() if row.created_at else None, + } for row in rows] + +def edit_schema_proposal( + proposal_id: str | uuid.UUID, *, payload: dict, + evidence_workflow_ids: list[str], rationale: str, + editor: str, change_note: str, +) -> dict: + """Save an attributed proposal draft revision and revalidate it.""" + from sqlalchemy import func + from mkb.db.models import ( + CanonicalWorkflow, RawWorkflowExtraction, SchemaProposal, SchemaProposalRevision, + WorkflowSchemaVersion, + ) + from mkb.workflows.curator import validate_proposal + + pid = uuid.UUID(str(proposal_id)) + if not editor.strip() or not change_note.strip(): + return {"error": "editor and change_note are required"} + try: + evidence_uuids = [uuid.UUID(value) for value in evidence_workflow_ids] + except (TypeError, ValueError, AttributeError): + return {"error": "evidence_workflow_ids must contain canonicalization UUIDs"} + init_db() + with SyncSessionLocal() as session: + row = session.query(SchemaProposal).filter_by(proposal_id=pid).first() + if not row or row.status not in {"pending", "revision_requested"}: + return {"error": "Only pending or revision-requested proposals can be edited"} + current = session.query(WorkflowSchemaVersion).filter_by(status="active").order_by( + WorkflowSchemaVersion.version.desc() + ).first() + if not current: + return {"error": "Active schema library not found"} + known_evidence = { + str(value) for (value,) in session.query(CanonicalWorkflow.canonicalization_id).filter( + CanonicalWorkflow.canonicalization_id.in_(evidence_uuids) + ).all() + } if evidence_workflow_ids else set() + if evidence_workflow_ids: + known_evidence.update({ + str(value) for (value,) in session.query(RawWorkflowExtraction.extraction_id).filter( + RawWorkflowExtraction.extraction_id.in_(evidence_uuids) + ).all() + }) + errors = validate_proposal( + row.proposal_type, payload, evidence_workflow_ids, current.payload, + ) + missing = sorted(set(evidence_workflow_ids) - known_evidence) + if missing: + errors.append(f"unknown evidence workflows: {', '.join(missing)}") + row.payload = payload + row.evidence_workflow_ids = evidence_workflow_ids + row.rationale = rationale.strip() + row.base_schema_version = current.name + row.analysis = {**(row.analysis or {}), "validation_errors": errors} + row.status = "pending" if not errors else "revision_requested" + revision_number = int( + session.query(func.coalesce(func.max(SchemaProposalRevision.revision_number), 0)) + .filter_by(proposal_id=pid).scalar() + ) + 1 + session.add(SchemaProposalRevision( + proposal_id=pid, revision_number=revision_number, + payload=payload, evidence_workflow_ids=evidence_workflow_ids, + analysis=row.analysis, rationale=row.rationale, + author=editor.strip(), author_type="human", + change_note=change_note.strip(), validation_errors=errors, + )) + session.commit() + return { + "proposal_id": str(pid), "status": row.status, + "revision_number": revision_number, "validation_errors": errors, + } + +def get_workflow_schema_status() -> dict: + """Return global schema and curator queue summary for the frontend.""" + from sqlalchemy import func + from mkb.db.models import SchemaProposal, WorkflowMaintenanceTask, WorkflowSchemaVersion + from mkb.workflows.schema_library import get_schema_library_payload + + init_db() + with SyncSessionLocal() as session: + active = session.query(WorkflowSchemaVersion).filter_by(status="active").order_by( + WorkflowSchemaVersion.version.desc() + ).first() + payload = active.payload if active else get_schema_library_payload() + proposal_counts = dict( + session.query(SchemaProposal.status, func.count(SchemaProposal.proposal_id)) + .group_by(SchemaProposal.status).all() + ) + pending_recanonicalizations = session.query(func.count(WorkflowMaintenanceTask.task_id)).filter( + WorkflowMaintenanceTask.task_type == "recanonicalize", + WorkflowMaintenanceTask.status == "pending", + ).scalar() or 0 + return { + "schema_version": active.name if active else payload["schema_version"], + "version_number": active.version if active else 1, + "status": active.status if active else "seed", + "change_summary": active.change_summary if active else "Built-in seed schema", + "created_by": active.created_by if active else "system", + "created_at": active.created_at.isoformat() if active and active.created_at else None, + "object_schema_count": len(payload.get("object_schemas", {})), + "operation_template_count": len(payload.get("operation_templates", {})), + "card_count": len(payload.get("cards", {})), + "granularity_relation_count": len(payload.get("granularity_relations", [])), + "proposal_counts": proposal_counts, + "pending_recanonicalizations": int(pending_recanonicalizations), + } + +def review_schema_proposal( + proposal_id: str | uuid.UUID, *, approve: bool | None = None, + reviewer: str, decision: str | None = None, notes: str = "", +) -> dict: + """Validate and approve/reject a proposal; approval creates a schema snapshot.""" + from sqlalchemy import func + from mkb.db.models import ( + CanonicalWorkflow, RawWorkflowExtraction, SchemaProposal, SchemaProposalRevision, + WorkflowMaintenanceTask, WorkflowSchemaVersion, + ) + from mkb.workflows.curator import apply_proposal, validate_proposal + from mkb.workflows.maintenance import recanonicalization_reason_for_proposal + + decision = decision or ("approve" if approve else "reject") + if decision not in {"approve", "reject", "request_revision"}: + return {"error": f"Unsupported review decision: {decision}"} + if not reviewer.strip(): + return {"error": "reviewer is required"} + if decision == "request_revision" and not notes.strip(): + return {"error": "Revision requests require reviewer notes"} + init_db() + with SyncSessionLocal() as session: + row = session.query(SchemaProposal).filter_by(proposal_id=uuid.UUID(str(proposal_id))).first() + if not row or row.status not in {"pending", "revision_requested"}: + return {"error": "Reviewable proposal not found"} + current = session.query(WorkflowSchemaVersion).filter_by(status="active").order_by(WorkflowSchemaVersion.version.desc()).first() + if not current: + return {"error": "Schema library is not initialized; run the curator first"} + errors = validate_proposal(row.proposal_type, row.payload, row.evidence_workflow_ids, current.payload) + known_evidence = { + str(value) for (value,) in session.query(CanonicalWorkflow.canonicalization_id).filter( + CanonicalWorkflow.canonicalization_id.in_([ + uuid.UUID(value) for value in row.evidence_workflow_ids + ]) + ).all() + } if row.evidence_workflow_ids else set() + if row.evidence_workflow_ids: + known_evidence.update({ + str(value) for (value,) in session.query(RawWorkflowExtraction.extraction_id).filter( + RawWorkflowExtraction.extraction_id.in_([ + uuid.UUID(value) for value in row.evidence_workflow_ids + ]) + ).all() + }) + missing = sorted(set(row.evidence_workflow_ids) - known_evidence) + if missing: + errors.append(f"unknown evidence workflows: {', '.join(missing)}") + rebased_from = None + if decision == "approve" and row.base_schema_version != current.name: + rebased_from = row.base_schema_version + row.base_schema_version = current.name + row.analysis = { + **(row.analysis or {}), + "rebased_from_schema": rebased_from, + "rebased_to_schema": current.name, + } + if decision == "approve" and errors: + return {"error": "Schema validation failed", "details": errors} + row.reviewed_by = reviewer.strip() + row.reviewer_notes = notes.strip() or None + row.reviewed_at = datetime.now(timezone.utc) + revision_number = int( + session.query(func.coalesce(func.max(SchemaProposalRevision.revision_number), 0)) + .filter_by(proposal_id=row.proposal_id).scalar() + ) + 1 + session.add(SchemaProposalRevision( + proposal_id=row.proposal_id, revision_number=revision_number, + payload=row.payload, evidence_workflow_ids=row.evidence_workflow_ids, + analysis=row.analysis, rationale=row.rationale, + author=reviewer.strip(), author_type="human", + change_note=( + f"Automatically rebased {rebased_from} to {current.name}. " + if rebased_from else "" + ) + f"Review decision: {decision}. {notes.strip()}".strip(), + validation_errors=errors, + )) + if decision in {"reject", "request_revision"}: + row.status = "rejected" if decision == "reject" else "revision_requested" + session.commit() + return { + "proposal_id": str(row.proposal_id), "status": row.status, + "revision_number": revision_number, + } + next_version = current.version + 1 + next_name = f"workflow-schema/1.{next_version - 1}" + base_payload = {**current.payload, "schema_version": next_name} + payload = apply_proposal(base_payload, row.proposal_type, row.payload) + current.status = "superseded" + session.add(WorkflowSchemaVersion( + version=next_version, name=next_name, payload=payload, + change_summary=f"Applied proposal {row.proposal_id}: {row.proposal_type}", + created_by=reviewer, + )) + row.status = "approved" + affected = 0 + queues_created = 0 + queues_updated = 0 + duplicate_queues_removed = 0 + completed = session.query(CanonicalWorkflow).filter_by(status="COMPLETED").order_by( + CanonicalWorkflow.project_id, CanonicalWorkflow.version.desc() + ).all() + latest_by_project = {} + for canonical in completed: + latest_by_project.setdefault(canonical.project_id, canonical) + # Every immutable schema snapshot has a new version. Even when a + # proposal directly cites only a subset, each latest project view is + # queued so its canonical graph can explicitly target that version. + for canonical in latest_by_project.values(): + canonical.provenance = { + **(canonical.provenance or {}), + "recanonicalization_required": True, + "target_schema_version": next_name, + } + pending_tasks = session.query(WorkflowMaintenanceTask).filter_by( + project_id=canonical.project_id, + task_type="recanonicalize", + status="pending", + ).order_by(WorkflowMaintenanceTask.created_at).all() + proposal_ids = [str(row.proposal_id)] + if pending_tasks: + task = pending_tasks[0] + previous_ids = (task.scope or {}).get("schema_proposal_ids", []) + task.scope = { + **(task.scope or {}), + "schema_proposal_ids": list(dict.fromkeys([ + *previous_ids, *proposal_ids, + ])), + } + task.reason = "schema_version_changed" + task.source_raw_extraction_id = canonical.raw_extraction_id + task.source_canonicalization_id = canonical.canonicalization_id + task.target_schema_version = next_name + task.requested_by = reviewer.strip() + for duplicate in pending_tasks[1:]: + session.delete(duplicate) + duplicate_queues_removed += 1 + queues_updated += 1 + else: + session.add(WorkflowMaintenanceTask( + project_id=canonical.project_id, + task_type="recanonicalize", + reason=recanonicalization_reason_for_proposal(row.proposal_type), + source_raw_extraction_id=canonical.raw_extraction_id, + source_canonicalization_id=canonical.canonicalization_id, + target_schema_version=next_name, + requested_by=reviewer.strip(), + scope={"schema_proposal_ids": proposal_ids}, + )) + queues_created += 1 + affected += 1 + session.commit() + return { + "proposal_id": str(row.proposal_id), "status": "approved", + "schema_version": next_name, + "rebased_from_schema": rebased_from, + "recanonicalization_scheduled": affected, + "queues_created": queues_created, + "queues_updated": queues_updated, + "duplicate_queues_removed": duplicate_queues_removed, + } diff --git a/src/mkb/services/workflows/serialization.py b/src/mkb/services/workflows/serialization.py new file mode 100644 index 0000000..cec5b3e --- /dev/null +++ b/src/mkb/services/workflows/serialization.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +from mkb.workflows.review import audit_raw_graph + + +def _review_flags(row) -> list[dict]: + flags = list(row.review_flags or []) + if row.graph: + seen = {repr(flag) for flag in flags} + for flag in audit_raw_graph(row.graph): + key = repr(flag) + if key not in seen: + flags.append(flag) + seen.add(key) + return flags + + +def serialize_raw_workflow(row, include_graph: bool) -> dict: + payload = { + "extraction_id": str(row.extraction_id), + "project_id": str(row.project_id), + "version": row.version, + "schema_version": row.schema_version, + "extractor_version": row.extractor_version, + "model": row.model, + "status": row.status, + "record_status": row.record_status, + "supersedes_extraction_id": ( + str(row.supersedes_extraction_id) if row.supersedes_extraction_id else None + ), + "correction_reason": row.correction_reason, + "correction_author": row.correction_author, + "correction_details": row.correction_details or {}, + "review_flags": _review_flags(row), + "provenance": row.provenance or {}, + "error": row.error, + "extracted_at": row.extracted_at.isoformat() if row.extracted_at else None, + "created_at": row.created_at.isoformat() if row.created_at else None, + "has_checkpoint": bool(row.checkpoint), + "checkpoint_summary": (row.checkpoint or {}).get("summary"), + "checkpoint_updated_at": ( + row.checkpoint_updated_at.isoformat() if row.checkpoint_updated_at else None + ), + "resumable": row.graph is None and row.status in {"IN_PROGRESS", "FAILED"}, + } + if include_graph: + payload["graph"] = row.graph + elif row.graph: + payload["node_count"] = len(row.graph.get("nodes", [])) + payload["edge_count"] = len(row.graph.get("edges", [])) + return payload + +def serialize_canonical_workflow(row, include_graph: bool) -> dict: + payload = { + "canonicalization_id": str(row.canonicalization_id), + "project_id": str(row.project_id), + "raw_extraction_id": str(row.raw_extraction_id), + "version": row.version, + "schema_version": row.schema_version, + "canonicalizer_version": row.canonicalizer_version, + "model": row.model, + "status": row.status, + "provenance": row.provenance or {}, + "error": row.error, + "canonicalized_at": row.canonicalized_at.isoformat() if row.canonicalized_at else None, + "created_at": row.created_at.isoformat() if row.created_at else None, + "has_checkpoint": bool(row.checkpoint), + "checkpoint_summary": (row.checkpoint or {}).get("summary"), + "checkpoint_updated_at": ( + row.checkpoint_updated_at.isoformat() if row.checkpoint_updated_at else None + ), + "resumable": row.graph is None and row.status in {"IN_PROGRESS", "FAILED"}, + } + if include_graph: + payload["graph"] = row.graph + elif row.graph: + payload.update( + node_count=len(row.graph.get("nodes", [])), + edge_count=len(row.graph.get("edges", [])), + ) + return payload diff --git a/src/mkb/web/_models.py b/src/mkb/web/_models.py index dd6c8ab..00211e7 100644 --- a/src/mkb/web/_models.py +++ b/src/mkb/web/_models.py @@ -89,10 +89,6 @@ class ProjectUpdateRequest(BaseModel): label: str -class WorkflowCanonicalizeRequest(BaseModel): - raw_extraction_id: str | None = None - - class WorkflowReextractionRequest(BaseModel): reason: str requested_by: str = "api" diff --git a/src/mkb/web/job_actions.py b/src/mkb/web/job_actions.py index cae59d4..2c462ba 100644 --- a/src/mkb/web/job_actions.py +++ b/src/mkb/web/job_actions.py @@ -215,14 +215,6 @@ def _upload_ingest_kwargs(**kwargs) -> dict[str, Any]: conflict_policy="project_kind", validate=_require_keys("project_id"), ), - "canonicalize_workflow": JobAction( - "canonicalize_workflow", - "canonical_workflow", - "Canonicalize Workflow", - "canonicalize_workflow", - conflict_policy="project_kind", - validate=_require_keys("project_id"), - ), "review_feedback": JobAction( "review_feedback", "feedback_review", diff --git a/src/mkb/web/routers/projects.py b/src/mkb/web/routers/projects.py index 58edea5..f40014e 100644 --- a/src/mkb/web/routers/projects.py +++ b/src/mkb/web/routers/projects.py @@ -22,7 +22,6 @@ SchemaCurateRequest, SchemaProposalEditRequest, SchemaProposalReviewRequest, - WorkflowCanonicalizeRequest, WorkflowRecanonicalizationRequest, WorkflowReextractionRequest, ) @@ -221,21 +220,6 @@ def latest_project_workflow(project_id: str): return result -@router.post("/api/projects/{project_id}/workflows/canonicalize") -def canonicalize_project_workflow(project_id: str, body: WorkflowCanonicalizeRequest): - _parse_uuid(project_id, "project_id") - if body.raw_extraction_id: - _parse_uuid(body.raw_extraction_id, "raw_extraction_id") - job_id = start_web_job_action( - jobs, - "canonicalize_workflow", - job_project_id=project_id, - project_id=project_id, - raw_extraction_id=body.raw_extraction_id, - ) - return {"job_id": job_id} - - @router.get("/api/projects/{project_id}/workflows/{version}") def project_workflow_version(project_id: str, version: int): _parse_uuid(project_id, "project_id") diff --git a/src/mkb/workflows/editing.py b/src/mkb/workflows/editing.py new file mode 100644 index 0000000..992578f --- /dev/null +++ b/src/mkb/workflows/editing.py @@ -0,0 +1,442 @@ +"""Low-level workflow editing helpers shared by agent tool adapters.""" + +from __future__ import annotations + +from copy import deepcopy +from typing import Any + +from mkb.workflows.schema_library import get_schema_library_payload + +SEARCHABLE_NODE_KINDS = {None, "", "object", "operation", "planning", "reasoning", "unknown"} +NODE_KINDS = {"object", "operation", "planning", "reasoning", "unknown"} +RELATION_ALIASES = { + "input": "input_to", + "input_to": "input_to", + "produces": "produces", + "output": "produces", + "output_of": "produces", + "same_as": "same_as", + "part_of": "part_of", + "has_part": "has_part", + "expands_to": "expands_to", + "summarized_by": "summarized_by", + "motivates": "motivates", + "leads_to": "leads_to", +} +STRUCTURAL_RELATIONS = {"same_as", "part_of", "has_part", "expands_to", "summarized_by"} +ENDPOINT_ALIASES = { + "source": ( + "source_node", + "source", + "source_id", + "source_node_id", + "from", + "from_node", + "from_node_id", + "source_label", + "from_label", + ), + "target": ( + "target_node", + "target", + "target_id", + "target_node_id", + "to", + "to_node", + "to_node_id", + "target_label", + "to_label", + ), +} + + +def dict_or_empty(value: Any) -> dict: + return value if isinstance(value, dict) else {} + + +def list_or_empty(value: Any) -> list: + return value if isinstance(value, list) else [] + + +def normalize_node_kind(value: Any) -> str: + kind = str(value or "unknown").strip().casefold().replace("-", "_") + return kind if kind in NODE_KINDS else "unknown" + + +def infer_relation_type(source_kind: str | None, target_kind: str | None, requested: str | None = None) -> str: + """Infer deterministic workflow relations from endpoint node kinds.""" + requested_key = str(requested or "").strip().casefold().replace("-", "_") + requested = RELATION_ALIASES.get(requested_key, requested_key) + if requested in STRUCTURAL_RELATIONS: + return requested + if source_kind == "object" and target_kind == "operation": + return "input_to" + if source_kind == "operation" and target_kind == "object": + return "produces" + if source_kind in {"planning", "reasoning"}: + return "leads_to" if requested == "leads_to" else "motivates" + return requested + + +def _edge_endpoint(edge: dict[str, Any], side: str) -> Any: + for key in ENDPOINT_ALIASES[side]: + if key in edge and edge.get(key) not in (None, ""): + return edge.get(key) + return None + + +def _endpoint_refs(value: Any) -> list[str]: + if isinstance(value, dict): + refs = [] + for key in ("node_id", "id", "name", "label", "raw_name", "canonical_name"): + if value.get(key) not in (None, ""): + refs.append(str(value[key]).strip()) + return refs + if isinstance(value, int): + return [str(value), f"n{value}", f"node_{value}", f"node-{value}", f"node {value}"] + if value is None: + return [] + text = str(value).strip() + refs = [text] + if text.isdigit(): + refs.extend([f"n{text}", f"node_{text}", f"node-{text}", f"node {text}"]) + return refs + + +def _resolve_node_ref(value: Any, old_node_refs: dict[str, str]) -> str | None: + for ref in _endpoint_refs(value): + if ref in old_node_refs: + return old_node_refs[ref] + refs = _endpoint_refs(value) + return refs[0] if refs else None + + +def compact_value( + value: Any, + *, + string_limit: int = 1000, + list_limit: int = 30, + dict_limit: int = 30, +) -> Any: + if isinstance(value, str): + return value if len(value) <= string_limit else f"{value[:string_limit]}... [truncated]" + if isinstance(value, list): + items = [ + compact_value( + item, + string_limit=string_limit, + list_limit=list_limit, + dict_limit=dict_limit, + ) + for item in value[:list_limit] + ] + if len(value) > list_limit: + items.append({"omitted_items": len(value) - list_limit}) + return items + if isinstance(value, dict): + result = {} + for index, (key, item) in enumerate(value.items()): + if index >= dict_limit: + result["omitted_keys"] = len(value) - dict_limit + break + result[key] = compact_value( + item, + string_limit=string_limit, + list_limit=list_limit, + dict_limit=dict_limit, + ) + return result + return value + + +def replace_by_id(items: list[dict[str, Any]], id_key: str, item: dict[str, Any]) -> tuple[list[dict[str, Any]], str]: + item_id = item.get(id_key) + if not isinstance(item_id, str) or not item_id.strip(): + raise ValueError(f"{id_key} is required") + for index, existing in enumerate(items): + if existing.get(id_key) == item_id: + items[index] = item + return items, "updated" + items.append(item) + return items, "added" + + +def replace_by_raw_ids(items: list[dict[str, Any]], item: dict[str, Any]) -> tuple[list[dict[str, Any]], str]: + raw_ids = tuple(item.get("raw_node_ids") or []) + if not raw_ids: + raise ValueError("raw_node_ids is required") + for index, existing in enumerate(items): + if tuple(existing.get("raw_node_ids") or []) == raw_ids: + items[index] = item + return items, "updated" + items.append(item) + return items, "added" + + +def normalize_raw_graph_payload(graph: dict, row, extraction_id) -> tuple[dict, dict]: + """Fill app-owned workflow envelope fields and tolerate common LLM aliases.""" + payload = deepcopy(graph if isinstance(graph, dict) else {}) + changes = { + "filled_graph_fields": [], + "assigned_node_ids": 0, + "assigned_edge_ids": 0, + "inferred_edge_relations": 0, + "overrode_edge_relations": 0, + "normalized_nodes": 0, + "normalized_edges": 0, + } + for key, value in { + "schema_version": row.schema_version, + "paper_id": str(row.project_id), + "extraction_id": str(extraction_id), + }.items(): + if payload.get(key) != value: + payload[key] = value + changes["filled_graph_fields"].append(key) + + payload["nodes"] = payload.get("nodes") if isinstance(payload.get("nodes"), list) else [] + payload["edges"] = payload.get("edges") if isinstance(payload.get("edges"), list) else [] + payload["unresolved_information"] = [ + item if isinstance(item, dict) else {"description": str(item)} + for item in list_or_empty(payload.get("unresolved_information")) + ] + if not isinstance(payload.get("reproducibility"), dict): + payload.pop("reproducibility", None) + + old_node_refs: dict[str, str] = {} + for index, node in enumerate(payload["nodes"], 1): + if not isinstance(node, dict): + node = {"raw_name": str(node), "evidence_text": str(node)} + payload["nodes"][index - 1] = node + original_refs = { + str(value).strip() + for value in ( + node.get("node_id"), + node.get("id"), + node.get("name"), + node.get("label"), + node.get("raw_name"), + node.get("canonical_name"), + ) + if value is not None and str(value).strip() + } + expected_id = f"raw:{extraction_id}:n{index:04d}" + node_id = str(node.get("node_id") or node.get("id") or "").strip() + if not node_id or not node_id.startswith(f"raw:{extraction_id}:n"): + node["node_id"] = expected_id + changes["assigned_node_ids"] += 1 + kind = normalize_node_kind(node.get("node_kind") or node.get("node_kind_guess") or node.get("kind")) + node["node_kind"] = kind + node["node_kind_guess"] = kind + node["raw_name"] = str(node.get("raw_name") or node.get("canonical_name") or node.get("label") or node["node_id"]) + node.setdefault("canonical_name", node.get("raw_name")) + node.setdefault("semantic_type", kind) + for key in ("parameters", "identity", "state", "role", "context", "attributes_explicitly_mentioned", "paper_location"): + node[key] = dict_or_empty(node.get(key)) + node["unparsed_modifiers"] = list_or_empty(node.get("unparsed_modifiers")) + node["aliases_observed"] = list_or_empty(node.get("aliases_observed")) + status = str(node.get("ontology_status") or "unmapped").strip().casefold() + node["ontology_status"] = "matched" if status == "mapped" else status if status in {"matched", "candidate", "unmapped"} else "unmapped" + node["evidence_text"] = str(node.get("evidence_text") or node.get("evidence") or node.get("raw_name")) + try: + node["confidence"] = float(node.get("confidence", 0.5)) + except (TypeError, ValueError): + node["confidence"] = 0.5 + node["confidence"] = max(0.0, min(1.0, node["confidence"])) + for key in ("id", "name", "label", "kind", "evidence"): + node.pop(key, None) + index_refs = (str(index), f"n{index}", f"node_{index}", f"node-{index}", f"node {index}") + for ref in (*original_refs, expected_id, *index_refs): + old_node_refs[ref] = node["node_id"] + changes["normalized_nodes"] += 1 + + node_kinds = {node.get("node_id"): node.get("node_kind") for node in payload["nodes"] if isinstance(node, dict)} + for index, edge in enumerate(payload["edges"], 1): + if not isinstance(edge, dict): + edge = {"evidence_text": str(edge)} + payload["edges"][index - 1] = edge + edge_id = str(edge.get("edge_id") or edge.get("id") or "").strip() + if not edge_id or not edge_id.startswith(f"raw:{extraction_id}:e"): + edge["edge_id"] = f"raw:{extraction_id}:e{index:04d}" + changes["assigned_edge_ids"] += 1 + edge.pop("id", None) + source = _edge_endpoint(edge, "source") + target = _edge_endpoint(edge, "target") + edge["source_node"] = _resolve_node_ref(source, old_node_refs) or "" + edge["target_node"] = _resolve_node_ref(target, old_node_refs) or "" + for key in (*ENDPOINT_ALIASES["source"], *ENDPOINT_ALIASES["target"]): + if key not in {"source_node", "target_node"}: + edge.pop(key, None) + relation = str(edge.get("relation_type") or edge.get("kind") or edge.get("relation") or "").strip().casefold().replace("-", "_") + normalized_relation = RELATION_ALIASES.get(relation, relation) + inferred_relation = infer_relation_type( + node_kinds.get(edge["source_node"]), + node_kinds.get(edge["target_node"]), + normalized_relation, + ) + edge["relation_type"] = inferred_relation if inferred_relation in RELATION_ALIASES.values() else "input_to" + if inferred_relation and inferred_relation != normalized_relation: + if normalized_relation: + changes["overrode_edge_relations"] += 1 + else: + changes["inferred_edge_relations"] += 1 + edge["attributes"] = dict_or_empty(edge.get("attributes")) + edge["evidence_text"] = str( + edge.get("evidence_text") + or edge.get("evidence") + or edge.get("evidence_quote") + or edge.get("evidence_snippet") + or f"{source} -> {target}" + ) + for key in ("kind", "relation", "evidence", "evidence_quote", "evidence_snippet"): + edge.pop(key, None) + if edge.get("paper_location") is not None: + edge["paper_location"] = dict_or_empty(edge.get("paper_location")) + try: + edge["confidence"] = float(edge.get("confidence", 0.5)) + except (TypeError, ValueError): + edge["confidence"] = 0.5 + edge["confidence"] = max(0.0, min(1.0, edge["confidence"])) + changes["normalized_edges"] += 1 + + return payload, changes + + +def compact_raw_checkpoint_manifest(graph: dict | None) -> dict: + if not isinstance(graph, dict): + return {"counts": {"nodes": 0, "edges": 0}, "nodes": [], "edges": []} + nodes = graph.get("nodes") if isinstance(graph.get("nodes"), list) else [] + edges = graph.get("edges") if isinstance(graph.get("edges"), list) else [] + return { + "counts": { + "nodes": len(nodes), + "edges": len(edges), + "unresolved_information": len(graph.get("unresolved_information") or []), + }, + "nodes": [ + { + "node_id": node.get("node_id"), + "raw_name": node.get("raw_name") or node.get("canonical_name") or node.get("label"), + "node_kind": node.get("node_kind") or node.get("node_kind_guess") or node.get("kind"), + } + for node in nodes[:80] + if isinstance(node, dict) + ], + "edges": [ + { + "edge_id": edge.get("edge_id"), + "source_node": edge.get("source_node") or edge.get("source"), + "target_node": edge.get("target_node") or edge.get("target"), + "relation_type": edge.get("relation_type") or edge.get("kind") or edge.get("relation"), + } + for edge in edges[:120] + if isinstance(edge, dict) + ], + "note": "The full checkpoint graph remains server-side. Continue from this manifest and save/checkpoint only changed draft content.", + } + + +def get_active_workflow_card_library(max_cards: int = 40, max_templates: int = 40) -> dict: + """Return a bounded view of the newest workflow card/schema library.""" + library = get_schema_library_payload() + cards = library.get("cards", {}) + templates = library.get("operation_templates", {}) + card_items = list(cards.items())[: max(1, min(int(max_cards), 200))] + template_items = list(templates.items())[: max(1, min(int(max_templates), 200))] + return { + "schema_version": library.get("schema_version"), + "cards": [ + { + "card_id": card_id, + "canonical_name": payload.get("canonical_name"), + "kind": payload.get("kind"), + "aliases": payload.get("aliases", []), + "parameter_slots": payload.get("parameter_slots", []), + "status": payload.get("status", "active"), + "replaced_by": payload.get("replaced_by"), + } + for card_id, payload in card_items + ], + "operation_templates": [ + { + "template_id": template_id, + "label": payload.get("label"), + "aliases": payload.get("aliases", []), + "slots": payload.get("slots", []), + "parameters": payload.get("parameters", {}), + "deprecated": bool(payload.get("deprecated")), + } + for template_id, payload in template_items + ], + } + + +def search_workflow_cards(query: str, node_kind: str | None = None, limit: int = 10) -> dict: + """Search the newest card base/templates before instantiating workflow nodes.""" + text = str(query or "").strip().casefold() + if not text: + return {"error": "query is required"} + if node_kind not in SEARCHABLE_NODE_KINDS: + return {"error": "node_kind must be one of object, operation, planning, reasoning, unknown, or omitted"} + + library = get_schema_library_payload() + results: list[dict] = [] + effective_limit = max(1, min(int(limit), 50)) + + for card_id, payload in (library.get("cards", {}) or {}).items(): + kind = payload.get("kind") + if node_kind and kind != node_kind: + continue + name = str(payload.get("canonical_name") or "") + aliases = [str(value) for value in payload.get("aliases", []) if value] + haystacks = [card_id, name, *aliases] + score = sum(3 for item in haystacks if text == item.casefold()) + score += sum(1 for item in haystacks if text in item.casefold()) + if score <= 0: + continue + results.append({ + "match_type": "card", + "score": score, + "card_id": card_id, + "canonical_name": name, + "kind": kind, + "aliases": aliases, + "parameter_slots": payload.get("parameter_slots", []), + "status": payload.get("status", "active"), + "replaced_by": payload.get("replaced_by"), + }) + + for template_id, payload in (library.get("operation_templates", {}) or {}).items(): + if node_kind and node_kind != "operation": + continue + label = str(payload.get("label") or "") + aliases = [str(value) for value in payload.get("aliases", []) if value] + haystacks = [template_id, label, *aliases] + score = sum(3 for item in haystacks if text == item.casefold()) + score += sum(1 for item in haystacks if text in item.casefold()) + if score <= 0: + continue + results.append({ + "match_type": "operation_template", + "score": score, + "template_id": template_id, + "label": label, + "kind": "operation", + "aliases": aliases, + "slots": payload.get("slots", []), + "parameters": payload.get("parameters", {}), + "deprecated": bool(payload.get("deprecated")), + }) + + results.sort( + key=lambda item: ( + -int(item.get("score", 0)), + str(item.get("canonical_name") or item.get("label") or item.get("card_id") or item.get("template_id")), + ) + ) + return { + "schema_version": library.get("schema_version"), + "query": query, + "node_kind": node_kind or "any", + "results": results[:effective_limit], + } diff --git a/src/mkb/workflows/review.py b/src/mkb/workflows/review.py index adaf1e7..5a55dcd 100644 --- a/src/mkb/workflows/review.py +++ b/src/mkb/workflows/review.py @@ -19,6 +19,12 @@ def audit_raw_graph(graph: dict, *, low_confidence_threshold: float = 0.5, later nodes = graph.get("nodes", []) edges = graph.get("edges", []) by_id = {node.get("node_id"): node for node in nodes} + if len(nodes) > 1 and not edges: + flags.append({ + "type": "missing_workflow_edges", + "item_type": "graph", + "item_id": graph.get("extraction_id"), + }) for item_type, items, id_key in (("node", nodes, "node_id"), ("edge", edges, "edge_id")): for item in items: if float(item.get("confidence", 0)) < low_confidence_threshold: diff --git a/tests/test_workflow_editing.py b/tests/test_workflow_editing.py new file mode 100644 index 0000000..3f33c7a --- /dev/null +++ b/tests/test_workflow_editing.py @@ -0,0 +1,186 @@ +import uuid +from types import SimpleNamespace + +from mkb.workflows.contract import RAW_WORKFLOW_SCHEMA_VERSION, RawWorkflowGraph +from mkb.workflows.editing import normalize_raw_graph_payload + + +def _row(project_id: uuid.UUID): + return SimpleNamespace(schema_version=RAW_WORKFLOW_SCHEMA_VERSION, project_id=project_id) + + +def test_normalize_raw_graph_payload_infers_relation_from_endpoint_kinds(): + eid = uuid.uuid4() + payload, normalization = normalize_raw_graph_payload( + { + "nodes": [ + {"raw_name": "powder", "node_kind": "object", "evidence_text": "powder"}, + {"raw_name": "anneal", "node_kind": "operation", "evidence_text": "anneal"}, + {"raw_name": "sample", "node_kind": "object", "evidence_text": "sample"}, + ], + "edges": [ + { + "source": "powder", + "target": "anneal", + "evidence_text": "powder was annealed", + }, + { + "source": "anneal", + "target": "sample", + "evidence_text": "annealing produced sample", + }, + ], + }, + _row(uuid.uuid4()), + eid, + ) + + graph = RawWorkflowGraph.model_validate(payload) + + assert [edge.relation_type for edge in graph.edges] == ["input_to", "produces"] + assert normalization["inferred_edge_relations"] == 2 + + +def test_normalize_raw_graph_payload_overrides_wrong_workflow_relation(): + eid = uuid.uuid4() + payload, normalization = normalize_raw_graph_payload( + { + "nodes": [ + {"raw_name": "powder", "node_kind": "object", "evidence_text": "powder"}, + {"raw_name": "anneal", "node_kind": "operation", "evidence_text": "anneal"}, + ], + "edges": [ + { + "source": "powder", + "target": "anneal", + "relation_type": "produces", + "evidence_text": "powder was annealed", + }, + ], + }, + _row(uuid.uuid4()), + eid, + ) + + graph = RawWorkflowGraph.model_validate(payload) + + assert graph.edges[0].relation_type == "input_to" + assert normalization["overrode_edge_relations"] == 1 + + +def test_normalize_raw_graph_payload_infers_planning_relation(): + eid = uuid.uuid4() + payload, normalization = normalize_raw_graph_payload( + { + "nodes": [ + { + "raw_name": "screen stable phases", + "node_kind": "planning", + "evidence_text": "screen stable phases", + }, + {"raw_name": "anneal", "node_kind": "operation", "evidence_text": "anneal"}, + ], + "edges": [ + { + "source": "screen stable phases", + "target": "anneal", + "relation_type": "input_to", + "evidence_text": "screening motivated annealing", + }, + ], + }, + _row(uuid.uuid4()), + eid, + ) + + graph = RawWorkflowGraph.model_validate(payload) + + assert graph.edges[0].relation_type == "motivates" + assert normalization["overrode_edge_relations"] == 1 + + +def test_normalize_raw_graph_payload_preserves_structural_relation(): + eid = uuid.uuid4() + payload, _normalization = normalize_raw_graph_payload( + { + "nodes": [ + { + "raw_name": "phase diagram", + "node_kind": "object", + "evidence_text": "phase diagram", + }, + { + "raw_name": "stability region", + "node_kind": "object", + "evidence_text": "stability region", + }, + ], + "edges": [ + { + "source": "stability region", + "target": "phase diagram", + "relation_type": "part_of", + "evidence_text": "the region is part of the phase diagram", + }, + ], + }, + _row(uuid.uuid4()), + eid, + ) + + graph = RawWorkflowGraph.model_validate(payload) + + assert graph.edges[0].relation_type == "part_of" + + +def test_normalize_raw_graph_payload_accepts_endpoint_aliases_and_numeric_refs(): + eid = uuid.uuid4() + payload, normalization = normalize_raw_graph_payload( + { + "nodes": [ + {"id": "node-a", "name": "powder", "kind": "object", "evidence": "powder"}, + {"id": "node-b", "name": "anneal", "kind": "operation", "evidence": "anneal"}, + {"id": "node-c", "name": "sample", "kind": "object", "evidence": "sample"}, + ], + "edges": [ + {"from_node": 1, "to_node": 2, "relation": "connection"}, + {"from": {"id": "node-b"}, "to": {"name": "sample"}, "kind": "edge"}, + ], + }, + _row(uuid.uuid4()), + eid, + ) + + graph = RawWorkflowGraph.model_validate(payload) + + assert [edge.relation_type for edge in graph.edges] == ["input_to", "produces"] + assert all(edge.evidence_text for edge in graph.edges) + assert normalization["inferred_edge_relations"] == 0 + assert normalization["overrode_edge_relations"] == 2 + + +def test_normalize_raw_graph_payload_rewrites_invalid_relation_when_endpoints_are_known(): + eid = uuid.uuid4() + payload, normalization = normalize_raw_graph_payload( + { + "nodes": [ + {"raw_name": "screen stable phases", "node_kind": "reasoning", "evidence_text": "screen"}, + {"raw_name": "anneal", "node_kind": "operation", "evidence_text": "anneal"}, + ], + "edges": [ + { + "source_label": "screen stable phases", + "target_label": "anneal", + "relation_type": "depends_on", + }, + ], + }, + _row(uuid.uuid4()), + eid, + ) + + graph = RawWorkflowGraph.model_validate(payload) + + assert graph.edges[0].relation_type == "motivates" + assert graph.edges[0].evidence_text == "screen stable phases -> anneal" + assert normalization["overrode_edge_relations"] == 1 diff --git a/tests/test_workflow_review.py b/tests/test_workflow_review.py new file mode 100644 index 0000000..e4cbf70 --- /dev/null +++ b/tests/test_workflow_review.py @@ -0,0 +1,41 @@ +import uuid + +from mkb.workflows.review import audit_raw_graph + + +def test_audit_flags_node_only_workflow_graph(): + eid = str(uuid.uuid4()) + graph = { + "schema_version": "workflow-cards/2.0", + "paper_id": str(uuid.uuid4()), + "extraction_id": eid, + "nodes": [ + { + "node_id": f"raw:{eid}:n0001", + "raw_name": "input structure", + "node_kind": "object", + "node_kind_guess": "object", + "evidence_text": "input structure", + "paper_location": {}, + "confidence": 0.9, + }, + { + "node_id": f"raw:{eid}:n0002", + "raw_name": "DFT calculation", + "node_kind": "operation", + "node_kind_guess": "operation", + "evidence_text": "DFT calculation", + "paper_location": {}, + "confidence": 0.9, + }, + ], + "edges": [], + } + + flags = audit_raw_graph(graph) + + assert { + "type": "missing_workflow_edges", + "item_type": "graph", + "item_id": eid, + } in flags diff --git a/tests/test_workflow_review_curator.py b/tests/test_workflow_review_curator.py index 5b9af95..4d435a2 100644 --- a/tests/test_workflow_review_curator.py +++ b/tests/test_workflow_review_curator.py @@ -7,6 +7,7 @@ from mkb.workflows.curator import ( apply_proposal, validate_proposal, ) +from mkb.workflows import editing as workflow_editing from mkb.workflows.review import audit_raw_graph, rebase_graph from mkb.workflows.contract import RawWorkflowGraph from mkb.workflows.validation import json_safe_validation_errors @@ -153,7 +154,7 @@ def test_object_and_operation_cards_evolve_symmetrically(): def test_extractor_card_search_uses_newest_library(monkeypatch): monkeypatch.setattr( - workflow_tools, + workflow_editing, "get_schema_library_payload", lambda: { "schema_version": "workflow-schema/9.9", diff --git a/tests/test_workflow_serialization.py b/tests/test_workflow_serialization.py new file mode 100644 index 0000000..ae5de74 --- /dev/null +++ b/tests/test_workflow_serialization.py @@ -0,0 +1,59 @@ +import uuid +from types import SimpleNamespace + +from mkb.services.workflows.serialization import serialize_raw_workflow + + +def test_serialize_raw_workflow_includes_dynamic_review_flags(): + eid = str(uuid.uuid4()) + row = SimpleNamespace( + extraction_id=eid, + project_id=uuid.uuid4(), + version=2, + schema_version="workflow-cards/2.0", + extractor_version="workflow-extractor/2.0", + model=None, + status="COMPLETED", + record_status="active", + supersedes_extraction_id=None, + correction_reason=None, + correction_author=None, + correction_details={}, + review_flags=[], + provenance={}, + error=None, + extracted_at=None, + created_at=None, + checkpoint=None, + checkpoint_updated_at=None, + graph={ + "schema_version": "workflow-cards/2.0", + "paper_id": str(uuid.uuid4()), + "extraction_id": eid, + "nodes": [ + { + "node_id": f"raw:{eid}:n0001", + "raw_name": "input", + "node_kind": "object", + "node_kind_guess": "object", + "evidence_text": "input", + "paper_location": {}, + "confidence": 0.9, + }, + { + "node_id": f"raw:{eid}:n0002", + "raw_name": "calculate", + "node_kind": "operation", + "node_kind_guess": "operation", + "evidence_text": "calculate", + "paper_location": {}, + "confidence": 0.9, + }, + ], + "edges": [], + }, + ) + + payload = serialize_raw_workflow(row, include_graph=False) + + assert any(flag["type"] == "missing_workflow_edges" for flag in payload["review_flags"]) From eec03ea0b300147cd8c5c83d9ad9cd4fd6ae5e83 Mon Sep 17 00:00:00 2001 From: theAfish Date: Wed, 1 Jul 2026 14:04:58 +0800 Subject: [PATCH 04/26] feat: workflow showing refinement --- frontend/package-lock.json | 16 + frontend/package.json | 1 + .../components/projects/WorkflowCanvas.tsx | 573 ++++++------------ 3 files changed, 204 insertions(+), 386 deletions(-) diff --git a/frontend/package-lock.json b/frontend/package-lock.json index a330b99..044d9ec 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -8,6 +8,7 @@ "name": "mkb-frontend", "version": "0.1.0", "dependencies": { + "@dagrejs/dagre": "^3.0.0", "@tanstack/react-table": "^8.21.3", "axios": "^1.15.2", "pdfjs-dist": "^6.0.227", @@ -28,6 +29,21 @@ "vite": "^8.0.10" } }, + "node_modules/@dagrejs/dagre": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@dagrejs/dagre/-/dagre-3.0.0.tgz", + "integrity": "sha512-ZzhnTy1rfuoew9Ez3EIw4L2znPGnYYhfn8vc9c4oB8iw6QAsszbiU0vRhlxWPFnmmNSFAkrYeF1PhM5m4lAN0Q==", + "license": "MIT", + "dependencies": { + "@dagrejs/graphlib": "4.0.1" + } + }, + "node_modules/@dagrejs/graphlib": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@dagrejs/graphlib/-/graphlib-4.0.1.tgz", + "integrity": "sha512-IvcV6FduIIAmLwnH+yun+QtV36SC7mERqa86aClNqmMN09WhmPPYU8ckHrZBozErf+UvHPWOTJYaGYiIcs0DgA==", + "license": "MIT" + }, "node_modules/@egjs/hammerjs": { "version": "2.0.17", "resolved": "https://registry.npmjs.org/@egjs/hammerjs/-/hammerjs-2.0.17.tgz", diff --git a/frontend/package.json b/frontend/package.json index a06017b..8a11823 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -10,6 +10,7 @@ "preview": "vite preview" }, "dependencies": { + "@dagrejs/dagre": "^3.0.0", "@tanstack/react-table": "^8.21.3", "axios": "^1.15.2", "pdfjs-dist": "^6.0.227", diff --git a/frontend/src/components/projects/WorkflowCanvas.tsx b/frontend/src/components/projects/WorkflowCanvas.tsx index 288dd05..5b64969 100644 --- a/frontend/src/components/projects/WorkflowCanvas.tsx +++ b/frontend/src/components/projects/WorkflowCanvas.tsx @@ -1,13 +1,17 @@ import { useEffect, useState } from 'react' +import dagre from '@dagrejs/dagre' import ReactFlow, { Background, + BaseEdge, Controls, + EdgeLabelRenderer, Handle, MarkerType, MiniMap, Position, applyNodeChanges, type Edge, + type EdgeProps, type Node, type NodeChange, type NodeProps, @@ -42,21 +46,20 @@ interface WorkflowNodeData { details?: Record } +interface WorkflowEdgeData { + title?: string + points?: Array<{ x: number; y: number }> + sourceSide?: AnchorSide + targetSide?: AnchorSide +} + const XML_NS = 'http://www.w3.org/2000/svg' const OP_WIDTH = 190 const OBJ_WIDTH = 190 const CONTEXT_WIDTH = 210 const NODE_HEIGHT = 74 -const X_GAP = 300 -const Y_GAP = 240 -const OBJECT_OFFSET = 145 -const MIN_ROW_SPACING = 235 -const COMPONENT_GAP_X = 360 -const COMPONENT_GAP_Y = 150 -const BRANCH_GAP = 320 type AnchorSide = 'top' | 'right' | 'bottom' | 'left' -type PositionedNode = WorkflowCanvasNode & { x: number; y: number } function NodeHandles() { const handles: AnchorSide[] = ['top', 'right', 'bottom', 'left'] @@ -172,10 +175,6 @@ function UnknownNode({ data }: NodeProps) { ) } -function average(values: number[]) { - return values.length ? values.reduce((sum, value) => sum + value, 0) / values.length : 0 -} - function sideToPosition(side: AnchorSide) { switch (side) { case 'top': @@ -311,7 +310,17 @@ function chooseAnchorSides(source: Node, target: Node 0) score -= 130 if (mostlyHorizontal && sourceSide === 'left' && targetSide === 'right' && dx < 0) score -= 130 - if (sourceSide === targetSide) score += 80 + if (sourceSide === targetSide) score += 1000 + + if (sourceSide === 'bottom') score -= 90 + if (sourceSide === 'right') score -= 35 + if (sourceSide === 'top') score += 220 + if (sourceSide === 'left') score += 70 + + if (targetSide === 'top') score -= 90 + if (targetSide === 'left') score -= 35 + if (targetSide === 'bottom') score += 220 + if (targetSide === 'right') score += 70 if (score < best.score) { best = { sourceSide, targetSide, score } @@ -346,380 +355,125 @@ function downloadFile(filename: string, mimeType: string, content: string) { URL.revokeObjectURL(url) } -function spreadRow(items: T[], minSpacing: number) { - if (items.length <= 1) return - items.sort((a, b) => a.x - b.x) - for (let index = 1; index < items.length; index += 1) { - if (items[index].x - items[index - 1].x < minSpacing) { - items[index].x = items[index - 1].x + minSpacing - } - } - - const midpoint = (items[0].x + items[items.length - 1].x) / 2 - const targetCenter = average(items.map(item => item.x)) - const shift = midpoint - targetCenter - items.forEach(item => { - item.x -= shift +function dedupePoints(points: Array<{ x: number; y: number }>) { + return points.filter((point, index) => { + const previous = points[index - 1] + return !previous || Math.abs(previous.x - point.x) > 1 || Math.abs(previous.y - point.y) > 1 }) } -function connectedComponents(nodes: WorkflowCanvasNode[], edges: WorkflowCanvasEdge[]) { - const nodeIds = new Set(nodes.map(node => node.id)) - const adjacency = new Map>() - nodes.forEach(node => adjacency.set(node.id, new Set())) - edges.forEach(edge => { - if (!nodeIds.has(edge.source) || !nodeIds.has(edge.target)) return - adjacency.get(edge.source)?.add(edge.target) - adjacency.get(edge.target)?.add(edge.source) - }) - - const seen = new Set() - const components: string[][] = [] - nodes.forEach(node => { - if (seen.has(node.id)) return - const queue = [node.id] - const component: string[] = [] - seen.add(node.id) - while (queue.length) { - const current = queue.shift()! - component.push(current) - adjacency.get(current)?.forEach(next => { - if (seen.has(next)) return - seen.add(next) - queue.push(next) - }) - } - components.push(component) - }) - return components +function distance(a: { x: number; y: number }, b: { x: number; y: number }) { + return Math.hypot(b.x - a.x, b.y - a.y) } -function rowItemsFromAnchors(ids: string[], anchors: Map, fallbackGap: number) { - const rowWidth = (ids.length - 1) * fallbackGap - return ids.map((id, index) => ({ - id, - x: anchors.has(id) ? anchors.get(id)! : index * fallbackGap - rowWidth / 2, - })) -} - -function centeredOffsets(count: number, gap: number) { - const center = (count - 1) / 2 - return Array.from({ length: count }, (_, index) => (index - center) * gap) -} - -function workflowLayoutGroups(nodes: WorkflowCanvasNode[], edges: WorkflowCanvasEdge[]) { - return connectedComponents(nodes, edges) -} - -function layoutComponent(nodes: WorkflowCanvasNode[], edges: WorkflowCanvasEdge[]) { - const nodeMap = new Map(nodes.map(node => [node.id, node])) - const operationIds = nodes.filter(node => node.kind === 'operation').map(node => node.id) - const objectIds = nodes.filter(node => node.kind === 'object').map(node => node.id) - const contextIds = nodes.filter(node => !['object', 'operation'].includes(node.kind)).map(node => node.id) - - const producerMap = new Map() - const consumerMap = new Map() - objectIds.forEach(id => { - producerMap.set(id, []) - consumerMap.set(id, []) - }) - - edges.forEach(edge => { - const sourceKind = nodeMap.get(edge.source)?.kind - const targetKind = nodeMap.get(edge.target)?.kind - if (sourceKind === 'operation' && targetKind === 'object') { - producerMap.get(edge.target)?.push(edge.source) - } - if (sourceKind === 'object' && targetKind === 'operation') { - consumerMap.get(edge.source)?.push(edge.target) - } - }) - - const opChildren = new Map>() - const opParents = new Map>() - const indegree = new Map() - operationIds.forEach(id => { - opChildren.set(id, new Set()) - opParents.set(id, new Set()) - indegree.set(id, 0) - }) - - objectIds.forEach(objectId => { - const producers = producerMap.get(objectId) ?? [] - const consumers = consumerMap.get(objectId) ?? [] - producers.forEach(producerId => { - consumers.forEach(consumerId => { - if (producerId === consumerId || opChildren.get(producerId)?.has(consumerId)) return - opChildren.get(producerId)?.add(consumerId) - opParents.get(consumerId)?.add(producerId) - indegree.set(consumerId, (indegree.get(consumerId) ?? 0) + 1) - }) - }) - }) - - const queue = operationIds - .filter(id => (indegree.get(id) ?? 0) === 0) - .sort((a, b) => (nodeMap.get(a)?.label ?? '').localeCompare(nodeMap.get(b)?.label ?? '')) - - const opLevel = new Map() - operationIds.forEach(id => opLevel.set(id, 0)) - - while (queue.length > 0) { - const currentId = queue.shift()! - const currentLevel = opLevel.get(currentId) ?? 0 - Array.from(opChildren.get(currentId) ?? []).forEach(childId => { - opLevel.set(childId, Math.max(opLevel.get(childId) ?? 0, currentLevel + 1)) - indegree.set(childId, (indegree.get(childId) ?? 1) - 1) - if ((indegree.get(childId) ?? 0) === 0) { - queue.push(childId) - } - }) - queue.sort((a, b) => (nodeMap.get(a)?.label ?? '').localeCompare(nodeMap.get(b)?.label ?? '')) - } - - const levels = new Map() - operationIds.forEach(id => { - const level = opLevel.get(id) ?? 0 - if (!levels.has(level)) levels.set(level, []) - levels.get(level)!.push(id) - }) - - const sortedLevels = Array.from(levels.keys()).sort((a, b) => a - b) - const levelOrder = new Map() - sortedLevels.forEach(level => { - levelOrder.set(level, [...(levels.get(level) ?? [])].sort((a, b) => (nodeMap.get(a)?.label ?? '').localeCompare(nodeMap.get(b)?.label ?? ''))) - }) - - const opX = new Map() - for (let pass = 0; pass < 6; pass += 1) { - sortedLevels.forEach(level => { - const ids = levelOrder.get(level) ?? [] - const anchors = new Map() - ids.forEach(id => { - const parents = Array.from(opParents.get(id) ?? []).filter(parent => opX.has(parent)) - if (parents.length) anchors.set(id, average(parents.map(parent => opX.get(parent)!))) - }) - const items = rowItemsFromAnchors(ids, anchors, X_GAP) - spreadRow(items, MIN_ROW_SPACING) - items.forEach(item => opX.set(item.id, item.x)) - levelOrder.set(level, items.sort((a, b) => a.x - b.x).map(item => item.id)) - }) +function midpointOnPolyline(points: Array<{ x: number; y: number }>) { + if (points.length === 0) return { x: 0, y: 0 } + if (points.length === 1) return points[0] - sortedLevels.slice().reverse().forEach(level => { - const ids = levelOrder.get(level) ?? [] - const anchors = new Map() - ids.forEach(id => { - const children = Array.from(opChildren.get(id) ?? []).filter(child => opX.has(child)) - if (children.length) anchors.set(id, average(children.map(child => opX.get(child)!))) - }) - const items = rowItemsFromAnchors(ids, anchors, X_GAP) - spreadRow(items, MIN_ROW_SPACING) - items.forEach(item => opX.set(item.id, item.x)) - levelOrder.set(level, items.sort((a, b) => a.x - b.x).map(item => item.id)) - }) - } - - if (operationIds.length === 0) { - objectIds.forEach((id, index) => opX.set(id, index * MIN_ROW_SPACING)) - } - - const branchOffsets = new Map() - const addBranchOffset = (operationId: string, offset: number) => { - const values = branchOffsets.get(operationId) ?? [] - values.push(offset) - branchOffsets.set(operationId, values) - } - - objectIds.forEach(objectId => { - const consumers = [...(consumerMap.get(objectId) ?? [])].sort((a, b) => { - const levelDelta = (opLevel.get(a) ?? 0) - (opLevel.get(b) ?? 0) - if (levelDelta !== 0) return levelDelta - return (nodeMap.get(a)?.label ?? '').localeCompare(nodeMap.get(b)?.label ?? '') - }) - if (consumers.length > 1) { - centeredOffsets(consumers.length, BRANCH_GAP).forEach((offset, index) => { - addBranchOffset(consumers[index], offset) - }) - } - - const producers = [...(producerMap.get(objectId) ?? [])].sort((a, b) => { - const levelDelta = (opLevel.get(a) ?? 0) - (opLevel.get(b) ?? 0) - if (levelDelta !== 0) return levelDelta - return (nodeMap.get(a)?.label ?? '').localeCompare(nodeMap.get(b)?.label ?? '') - }) - if (producers.length > 1) { - centeredOffsets(producers.length, BRANCH_GAP).forEach((offset, index) => { - addBranchOffset(producers[index], offset) - }) - } - }) - - branchOffsets.forEach((offsets, operationId) => { - opX.set(operationId, (opX.get(operationId) ?? 0) + average(offsets)) - }) + const total = points.slice(1).reduce((sum, point, index) => sum + distance(points[index], point), 0) + const target = total / 2 + let traversed = 0 - const levelRows = new Map>() - operationIds.forEach(id => { - const level = opLevel.get(id) ?? 0 - if (!levelRows.has(level)) levelRows.set(level, []) - levelRows.get(level)!.push({ id, x: opX.get(id) ?? 0 }) - }) - levelRows.forEach(items => { - spreadRow(items, MIN_ROW_SPACING) - items.forEach(item => opX.set(item.id, item.x)) - }) - - const objectLayout = objectIds.map(id => { - const producers = producerMap.get(id) ?? [] - const consumers = consumerMap.get(id) ?? [] - - if (producers.length > 0 && consumers.length > 0) { - const latestLevel = Math.max(...producers.map(producerId => opLevel.get(producerId) ?? 0)) - const earliestLevel = Math.min(...consumers.map(consumerId => opLevel.get(consumerId) ?? 0)) - const nearbyProducers = producers.filter(producerId => (opLevel.get(producerId) ?? 0) === latestLevel) - const nearbyConsumers = consumers.filter(consumerId => (opLevel.get(consumerId) ?? 0) === earliestLevel) - return { - id, - x: average([...nearbyProducers, ...nearbyConsumers].map(opId => opX.get(opId) ?? 0)), - y: ((latestLevel + earliestLevel) / 2) * Y_GAP, - } - } - - if (consumers.length > 0) { - const earliestLevel = Math.min(...consumers.map(consumerId => opLevel.get(consumerId) ?? 0)) - const earliestConsumers = consumers.filter(consumerId => (opLevel.get(consumerId) ?? 0) === earliestLevel) + for (let index = 1; index < points.length; index += 1) { + const start = points[index - 1] + const end = points[index] + const segment = distance(start, end) + if (traversed + segment >= target) { + const ratio = segment === 0 ? 0 : (target - traversed) / segment return { - id, - x: average(earliestConsumers.map(consumerId => opX.get(consumerId) ?? 0)), - y: earliestLevel * Y_GAP - OBJECT_OFFSET, + x: start.x + (end.x - start.x) * ratio, + y: start.y + (end.y - start.y) * ratio, } } + traversed += segment + } - if (producers.length > 0) { - const latestLevel = Math.max(...producers.map(producerId => opLevel.get(producerId) ?? 0)) - const latestProducers = producers.filter(producerId => (opLevel.get(producerId) ?? 0) === latestLevel) - return { - id, - x: average(latestProducers.map(producerId => opX.get(producerId) ?? 0)), - y: latestLevel * Y_GAP + OBJECT_OFFSET, - } - } + return points[points.length - 1] +} - const isolatedIndex = objectIds.indexOf(id) - return { id, x: isolatedIndex * MIN_ROW_SPACING, y: -OBJECT_OFFSET } - }) +function controlDistance(start: { x: number; y: number }, end: { x: number; y: number }) { + return Math.max(44, Math.min(150, distance(start, end) / 2)) +} - const objectRows = new Map>() - objectLayout.forEach(item => { - const rowKey = Math.round(item.y) - if (!objectRows.has(rowKey)) objectRows.set(rowKey, []) - objectRows.get(rowKey)!.push(item) - }) - objectRows.forEach(items => spreadRow(items, MIN_ROW_SPACING)) +function smoothBezierPath(points: Array<{ x: number; y: number }>, sourceSide?: AnchorSide, targetSide?: AnchorSide) { + if (points.length === 0) return '' + if (points.length === 1) return `M ${points[0].x} ${points[0].y}` + if (points.length === 2) { + const [start, end] = points + const sourceVector = sideVector(sourceSide ?? 'bottom') + const targetVector = sideVector(targetSide ?? 'top') + const offset = controlDistance(start, end) + const c1 = { x: start.x + sourceVector.x * offset, y: start.y + sourceVector.y * offset } + const c2 = { x: end.x + targetVector.x * offset, y: end.y + targetVector.y * offset } + return `M ${start.x} ${start.y} C ${c1.x} ${c1.y}, ${c2.x} ${c2.y}, ${end.x} ${end.y}` + } - const basePositions = new Map() - nodes.forEach(node => { - if (node.kind === 'operation') { - basePositions.set(node.id, { ...node, x: opX.get(node.id) ?? 0, y: (opLevel.get(node.id) ?? 0) * Y_GAP }) - return + const parts = [`M ${points[0].x} ${points[0].y}`] + for (let index = 0; index < points.length - 1; index += 1) { + const previous = points[index - 1] ?? points[index] + const start = points[index] + const end = points[index + 1] + const next = points[index + 2] ?? end + const sourceVector = sourceSide ? sideVector(sourceSide) : null + const targetVector = targetSide ? sideVector(targetSide) : null + let c1 = { + x: start.x + (end.x - previous.x) / 6, + y: start.y + (end.y - previous.y) / 6, } - if (node.kind === 'object') { - const objectPosition = objectLayout.find(item => item.id === node.id) ?? { x: 0, y: -OBJECT_OFFSET } - basePositions.set(node.id, { ...node, x: objectPosition.x, y: objectPosition.y }) + let c2 = { + x: end.x - (next.x - start.x) / 6, + y: end.y - (next.y - start.y) / 6, } - }) - - const incoming = new Map() - const outgoing = new Map() - contextIds.forEach(id => { - incoming.set(id, []) - outgoing.set(id, []) - }) - edges.forEach(edge => { - if (contextIds.includes(edge.source)) outgoing.get(edge.source)?.push(edge.target) - if (contextIds.includes(edge.target)) incoming.get(edge.target)?.push(edge.source) - }) - - const contextLayout = contextIds.map((id, index) => { - const downstream = (outgoing.get(id) ?? []).map(targetId => basePositions.get(targetId)).filter(Boolean) as PositionedNode[] - const upstream = (incoming.get(id) ?? []).map(sourceId => basePositions.get(sourceId)).filter(Boolean) as PositionedNode[] - const anchors = downstream.length ? downstream : upstream - if (anchors.length) { - return { - id, - x: average(anchors.map(node => node.x)), - y: average(anchors.map(node => node.y)) - OBJECT_OFFSET, - } + if (index === 0 && sourceVector) { + const offset = controlDistance(start, end) + c1 = { x: start.x + sourceVector.x * offset, y: start.y + sourceVector.y * offset } } - return { id, x: index * MIN_ROW_SPACING, y: -OBJECT_OFFSET * 2 } - }) - const contextRows = new Map>() - contextLayout.forEach(item => { - const rowKey = Math.round(item.y) - if (!contextRows.has(rowKey)) contextRows.set(rowKey, []) - contextRows.get(rowKey)!.push(item) - }) - contextRows.forEach(items => spreadRow(items, MIN_ROW_SPACING)) - contextLayout.forEach(item => { - const node = nodeMap.get(item.id) - if (node) basePositions.set(item.id, { ...node, x: item.x, y: item.y }) - }) - - const positioned: PositionedNode[] = nodes.map(node => basePositions.get(node.id) ?? { ...node, x: 0, y: 0 }) - - return positioned -} - -function packComponents(components: PositionedNode[][]) { - const packed: PositionedNode[] = [] - let cursorX = 0 - let cursorY = 0 - let rowHeight = 0 - const maxRowWidth = Math.max(1200, Math.ceil(Math.sqrt(components.length || 1)) * 900) - - const sorted = [...components].sort((a, b) => b.length - a.length) - sorted.forEach(component => { - const bounds = component.reduce( - (acc, node) => { - const width = nodeWidth(node.kind) - return { - minX: Math.min(acc.minX, node.x), - minY: Math.min(acc.minY, node.y), - maxX: Math.max(acc.maxX, node.x + width), - maxY: Math.max(acc.maxY, node.y + NODE_HEIGHT), - } - }, - { minX: Number.POSITIVE_INFINITY, minY: Number.POSITIVE_INFINITY, maxX: Number.NEGATIVE_INFINITY, maxY: Number.NEGATIVE_INFINITY }, - ) - const width = bounds.maxX - bounds.minX - const height = bounds.maxY - bounds.minY - if (cursorX > 0 && cursorX + width > maxRowWidth) { - cursorX = 0 - cursorY += rowHeight + COMPONENT_GAP_Y - rowHeight = 0 + if (index === points.length - 2 && targetVector) { + const offset = controlDistance(start, end) + c2 = { x: end.x + targetVector.x * offset, y: end.y + targetVector.y * offset } } + parts.push(`C ${c1.x} ${c1.y}, ${c2.x} ${c2.y}, ${end.x} ${end.y}`) + } + return parts.join(' ') +} + +function WorkflowEdge({ + id, + sourceX, + sourceY, + targetX, + targetY, + markerEnd, + style, + data, + label, +}: EdgeProps) { + const points = dedupePoints([ + { x: sourceX, y: sourceY }, + ...(data?.points ?? []), + { x: targetX, y: targetY }, + ]) + const path = smoothBezierPath(points, data?.sourceSide, data?.targetSide) + const labelPoint = midpointOnPolyline(points) - component.forEach(node => { - packed.push({ - ...node, - x: node.x - bounds.minX + cursorX, - y: node.y - bounds.minY + cursorY, - }) - }) - - cursorX += width + COMPONENT_GAP_X - rowHeight = Math.max(rowHeight, height) - }) - - const bounds = packed.reduce( - (acc, node) => ({ - minX: Math.min(acc.minX, node.x), - maxX: Math.max(acc.maxX, node.x + nodeWidth(node.kind)), - }), - { minX: Number.POSITIVE_INFINITY, maxX: Number.NEGATIVE_INFINITY }, + return ( + <> + + {label ? ( + +
+ {label} +
+ + ) : null} + ) - const centerShift = (bounds.minX + bounds.maxX) / 2 - return packed.map(node => ({ ...node, x: node.x - centerShift })) } function anchorEdges(edgeList: Edge[], nodeList: Node[]) { @@ -732,6 +486,7 @@ function anchorEdges(edgeList: Edge[], nodeList: Node[]) { ...edge, sourceHandle: anchors ? `source-${anchors.sourceSide}` : edge.sourceHandle, targetHandle: anchors ? `target-${anchors.targetSide}` : edge.targetHandle, + data: anchors ? { ...(edge.data ?? {}), sourceSide: anchors.sourceSide, targetSide: anchors.targetSide } : edge.data, } }) } @@ -741,35 +496,73 @@ function buildLayout(nodes: WorkflowCanvasNode[], edges: WorkflowCanvasEdge[]): const nodeMap = new Map(nodes.map(node => [node.id, node])) const validEdges = edges.filter(edge => nodeMap.has(edge.source) && nodeMap.has(edge.target)) - const components = workflowLayoutGroups(nodes, validEdges).map(componentIds => { - const idSet = new Set(componentIds) - const componentNodes = nodes.filter(node => idSet.has(node.id)) - const componentEdges = validEdges.filter(edge => idSet.has(edge.source) && idSet.has(edge.target)) - return layoutComponent(componentNodes, componentEdges) + const dagreGraph = new dagre.graphlib.Graph({ multigraph: true }) + dagreGraph.setDefaultEdgeLabel(() => ({})) + dagreGraph.setGraph({ + rankdir: 'TB', + align: 'UL', + nodesep: 62, + edgesep: 34, + ranksep: 96, + marginx: 32, + marginy: 32, + acyclicer: 'greedy', + ranker: 'network-simplex', + }) + + ;[...nodes] + .sort((a, b) => a.label.localeCompare(b.label)) + .forEach(node => { + dagreGraph.setNode(node.id, { + width: nodeWidth(node.kind), + height: NODE_HEIGHT, + }) + }) + + ;[...validEdges] + .sort((a, b) => String(a.id).localeCompare(String(b.id))) + .forEach(edge => { + dagreGraph.setEdge( + edge.source, + edge.target, + { + weight: 1, + minlen: 1, + width: edge.label ? Math.max(40, edge.label.length * 6) : 0, + height: edge.label ? 18 : 0, + }, + edge.id, + ) + }) + + dagre.layout(dagreGraph) + + const flowNodes: Node[] = nodes.map(node => { + const layoutNode = dagreGraph.node(node.id) as { x?: number; y?: number } | undefined + const width = nodeWidth(node.kind) + return { + id: node.id, + type: node.kind, + position: { + x: (layoutNode?.x ?? 0) - width / 2, + y: (layoutNode?.y ?? 0) - NODE_HEIGHT / 2, + }, + data: { id: node.id, kind: node.kind, label: node.label, title: node.title, details: node.details }, + } }) - const packedNodes = packComponents(components) - const flowNodes: Node[] = packedNodes.map(node => ({ - id: node.id, - type: node.kind, - position: { x: node.x, y: node.y }, - data: { id: node.id, kind: node.kind, label: node.label, title: node.title, details: node.details }, - })) const flowEdges: Edge[] = validEdges.map(edge => { + const layoutEdge = dagreGraph.edge({ v: edge.source, w: edge.target, name: edge.id }) as { points?: Array<{ x: number; y: number }> } | undefined return { id: edge.id, source: edge.source, target: edge.target, - type: 'smoothstep', + type: 'workflow', label: edge.label, - data: edge.title ? { title: edge.title } : undefined, + data: { title: edge.title, points: layoutEdge?.points ?? [] }, animated: false, markerEnd: { type: MarkerType.ArrowClosed, color: '#64748b' }, style: { stroke: '#64748b', strokeWidth: 1.5 }, - labelStyle: { fill: '#94a3b8', fontSize: 11 }, - labelBgStyle: { fill: 'rgba(9, 9, 11, 0.92)', fillOpacity: 1 }, - labelBgPadding: [6, 2], - labelBgBorderRadius: 6, } }) @@ -784,6 +577,10 @@ const nodeTypes = { unknown: UnknownNode, } +const edgeTypes = { + workflow: WorkflowEdge, +} + function nodeColor(kind: WorkflowNodeKind) { switch (kind) { case 'operation': @@ -824,7 +621,10 @@ export default function WorkflowCanvas({ const handleNodesChange = (changes: NodeChange[]) => { setFlowNodes(currentNodes => { const nextNodes = applyNodeChanges(changes, currentNodes) - setFlowEdges(currentEdges => anchorEdges(currentEdges, nextNodes)) + setFlowEdges(currentEdges => anchorEdges(currentEdges.map(edge => ({ + ...edge, + data: { ...(edge.data ?? {}), points: [] }, + })), nextNodes)) return nextNodes }) } @@ -963,6 +763,7 @@ export default function WorkflowCanvas({ nodes={flowNodes} edges={flowEdges} nodeTypes={nodeTypes} + edgeTypes={edgeTypes} onNodesChange={handleNodesChange} onEdgesChange={onEdgesChange} onNodeClick={(_, node) => setSelectedNodeId(node.id)} From 3532db6904af1573e404ea3eb95c6edc7b10ca66 Mon Sep 17 00:00:00 2001 From: theAfish Date: Thu, 16 Jul 2026 12:06:10 +0800 Subject: [PATCH 05/26] feat: postprocessing with scripts --- .gitignore | 2 + README.md | 746 +++++------------- .../versions/0021_post_processor_scripts.py | 31 + examples/basic_usage.py | 6 +- examples/skills/sequence_normalizer/SKILL.md | 9 + .../sequence_normalizer.py | 185 +++++ .../spaces/biomineralization_templates.json | 325 ++++++-- .../spaces/computational_materials_qa.json | 7 +- frontend/src/api/postProcessorScripts.ts | 13 + frontend/src/components/SpaceDraftCard.tsx | 45 +- .../components/projections/SectionTable.tsx | 16 +- .../src/components/projects/BrowseTab.tsx | 78 +- frontend/src/pages/ProjectionsPage.tsx | 4 +- frontend/src/pages/SpacesPage.tsx | 196 +++-- frontend/src/types/index.ts | 14 +- src/mkb/agents/projection.py | 9 +- src/mkb/agents/projection_reviewer.py | 253 +++++- src/mkb/agents/prompts/orchestrator.py | 5 +- src/mkb/agents/prompts/projection.py | 12 +- src/mkb/agents/prompts/workflow_extraction.py | 265 ++++--- src/mkb/agents/tools/orchestrator_tools.py | 8 +- src/mkb/cli.py | 2 +- src/mkb/db/models.py | 17 + src/mkb/knowledge_graph.py | 9 +- src/mkb/post_processors/__init__.py | 1 + src/mkb/post_processors/registry.py | 122 +++ src/mkb/services/feedback.py | 2 +- src/mkb/services/spaces.py | 3 +- src/mkb/spaces/registry.py | 90 ++- src/mkb/spaces/schema_utils.py | 33 + src/mkb/web/_models.py | 2 +- src/mkb/web/api_server.py | 2 + src/mkb/web/routers/post_processor_scripts.py | 26 + src/mkb/workflows/contract.py | 20 +- src/mkb/workflows/editing.py | 28 +- tests/test_space_post_processors.py | 132 ++++ tests/test_space_schema_utils.py | 29 + tests/test_workflow_contract.py | 56 ++ tests/test_workflow_editing.py | 70 ++ 39 files changed, 2037 insertions(+), 836 deletions(-) create mode 100644 alembic/versions/0021_post_processor_scripts.py create mode 100644 examples/skills/sequence_normalizer/SKILL.md create mode 100644 examples/skills/sequence_normalizer/sequence_normalizer.py create mode 100644 frontend/src/api/postProcessorScripts.ts create mode 100644 src/mkb/post_processors/__init__.py create mode 100644 src/mkb/post_processors/registry.py create mode 100644 src/mkb/web/routers/post_processor_scripts.py create mode 100644 tests/test_space_schema_utils.py diff --git a/.gitignore b/.gitignore index 0ecc25f..f23b2a3 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,8 @@ build/ *.swo *~ +.vscode/ + # Environment .env diff --git a/README.md b/README.md index 4a6932c..4e259e4 100644 --- a/README.md +++ b/README.md @@ -1,149 +1,81 @@ # mat-know-base -A self-hosted system for ingesting scientific papers and related data into a structured knowledge base. Files are stored immutably using content-addressable storage (SHA256 deduplication), with metadata tracked in PostgreSQL + pgvector and raw binaries in MinIO (S3-compatible). A processing pipeline converts raw files into LLM-readable formats. An LLM agent then extracts structured **knowledge frames** — one per research project — with flexible, agent-decided structure capturing all scientific knowledge from the source material. - -## Architecture - -``` -data/papers/smith2024/ Research package (paper + supplementary) - │ - ▼ -┌──────────────┐ SHA256 ┌─────────────────────────────────┐ -│ Ingestion ├──────────►│ MinIO (S3) │ -│ Worker │ │ raw/ ← original files │ -└──────┬───────┘ │ processed/ ← converted outputs │ - │ metadata └─────────────────────────────────┘ - ▼ ▲ -┌─────────────────┐ │ upload converted files -│ PostgreSQL │ ┌────────┴──────────┐ -│ + pgvector │ │ Processing Pipeline│ -│ │ ◄────┤ PDF → .md │ -│ assets │ │ DOCX → .md │ -│ processed_assets│ │ CSV → .parquet │ -│ project_assets │ │ IMG → .json │ -│ │ └───────────────────┘ -│ knowledge_frames│ -│ extraction_passes│ ◄── LLM extraction agent (multi-pass) -│ spaces │ -│ projections │ ◄── Projection agent (space-specific) -│ feedbacks │ ◄── Feedback loop between agents -└─────────────────┘ -``` - -### Data Flow - -1. **Ingest** — Raw files are SHA256-deduplicated, uploaded to MinIO, registered in PostgreSQL as a project -2. **Process** — Raw files are converted to LLM-readable formats (Markdown, Parquet, JSON metadata) -3. **Extract** — An LLM agent reads processed data and produces one **knowledge frame** per project (with optional multi-pass review) -4. **Project** — Domain-specific "spaces" define structured extraction schemas; projection agents extract targeted data from knowledge frames -5. **Feedback** — Projection agents flag unclear data; KB agents review and resolve feedback on user activation - -### Knowledge Frame - -Each research project produces one knowledge frame with: - -- **Paper metadata** (fixed) — title, authors, journal, year, DOI -- **Domain** (fixed) — research domain string -- **Free-form sections** (agent-decided) — the agent chooses what categories best represent the paper's knowledge (e.g., materials, experimental_data, synthesis_routes, mechanisms, etc.) - -Every extracted item is tagged with an **evidence level**: -- **Level 1**: Causal experimental evidence -- **Level 2**: Direct experimental observation -- **Level 3**: Correlative evidence -- **Level 4**: Predicted / inferred - -### Spaces & Projections - -A **Space** defines a domain-specific extraction schema (e.g., "biomineralization templates"). A **Projection** is the result of applying a Space to a knowledge frame — extracting structured data per the schema definition. - -### Agentic Feedback - -Projection agents can flag ambiguous or missing data. The KB extraction agent can review these feedback items (on user activation) and update the knowledge frame accordingly. - -### Projection Review (Multi-Agent) - -A strict **Projection Reviewer** agent consolidates and corrects projection data through a multi-agent review process: - -``` -User activates review - │ - ▼ -┌─────────────────────────┐ -│ Projection Reviewer │ reads all projections + frame + source -│ (strict data auditor) │ -└────────┬────────────────┘ - │ delegates verification - ▼ -┌─────────────────────────┐ -│ Projection Fixer │ re-reads source material -│ (sub-agent) │ returns corrections -└─────────────────────────┘ - │ - ▼ - Single reviewed projection - (consolidated, corrected) -``` - -The reviewer: -1. Loads all projection runs (from single or multiple extraction events) -2. Cross-references against the knowledge frame and original source files -3. Delegates complex verification to the fixer sub-agent -4. Produces a single **reviewed projection** — consolidated, corrected, deduplicated - -### Knowledge Graph Construction (Global Concept Graph) - -Knowledge graph construction uses a dedicated KG extraction agent with one shared global space across all domains. - -Design goals: -- **One global graph space**: all projects contribute to a shared graph so inter-domain links can emerge. -- **Concept-only nodes**: only scientific concepts become nodes. -- **Concept relations as edges**: edges encode directed concept-to-concept relations. -- **Details in references**: values/conditions/metadata are stored as references back to frame/database context, not turned into extra nodes. -- **Redundancy-aware build**: the agent checks existing graph content to reduce duplicate concepts/edges. - -Recommended usage flow: -1. Extract knowledge frames first. -2. Optionally clear old KG outputs. -3. Run KG extraction. -4. Inspect merged graph output. - -## Prerequisites +Materials Knowledge Base is a local-first tool for turning scientific papers and +supplementary files into structured knowledge. + +It can: + +- ingest a folder of papers and data files as a research project +- store raw files in MinIO and metadata in PostgreSQL/pgvector +- process PDFs, text files, spreadsheets, and images into LLM-readable outputs +- extract project-level knowledge frames with an LLM agent +- project frames into domain-specific schemas called spaces +- build and review a shared concept knowledge graph +- browse and manage the library through a React web UI + +## Requirements - Python 3.10+ -- Docker & Docker Compose -- `libmagic` (usually pre-installed on Linux; `brew install libmagic` on macOS) +- Node.js and npm +- Docker and Docker Compose +- `libmagic` + +On macOS, install `libmagic` with: + +```bash +brew install libmagic +``` + +## First-Time Setup -## Quick Start +Clone the repo, then run: ```bash -# 1. Create virtual environment and install python3 -m venv .venv source .venv/bin/activate pip install -e ".[dev]" +``` + +Create your local environment file: -# 2. Copy environment config +```bash cp .env.example .env +``` + +Edit `.env` and add your LLM credentials: + +```bash +MKB_EXTRACTION_MODEL=openai/qwen-plus +OPENAI_API_KEY=your_key_here +OPENAI_API_BASE=your_openai_compatible_base_url +``` + +For OpenAI directly, `OPENAI_API_KEY` is usually enough. For other +OpenAI-compatible providers, set both `OPENAI_API_KEY` and `OPENAI_API_BASE`. -# 3. Start infrastructure (PostgreSQL + MinIO) +Start PostgreSQL, pgvector, and MinIO: + +```bash make up +``` + +Create the database tables: -# 4. Create database tables +```bash mkb setup +``` -# 5. Install frontend dependencies once +Install the frontend dependencies: + +```bash cd frontend npm install cd .. - -# 6. Start backend API + React frontend together -bash scripts/dev.sh ``` -Open http://127.0.0.1:5173. +## Run The App -## Easiest Daily Usage (React + API) - -After first-time setup, day-to-day startup is just: +Start the backend API and React frontend together: ```bash source .venv/bin/activate @@ -151,503 +83,263 @@ make up bash scripts/dev.sh ``` -`scripts/dev.sh` starts both services in one terminal: -- backend API: `make server` (http://127.0.0.1:8503) -- frontend dev server: `cd frontend && npm run dev` (http://127.0.0.1:5173) +Open: + +- React UI: http://127.0.0.1:5173 +- Backend API: http://127.0.0.1:8503 +- MinIO console: http://127.0.0.1:9001 -Press `Ctrl+C` once to stop both. +MinIO local login: -If the UI appears empty: +```text +minioadmin / minioadmin +``` + +Press `Ctrl+C` in the `scripts/dev.sh` terminal to stop the app servers. + +Stop Docker services with: ```bash -mkb projects -mkb frames -curl http://127.0.0.1:8503/api/projects?limit=5 +make down ``` -- If `mkb projects` has rows but the UI is empty, the API is likely not running on port 8503. -- If both CLI and API are empty, ingest data first (`mkb ingest ...` or upload from the Projects page). +## Basic Workflow -## Python API (Primary Interface) +Put one paper and its supplementary files in a folder, for example: -The recommended interface is `mkb.api`. See `examples/basic_usage.py` for a complete walkthrough. +```text +data/papers/smith2024/ + paper.pdf + supplement.csv + notes.txt +``` -```python -from mkb import api +Then run: -# Setup -api.setup() +```bash +mkb ingest ./data/papers/smith2024 --label "Smith 2024" +mkb process +mkb extract --max-passes 2 +``` -# Ingest & process -result = api.ingest("./data/papers/smith2024", label="Smith 2024") -api.process() +Check the result: -# Extract with multi-pass review -api.extract(max_passes=2, verbose=True) +```bash +mkb projects +mkb frames +mkb frame +``` -# Query frames -frame = api.get_frame(project_id="...") -print(frame["content"]["paper"]) -print(frame["content"].keys()) # agent-decided sections +You can also do the same from the React UI at http://127.0.0.1:5173. -# Spaces & projections -api.create_space( - name="biomineralization", - domain="biomineralization", - extraction_schema={...}, - system_prompt="...", - field_descriptions={...}, -) -api.project(space_id="...", project_id="...") +## Python API -# Feedback -api.list_feedback(project_id="...", status="OPEN") -api.review_feedback(project_id="...") +The main Python interface is `mkb.api`. -# Projection review (multi-agent) -api.review_projections(space_id="...", project_id="...") -api.review_projections_all(space_id="...") -api.list_reviewed_projections(space_id="...") -api.get_reviewed_projection(reviewed_projection_id="...") +```python +from mkb import api -# Knowledge graph construction (global concept graph) -api.clear_knowledge_graphs() -api.extract_knowledge_graph(project_id="...") -kg = api.get_knowledge_graph() -print(len(kg["graph"]["concepts"]), len(kg["graph"]["relations"])) +api.setup() -# Knowledge graph review (deduplication + quality cleanup) -api.review_knowledge_graph() # auto mode (random global or local) -api.review_knowledge_graph(mode="global", verbose=True) # full graph: standardize + dedup -api.review_knowledge_graph(mode="local", seed_count=15) # neighborhood review (least-reviewed first) -counts = api.get_graph_review_counts() # per-element review counters +project = api.ingest("./data/papers/smith2024", label="Smith 2024") +api.process(project_id=project["project_id"]) +api.extract(project_id=project["project_id"], max_passes=2) -# Search papers and data -results = api.search_library("enamel mineralization") -print(results["projects"]) -print(results["assets"]) +frame = api.get_frame(project["project_id"]) +print(frame["content"].keys()) ``` -## CLI +See [examples/basic_usage.py](examples/basic_usage.py) for a longer walkthrough. + +## Common CLI Commands + +Database: ```bash -# Database mkb setup mkb reset-db +``` -# Ingestion +Ingest and process: + +```bash mkb ingest ./data/papers/smith2024 --label "Smith 2024" mkb sync --root-dir ./data/papers - -# Processing mkb process -mkb process --project-id +mkb process --project-id +mkb processed --project-id +``` -# Knowledge extraction -mkb extract # all pending -mkb extract --project-id # one project -mkb extract --max-passes 3 # multi-pass -mkb extract --model openai/gpt-4o # override model -mkb extraction-history # view pass history +Extract and inspect knowledge frames: -# Listing +```bash +mkb extract +mkb extract --project-id --max-passes 2 mkb projects -mkb assets --project-id -mkb search "enamel mineralization" -mkb search "csv supplement" --project-id +mkb assets --project-id mkb frames mkb frame - -# Spaces & Projections -mkb space create --name catalysis --domain catalysis --schema-file schema.json -mkb space load space_definition.json -mkb space list -mkb space show catalysis -mkb project-run --space --project-id -mkb project-run --space --all -mkb projections --space-id - -# Feedback -mkb feedback --project-id --status OPEN -mkb review-feedback --project-id -mkb resolve-feedback --status RESOLVED --notes "Fixed" - -# Projection Review (multi-agent consolidation) -mkb review-projections --space --project-id -mkb review-projections --space --all -mkb reviewed-projections --space-id -mkb reviewed-projection - -# Knowledge graph construction (global concept graph) -mkb kg-clear # clear old KG projections (and legacy frame-graph sections) -mkb kg-extract # build KG for all completed frames -mkb kg-extract --project-id # build KG for one project -mkb kg-extract --frame-id # build KG for one frame -mkb kg-show # show merged global concept graph -mkb kg-show --project-id # merged graph filtered to one project - -# Start the FastAPI backend (serves the React UI in production) -make server - -# Start legacy Streamlit UI (optional) -mkb ui --port 8501 +mkb extraction-history +mkb search "hydroxyapatite nucleation" ``` -## Search +Run the web services separately: -Keyword search is available in all three interfaces: +```bash +mkb api --host 127.0.0.1 --port 8503 +cd frontend && npm run dev +``` -- **UI**: The **Research Projects → Browse** view includes a search box for papers and ingested data assets. -- **Python API**: `api.search_library(query, limit=25, project_id=None)` returns matching projects and assets. -- **CLI**: `mkb search "keywords"` prints matching projects and assets, with optional `--project-id` scoping. +The legacy Streamlit UI is still available: -Search behavior: +```bash +mkb ui --port 8501 +``` -- Queries are split into whitespace-separated keyword tokens. -- All tokens must match somewhere in a result. -- Project matches use project label and source path. -- Asset matches use filename, MIME type, and selected asset metadata fields. +## Spaces And Projections -## Knowledge Graph Quickstart +A space defines a domain-specific schema. A projection applies that schema to a +knowledge frame. -```bash -# 1) Make sure knowledge frames exist -mkb extract +Create a space from a JSON definition: -# 2) Optional clean rebuild -mkb kg-clear +```bash +mkb space load examples/spaces/computational_materials_qa.json +``` -# 3) Construct concept graph projections -mkb kg-extract +List and inspect spaces: -# 4) Inspect merged graph -mkb kg-show +```bash +mkb space list +mkb space show ``` -Project-scoped example: +Run projections: ```bash -mkb kg-clear --project-id -mkb kg-extract --project-id -mkb kg-show --project-id +mkb project-run --space --project-id +mkb project-run --space --all +mkb projections --space-id +mkb projection ``` -Notes: -- `kg-extract` clears existing KG projections for target frame(s) by default. Use `--no-clear-existing` to keep prior projection history. -- Legacy graph-like sections inside frame content are removed by default during cleanup/extraction. Use `--keep-legacy-frame-graphs` to skip that behavior. +Review and consolidate projections: -## Knowledge Graph Review +```bash +mkb review-projections --space --project-id +mkb review-projections --space --all +``` -After building the graph, a dedicated **Graph Review Agent** deduplicates concepts, standardizes relation naming, and prunes low-quality entries. It runs in two modes: +## Knowledge Graph -| Mode | What it does | -|------|-------------| -| **global** | Analyzes the full graph: groups similar relation names and standardizes them; finds and merges synonymous concept nodes across all projections | -| **local** | Selects the least-reviewed concepts as starting points, explores their neighborhood, verifies ambiguous entries against source knowledge frames, and fixes local issues | +Build a shared concept graph from completed knowledge frames: -Each run tracks how many times each node/edge was examined and modified in the `graph_element_reviews` table — always incremented by the orchestration script, never by the agent itself. +```bash +mkb kg-clear +mkb kg-extract +mkb kg-show +``` -### Python API +For one project: -```python -# Global mode: relation standardization + concept deduplication -api.review_knowledge_graph(mode="global", verbose=True) - -# Local mode: deep-dive on least-reviewed concepts -api.review_knowledge_graph(mode="local", seed_count=15, verbose=True) - -# Auto: randomly picks global or local each time (default) -api.review_knowledge_graph() - -# Inspect per-element review counts -counts = api.get_graph_review_counts() -# counts["concepts"]["hydroxyapatite"] → {"times_examined": 3, "times_modified": 1, ...} -# counts["relations"]["amelotin||promotes||hydroxyapatite nucleation"] → {...} -``` - -### Review tools available to the agent - -| Tool | Mode | Description | -|------|------|-------------| -| `get_concept_details` | both | Full concept record + all incoming/outgoing relations | -| `get_concept_neighbors` | both | Concept + 1-hop neighbors + relations | -| `get_relation_type_distribution` | both | Count of each distinct relation label | -| `search_graph_elements` | both | Keyword search across concept labels, aliases, relation names | -| `find_similar_concepts` | both | Token-overlap similarity search for near-duplicate concepts | -| `merge_concepts` | both | Merge N concepts into one canonical node across all projections | -| `standardize_relation_name` | both | Rename relation type(s) to a canonical form everywhere | -| `delete_concept` | both | Delete an isolated concept (rejects if relations still exist) | -| `delete_relation` | both | Delete a specific directed relation | -| `get_frame_content` | local | Read a source knowledge frame for concept verification | - -### Knowledge Graph Output Shape - -`mkb kg-show` and `api.get_knowledge_graph()` return a normalized concept graph: - -```json -{ - "graph": { - "concepts": [ - { - "label": "Amelotin", - "aliases": ["AMTN"], - "source_project_ids": ["..."], - "source_frame_ids": ["..."], - "knowledge_refs": [ - { - "project_id": "...", - "frame_id": "...", - "field_path": "...", - "snippet": "..." - } - ] - } - ], - "relations": [ - { - "source": "Amelotin", - "relation": "promotes", - "target": "Hydroxyapatite nucleation", - "evidence_level": 2, - "source_project_id": "...", - "source_frame_id": "...", - "knowledge_ref": { - "project_id": "...", - "frame_id": "...", - "field_path": "...", - "snippet": "..." - } - } - ] - } -} -``` - -## LLM Configuration - -Knowledge extraction uses google-adk with LiteLLM. Configure in `.env`: - -```bash -MKB_EXTRACTION_MODEL=openai/deepseek-v4-pro-guan -LLM_API_KEY= -LLM_API_BASE= -``` - -For OpenAI itself, `OPENAI_API_KEY` / `OPENAI_API_BASE` still work. For -non-OpenAI models exposed through an OpenAI-compatible endpoint, prefer the -provider-agnostic `LLM_API_KEY` / `LLM_API_BASE` names above, and use -`openai/` as the model id. - -## Project Structure - -``` -src/mkb/ -├── api.py # Primary Python interface -├── cli.py # CLI (thin wrapper around api) -├── config.py # Settings from .env -├── db/ -│ ├── engine.py # SQLAlchemy engine + init_db() -│ └── models.py # ORM models (12 tables + enums) -├── storage/ -│ └── s3.py # MinIO upload/download/exists/delete -├── ingest/ -│ └── worker.py # CAS ingestion (SHA256, MIME, batching) -├── processors/ -│ ├── base.py # Abstract Processor + ProcessingResult -│ ├── coordinator.py # Routes assets to processors -│ ├── pdf_processor.py -│ ├── text_processor.py -│ ├── dataframe_processor.py -│ └── image_processor.py -├── agents/ -│ ├── extraction.py # KB extraction agent + multi-pass orchestration -│ ├── review.py # Review agent for multi-turn extraction -│ ├── projection.py # Projection agent (space-specific extraction) -│ ├── knowledge_graph.py # KG agent (global concept graph extraction) -│ ├── projection_reviewer.py # Projection reviewer (multi-agent consolidation) -│ ├── projection_fixer.py # Fixer sub-agent (source verification) -│ ├── feedback_reviewer.py # Feedback review agent -│ ├── dev_agent.py # Dev agent interface (design only) -│ ├── runner.py # Generic AgentRunner wrapper -│ ├── prompts/ # Agent prompts -│ │ ├── kb_extraction.py # Flexible KB extraction prompt -│ │ ├── review.py # Review pass prompt -│ │ ├── projection.py # Projection prompt builder -│ │ ├── knowledge_graph.py # Concept-graph extraction prompt -│ │ ├── projection_review.py # Projection reviewer prompt -│ │ ├── projection_fixer.py # Fixer sub-agent prompt -│ │ └── feedback_review.py # Feedback review prompt -│ └── tools/ # Agent tool functions -│ ├── reading.py # Reading tools (markdown, dataframe, image, search) -│ ├── frames.py # Frame save/get/update tools -│ ├── projection.py # Projection save + flag_for_feedback -│ ├── knowledge_graph.py # Concept-graph tools + redundancy checks -│ ├── projection_review.py # Projection review + re-extraction tools -│ └── feedback.py # Feedback query + resolve tools -├── spaces/ -│ └── registry.py # Space CRUD operations -├── feedback/ -│ └── manager.py # Feedback CRUD + resolution -└── ui/ - ├── app.py # Streamlit entry point - ├── pages/ # UI pages - │ ├── projects.py - │ ├── frames.py - │ ├── projections.py - │ └── feedback.py - └── components/ # Reusable UI components - ├── frame_viewer.py - └── graph_viz.py -``` - -## Database Tables - -| Table | Purpose | -|---|---| -| `research_projects` | One per research package (paper + supplementary) | -| `assets` | One row per unique raw file (SHA256 deduplicated) | -| `project_assets` | Many-to-many link between projects and assets | -| `processed_assets` | One row per successful conversion output | -| `processing_logs` | Audit trail for processing attempts | -| `knowledge_frames` | One structured frame per project (JSONB content + metadata) | -| `extraction_passes` | Audit trail for each extraction/review pass | -| `spaces` | Domain-specific extraction configurations | -| `projections` | Results of projecting frames through spaces | -| `feedbacks` | Feedback items between agents | -| `graph_element_reviews` | Per-element review counts (`times_examined`, `times_modified`) for graph nodes and edges | - -## Data Sharing - -Because the database and files are developed locally (papers, processed outputs, PostgreSQL, MinIO) and cannot be committed to git, two helper scripts let you snapshot and restore the entire local state. - -### Pack (create a snapshot) - -```bash -# Auto-named: mkb_data_YYYYMMDD_HHMMSS.tar.gz -make pack +```bash +mkb kg-clear --project-id +mkb kg-extract --project-id +mkb kg-show --project-id +``` -# Custom filename -make pack out=my_dataset_v1.tar.gz +Review the graph: -# Or run directly -bash scripts/pack_data.sh my_snapshot.tar.gz +```bash +mkb kg-review --mode global +mkb kg-review --mode local --seed-count 10 +mkb kg-review-counts ``` -What gets bundled: -- **PostgreSQL** — full `pg_dump` of the `mkb` database (schema + data) -- **MinIO buckets** — `raw`, `processed`, `archive`, `temp` -- **Local dirs** — `data/papers/`, `data/processed/`, `data/uploads/`, `data/inbox/` -- **manifest.json** — records timestamp, database name, and bucket list - -Requirements: Docker (already needed), `tar`, `python3`. +## Workflow Extraction -### Unpack (restore from a snapshot) +The repo also includes workflow extraction, review, schema curation, and +maintenance commands: ```bash -# Full restore (interactive confirmation before dropping the DB) -make unpack file=mkb_data_20260429_120000.tar.gz - -# Or run directly -bash scripts/unpack_data.sh mkb_data_20260429_120000.tar.gz +mkb workflow-review --status active +mkb workflow-correct corrected_graph.json --reason "..." --author "..." --evidence "..." +mkb schema-curate +mkb schema-proposals +mkb schema-review --approve --reviewer "..." +mkb workflow-reextract --reason low_quality_extraction --run +mkb workflow-recanonicalize --run +mkb workflow-index +mkb workflow-search --source "..." --operation "..." ``` -Partial restore flags: +For design details, see: -| Flag | Effect | -|------|--------| -| `--pg-only` | Restore PostgreSQL only | -| `--minio-only` | Restore MinIO buckets only | -| `--local-only` | Restore local data dirs only | -| `--no-pg` | Skip PostgreSQL restore | -| `--no-minio` | Skip MinIO buckets | -| `--no-local` | Skip local data dirs | +- [docs/workflow-lifecycle-policy.md](docs/workflow-lifecycle-policy.md) +- [docs/workflow-card-architecture.md](docs/workflow-card-architecture.md) +- [docs/architecture-map.md](docs/architecture-map.md) -After unpacking, run `alembic upgrade head` if the schema migration level differs between the snapshot and your current codebase. +## Data Snapshots -### Typical workflow for onboarding a new developer +The database and MinIO files are local runtime state, so they are not committed +to git. Use the helper scripts to move a local dataset between machines. -```bash -# 1. Clone repo and install -python3 -m venv .venv && source .venv/bin/activate -pip install -e ".[dev]" +Create a snapshot: -# 2. Start services -make up +```bash +make pack +``` -# 3. Restore a shared snapshot -make unpack file=mkb_data_20260429_120000.tar.gz +Restore a snapshot: -# 4. Apply any pending migrations -alembic upgrade head +```bash +make unpack file=mkb_data_YYYYMMDD.tar.gz ``` -## Frontend (React + Vite) +## Development -A React 19 + Vite + TypeScript UI lives in `frontend/`. It replaces the legacy Streamlit UI and communicates with the FastAPI backend through a Vite dev-server proxy. +Run Python tests: -### Prerequisites - -- Node.js 20+ and npm 10+ -- Backend running (`make server`) +```bash +pytest +``` -### Installation +Run frontend type checks and build: ```bash cd frontend -npm install +npm run build ``` -### Development +Run the project checks: ```bash -# In one terminal — start the backend -make server - -# In another terminal — start the Vite dev server -cd frontend -npm run dev +make check ``` -Open http://localhost:5173 (or the port shown in the Vite output). +Useful docs: + +- [docs/development.md](docs/development.md) +- [src/mkb/ui/README.md](src/mkb/ui/README.md) -All `/api/*` requests are proxied to `http://127.0.0.1:8503` by Vite, so no CORS configuration is needed during development. +## Troubleshooting -### Production build +If the UI is empty, check that the API is running: ```bash -cd frontend -npm run build # output goes to frontend/dist/ -npm run preview # serve the production build locally +curl http://127.0.0.1:8503/api/projects?limit=5 ``` -### Pages - -This frontend is a single-page app with sidebar navigation (not URL routes). - -| Page | Description | -|------|-------------| -| Projects | Browse, upload, and manage research packages | -| Knowledge Frames | View extracted frames; run processing, extraction, projection, and graph pipelines per project | -| Projections | Aggregated projection table across papers for a selected space | -| Dataset Graph | Interactive force-directed concept graph (vis-network, Barnes-Hut physics); supports node/edge coloring by evidence level, review coverage, modification heat, and connectivity | -| Assistant | LLM assistant interface | -| Feedback | Review and resolve feedback items | - -### Tech stack +If the API has no projects, ingest data first: -- **React 19** + **TypeScript** -- **Vite 8** (build tool + dev proxy) -- **Tailwind CSS v4** (CSS-first config, no `tailwind.config.js`) -- **vis-network 10** — knowledge graph visualization (same library as pyvis) -- **Zustand v4** — UI state management -- **Axios v1** — HTTP client +```bash +mkb ingest ./data/papers/smith2024 --label "Smith 2024" +``` -## Services +If `scripts/dev.sh` fails, make sure both dependency installs completed: -| Service | URL | Credentials | -|---------|-----|------------| -| MinIO Console | http://localhost:9001 | minioadmin / minioadmin | -| MinIO S3 API | http://localhost:9000 | minioadmin / minioadmin | -| PostgreSQL | localhost:5432 | mkb / mkb_dev | -| FastAPI backend | http://localhost:8503 | — | -| React UI (dev) | http://localhost:5173 | — | +```bash +source .venv/bin/activate +pip install -e ".[dev]" +cd frontend && npm install +``` diff --git a/alembic/versions/0021_post_processor_scripts.py b/alembic/versions/0021_post_processor_scripts.py new file mode 100644 index 0000000..55b6793 --- /dev/null +++ b/alembic/versions/0021_post_processor_scripts.py @@ -0,0 +1,31 @@ +"""Add deterministic post-processor scripts. + +Revision ID: 0021_post_processor_scripts +Revises: 0020_custom_skills +Create Date: 2026-07-16 +""" + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + + +revision = "0021_post_processor_scripts" +down_revision = "0020_custom_skills" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "post_processor_scripts", + sa.Column("script_id", postgresql.UUID(as_uuid=True), primary_key=True), + sa.Column("name", sa.String(length=255), nullable=False), + sa.Column("filename", sa.String(length=255), nullable=False), + sa.Column("storage_path", sa.Text(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + ) + + +def downgrade() -> None: + op.drop_table("post_processor_scripts") \ No newline at end of file diff --git a/examples/basic_usage.py b/examples/basic_usage.py index d56f062..31a4ff4 100644 --- a/examples/basic_usage.py +++ b/examples/basic_usage.py @@ -109,6 +109,7 @@ extraction_schema={ "catalysts": { "type": "list", + "description": "All catalyst materials studied, including composition and performance metrics.", "item_schema": { "name": {"type": "string", "required": True}, "composition": {"type": "string", "required": True}, @@ -120,6 +121,7 @@ }, "reactions": { "type": "list", + "description": "All chemical reactions described, with reactants, products, and conditions.", "item_schema": { "name": {"type": "string", "required": True}, "reactants": {"type": "list", "required": True}, @@ -130,10 +132,6 @@ }, }, system_prompt="Extract catalyst materials and reactions from this paper.", - field_descriptions={ - "catalysts": "All catalyst materials studied, including composition and performance metrics.", - "reactions": "All chemical reactions described, with reactants, products, and conditions.", - }, description="Heterogeneous catalysis data extraction", ) print(f"Space created: {space_result}") diff --git a/examples/skills/sequence_normalizer/SKILL.md b/examples/skills/sequence_normalizer/SKILL.md new file mode 100644 index 0000000..8889757 --- /dev/null +++ b/examples/skills/sequence_normalizer/SKILL.md @@ -0,0 +1,9 @@ +# Sequence Normalizer + +Normalizes amino-acid sequence strings in biomineralization projection rows. + +Run `sequence_normalizer.py` as a post-processor script. It accepts the standard +post-processor JSON payload on stdin and prints one JSON result. It processes the +selected live projection, writes `normalized_sequence` and +`sequence_normalization` fields for successful normalizations, and asks the +reviewer agent to resolve uncertain sequences. \ No newline at end of file diff --git a/examples/skills/sequence_normalizer/sequence_normalizer.py b/examples/skills/sequence_normalizer/sequence_normalizer.py new file mode 100644 index 0000000..2949c9a --- /dev/null +++ b/examples/skills/sequence_normalizer/sequence_normalizer.py @@ -0,0 +1,185 @@ +"""Script-first amino-acid sequence normalization post-processor.""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass, field +import json +import re +import sys + + +CANONICAL = set("ACDEFGHIKLMNPQRSTVWY") +AA3 = { + "ALA": "A", "ARG": "R", "ASN": "N", "ASP": "D", "CYS": "C", + "GLN": "Q", "GLU": "E", "GLY": "G", "HIS": "H", "ILE": "I", + "LEU": "L", "LYS": "K", "MET": "M", "PHE": "F", "PRO": "P", + "SER": "S", "THR": "T", "TRP": "W", "TYR": "Y", "VAL": "V", +} +AA_FULL = { + "ALANINE": "A", "ARGININE": "R", "ASPARAGINE": "N", "ASPARTATE": "D", + "ASPARTICACID": "D", "CYSTEINE": "C", "GLUTAMINE": "Q", "GLUTAMATE": "E", + "GLUTAMICACID": "E", "GLYCINE": "G", "HISTIDINE": "H", "ISOLEUCINE": "I", + "LEUCINE": "L", "LYSINE": "K", "METHIONINE": "M", "PHENYLALANINE": "F", + "PROLINE": "P", "SERINE": "S", "THREONINE": "T", "TRYPTOPHAN": "W", + "TYROSINE": "Y", "VALINE": "V", +} +PTMS = { + "S": r"\[pS\]|\(pS\)|pSer|phosphoserine|pS", + "T": r"\[pT\]|\(pT\)|pThr|phosphothreonine|pT", + "Y": r"\[pY\]|\(pY\)|pTyr|phosphotyrosine|pY", +} +N_TERMINI = [("Ace-", "Acetylation"), ("Ac-", "Acetylation"), ("Acetyl-", "Acetylation"), ("NH2-", "Free amine")] +C_TERMINI = [("-CONH2", "Amidation"), ("-NH2", "Amidation"), ("-Amide", "Amidation")] + + +@dataclass +class NormalizeResult: + normalized_sequence: str + need_normalization: bool + success: bool + modifications: list = field(default_factory=list) + terminal_modifications: dict = field(default_factory=lambda: {"n_terminal": [], "c_terminal": []}) + removed_groups: list = field(default_factory=list) + removed_noncanonical_residues: list = field(default_factory=list) + unknown_tokens: list = field(default_factory=list) + warnings: list = field(default_factory=list) + confidence: float = 1.0 + + +def _expand_repeats(text: str, result: NormalizeResult) -> str: + stack = [("", None)] + last_closed = "" + index = 0 + while index < len(text): + char = text[index] + if char in "([": + stack.append(("", char)) + index += 1 + continue + if char in ")]": + close = ")" if char == ")" else "]" + opener = "(" if close == ")" else "[" + end = index + 1 + while end < len(text) and text[end].isdigit(): + end += 1 + count_text = text[index + 1:end] + count = int(count_text) if count_text else 1 + if len(stack) > 1 and stack[-1][1] == opener: + content, _ = stack.pop() + last_closed = content * count + stack[-1] = (stack[-1][0] + last_closed, stack[-1][1]) + elif count_text and last_closed: + stack[-1] = (stack[-1][0] + last_closed * count, stack[-1][1]) + result.warnings.append("Unmatched closing repeat bracket interpreted as implicit outer repeat.") + else: + stack[-1] = (stack[-1][0] + text[index:end], stack[-1][1]) + result.warnings.append("Unmatched closing bracket could not be expanded.") + index = end + continue + stack[-1] = (stack[-1][0] + char, stack[-1][1]) + index += 1 + if len(stack) > 1: + result.warnings.append("Unclosed repeat brackets detected; content was kept literally.") + return "".join(buffer for buffer, _ in stack) + + +def normalize_sequence(sequence: str) -> NormalizeResult: + raw = str(sequence or "").strip() + if re.fullmatch(r"[ACDEFGHIKLMNPQRSTVWY]+", raw): + return NormalizeResult(raw, False, True) + result = NormalizeResult("", True, True) + text = raw + for token, label in N_TERMINI: + if text.lower().startswith(token.lower()): + result.terminal_modifications["n_terminal"].append(label) + text = text[len(token):] + break + for token, label in C_TERMINI: + if text.lower().endswith(token.lower()): + result.terminal_modifications["c_terminal"].append(label) + text = text[:-len(token)] + break + text = text.replace("-", " ") + for match in reversed(list(re.finditer(r"\(([^()]*)\)", text))): + words = re.findall(r"[A-Za-z]+", match.group(1)) + if len(words) >= 3 and sum(any(char.islower() for char in word) for word in words) >= 3: + result.removed_groups.append({"type": "annotation", "text": match.group(1)}) + text = text[:match.start()] + text[match.end():] + text = _expand_repeats(text, result) + ptm_pattern = "|".join(f"(?:{pattern})" for pattern in PTMS.values()) + pieces = [] + for fragment in re.split(f"({ptm_pattern})", text, flags=re.I): + if not fragment: + continue + if re.fullmatch(ptm_pattern, fragment, flags=re.I): + pieces.append(fragment) + else: + pieces.extend(re.findall(r"[A-Za-z0-9]+|[\[\]\(\)]", fragment)) + output = [] + for token in pieces: + for residue, pattern in PTMS.items(): + if re.fullmatch(pattern, token, flags=re.I): + output.append(residue) + result.modifications.append({"position": len(output), "residue": residue, "type": "phosphorylation", "original": token}) + break + else: + upper = token.upper() + if upper in AA3: + output.append(AA3[upper]) + elif upper in AA_FULL: + output.append(AA_FULL[upper]) + elif token == upper and re.fullmatch(r"[ACDEFGHIKLMNPQRSTVWY]+", token): + output.extend(token) + elif re.fullmatch(r"[A-Za-z]+", token): + result.removed_groups.append({"type": "annotation", "token": token}) + else: + result.unknown_tokens.append(token) + result.normalized_sequence = "".join(output) + if result.unknown_tokens: + result.success, result.confidence = False, 0.5 + elif result.removed_groups: + result.success, result.confidence = False, 0.6 + result.warnings.append("Removed annotation text may contain sequence-relevant information; manual review required.") + return result + + +def main() -> None: + payload = json.load(sys.stdin) + projections = payload.get("projections") or [] + if not projections: + print(json.dumps({"run_agent": False, "context": {"message": "No projections to normalize."}})) + return + projection = projections[0] + data = projection.get("data") or {} + updates, reports, failures = [], [], [] + for collection, source_field in (("templates", "sequence"), ("functional_modules", "amino_acid_sequence")): + for index, row in enumerate(data.get(collection) or []): + raw = row.get(source_field) if isinstance(row, dict) else None + if not isinstance(raw, str) or not raw.strip(): + continue + result = normalize_sequence(raw) + report = asdict(result) + base = f"{collection}[{index}]" + updates.append({"path": f"{base}.sequence_normalization", "value": report}) + if result.success: + updates.append({"path": f"{base}.normalized_sequence", "value": result.normalized_sequence}) + entry = {"path": base, "source_field": source_field, "original_sequence": raw, "result": report} + reports.append(entry) + if not result.success: + failures.append(entry) + patch = None + if updates: + patch = { + "winning_projection_id": projection["projection_id"], + "updates": updates, + "review_notes": f"Sequence normalizer processed {len(reports)} sequence field(s); {len(failures)} require review.", + } + print(json.dumps({ + "run_agent": bool(failures), + "patch": patch, + "context": {"kind": "sequence_normalization", "processed": reports, "failed": failures}, + })) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/examples/spaces/biomineralization_templates.json b/examples/spaces/biomineralization_templates.json index 9ba59f5..eb1f77f 100644 --- a/examples/spaces/biomineralization_templates.json +++ b/examples/spaces/biomineralization_templates.json @@ -5,80 +5,293 @@ "extraction_schema": { "templates": { "type": "list", - "description": "Biomineralization template entries — proteins, peptides, or other molecules that directly participate in or regulate mineralization.", - "filter": {"field": "experimental_role", "equals": "primary_template"}, + "description": "Biomineralization template entries — proteins, peptides, or other molecules that directly participate in or regulate mineralization.\n\nExtraction guidance: Extract only biomineralization-relevant template molecules that meet the inclusion criteria. Exclude controls, comparison proteins, and background references unless the paper directly demonstrates a mineralization role. Preserve all reported species and functional roles as lists. Example: Amelogenin from Homo sapiens and Mus musculus enamel studies, a full-length protein acting on calcium phosphate with roles in nucleation promotion and crystal orientation control.", + "filter": { + "field": "experimental_role", + "equals": "primary_template" + }, "item_schema": { - "template_name": {"type": "string", "required": true, "description": "Name of the template molecule (e.g., Amelogenin, Osteopontin)"}, - "source_species": {"type": "list", "item_type": "string", "required": true, "description": "All species involved in the experiments or molecular origin (e.g., Homo sapiens, Mus musculus, Pinctada fucata)"}, - "source_system": {"type": "string", "required": true, "description": "Biological system where the template operates (e.g., Enamel matrix, Nacre layer, Bone matrix)"}, - "molecule_type": {"type": "string", "required": true, "description": "Type of molecule (e.g., Full-length protein, Matrix protein, Peptide, Glycoprotein)"}, - "template_role": {"type": "string", "required": false, "description": "Role/class of the tested molecule: native_protein, derived_peptide, synthetic_peptide, or mutant"}, - "experimental_role": {"type": "string", "required": false, "description": "Role in the experiment: primary_template, control, comparison, or background_reference"}, - "mineralization_system": {"type": "string", "required": true, "description": "The mineral system influenced (e.g., Calcium phosphate, Calcium carbonate, Silica)"}, - "functional_tags": {"type": "list", "item_type": "string", "required": true, "description": "List of functional roles (e.g., Nucleation promotion, Crystal orientation control, Crystal growth regulation, Ion enrichment)"}, - "sequence": {"type": "string", "required": false, "description": "Amino acid sequence if reported"}, - "molecular_weight_kda": {"type": "number", "required": false, "description": "Molecular weight in kDa if reported"}, - "isoelectric_point": {"type": "number", "required": false, "description": "pI value if reported"}, - "evidence_level": {"type": "integer", "required": true, "description": "Highest applicable evidence level: 1 = in vivo functional validation, 2 = in vitro mineralization experiment, 3 = indirect experimental evidence, 4 = prediction/hypothesis/inference"}, - "references": {"type": "string", "required": false, "description": "PMID, DOI, or other literature identifier"}, - "notes": {"type": "string", "required": false, "description": "Additional notes or observations"} + "template_name": { + "type": "string", + "required": true, + "description": "Name of the template molecule (e.g., Amelogenin, Osteopontin)" + }, + "source_species": { + "type": "list", + "item_type": "string", + "required": true, + "description": "All species involved in the experiments or molecular origin (e.g., Homo sapiens, Mus musculus, Pinctada fucata)" + }, + "source_system": { + "type": "string", + "required": true, + "description": "Biological system where the template operates (e.g., Enamel matrix, Nacre layer, Bone matrix)" + }, + "molecule_type": { + "type": "string", + "required": true, + "description": "Type of molecule (e.g., Full-length protein, Matrix protein, Peptide, Glycoprotein)" + }, + "template_role": { + "type": "string", + "required": false, + "description": "Role/class of the tested molecule: native_protein, derived_peptide, synthetic_peptide, or mutant" + }, + "experimental_role": { + "type": "string", + "required": false, + "description": "Role in the experiment: primary_template, control, comparison, or background_reference" + }, + "mineralization_system": { + "type": "string", + "required": true, + "description": "The mineral system influenced (e.g., Calcium phosphate, Calcium carbonate, Silica)" + }, + "functional_tags": { + "type": "list", + "item_type": "string", + "required": true, + "description": "List of functional roles (e.g., Nucleation promotion, Crystal orientation control, Crystal growth regulation, Ion enrichment)" + }, + "sequence": { + "type": "string", + "required": false, + "description": "Amino acid sequence if reported" + }, + "normalized_sequence": { + "type": "string", + "required": false, + "description": "Canonical one-letter amino-acid sequence produced by the seq_norm post-processor. Preserve the reported sequence in sequence." + }, + "sequence_normalization": { + "type": "object", + "required": false, + "description": "Structured seq_norm result: original sequence, candidate normalized sequence, success flag, confidence, modifications, warnings, and unresolved tokens." + }, + "molecular_weight_kda": { + "type": "number", + "required": false, + "description": "Molecular weight in kDa if reported" + }, + "isoelectric_point": { + "type": "number", + "required": false, + "description": "pI value if reported" + }, + "evidence_level": { + "type": "integer", + "required": true, + "description": "Highest applicable evidence level: 1 = in vivo functional validation, 2 = in vitro mineralization experiment, 3 = indirect experimental evidence, 4 = prediction/hypothesis/inference" + }, + "references": { + "type": "string", + "required": false, + "description": "PMID, DOI, or other literature identifier" + }, + "notes": { + "type": "string", + "required": false, + "description": "Additional notes or observations" + } } }, "functional_modules": { "type": "list", - "description": "Functional fragments or domains within biomineralization templates that have specific mineralization-related activity.", + "description": "Functional fragments or domains within biomineralization templates that have specific mineralization-related activity.\n\nExtraction guidance: Extract specific fragments, domains, or motifs within templates that have demonstrated or predicted mineralization activity. Include position, sequence (if available), structural features, and functional role. Example: N-terminal acidic fragment of Amelogenin (positions 1-25), Ser-rich/acidic, promotes nucleation.", "item_schema": { - "parent_protein": {"type": "string", "required": true, "description": "Name of the parent protein/template"}, - "fragment_name": {"type": "string", "required": true, "description": "Name or descriptor of the fragment (e.g., N-terminal acidic fragment, Asp-rich motif)"}, - "start_end_position": {"type": "string", "required": false, "description": "Amino acid position range (e.g., 1-25, 120-145)"}, - "amino_acid_sequence": {"type": "string", "required": false, "description": "Sequence of the fragment if reported"}, - "length": {"type": "string", "required": false, "description": "Length with unit (e.g., 25 aa, 26 aa)"}, - "key_features": {"type": "string", "required": true, "description": "Structural or compositional features (e.g., Ser-rich / acidic, Asp-rich, Ca2+-binding motif)"}, - "functional_tag": {"type": "string", "required": true, "description": "Primary functional role (e.g., Nucleation promotion, Ion enrichment, Crystal face binding)"}, - "mineralization_effect": {"type": "string", "required": false, "description": "Specific effect on mineralization (e.g., promotes hydroxyapatite nucleation, inhibits calcite growth)"}, - "evidence_level": {"type": "integer", "required": true, "description": "Evidence level 1-4"}, - "references": {"type": "string", "required": false, "description": "PMID or DOI references"}, - "notes": {"type": "string", "required": false, "description": "Additional notes"} + "parent_protein": { + "type": "string", + "required": true, + "description": "Name of the parent protein/template" + }, + "fragment_name": { + "type": "string", + "required": true, + "description": "Name or descriptor of the fragment (e.g., N-terminal acidic fragment, Asp-rich motif)" + }, + "start_end_position": { + "type": "string", + "required": false, + "description": "Amino acid position range (e.g., 1-25, 120-145)" + }, + "amino_acid_sequence": { + "type": "string", + "required": false, + "description": "Sequence of the fragment if reported" + }, + "normalized_sequence": { + "type": "string", + "required": false, + "description": "Canonical one-letter amino-acid sequence produced by the seq_norm post-processor. Preserve the reported sequence in amino_acid_sequence." + }, + "sequence_normalization": { + "type": "object", + "required": false, + "description": "Structured seq_norm result: original sequence, candidate normalized sequence, success flag, confidence, modifications, warnings, and unresolved tokens." + }, + "length": { + "type": "string", + "required": false, + "description": "Length with unit (e.g., 25 aa, 26 aa)" + }, + "key_features": { + "type": "string", + "required": true, + "description": "Structural or compositional features (e.g., Ser-rich / acidic, Asp-rich, Ca2+-binding motif)" + }, + "functional_tag": { + "type": "string", + "required": true, + "description": "Primary functional role (e.g., Nucleation promotion, Ion enrichment, Crystal face binding)" + }, + "mineralization_effect": { + "type": "string", + "required": false, + "description": "Specific effect on mineralization (e.g., promotes hydroxyapatite nucleation, inhibits calcite growth)" + }, + "evidence_level": { + "type": "integer", + "required": true, + "description": "Evidence level 1-4" + }, + "references": { + "type": "string", + "required": false, + "description": "PMID or DOI references" + }, + "notes": { + "type": "string", + "required": false, + "description": "Additional notes" + } } }, "mineralization_conditions": { "type": "list", - "description": "Experimental conditions under which biomineralization templates were studied.", + "description": "Experimental conditions under which biomineralization templates were studied.\n\nExtraction guidance: Extract experimental conditions for mineralization studies: pH, temperature, ion concentrations, template concentrations, incubation times. Include resulting crystal phase, morphology, and characterization methods used.", "item_schema": { - "template_name": {"type": "string", "required": true, "description": "Template being studied"}, - "mineral_phase": {"type": "string", "required": true, "description": "Mineral phase formed (e.g., hydroxyapatite, calcite, aragonite, vaterite)"}, - "ph": {"type": "number", "required": false, "description": "pH of the mineralization solution"}, - "temperature_c": {"type": "number", "required": false, "description": "Temperature in Celsius"}, - "calcium_concentration_mm": {"type": "number", "required": false, "description": "Ca2+ concentration in mM"}, - "phosphate_concentration_mm": {"type": "number", "required": false, "description": "Phosphate concentration in mM (for calcium phosphate systems)"}, - "carbonate_concentration_mm": {"type": "number", "required": false, "description": "Carbonate concentration in mM (for calcium carbonate systems)"}, - "template_concentration": {"type": "string", "required": false, "description": "Template concentration with units"}, - "incubation_time": {"type": "string", "required": false, "description": "Duration of mineralization experiment"}, - "crystal_morphology": {"type": "string", "required": false, "description": "Observed crystal morphology"}, - "crystal_size": {"type": "string", "required": false, "description": "Crystal size with units"}, - "characterization_methods": {"type": "list", "item_type": "string", "required": false, "description": "Methods used (e.g., XRD, TEM, SEM, FTIR, AFM)"}, - "evidence_level": {"type": "integer", "required": true, "description": "Evidence level 1-4"}, - "notes": {"type": "string", "required": false, "description": "Additional notes"} + "template_name": { + "type": "string", + "required": true, + "description": "Template being studied" + }, + "mineral_phase": { + "type": "string", + "required": true, + "description": "Mineral phase formed (e.g., hydroxyapatite, calcite, aragonite, vaterite)" + }, + "ph": { + "type": "number", + "required": false, + "description": "pH of the mineralization solution" + }, + "temperature_c": { + "type": "number", + "required": false, + "description": "Temperature in Celsius" + }, + "calcium_concentration_mm": { + "type": "number", + "required": false, + "description": "Ca2+ concentration in mM" + }, + "phosphate_concentration_mm": { + "type": "number", + "required": false, + "description": "Phosphate concentration in mM (for calcium phosphate systems)" + }, + "carbonate_concentration_mm": { + "type": "number", + "required": false, + "description": "Carbonate concentration in mM (for calcium carbonate systems)" + }, + "template_concentration": { + "type": "string", + "required": false, + "description": "Template concentration with units" + }, + "incubation_time": { + "type": "string", + "required": false, + "description": "Duration of mineralization experiment" + }, + "crystal_morphology": { + "type": "string", + "required": false, + "description": "Observed crystal morphology" + }, + "crystal_size": { + "type": "string", + "required": false, + "description": "Crystal size with units" + }, + "characterization_methods": { + "type": "list", + "item_type": "string", + "required": false, + "description": "Methods used (e.g., XRD, TEM, SEM, FTIR, AFM)" + }, + "evidence_level": { + "type": "integer", + "required": true, + "description": "Evidence level 1-4" + }, + "notes": { + "type": "string", + "required": false, + "description": "Additional notes" + } } }, "structure_activity_relationships": { "type": "list", - "description": "Relationships between template structure and mineralization activity.", + "description": "Relationships between template structure and mineralization activity.\n\nExtraction guidance: Extract relationships between structural features of templates and their effects on mineralization. Include the structural feature, the functional effect, and any proposed mechanism.", "item_schema": { - "template_name": {"type": "string", "required": true}, - "structural_feature": {"type": "string", "required": true, "description": "Structural feature (e.g., beta-sheet content, acidic residue density, self-assembly)"}, - "functional_effect": {"type": "string", "required": true, "description": "Effect on mineralization (e.g., promotes oriented nucleation, controls crystal polymorph)"}, - "mechanism": {"type": "string", "required": false, "description": "Proposed mechanism if discussed"}, - "evidence_level": {"type": "integer", "required": true}, - "notes": {"type": "string", "required": false} + "template_name": { + "type": "string", + "required": true + }, + "structural_feature": { + "type": "string", + "required": true, + "description": "Structural feature (e.g., beta-sheet content, acidic residue density, self-assembly)" + }, + "functional_effect": { + "type": "string", + "required": true, + "description": "Effect on mineralization (e.g., promotes oriented nucleation, controls crystal polymorph)" + }, + "mechanism": { + "type": "string", + "required": false, + "description": "Proposed mechanism if discussed" + }, + "evidence_level": { + "type": "integer", + "required": true + }, + "notes": { + "type": "string", + "required": false + } } } }, "system_prompt": "You are extracting structured data for a High-Activity Biomineralization Template Database. Focus on identifying biomineralization templates (proteins, peptides, and other molecules that directly participate in or regulate biomineralization processes), their functional modules (active fragments/domains), experimental mineralization conditions, and structure-activity relationships.\n\nInclusion criteria for template entries:\n- the molecule is the primary research object of the study, or\n- the paper demonstrates that it regulates mineralization, or\n- the authors describe it as a biomineralization template or matrix molecule.\n\nExclusion criteria for template entries:\n- negative controls\n- comparison proteins used only for benchmarking\n- background examples mentioned from prior literature\n- standard reference proteins without direct mineralization evidence in the paper.\n\nPay special attention to:\n- Template identity: exact protein/peptide names, all species involved, and biological system\n- Functional roles: nucleation promotion, crystal orientation control, crystal growth regulation, ion enrichment, polymorph selection\n- Active fragments: specific domains or motifs with mineralization activity, their sequences and positions\n- Mineralization conditions: pH, temperature, ion concentrations, incubation times\n- Structure-activity links: how structural features relate to mineralization function\n- Role classification: use template_role to distinguish native proteins, derived peptides, synthetic peptides, and mutants; use experimental_role to separate primary templates from controls or comparisons.\n\nFor each entry, assign the highest applicable evidence level reported in the paper:\n- Level 1: in vivo validation with functional evidence\n- Level 2: in vitro mineralization experiments\n- Level 3: indirect experimental evidence\n- Level 4: prediction, hypothesis, or inference\n\nKeep multi-value fields such as source_species and functional_tags as JSON lists internally.", - "field_descriptions": { - "templates": "Extract only biomineralization-relevant template molecules that meet the inclusion criteria. Exclude controls, comparison proteins, and background references unless the paper directly demonstrates a mineralization role. Preserve all reported species and functional roles as lists. Example: Amelogenin from Homo sapiens and Mus musculus enamel studies, a full-length protein acting on calcium phosphate with roles in nucleation promotion and crystal orientation control.", - "functional_modules": "Extract specific fragments, domains, or motifs within templates that have demonstrated or predicted mineralization activity. Include position, sequence (if available), structural features, and functional role. Example: N-terminal acidic fragment of Amelogenin (positions 1-25), Ser-rich/acidic, promotes nucleation.", - "mineralization_conditions": "Extract experimental conditions for mineralization studies: pH, temperature, ion concentrations, template concentrations, incubation times. Include resulting crystal phase, morphology, and characterization methods used.", - "structure_activity_relationships": "Extract relationships between structural features of templates and their effects on mineralization. Include the structural feature, the functional effect, and any proposed mechanism." - } + "post_processors": [ + { + "id": "seq_norm", + "name": "Sequence normalization", + "description": "Runs a deterministic amino-acid sequence normalizer first. Only uncertain sequences are sent to the reviewer.", + "prompt": "You are resolving only the sequence records listed in the seq_norm script context. The script has already saved its structured diagnostic output and every confident normalized_sequence value. Do not alter raw sequence or amino_acid_sequence fields. For each failed record, use the raw sequence and the script report as a starting point, verify against the paper when needed, then write normalized_sequence and the full sequence_normalization object in the same row. sequence_normalization must record the resolved normalized_sequence, success, confidence, modifications including PTMs and terminal modifications when supported, warnings, and unresolved tokens. Never create or write processed_sequence. Preserve successful script results unless source evidence proves them wrong. Do not infer residues hidden by prose annotations, partial-sequence statements, unknown tokens, or unsupported modifications. If the source cannot resolve a failed sequence, leave normalized_sequence empty and record success=false plus a concise warning in sequence_normalization. Use save_reviewed_projection_patch for these field-level changes and confirm the saved changed paths.", + "tool_groups": ["reading"], + "skill_ids": [], + "script": null, + "output_columns": [ + {"name": "normalized_sequence", "description": "Canonical one-letter amino-acid sequence."}, + {"name": "sequence_normalization", "description": "Structured normalization result and diagnostics."} + ], + "enabled": true + } + ] } diff --git a/examples/spaces/computational_materials_qa.json b/examples/spaces/computational_materials_qa.json index 41cf038..a7651a2 100644 --- a/examples/spaces/computational_materials_qa.json +++ b/examples/spaces/computational_materials_qa.json @@ -6,7 +6,7 @@ "extraction_schema": { "questions": { "type": "list", - "description": "Self-contained agent benchmark tasks. Each item must be runnable in isolation — never cross-reference other items in this list. One projected `questions[i]` corresponds to exactly one task YAML in mat_agent_bench's question_bank.", + "description": "Self-contained agent benchmark tasks. Each item must be runnable in isolation — never cross-reference other items in this list. One projected `questions[i]` corresponds to exactly one task YAML in mat_agent_bench's question_bank.\n\nExtraction guidance: Extract one self-contained agent task per concrete computational workflow described in the paper. Do not bundle multiple unrelated workflows into a single item, and do not split one workflow across items. Each `questions[i]` must round-trip cleanly to a standalone mat_agent_bench YAML file under `question_bank//.yaml`. Keep the list short and high-signal: 1-5 items per paper is typical; emit `[]` if nothing in the frame is benchmark-worthy.", "item_schema": { "id": { "type": "string", @@ -62,8 +62,5 @@ } } }, - "system_prompt": "You are projecting research-paper knowledge frames into agent-task benchmark items in the mat_agent_bench format (https://github.com/ruoyuwang1995nya/mat_agent_bench). Each item in `questions` must be a self-contained task that an autonomous coding agent could run end-to-end without seeing any sibling task.\n\nHard rules for isolation:\n1. NEVER reference another `questions[i]` (no 'as in question X', no shared state).\n2. Every file the agent needs must appear in this item's own `data_files`. If two tasks happen to consume the same physical file, copy the entry — do not share by reference.\n3. Reference answers must be verifiable from ONLY this task's deliverables. A grader looking at one YAML in isolation must be able to score it.\n4. `id` must be globally unique. Generate one using the convention `___` where CAP is the capability prefix (IG/SR/SC/WF/BP/DD/EC/SA/SF), short_domain is a slug like `abacus` or `vasp`, NNN is a 3-digit counter scoped to (capability, domain), and the date is today.\n\nWhat to extract:\n- Walk the knowledge frame and identify concrete, reproducible computational workflows the paper describes (input deck generation, structure construction, post-processing, etc.). Skip narrative-only passages.\n- For each candidate workflow, formulate ONE focused task that exercises a single capability. Prefer narrow, testable prompts (e.g. 'generate an ABACUS SCF INPUT with dipole correction for the provided slab') over broad ones ('reproduce the whole paper').\n- Map the workflow to the closest `capability` value. If unsure between two, choose the more specific one and add the other as a tag.\n- Write `human_prompt_seed` as if you were the end-user assigning the task. Be explicit about deliverable filenames and the working directory. Multilingual prompts are allowed; match the source paper's language when natural.\n- Populate `data_files` with everything the agent will need at runtime. Use stable filenames (snake_case). Leave `oss_url` as an empty string — uploads happen at bank-registration time.\n- Build `reference_answers` so each entry is independently checkable. Prefer cheap verifiers (text_file_contains_all, text_file_regex, artifact_exists, numeric ranges) over llm_binary_judge. Always include at least one `artifact_exists` entry per deliverable file.\n- Build `scoring_checklist` to mirror `reference_answers`: every non-budget reference key gets a checklist entry whose `id` matches. Prefix the `criterion` text with `[Must]` (strict requirement), `[Suggested]` (benchmark-tuned numeric range), or `[Variable]` (acceptable-but-optional knob). Always add the four efficiency items at the end: `turn_budget`, `no_retries`, `duration_budget`, `token_budget_total` (with matching budget entries in reference_answers).\n- Include a `grounding_source` reference_answer pointing at the seed data file plus an `llm_binary_judge` checklist item that verifies the answer is grounded in the source paper.\n- Record the paper-side justification in `source_evidence` (one short quote or `section: ...` pointer) so reviewers can trust the task.\n\nNumeric-range guidance for `text_file_numeric_range`:\n- Use `min`/`max` for tunable knobs (cutoffs, thresholds) with realistic ranges drawn from the paper or the engine's defaults.\n- Use `expected` + `tolerance: 0` for hard-required integer flags.\n- Set `allow_missing_key: true` on knobs where omission is acceptable.\n\nIf the frame does not contain enough material for a high-quality benchmark item, emit zero items rather than fabricating one — the projection may legitimately return an empty `questions: []` for review-only papers.", - "field_descriptions": { - "questions": "Extract one self-contained agent task per concrete computational workflow described in the paper. Do not bundle multiple unrelated workflows into a single item, and do not split one workflow across items. Each `questions[i]` must round-trip cleanly to a standalone mat_agent_bench YAML file under `question_bank//.yaml`. Keep the list short and high-signal: 1-5 items per paper is typical; emit `[]` if nothing in the frame is benchmark-worthy." - } + "system_prompt": "You are projecting research-paper knowledge frames into agent-task benchmark items in the mat_agent_bench format (https://github.com/ruoyuwang1995nya/mat_agent_bench). Each item in `questions` must be a self-contained task that an autonomous coding agent could run end-to-end without seeing any sibling task.\n\nHard rules for isolation:\n1. NEVER reference another `questions[i]` (no 'as in question X', no shared state).\n2. Every file the agent needs must appear in this item's own `data_files`. If two tasks happen to consume the same physical file, copy the entry — do not share by reference.\n3. Reference answers must be verifiable from ONLY this task's deliverables. A grader looking at one YAML in isolation must be able to score it.\n4. `id` must be globally unique. Generate one using the convention `___` where CAP is the capability prefix (IG/SR/SC/WF/BP/DD/EC/SA/SF), short_domain is a slug like `abacus` or `vasp`, NNN is a 3-digit counter scoped to (capability, domain), and the date is today.\n\nWhat to extract:\n- Walk the knowledge frame and identify concrete, reproducible computational workflows the paper describes (input deck generation, structure construction, post-processing, etc.). Skip narrative-only passages.\n- For each candidate workflow, formulate ONE focused task that exercises a single capability. Prefer narrow, testable prompts (e.g. 'generate an ABACUS SCF INPUT with dipole correction for the provided slab') over broad ones ('reproduce the whole paper').\n- Map the workflow to the closest `capability` value. If unsure between two, choose the more specific one and add the other as a tag.\n- Write `human_prompt_seed` as if you were the end-user assigning the task. Be explicit about deliverable filenames and the working directory. Multilingual prompts are allowed; match the source paper's language when natural.\n- Populate `data_files` with everything the agent will need at runtime. Use stable filenames (snake_case). Leave `oss_url` as an empty string — uploads happen at bank-registration time.\n- Build `reference_answers` so each entry is independently checkable. Prefer cheap verifiers (text_file_contains_all, text_file_regex, artifact_exists, numeric ranges) over llm_binary_judge. Always include at least one `artifact_exists` entry per deliverable file.\n- Build `scoring_checklist` to mirror `reference_answers`: every non-budget reference key gets a checklist entry whose `id` matches. Prefix the `criterion` text with `[Must]` (strict requirement), `[Suggested]` (benchmark-tuned numeric range), or `[Variable]` (acceptable-but-optional knob). Always add the four efficiency items at the end: `turn_budget`, `no_retries`, `duration_budget`, `token_budget_total` (with matching budget entries in reference_answers).\n- Include a `grounding_source` reference_answer pointing at the seed data file plus an `llm_binary_judge` checklist item that verifies the answer is grounded in the source paper.\n- Record the paper-side justification in `source_evidence` (one short quote or `section: ...` pointer) so reviewers can trust the task.\n\nNumeric-range guidance for `text_file_numeric_range`:\n- Use `min`/`max` for tunable knobs (cutoffs, thresholds) with realistic ranges drawn from the paper or the engine's defaults.\n- Use `expected` + `tolerance: 0` for hard-required integer flags.\n- Set `allow_missing_key: true` on knobs where omission is acceptable.\n\nIf the frame does not contain enough material for a high-quality benchmark item, emit zero items rather than fabricating one — the projection may legitimately return an empty `questions: []` for review-only papers." } diff --git a/frontend/src/api/postProcessorScripts.ts b/frontend/src/api/postProcessorScripts.ts new file mode 100644 index 0000000..77b1cc9 --- /dev/null +++ b/frontend/src/api/postProcessorScripts.ts @@ -0,0 +1,13 @@ +import client from './client' +import type { PostProcessorScript } from '../types' + +export const listPostProcessorScripts = () => + client.get('/post-processor-scripts').then(r => r.data) + +export const uploadPostProcessorScript = (file: File) => { + const form = new FormData() + form.append('file', file) + return client.post('/post-processor-scripts/upload', form, { + headers: { 'Content-Type': 'multipart/form-data' }, timeout: 120_000, + }).then(r => r.data) +} \ No newline at end of file diff --git a/frontend/src/components/SpaceDraftCard.tsx b/frontend/src/components/SpaceDraftCard.tsx index c0c0b69..a68d493 100644 --- a/frontend/src/components/SpaceDraftCard.tsx +++ b/frontend/src/components/SpaceDraftCard.tsx @@ -9,7 +9,7 @@ export interface SpaceDraft { description?: string extraction_schema: Record system_prompt: string - field_descriptions: Record + field_descriptions?: Record } interface Props { @@ -43,9 +43,9 @@ export default function SpaceDraftCard({ draft, existing, onSaved }: Props) { domain: draft.domain, purpose: draft.purpose, description: draft.description ?? '', - extraction_schema: draft.extraction_schema, + extraction_schema: mergeLegacyFieldDescriptions(draft.extraction_schema, draft.field_descriptions), system_prompt: draft.system_prompt, - field_descriptions: draft.field_descriptions, + field_descriptions: {}, } const res = await createSpace(payload) if ((res as unknown as { error?: string }).error) { @@ -58,9 +58,9 @@ export default function SpaceDraftCard({ draft, existing, onSaved }: Props) { domain: draft.domain, purpose: draft.purpose, description: draft.description ?? '', - extraction_schema: draft.extraction_schema, + extraction_schema: mergeLegacyFieldDescriptions(draft.extraction_schema, draft.field_descriptions), system_prompt: draft.system_prompt, - field_descriptions: draft.field_descriptions, + field_descriptions: {}, }) setSavedAs(existing.space_id) onSaved?.({ space_id: existing.space_id, name: `${existing.name} (v${res.version})` }) @@ -173,9 +173,9 @@ export function extractDraftsFromText(text: string): SpaceDraft[] { domain: obj.domain ?? '', purpose: obj.purpose ?? 'tabular_database', description: obj.description ?? '', - extraction_schema: obj.extraction_schema, + extraction_schema: mergeLegacyFieldDescriptions(obj.extraction_schema, obj.field_descriptions), system_prompt: obj.system_prompt ?? '', - field_descriptions: obj.field_descriptions ?? {}, + field_descriptions: {}, }) } } catch { @@ -184,3 +184,34 @@ export function extractDraftsFromText(text: string): SpaceDraft[] { } return drafts } + +function stringifyLegacyDescription(value: unknown) { + return typeof value === 'string' ? value : value == null ? '' : JSON.stringify(value, null, 2) +} + +function mergeLegacyFieldDescriptions( + schemaValue: unknown, + descriptionsValue: unknown, +): Record { + if (!schemaValue || typeof schemaValue !== 'object' || Array.isArray(schemaValue)) return {} + const schema = { ...(schemaValue as Record) } + const descriptions = + descriptionsValue && typeof descriptionsValue === 'object' && !Array.isArray(descriptionsValue) + ? (descriptionsValue as Record) + : {} + + for (const [key, rawDescription] of Object.entries(descriptions)) { + const description = stringifyLegacyDescription(rawDescription).trim() + const node = schema[key] + if (!description || !node || typeof node !== 'object' || Array.isArray(node)) continue + const section = { ...(node as Record) } + const existing = typeof section.description === 'string' ? section.description.trim() : '' + if (!existing.includes(description)) { + section.description = existing + ? `${existing}\n\nExtraction guidance: ${description}` + : description + } + schema[key] = section + } + return schema +} diff --git a/frontend/src/components/projections/SectionTable.tsx b/frontend/src/components/projections/SectionTable.tsx index beda788..773315c 100644 --- a/frontend/src/components/projections/SectionTable.tsx +++ b/frontend/src/components/projections/SectionTable.tsx @@ -150,13 +150,21 @@ export default function SectionTable({ useEffect(() => { setPrefs(p => { - const merged = Array.from(new Set([...p.known, ...allCols])) - if (merged.length === p.known.length) return p - const next = { ...p, known: merged } + const added = allCols.filter(column => !p.known.includes(column)) + if (added.length === 0) return p + const known = [...p.known, ...added] + // New fields can be introduced by a post-processor after the user has + // saved column preferences. Reveal them once without re-enabling any + // columns the user previously hid. + const visible = Array.from(new Set([ + ...p.visible, + ...defaultColumns(added, schemaOrder).filter(column => !p.visible.includes(column)), + ])) + const next = { ...p, known, visible } saveColPrefs(name, next) return next }) - }, [allCols, name]) + }, [allCols, name, schemaOrder]) const visibleCols = prefs.visible.filter(c => prefs.known.includes(c)) diff --git a/frontend/src/components/projects/BrowseTab.tsx b/frontend/src/components/projects/BrowseTab.tsx index a238140..330aec9 100644 --- a/frontend/src/components/projects/BrowseTab.tsx +++ b/frontend/src/components/projects/BrowseTab.tsx @@ -1,12 +1,47 @@ import { useCallback, useEffect, useState } from 'react' import { JOB_FINISHED_EVENT } from '../../api/jobPolling' -import { listProjects } from '../../api/projects' +import { getProject, listProjects } from '../../api/projects' import ProjectGroupedList from '../ProjectGroupedList' import type { Job, Project, Space } from '../../types' import ProjectDetail from './ProjectDetail' import StatusLights from './StatusLights' +function projectsEqual(a: Project, b: Project): boolean { + return ( + a.project_id === b.project_id && + a.label === b.label && + a.source_path === b.source_path && + a.file_count === b.file_count && + a.asset_count === b.asset_count && + a.processing_status === b.processing_status && + a.frame_status === b.frame_status && + a.workflow_status === b.workflow_status && + a.workflow_version === b.workflow_version && + a.canonical_workflow_status === b.canonical_workflow_status && + a.canonical_workflow_version === b.canonical_workflow_version && + a.created_at === b.created_at && + (a.group_id ?? null) === (b.group_id ?? null) + ) +} + +function mergeProjectList(current: Project[], next: Project[]): Project[] { + const currentById = new Map(current.map(p => [p.project_id, p])) + let changed = current.length !== next.length + + const merged = next.map(project => { + const existing = currentById.get(project.project_id) + if (existing && projectsEqual(existing, project)) return existing + changed = true + return project + }) + + return changed ? merged : current +} + +function mergeProject(current: Project, next: Project): Project { + return projectsEqual(current, next) ? current : next +} export default function BrowseTab({ spaces }: { spaces: Space[] }) { const [projects, setProjects] = useState([]) @@ -17,15 +52,38 @@ export default function BrowseTab({ spaces }: { spaces: Space[] }) { try { // Don't blank the table during a background refresh — only show the // loading placeholder on the very first fetch. - setLoading(prev => projects.length === 0 ? true : prev) const data = await listProjects(5000) - setProjects(data) + setProjects(prev => mergeProjectList(prev, data)) + setSelected(prev => { + if (!prev) return prev + const updated = data.find(p => p.project_id === prev.project_id) + return updated ? mergeProject(prev, updated) : null + }) } finally { setLoading(false) } // eslint-disable-next-line react-hooks/exhaustive-deps }, []) + const refreshProject = useCallback(async (projectId: string) => { + const updated = await getProject(projectId) + setProjects(prev => prev.map(p => + p.project_id === updated.project_id ? mergeProject(p, updated) : p + )) + setSelected(prev => + prev?.project_id === updated.project_id ? mergeProject(prev, updated) : prev + ) + }, []) + + const handleProjectUpdated = useCallback((updated: Project) => { + setProjects(prev => prev.map(p => + p.project_id === updated.project_id ? mergeProject(p, updated) : p + )) + setSelected(prev => + prev?.project_id === updated.project_id ? mergeProject(prev, updated) : prev + ) + }, []) + useEffect(() => { load() }, [load]) useEffect(() => { @@ -45,12 +103,16 @@ export default function BrowseTab({ spaces }: { spaces: Space[] }) { 'upload', ].includes(job.kind) ) { - load() + if (job.project_id && job.project_id !== '__upload__') { + refreshProject(job.project_id).catch(() => load()) + } else { + load() + } } } window.addEventListener(JOB_FINISHED_EVENT, refreshOnFinishedJob) return () => window.removeEventListener(JOB_FINISHED_EVENT, refreshOnFinishedJob) - }, [load]) + }, [load, refreshProject]) const getStatus = useCallback( (id: string) => projects.find(p => p.project_id === id)?.frame_status ?? 'NO_FRAME', @@ -102,10 +164,8 @@ export default function BrowseTab({ spaces }: { spaces: Space[] }) { project={selected} spaces={spaces} onClose={() => setSelected(null)} - onJobComplete={load} - onProjectUpdated={updated => setProjects(prev => prev.map(p => - p.project_id === updated.project_id ? updated : p - ))} + onJobComplete={() => refreshProject(selected.project_id)} + onProjectUpdated={handleProjectUpdated} onDeleted={() => { setSelected(null); load() }} /> )} diff --git a/frontend/src/pages/ProjectionsPage.tsx b/frontend/src/pages/ProjectionsPage.tsx index 9d62b9b..3077321 100644 --- a/frontend/src/pages/ProjectionsPage.tsx +++ b/frontend/src/pages/ProjectionsPage.tsx @@ -311,7 +311,7 @@ export default function ProjectionsPage() {

Aggregated extraction results per space.

- +