diff --git a/backend/cli/package.json b/backend/cli/package.json index 7eb1ccdd..34212c66 100644 --- a/backend/cli/package.json +++ b/backend/cli/package.json @@ -102,6 +102,7 @@ "hono-openapi": "catalog:", "ignore": "7.0.5", "jsonc-parser": "3.3.1", + "marked": "catalog:", "minimatch": "10.0.3", "open": "10.1.2", "partial-json": "0.1.7", diff --git a/backend/cli/src/compute/jobs.ts b/backend/cli/src/compute/jobs.ts new file mode 100644 index 00000000..f0a49ffa --- /dev/null +++ b/backend/cli/src/compute/jobs.ts @@ -0,0 +1,757 @@ +import { spawn, type ChildProcess } from "node:child_process" +import crypto from "node:crypto" +import { createReadStream } from "node:fs" +import fs from "node:fs/promises" +import path from "node:path" +import z from "zod" +import { Global } from "../global" +import { OpenScience } from "../openscience" +import { Shell } from "../shell/shell" + +export namespace ComputeJobs { + export const Scheduler = z.enum(["none", "slurm", "pbs"]) + export type Scheduler = z.infer + + export const Host = z.object({ + id: z.string(), + label: z.string(), + host: z.string(), + user: z.string().optional(), + port: z.number().int().positive().optional(), + scheduler: Scheduler.default("none"), + workdir: z.string().optional(), + }) + export type Host = z.infer + + export const Probe = z.object({ + ok: z.boolean(), + host: z.string(), + latency_ms: z.number().nonnegative(), + hostname: z.string().optional(), + python: z.boolean(), + gpu: z.boolean(), + slurm: z.boolean(), + pbs: z.boolean(), + error: z.string().optional(), + }) + export type Probe = z.infer + + export const Target = z.discriminatedUnion("kind", [ + z.object({ kind: z.literal("local") }), + z.object({ kind: z.literal("ssh"), host_id: z.string() }), + ]) + export type Target = z.infer + + export const Resources = z.object({ + cpus: z.number().int().min(1).max(1024).optional(), + gpus: z.number().int().min(0).max(128).optional(), + memory_gb: z.number().min(0.1).max(100_000).optional(), + time_minutes: z + .number() + .int() + .min(1) + .max(60 * 24 * 30) + .optional(), + partition: z.string().trim().min(1).max(120).optional(), + }) + export type Resources = z.infer + + export const Artifact = z.object({ + path: z.string(), + size: z.number().int().nonnegative(), + sha256: z.string().regex(/^[a-f0-9]{64}$/), + modified_at: z.string(), + }) + export type Artifact = z.infer + + export const Reproducibility = z.object({ + captured_at: z.string(), + command: z.string(), + cwd: z.string(), + platform: z.string(), + arch: z.string(), + bun: z.string(), + node: z.string(), + python: z.string().optional(), + git: z + .object({ + branch: z.string().optional(), + commit: z.string().optional(), + dirty: z.boolean(), + }) + .optional(), + lockfiles: Artifact.array(), + resources: Resources.optional(), + }) + export type Reproducibility = z.infer + + export const Input = z.object({ + name: z.string().trim().min(1).max(120), + command: z.string().trim().min(1).max(100_000), + cwd: z.string().optional(), + target: Target, + resources: Resources.optional(), + modules: z.array(z.string().trim().min(1).max(240)).max(64).optional(), + container: z.string().trim().min(1).max(2_000).optional(), + artifacts: z.array(z.string().trim().min(1).max(2_000)).max(100).optional(), + checkpoint: z.string().trim().min(1).max(2_000).optional(), + }) + export type Input = z.infer + + export const Status = z.enum(["queued", "running", "succeeded", "failed", "cancelled", "interrupted"]) + export type Status = z.infer + + export const Job = z.object({ + id: z.string(), + name: z.string(), + command: z.string(), + cwd: z.string().optional(), + target: Target, + target_label: z.string(), + scheduler: Scheduler, + status: Status, + created_at: z.string(), + started_at: z.string().optional(), + completed_at: z.string().optional(), + exit_code: z.number().int().nullable().optional(), + pid: z.number().int().positive().optional(), + error: z.string().optional(), + resources: Resources.optional(), + modules: z.array(z.string()).optional(), + container: z.string().optional(), + artifact_patterns: z.array(z.string()).optional(), + artifacts: Artifact.array().optional(), + checkpoint_path: z.string().optional(), + checkpoint: Artifact.optional(), + reproducibility: Reproducibility.optional(), + capture_error: z.string().optional(), + }) + export type Job = z.infer + + type Options = { + root?: string + hosts?: Host[] + } + + type Runtime = { + process: ChildProcess + detached: boolean + host?: Host + } + + const active = new Map() + const locks = new Map>() + const terminal = new Set(["succeeded", "failed", "cancelled", "interrupted"]) + + const rootOf = (root?: string) => root ?? path.join(Global.Path.data, "compute") + const metaOf = (root: string) => path.join(root, "jobs.json") + const logsOf = (root: string) => path.join(root, "jobs") + const exitOf = (root: string, id: string) => path.join(logsOf(root), `${id}.exit`) + const keyOf = (root: string, id: string) => `${root}\0${id}` + + async function read(root: string): Promise { + const value = await Bun.file(metaOf(root)) + .json() + .catch(() => []) + const result = Job.array().safeParse(value) + return result.success ? result.data : [] + } + + async function write(root: string, jobs: Job[]): Promise { + await fs.mkdir(root, { recursive: true }) + await Bun.write(metaOf(root), JSON.stringify(jobs, null, 2), { mode: 0o600 }) + } + + async function change(root: string, edit: (jobs: Job[]) => T | Promise): Promise { + const prior = locks.get(root) ?? Promise.resolve() + const task = prior + .catch(() => undefined) + .then(async () => { + const jobs = await read(root) + const result = await edit(jobs) + await write(root, jobs) + return result + }) + locks.set( + root, + task.then( + () => undefined, + () => undefined, + ), + ) + return task + } + + async function patch(root: string, id: string, value: Partial): Promise { + return change(root, (jobs) => { + const index = jobs.findIndex((job) => job.id === id) + if (index < 0) throw new Error(`Compute job ${id} was not found`) + const next = Job.parse({ ...jobs[index], ...value }) + jobs[index] = next + return next + }) + } + + function alive(pid: number): boolean { + try { + process.kill(pid, 0) + return true + } catch { + return false + } + } + + async function sync(root: string): Promise { + const jobs = await read(root) + const updates = ( + await Promise.all( + jobs.map(async (job): Promise<{ id: string; value: Partial } | undefined> => { + if (terminal.has(job.status) || active.has(keyOf(root, job.id))) return + if (job.status === "queued" && Date.now() - Date.parse(job.created_at) < 5_000) return + const marker = await Bun.file(exitOf(root, job.id)) + .text() + .catch(() => undefined) + const exit = marker?.trim().match(/^-?\d+$/) ? Number(marker.trim()) : undefined + if (job.target.kind === "local" && exit !== undefined) { + return { + id: job.id, + value: { + status: exit === 0 ? "succeeded" : "failed", + completed_at: new Date().toISOString(), + exit_code: exit, + pid: undefined, + }, + } + } + if (job.target.kind === "local" && job.pid && alive(job.pid)) return + return { + id: job.id, + value: { + status: "interrupted", + completed_at: new Date().toISOString(), + exit_code: null, + pid: undefined, + error: + job.target.kind === "ssh" + ? "The app connection ended before this remote job reported a result. Check the remote scheduler before rerunning it." + : "The job process ended before it could report a result.", + }, + } + }), + ) + ).filter((item): item is { id: string; value: Partial } => !!item) + if (!updates.length) return + await change(root, (current) => { + for (const update of updates) { + const index = current.findIndex((job) => job.id === update.id) + if (index < 0 || terminal.has(current[index]!.status) || active.has(keyOf(root, update.id))) continue + current[index] = Job.parse({ ...current[index], ...update.value }) + } + }) + } + + export function quote(value: string): string { + return `'${value.replaceAll("'", `'\"'\"'`)}'` + } + + function name(value: string): string { + const clean = value + .toLowerCase() + .replace(/[^a-z0-9_-]+/g, "-") + .replace(/^-+|-+$/g, "") + .slice(0, 42) + return clean || "job" + } + + function clock(minutes: number): string { + const hours = Math.floor(minutes / 60) + const mins = minutes % 60 + return `${String(hours).padStart(2, "0")}:${String(mins).padStart(2, "0")}:00` + } + + function workload(input: { command: string; modules?: string[]; container?: string }): string { + const modules = input.modules?.length ? `module load ${input.modules.map(quote).join(" ")}` : undefined + const command = input.container + ? `apptainer exec ${quote(input.container)} bash -lc ${quote(input.command)}` + : input.command + return [modules, command].filter((part): part is string => !!part).join(" && ") + } + + function slurm(input: { resources?: Resources }): string[] { + const resources = input.resources + if (!resources) return [] + return [ + resources.cpus ? `--cpus-per-task=${resources.cpus}` : undefined, + resources.gpus ? `--gres=gpu:${resources.gpus}` : undefined, + resources.memory_gb ? `--mem=${resources.memory_gb}G` : undefined, + resources.time_minutes ? `--time=${clock(resources.time_minutes)}` : undefined, + resources.partition ? `--partition=${quote(resources.partition)}` : undefined, + ].filter((part): part is string => !!part) + } + + function pbs(input: { resources?: Resources }): string[] { + const resources = input.resources + if (!resources) return [] + const select = [ + "select=1", + resources.cpus ? `ncpus=${resources.cpus}` : undefined, + resources.gpus ? `ngpus=${resources.gpus}` : undefined, + resources.memory_gb ? `mem=${resources.memory_gb}gb` : undefined, + ] + .filter((part): part is string => !!part) + .join(":") + return [ + select === "select=1" ? undefined : `-l ${quote(select)}`, + resources.time_minutes ? `-l ${quote(`walltime=${clock(resources.time_minutes)}`)}` : undefined, + ].filter((part): part is string => !!part) + } + + function remote( + input: { + id: string + name: string + command: string + cwd?: string + resources?: Resources + modules?: string[] + container?: string + }, + host: Host, + ): string { + const cwd = input.cwd || host.workdir || "." + const job = `os-${input.id}` + const folder = `.openscience/jobs` + const log = `${folder}/${input.id}.log` + const enter = `cd ${quote(cwd)} && mkdir -p ${quote(folder)}` + const run = workload(input) + if (host.scheduler === "slurm") { + return [ + enter, + [ + "sbatch --wait --parsable", + `--job-name=${quote(job)}`, + `--output=${quote(log)}`, + `--error=${quote(log)}`, + ...slurm(input), + `--wrap=${quote(run)}`, + ].join(" "), + "code=$?", + `test -f ${quote(log)} && cat ${quote(log)}`, + "exit $code", + ].join("; ") + } + if (host.scheduler === "pbs") { + const script = `#!/usr/bin/env bash\nset -o pipefail\n${run}\n` + return [ + enter, + [ + `printf %s ${quote(script)} | qsub -W block=true`, + `-N ${quote(name(job))}`, + "-j oe", + `-o ${quote(log)}`, + ...pbs(input), + ].join(" "), + "code=$?", + `test -f ${quote(log)} && cat ${quote(log)}`, + "exit $code", + ].join("; ") + } + return `${enter} && exec bash -lc ${quote(run)}` + } + + function ssh(host: Host, script: string): string[] { + const destination = host.user ? `${host.user}@${host.host}` : host.host + if (destination.startsWith("-")) throw new Error("SSH destinations cannot begin with a hyphen") + const port = host.port ? ["-p", String(host.port)] : [] + return ["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=8", ...port, "--", destination, script] + } + + export function command( + input: { + id: string + name: string + command: string + cwd?: string + resources?: Resources + modules?: string[] + container?: string + }, + host?: Host, + ): { argv: string[]; scheduler: Scheduler; label: string } { + if (!host) { + return { + argv: [Shell.acceptable(), "-lc", input.command], + scheduler: "none", + label: "This computer", + } + } + return { + argv: ssh(host, remote(input, host)), + scheduler: host.scheduler, + label: host.label, + } + } + + async function output(argv: string[], cwd: string): Promise { + const proc = Bun.spawn(argv, { + cwd, + env: await OpenScience.subprocessEnv(process.env), + stdin: "ignore", + stdout: "pipe", + stderr: "ignore", + }) + const [code, text] = await Promise.all([proc.exited, new Response(proc.stdout).text()]) + if (code !== 0) return + return text.trim() || undefined + } + + function inside(root: string, file: string): string | undefined { + const target = path.resolve(root, file) + const relative = path.relative(root, target) + if (!relative || relative === ".." || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) return + return relative + } + + async function fingerprint(root: string, file: string): Promise { + const relative = inside(root, file) + if (!relative) return + const target = path.join(root, relative) + const stat = await fs.stat(target).catch(() => undefined) + if (!stat?.isFile()) return + const hash = new Bun.CryptoHasher("sha256") + for await (const chunk of createReadStream(target)) hash.update(chunk) + return Artifact.parse({ + path: relative.split(path.sep).join("/"), + size: stat.size, + sha256: hash.digest("hex"), + modified_at: stat.mtime.toISOString(), + }) + } + + async function artifacts(root: string, patterns: string[]): Promise { + const files = new Set() + for (const pattern of patterns) { + if (!inside(root, pattern.replaceAll("*", "x"))) continue + const glob = new Bun.Glob(pattern) + for await (const file of glob.scan({ cwd: root, dot: true, onlyFiles: true })) { + files.add(file) + if (files.size >= 200) break + } + if (files.size >= 200) break + } + const values = await Promise.all([...files].toSorted().map((file) => fingerprint(root, file))) + return values.filter((item): item is Artifact => !!item) + } + + const lockfiles = [ + "bun.lock", + "bun.lockb", + "package-lock.json", + "pnpm-lock.yaml", + "yarn.lock", + "uv.lock", + "poetry.lock", + "Pipfile.lock", + "requirements.txt", + "environment.yml", + "environment.yaml", + "renv.lock", + "Manifest.toml", + "Cargo.lock", + ] + + async function reproduce(job: Job): Promise { + const cwd = path.resolve(job.cwd ?? process.cwd()) + const [branch, commit, status, python, capturedLocks] = await Promise.all([ + output(["git", "rev-parse", "--abbrev-ref", "HEAD"], cwd), + output(["git", "rev-parse", "HEAD"], cwd), + output(["git", "status", "--porcelain"], cwd), + output(["python3", "--version"], cwd), + Promise.all(lockfiles.map((file) => fingerprint(cwd, file))), + ]) + const git = branch || commit || status !== undefined ? { branch, commit, dirty: !!status } : undefined + return Reproducibility.parse({ + captured_at: new Date().toISOString(), + command: job.command, + cwd, + platform: process.platform, + arch: process.arch, + bun: Bun.version, + node: process.version, + python, + git, + lockfiles: capturedLocks.filter((item): item is Artifact => !!item), + resources: job.resources, + }) + } + + async function capture(job: Job): Promise> { + const cwd = path.resolve(job.cwd ?? process.cwd()) + const [found, checkpoint, reproducibility] = await Promise.all([ + artifacts(cwd, job.artifact_patterns ?? []), + job.checkpoint_path ? fingerprint(cwd, job.checkpoint_path) : undefined, + reproduce(job), + ]) + return { + artifacts: found, + checkpoint, + reproducibility, + } + } + + export async function probe(host: Host): Promise { + const parsed = Host.parse(host) + const started = performance.now() + const script = [ + "printf 'connected=1\\n'", + "printf 'hostname='; hostname 2>/dev/null || true", + "command -v python3 >/dev/null 2>&1 && printf 'python=1\\n' || true", + "command -v nvidia-smi >/dev/null 2>&1 && printf 'gpu=1\\n' || true", + "command -v sbatch >/dev/null 2>&1 && printf 'slurm=1\\n' || true", + "command -v qsub >/dev/null 2>&1 && printf 'pbs=1\\n' || true", + ].join("; ") + const argv = ssh(parsed, script) + const proc = spawn(argv[0]!, argv.slice(1), { + env: await OpenScience.subprocessEnv(process.env), + windowsHide: true, + stdio: ["ignore", "pipe", "pipe"], + }) + const output: Buffer[] = [] + const errors: Buffer[] = [] + proc.stdout?.on("data", (chunk: Buffer) => output.push(chunk)) + proc.stderr?.on("data", (chunk: Buffer) => errors.push(chunk)) + const done = new Promise<{ code: number | null; error?: string }>((resolve) => { + proc.once("error", (error) => resolve({ code: null, error: error.message })) + proc.once("exit", (code) => resolve({ code })) + }) + const result = await Promise.race([ + done, + Bun.sleep(12_000).then(() => ({ code: null, error: "Connection timed out" })), + ]) + if (proc.exitCode === null) { + await Shell.killTree(proc, { + detached: false, + exited: () => proc.exitCode !== null, + }) + } + const text = Buffer.concat(output).toString("utf8") + const error = result.error || (result.code === 0 ? undefined : Buffer.concat(errors).toString("utf8").trim()) + return Probe.parse({ + ok: result.code === 0 && text.includes("connected=1"), + host: parsed.label, + latency_ms: Math.round(performance.now() - started), + hostname: text.match(/^hostname=(.+)$/m)?.[1]?.trim(), + python: text.includes("python=1"), + gpu: text.includes("gpu=1"), + slurm: text.includes("slurm=1"), + pbs: text.includes("pbs=1"), + error: error || undefined, + }) + } + + async function execute(job: Job, host: Host | undefined, root: string): Promise { + await fs.mkdir(logsOf(root), { recursive: true }) + await fs.rm(exitOf(root, job.id), { force: true }) + const wrapped = host + ? job.command + : `(${job.command}\n); code=$?; printf %s "$code" > ${quote(exitOf(root, job.id))}; exit "$code"` + const spec = command({ ...job, command: wrapped }, host) + const log = path.join(logsOf(root), `${job.id}.log`) + const output = await fs.open(log, "a", 0o600) + const env = await OpenScience.subprocessEnv(process.env) + const queued = await get(job.id, { root }) + if (queued?.status === "cancelled") { + await output.close() + return + } + const detached = process.platform !== "win32" + const proc = spawn(spec.argv[0]!, spec.argv.slice(1), { + cwd: host ? undefined : job.cwd, + env, + detached, + windowsHide: true, + stdio: ["ignore", output.fd, output.fd], + }) + const result = new Promise<{ code: number | null; error?: string }>((resolve) => { + proc.once("error", (error) => resolve({ code: null, error: error.message })) + proc.once("exit", (code) => resolve({ code })) + }) + await output.close() + const current = await get(job.id, { root }) + if (current?.status === "cancelled") { + await Shell.killTree(proc, { + detached, + exited: () => proc.exitCode !== null, + }) + return + } + active.set(keyOf(root, job.id), { process: proc, detached, host }) + await patch(root, job.id, { + status: "running", + started_at: new Date().toISOString(), + pid: proc.pid, + }) + const completed = await result + const final = await get(job.id, { root }) + if (final?.status === "cancelled") { + active.delete(keyOf(root, job.id)) + return + } + const captureResult = host + ? undefined + : await capture(job) + .then((value) => ({ ...value, capture_error: undefined })) + .catch((error) => ({ + capture_error: error instanceof Error ? error.message : String(error), + })) + await patch(root, job.id, { + status: completed.code === 0 ? "succeeded" : "failed", + completed_at: new Date().toISOString(), + exit_code: completed.code, + error: completed.error, + ...captureResult, + }).finally(() => active.delete(keyOf(root, job.id))) + } + + export async function start(input: Input, options: Options = {}): Promise { + const parsed = Input.parse(input) + const root = rootOf(options.root) + const hostId = parsed.target.kind === "ssh" ? parsed.target.host_id : undefined + const host = hostId ? options.hosts?.find((item) => item.id === hostId) : undefined + if (parsed.target.kind === "ssh" && !host) throw new Error("The selected SSH compute profile was not found") + const id = crypto.randomUUID().slice(0, 12) + const spec = command({ id, ...parsed }, host) + const job = Job.parse({ + id, + name: parsed.name, + command: parsed.command, + cwd: parsed.cwd || host?.workdir, + target: parsed.target, + target_label: spec.label, + scheduler: spec.scheduler, + status: "queued", + created_at: new Date().toISOString(), + resources: parsed.resources, + modules: parsed.modules, + container: parsed.container, + artifact_patterns: parsed.artifacts, + checkpoint_path: parsed.checkpoint, + }) + await change(root, (jobs) => { + jobs.push(job) + }) + void execute(job, host, root).catch(async (error) => { + await fs.mkdir(logsOf(root), { recursive: true }) + await fs + .appendFile( + path.join(logsOf(root), `${job.id}.log`), + `${error instanceof Error ? error.message : String(error)}\n`, + ) + .catch(() => {}) + await patch(root, job.id, { + status: "failed", + completed_at: new Date().toISOString(), + exit_code: null, + error: error instanceof Error ? error.message : String(error), + }).catch(() => {}) + }) + return job + } + + export async function list(options: Options = {}): Promise { + const root = rootOf(options.root) + await sync(root) + return (await read(root)).toSorted( + (a, b) => Date.parse(b.created_at) - Date.parse(a.created_at) || b.id.localeCompare(a.id), + ) + } + + export async function get(id: string, options: Options = {}): Promise { + return (await read(rootOf(options.root))).find((job) => job.id === id) + } + + export async function log(id: string, options: Options & { bytes?: number } = {}): Promise { + const job = await get(id, options) + if (!job) throw new Error(`Compute job ${id} was not found`) + const text = await Bun.file(path.join(logsOf(rootOf(options.root)), `${job.id}.log`)) + .text() + .catch(() => "") + return text.slice(-Math.max(1, options.bytes ?? 256_000)) + } + + export async function cancel(id: string, options: Options = {}): Promise { + const root = rootOf(options.root) + const job = await get(id, { root }) + if (!job) throw new Error(`Compute job ${id} was not found`) + if (terminal.has(job.status)) return job + const cancelled = await patch(root, id, { + status: "cancelled", + completed_at: new Date().toISOString(), + exit_code: null, + }) + const runtime = active.get(keyOf(root, id)) + if (runtime) { + await Shell.killTree(runtime.process, { + detached: runtime.detached, + exited: () => runtime.process.exitCode !== null, + }) + active.delete(keyOf(root, id)) + } else if (job.pid) { + try { + if (process.platform === "win32") process.kill(job.pid, "SIGTERM") + else process.kill(-job.pid, "SIGTERM") + } catch {} + } + const hostId = job.target.kind === "ssh" ? job.target.host_id : undefined + const host = hostId ? options.hosts?.find((item) => item.id === hostId) : undefined + if (host && host.scheduler !== "none") { + const action = + host.scheduler === "slurm" + ? `scancel --name ${quote(`os-${job.id}`)}` + : `qselect -N ${quote(name(`os-${job.id}`))} | xargs -r qdel` + const spec = command( + { id: job.id, name: job.name, command: action, cwd: host.workdir }, + { ...host, scheduler: "none" }, + ) + const proc = spawn(spec.argv[0]!, spec.argv.slice(1), { + env: await OpenScience.subprocessEnv(process.env), + windowsHide: true, + stdio: "ignore", + }) + await new Promise((resolve) => { + proc.once("error", () => resolve()) + proc.once("exit", () => resolve()) + }) + } + return cancelled + } + + export async function clear(options: Options = {}): Promise { + const root = rootOf(options.root) + const removed = await change(root, (jobs) => { + const done = jobs.filter((job) => terminal.has(job.status)).map((job) => job.id) + const keep = jobs.filter((job) => !terminal.has(job.status)) + jobs.splice(0, jobs.length, ...keep) + return done + }) + await Promise.all( + removed.flatMap((id) => [ + fs.rm(path.join(logsOf(root), `${id}.log`), { force: true }), + fs.rm(exitOf(root, id), { force: true }), + ]), + ) + return removed.length + } + + export async function wait(id: string, options: Options & { timeout?: number } = {}): Promise { + const started = Date.now() + const timeout = options.timeout ?? 30_000 + for (;;) { + const job = (await list(options)).find((item) => item.id === id) + if (!job) throw new Error(`Compute job ${id} was not found`) + if (terminal.has(job.status)) return job + if (Date.now() - started >= timeout) throw new Error(`Timed out waiting for compute job ${id}`) + await Bun.sleep(25) + } + } +} diff --git a/backend/cli/src/file/annotations.ts b/backend/cli/src/file/annotations.ts new file mode 100644 index 00000000..91340271 --- /dev/null +++ b/backend/cli/src/file/annotations.ts @@ -0,0 +1,249 @@ +import path from "node:path" +import { ulid } from "ulid" +import z from "zod" +import { Instance } from "../project/instance" +import { Storage } from "../storage/storage" + +export namespace ArtifactAnnotation { + export const Anchor = z.discriminatedUnion("kind", [ + z.object({ + kind: z.literal("artifact"), + label: z.string().trim().max(500).optional(), + }), + z.object({ + kind: z.literal("text"), + startLine: z.number().int().min(1), + endLine: z.number().int().min(1), + quote: z.string().max(10_000).optional(), + }), + z.object({ + kind: z.literal("notebook"), + cellId: z.string().trim().min(1).max(500), + line: z.number().int().min(1).optional(), + }), + z.object({ + kind: z.literal("molecule"), + selection: z.string().trim().min(1).max(2_000), + count: z.number().int().min(1).optional(), + }), + z.object({ + kind: z.literal("genome"), + chromosome: z.string().trim().min(1).max(200), + start: z.number().int().min(0), + end: z.number().int().min(0), + }), + ]) + export type Anchor = z.infer + + export const Message = z.object({ + id: z.string(), + body: z.string(), + author: z.string(), + createdAt: z.number(), + }) + export type Message = z.infer + + export const Revision = z.object({ + version: z.number().int().positive(), + event: z.enum(["created", "edited", "replied", "resolved", "reopened", "deleted"]), + actor: z.string(), + at: z.number(), + status: z.enum(["open", "resolved"]), + messages: Message.array(), + deletedAt: z.number().optional(), + }) + export type Revision = z.infer + + export const Info = z.object({ + id: z.string(), + projectID: z.string(), + path: z.string(), + artifactHash: z.string().regex(/^[a-f0-9]{64}$/), + anchor: Anchor, + messages: Message.array(), + status: z.enum(["open", "resolved"]), + version: z.number().int().positive(), + revisions: Revision.array(), + createdAt: z.number(), + updatedAt: z.number(), + deletedAt: z.number().optional(), + }) + export type Info = z.infer + const Legacy = Info.omit({ artifactHash: true, version: true, revisions: true, deletedAt: true }) + + export const Create = z.object({ + path: z.string().trim().min(1).max(10_000), + body: z.string().trim().min(1).max(100_000), + author: z.string().trim().min(1).max(200).optional(), + anchor: Anchor.default({ kind: "artifact" }), + }) + export type Create = z.infer + + export const Update = z + .object({ + status: z.enum(["open", "resolved"]).optional(), + body: z.string().trim().min(1).max(100_000).optional(), + reply: z.string().trim().min(1).max(100_000).optional(), + author: z.string().trim().min(1).max(200).optional(), + }) + .refine( + (value) => value.status !== undefined || value.body !== undefined || value.reply !== undefined, + "No annotation update supplied", + ) + export type Update = z.infer + + const prefix = () => ["artifact_annotation", Instance.project.id] + const key = (id: string) => [...prefix(), id] + + async function target(value: string) { + const absolute = path.resolve(Instance.directory, value) + if (!(await Instance.containsCanonicalPath(absolute))) { + throw new Error(`Annotation target is outside the project: ${value}`) + } + return { + absolute, + relative: path.relative(Instance.directory, absolute).replaceAll("\\", "/"), + } + } + + async function digest(file: string) { + const hasher = new Bun.CryptoHasher("sha256") + const reader = Bun.file(file).stream().getReader() + const feed = async (): Promise => { + const chunk = await reader.read() + if (chunk.done) return + hasher.update(chunk.value) + return feed() + } + await feed() + return hasher.digest("hex") + } + + function hash(value: string) { + const hasher = new Bun.CryptoHasher("sha256") + hasher.update(value) + return hasher.digest("hex") + } + + function revision(record: Info, event: Revision["event"], actor: string, at: number): Revision { + return { + version: record.version, + event, + actor, + at, + status: record.status, + messages: record.messages.map((message) => ({ ...message })), + ...(record.deletedAt ? { deletedAt: record.deletedAt } : {}), + } + } + + async function read(id: string) { + const stored = await Storage.read(key(id)) + const current = Info.safeParse(stored) + if (current.success) return current.data + const legacy = Legacy.parse(stored) + const location = await target(legacy.path) + const artifactHash = (await Bun.file(location.absolute).exists()) + ? await digest(location.absolute) + : hash(`missing:${legacy.path}`) + const record: Info = { + ...legacy, + artifactHash, + version: 1, + revisions: [], + } + record.revisions.push(revision(record, "created", record.messages[0]?.author ?? "You", record.createdAt)) + await Storage.write(key(id), record) + return record + } + + export async function list(filepath: string) { + const location = await target(filepath) + const keys = await Storage.list(prefix()) + const records = await Promise.all(keys.map((item) => read(item.at(-1)!))) + return records + .filter((item) => item.path === location.relative && !item.deletedAt) + .toSorted((a, b) => a.createdAt - b.createdAt) + } + + export async function create(input: Create) { + const now = Date.now() + const id = `ann_${ulid()}` + const location = await target(input.path) + if (!(await Bun.file(location.absolute).exists())) { + throw new Error(`Annotation target does not exist: ${input.path}`) + } + const record: Info = { + id, + projectID: Instance.project.id, + path: location.relative, + artifactHash: await digest(location.absolute), + anchor: input.anchor, + messages: [ + { + id: `msg_${ulid()}`, + body: input.body, + author: input.author ?? "You", + createdAt: now, + }, + ], + status: "open", + version: 1, + revisions: [], + createdAt: now, + updatedAt: now, + } + record.revisions.push(revision(record, "created", input.author ?? "You", now)) + await Storage.write(key(id), record) + return record + } + + export async function update(id: string, input: Update) { + await read(id) + return Storage.update(key(id), (record) => { + if (record.deletedAt) throw new Error(`Annotation ${id} has been deleted`) + const now = Date.now() + const actor = input.author ?? "You" + const event: Revision["event"] = input.body + ? "edited" + : input.reply + ? "replied" + : input.status === "resolved" + ? "resolved" + : "reopened" + if (input.status) record.status = input.status + if (input.body && record.messages[0]) { + record.messages[0].body = input.body + record.messages[0].author = actor + } + if (input.reply) { + record.messages.push({ + id: `msg_${ulid()}`, + body: input.reply, + author: actor, + createdAt: now, + }) + } + record.version += 1 + record.updatedAt = now + record.revisions.push(revision(record, event, actor, now)) + }) + } + + export async function remove(id: string) { + await read(id) + const record = await Storage.update(key(id), (record) => { + if (record.deletedAt) return + const now = Date.now() + record.deletedAt = now + record.updatedAt = now + record.version += 1 + record.revisions.push(revision(record, "deleted", "You", now)) + }) + return { deleted: true as const, version: record.version } + } + + export async function history(id: string) { + return read(id) + } +} diff --git a/backend/cli/src/file/artifacts.ts b/backend/cli/src/file/artifacts.ts new file mode 100644 index 00000000..12d2358a --- /dev/null +++ b/backend/cli/src/file/artifacts.ts @@ -0,0 +1,407 @@ +import { $ } from "bun" +import fs from "node:fs" +import path from "node:path" +import z from "zod" + +export namespace ArtifactFile { + export const Kind = z.enum([ + "notebook", + "dataset", + "figure", + "report", + "structure", + "sequence", + "genomics", + "spectrum", + "model", + "archive", + ]) + export type Kind = z.infer + + export const Info = z.object({ + name: z.string(), + path: z.string(), + kind: Kind, + format: z.string(), + size: z.number(), + modified: z.number(), + }) + export type Info = z.infer + + export const Provenance = z.object({ + path: z.string(), + tracked: z.boolean(), + dirty: z.boolean(), + status: z.enum(["clean", "modified", "added", "deleted", "untracked", "local"]), + branch: z.string().optional(), + commit: z + .object({ + sha: z.string(), + author: z.string(), + email: z.string(), + date: z.string(), + message: z.string(), + }) + .optional(), + }) + export type Provenance = z.infer + + export const AuditCheck = z.object({ + id: z.string(), + label: z.string(), + status: z.enum(["pass", "warn", "fail"]), + detail: z.string(), + weight: z.number().positive(), + }) + export type AuditCheck = z.infer + + export const Audit = z.object({ + generated_at: z.string(), + score: z.number().int().min(0).max(100), + status: z.enum(["ready", "warnings", "blocked"]), + git: z + .object({ + branch: z.string().optional(), + commit: z.string().optional(), + dirty: z.boolean(), + }) + .optional(), + lockfiles: z.string().array(), + environments: z.string().array(), + notebooks: z.object({ + total: z.number().int().nonnegative(), + valid: z.number().int().nonnegative(), + invalid: z.string().array(), + }), + artifacts: z.object({ + total: z.number().int().nonnegative(), + nonempty: z.number().int().nonnegative(), + bytes: z.number().int().nonnegative(), + }), + checks: AuditCheck.array(), + }) + export type Audit = z.infer + + export const ManifestArtifact = Info.extend({ + sha256: z.string().regex(/^[a-f0-9]{64}$/), + }) + export type ManifestArtifact = z.infer + + export const Manifest = z.object({ + format: z.literal("openscience.artifact-manifest.v1"), + generated_at: z.string(), + digest: z.string().regex(/^[a-f0-9]{64}$/), + artifacts: ManifestArtifact.array(), + }) + export type Manifest = z.infer + + const kinds: Record = { + notebook: ["ipynb"], + dataset: ["csv", "tsv", "jsonl", "parquet", "feather", "arrow", "xls", "xlsx", "h5", "hdf5", "h5ad", "loom"], + figure: ["png", "jpg", "jpeg", "svg", "webp", "tif", "tiff", "gif"], + report: ["pdf", "html", "htm", "md", "markdown", "docx", "tex", "latex"], + structure: ["pdb", "ent", "cif", "mmcif", "pdbqt", "gro", "xyz", "sdf", "mol", "mol2", "smi", "smiles"], + sequence: ["fa", "fasta", "faa", "fna", "ffn", "frn", "fastq", "fq"], + genomics: ["vcf", "bcf", "bam", "cram", "bed", "bedgraph", "gff", "gff3", "gtf", "bigwig", "bw"], + spectrum: ["mzml", "mzxml", "mgf", "cdf"], + model: ["pkl", "pickle", "joblib", "pt", "pth", "ckpt", "safetensors", "onnx", "pb"], + archive: ["zip", "tar", "gz", "bz2", "xz", "7z"], + } + const extensions = Object.fromEntries( + Object.entries(kinds).flatMap(([kind, values]) => values.map((value) => [value, kind])), + ) as Record + const excluded = new Set([ + ".git", + ".hg", + ".svn", + ".cache", + ".next", + ".turbo", + ".venv", + "node_modules", + "dist", + "build", + "target", + "vendor", + "__pycache__", + ]) + const LIMIT = 5_000 + const DEPTH = 16 + const lockfiles = [ + "bun.lock", + "bun.lockb", + "package-lock.json", + "pnpm-lock.yaml", + "yarn.lock", + "uv.lock", + "poetry.lock", + "Pipfile.lock", + "renv.lock", + "Cargo.lock", + "Manifest.toml", + ] + const environments = [ + "pyproject.toml", + "requirements.txt", + "environment.yml", + "environment.yaml", + "Dockerfile", + "compose.yml", + "docker-compose.yml", + "package.json", + "renv.lock", + "Project.toml", + ] + + export function classify(file: string): { kind: Kind; format: string } | undefined { + const format = path.extname(file).slice(1).toLowerCase() + const kind = extensions[format] + if (!kind) return + return { kind, format } + } + + export async function scan(root: string): Promise { + const artifacts: Info[] = [] + const walk = async (directory: string, relative: string, depth: number): Promise => { + if (depth > DEPTH || artifacts.length >= LIMIT) return + const entries = await fs.promises.readdir(directory, { withFileTypes: true }).catch(() => [] as fs.Dirent[]) + for (const entry of entries) { + if (artifacts.length >= LIMIT) return + if (entry.isDirectory()) { + if (excluded.has(entry.name) || entry.name.startsWith(".")) continue + await walk(path.join(directory, entry.name), path.join(relative, entry.name), depth + 1) + continue + } + if (!entry.isFile()) continue + const classified = classify(entry.name) + if (!classified) continue + const full = path.join(directory, entry.name) + const stat = await fs.promises.stat(full).catch(() => undefined) + if (!stat) continue + artifacts.push({ + name: entry.name, + path: path.join(relative, entry.name).replaceAll(path.sep, "/").replace(/^\.\//, ""), + kind: classified.kind, + format: classified.format, + size: stat.size, + modified: stat.mtimeMs, + }) + } + } + await walk(root, ".", 0) + return artifacts.toSorted((a, b) => b.modified - a.modified || a.path.localeCompare(b.path)) + } + + export async function provenance(root: string, file: string): Promise { + const inside = await $`git rev-parse --is-inside-work-tree`.cwd(root).quiet().nothrow() + if (inside.exitCode !== 0) { + return { path: file, tracked: false, dirty: false, status: "local" } + } + const [branchResult, trackedResult, statusResult, logResult] = await Promise.all([ + $`git branch --show-current`.cwd(root).quiet().nothrow().text(), + $`git ls-files --error-unmatch -- ${file}`.cwd(root).quiet().nothrow(), + $`git status --porcelain=v1 -- ${file}`.cwd(root).quiet().nothrow().text(), + $`git log -1 --format=%H%x00%an%x00%ae%x00%aI%x00%s -- ${file}`.cwd(root).quiet().nothrow().text(), + ]) + const tracked = trackedResult.exitCode === 0 + const code = statusResult.trim().slice(0, 2) + const status = statusOf(code, tracked) + const parts = logResult.trim().split("\0") + const commit = + parts.length >= 5 + ? { + sha: parts[0]!, + author: parts[1]!, + email: parts[2]!, + date: parts[3]!, + message: parts.slice(4).join("\0"), + } + : undefined + return { + path: file, + tracked, + dirty: status !== "clean", + status, + branch: branchResult.trim() || undefined, + commit, + } + } + + export async function audit(root: string): Promise { + const artifacts = await scan(root) + const notebooks = artifacts.filter((artifact) => artifact.kind === "notebook") + const invalid = ( + await Promise.all( + notebooks.map(async (notebook) => { + const value = await Bun.file(path.join(root, notebook.path)) + .json() + .catch(() => undefined) + if (!value || typeof value !== "object") return notebook.path + const record = value as Record + if (typeof record.nbformat !== "number" || !Array.isArray(record.cells)) return notebook.path + return undefined + }), + ) + ).filter((file): file is string => !!file) + const [branch, commit, status, presentLocks, presentEnvironments, readme] = await Promise.all([ + $`git branch --show-current`.cwd(root).quiet().nothrow().text(), + $`git rev-parse HEAD`.cwd(root).quiet().nothrow().text(), + $`git status --porcelain`.cwd(root).quiet().nothrow().text(), + Promise.all(lockfiles.map(async (file) => ((await Bun.file(path.join(root, file)).exists()) ? file : undefined))), + Promise.all( + environments.map(async (file) => ((await Bun.file(path.join(root, file)).exists()) ? file : undefined)), + ), + Promise.all(["README.md", "README.rst", "README.txt"].map((file) => Bun.file(path.join(root, file)).exists())), + ]) + const locks = presentLocks.filter((file): file is string => !!file) + const envs = presentEnvironments.filter((file): file is string => !!file) + const git = commit.trim() + ? { + branch: branch.trim() || undefined, + commit: commit.trim(), + dirty: Boolean(status.trim()), + } + : undefined + const checks = [ + check( + "git-repository", + "Version-controlled project", + git ? "pass" : "fail", + git ? "Git repository detected." : "Initialize Git so every result can point to exact code.", + 10, + ), + check( + "git-clean", + "Clean working tree", + !git ? "fail" : git.dirty ? "warn" : "pass", + !git + ? "No Git state is available." + : git.dirty + ? "Commit or stash local changes before a definitive run." + : "Working tree matches the captured commit.", + 15, + ), + check( + "git-commit", + "Reachable code snapshot", + git?.commit ? "pass" : "fail", + git?.commit ? `Current commit ${git.commit.slice(0, 12)}.` : "Create a commit before recording results.", + 10, + ), + check( + "environment-lock", + "Locked dependencies", + locks.length ? "pass" : "fail", + locks.length ? locks.join(", ") : "Add a lockfile such as uv.lock, renv.lock, bun.lock, or package-lock.json.", + 15, + ), + check( + "environment-spec", + "Environment specification", + envs.length ? "pass" : "warn", + envs.length + ? envs.join(", ") + : "Add pyproject.toml, environment.yml, requirements.txt, Dockerfile, or an equivalent spec.", + 10, + ), + check( + "notebooks", + "Executable notebook structure", + invalid.length ? "fail" : "pass", + invalid.length + ? `Invalid notebooks: ${invalid.join(", ")}` + : notebooks.length + ? `${notebooks.length} notebook${notebooks.length === 1 ? "" : "s"} passed structural validation.` + : "No notebooks need validation.", + 15, + ), + check( + "artifacts", + "Non-empty research artifacts", + !artifacts.length || artifacts.some((artifact) => artifact.size === 0) ? "warn" : "pass", + artifacts.length + ? `${artifacts.filter((artifact) => artifact.size > 0).length}/${artifacts.length} artifacts are non-empty.` + : "No generated research artifacts yet.", + 10, + ), + check( + "readme", + "Project instructions", + readme.some(Boolean) ? "pass" : "warn", + readme.some(Boolean) ? "README found." : "Add a README with setup, data, and execution instructions.", + 5, + ), + ] satisfies AuditCheck[] + const total = checks.reduce((sum, item) => sum + item.weight, 0) + const earned = checks.reduce( + (sum, item) => sum + (item.status === "pass" ? item.weight : item.status === "warn" ? item.weight / 2 : 0), + 0, + ) + const score = Math.round((earned / total) * 100) + return Audit.parse({ + generated_at: new Date().toISOString(), + score, + status: checks.some((item) => item.status === "fail") ? "blocked" : score >= 85 ? "ready" : "warnings", + git, + lockfiles: locks, + environments: envs, + notebooks: { + total: notebooks.length, + valid: notebooks.length - invalid.length, + invalid, + }, + artifacts: { + total: artifacts.length, + nonempty: artifacts.filter((artifact) => artifact.size > 0).length, + bytes: artifacts.reduce((sum, artifact) => sum + artifact.size, 0), + }, + checks, + }) + } + + export async function manifest(root: string): Promise { + const artifacts = await scan(root) + const sorted = artifacts.toSorted((a, b) => a.path.localeCompare(b.path)) + const batches = Array.from({ length: Math.ceil(sorted.length / 16) }, (_, index) => + sorted.slice(index * 16, (index + 1) * 16), + ) + const hashed: ManifestArtifact[] = [] + for (const batch of batches) { + hashed.push( + ...(await Promise.all( + batch.map(async (artifact) => + ManifestArtifact.parse({ ...artifact, sha256: await hash(path.join(root, artifact.path)) }), + ), + )), + ) + } + const digest = await crypto.subtle.digest( + "SHA-256", + new TextEncoder().encode(hashed.map((artifact) => `${artifact.sha256} ${artifact.path}`).join("\n")), + ) + return Manifest.parse({ + format: "openscience.artifact-manifest.v1", + generated_at: new Date().toISOString(), + digest: [...new Uint8Array(digest)].map((byte) => byte.toString(16).padStart(2, "0")).join(""), + artifacts: hashed, + }) + } + + function check(id: string, label: string, status: AuditCheck["status"], detail: string, weight: number): AuditCheck { + return { id, label, status, detail, weight } + } + + async function hash(file: string): Promise { + const digest = new Bun.CryptoHasher("sha256") + for await (const chunk of fs.createReadStream(file)) digest.update(chunk) + return digest.digest("hex") + } + + function statusOf(code: string, tracked: boolean): Provenance["status"] { + if (code === "??") return "untracked" + if (code.includes("D")) return "deleted" + if (code.includes("A")) return "added" + if (code) return "modified" + return tracked ? "clean" : "local" + } +} diff --git a/backend/cli/src/file/index.ts b/backend/cli/src/file/index.ts index 5db6e9b3..923a22b0 100644 --- a/backend/cli/src/file/index.ts +++ b/backend/cli/src/file/index.ts @@ -15,9 +15,15 @@ import fuzzysort from "fuzzysort" import { Global } from "../global" import { FileWatcher } from "./watcher" import { createSearchCache } from "./search-cache" +import { ScienceFile } from "./science" +import { ArtifactFile } from "./artifacts" +import { StarterFile } from "./starters" +import { PublicationFile } from "./publication" +import { PublicationReview } from "./review" export namespace File { const log = Log.create({ service: "file" }) + const preview = 8 * 1024 * 1024 export const Info = z .object({ @@ -73,6 +79,8 @@ export namespace File { .optional(), encoding: z.literal("base64").optional(), mimeType: z.string().optional(), + size: z.number().optional(), + truncated: z.boolean().optional(), }) .meta({ ref: "FileContent", @@ -212,6 +220,12 @@ export namespace File { state() } + async function contained(file: string): Promise { + const full = path.join(Instance.directory, file) + if (await Instance.containsCanonicalPath(full)) return full + throw new Error(`Access denied: path escapes project directory`) + } + export async function status() { const project = Instance.project if (project.vcs !== "git") return [] @@ -289,13 +303,7 @@ export namespace File { export async function read(file: string): Promise { using _ = log.time("read", { file }) const project = Instance.project - const full = path.join(Instance.directory, file) - - // TODO: Filesystem.contains is lexical only - symlinks inside the project can escape. - // TODO: On Windows, cross-drive paths bypass this check. Consider realpath canonicalization. - if (!Instance.containsPath(full)) { - throw new Error(`Access denied: path escapes project directory`) - } + const full = await contained(file) const bunFile = Bun.file(full) @@ -303,19 +311,37 @@ export namespace File { return { type: "text", content: "" } } - const encode = await shouldEncode(bunFile) + const encode = ScienceFile.binary(file) || (await shouldEncode(bunFile)) if (encode) { + if (bunFile.size > 16 * 1024 * 1024) { + return { + type: "text", + content: "", + mimeType: bunFile.type || "application/octet-stream", + encoding: "base64", + size: bunFile.size, + truncated: true, + } + } const buffer = await bunFile.arrayBuffer().catch(() => new ArrayBuffer(0)) const content = Buffer.from(buffer).toString("base64") const mimeType = bunFile.type || "application/octet-stream" - return { type: "text", content, mimeType, encoding: "base64" } + return { type: "text", content, mimeType, encoding: "base64", size: bunFile.size } } - // Return the file content verbatim — callers like the web editor write - // it back, so trimming here would silently strip leading/trailing - // whitespace and the trailing newline on the first save. - const content = await bunFile.text().catch(() => "") + const truncated = bunFile.size > preview + // Keep scientific/text previews bounded. The UI treats this response as + // read-only, so a partial preview can never overwrite the source file. + const content = await (truncated ? bunFile.slice(0, preview) : bunFile).text().catch(() => "") + if (truncated) { + return { + type: "text", + content, + size: bunFile.size, + truncated: true, + } + } if (project.vcs === "git") { let diff = await $`git diff ${file}`.cwd(Instance.directory).quiet().nothrow().text() @@ -333,12 +359,77 @@ export namespace File { return { type: "text", content } } + export async function inspect(file: string): Promise { + const full = await contained(file) + return ScienceFile.inspect(full, file) + } + + export async function raw(file: string): Promise { + const full = await contained(file) + const content = Bun.file(full) + if (!(await content.exists())) throw new HTTPException(404, { message: `File not found: ${file}` }) + return content + } + + export async function artifacts(): Promise { + return ArtifactFile.scan(Instance.directory) + } + + export async function provenance(file: string): Promise { + await contained(file) + return ArtifactFile.provenance(Instance.directory, file) + } + + export async function reproducibility(): Promise { + return ArtifactFile.audit(Instance.directory) + } + + export async function manifest(): Promise { + return ArtifactFile.manifest(Instance.directory) + } + + export async function starter(template: StarterFile.Template): Promise { + return StarterFile.create(Instance.directory, template) + } + + export async function publicationCapabilities(): Promise { + return PublicationFile.capabilities() + } + + export async function publication(input: PublicationFile.Input): Promise { + return PublicationFile.render(Instance.directory, input) + } + + export async function review(input: PublicationReview.RunInput): Promise { + return PublicationReview.run(input) + } + + export async function reviewCurrent(file: string): Promise { + return PublicationReview.current(file) + } + + export async function reviewHistory(file: string): Promise { + return PublicationReview.history(file) + } + + export async function reviewResolve( + id: string, + finding: string, + input: PublicationReview.ResolveInput, + ): Promise { + return PublicationReview.resolve(id, finding, input) + } + + export async function reviewFinalize( + id: string, + input: PublicationReview.FinalizeInput, + ): Promise { + return PublicationReview.finalize(id, input) + } + export async function write(file: string, content: string): Promise { using _ = log.time("write", { file }) - const full = path.join(Instance.directory, file) - if (!Instance.containsPath(full)) { - throw new Error(`Access denied: path escapes project directory`) - } + const full = await contained(file) const exists = await Bun.file(full).exists() await Bun.write(full, content) @@ -369,10 +460,7 @@ export namespace File { ignored = ig.ignores.bind(ig) } const resolved = dir ? path.join(Instance.directory, dir) : Instance.directory - - // TODO: Filesystem.contains is lexical only - symlinks inside the project can escape. - // TODO: On Windows, cross-drive paths bypass this check. Consider realpath canonicalization. - if (!Instance.containsPath(resolved)) { + if (!(await Instance.containsCanonicalPath(resolved))) { throw new Error(`Access denied: path escapes project directory`) } diff --git a/backend/cli/src/file/publication.ts b/backend/cli/src/file/publication.ts new file mode 100644 index 00000000..84fe5060 --- /dev/null +++ b/backend/cli/src/file/publication.ts @@ -0,0 +1,244 @@ +import fs from "node:fs/promises" +import path from "node:path" +import { marked, Renderer } from "marked" +import z from "zod" +import { OpenScience } from "../openscience" +import { Filesystem } from "../util/filesystem" +import { escapeHtml } from "../util/html" +import { PublicationReview } from "./review" + +export namespace PublicationFile { + export const Format = z.enum(["html", "pdf", "docx", "latex", "pptx"]) + export type Format = z.infer + + export const Input = z + .object({ + path: z.string().trim().min(1).max(4_000), + format: Format, + readiness: z.enum(["draft", "reviewed"]).default("draft"), + review_id: z.string().startsWith("review_").optional(), + }) + .superRefine((input, context) => { + if (input.readiness !== "reviewed" || input.review_id) return + context.addIssue({ + code: "custom", + path: ["review_id"], + message: "A reviewed publication export requires a finalized review report", + }) + }) + export type Input = z.input + + export const Capabilities = z.object({ + pandoc: z.boolean(), + pdf_engine: z.string().optional(), + formats: z.record(Format, z.boolean()), + }) + export type Capabilities = z.infer + + export const Result = z.object({ + path: z.string(), + format: Format, + size: z.number().int().nonnegative(), + created_at: z.string(), + engine: z.string(), + readiness: z.enum(["draft", "reviewed"]), + review_id: z.string().optional(), + }) + export type Result = z.infer + + const extensions: Record = { + html: "html", + pdf: "pdf", + docx: "docx", + latex: "tex", + pptx: "pptx", + } + + export async function capabilities(): Promise { + const options = { PATH: process.env.PATH } + const pandoc = Boolean(Bun.which("pandoc", options)) + const pdf = + Bun.which("xelatex", options) ?? Bun.which("pdflatex", options) ?? Bun.which("typst", options) ?? undefined + return Capabilities.parse({ + pandoc, + pdf_engine: pdf ? path.basename(pdf) : undefined, + formats: { + html: true, + pdf: pandoc && Boolean(pdf), + docx: pandoc, + latex: pandoc, + pptx: pandoc, + }, + }) + } + + export async function render(root: string, input: Input): Promise { + const parsed = Input.parse(input) + const source = resolve(root, parsed.path) + if (![".md", ".markdown"].includes(path.extname(source).toLowerCase())) { + throw new Error("Publication export currently requires a Markdown report") + } + if (!(await Filesystem.containsCanonical(root, source))) { + throw new Error("Publication path escapes the project directory") + } + if (!(await Bun.file(source).exists())) throw new Error(`Report not found: ${parsed.path}`) + const snapshot = await Bun.file(source).arrayBuffer() + const markdown = new TextDecoder().decode(snapshot) + const artifactHash = await hash(snapshot) + const review = + parsed.readiness === "reviewed" + ? await PublicationReview.assertReady(parsed.path, parsed.review_id!, artifactHash) + : undefined + const support = await capabilities() + if (!support.formats[parsed.format]) { + throw new Error( + parsed.format === "pdf" + ? "PDF export requires Pandoc and a local TeX or Typst engine" + : `${parsed.format.toUpperCase()} export requires Pandoc`, + ) + } + const folder = path.join(root, "exports") + await fs.mkdir(folder, { recursive: true }) + const stamp = new Date().toISOString().replace(/\D/g, "").slice(0, 17) + const nonce = crypto.randomUUID().slice(0, 8) + const stem = + path + .basename(source, path.extname(source)) + .toLowerCase() + .replace(/[^a-z0-9_-]+/g, "-") + .replace(/^-+|-+$/g, "") || "report" + const relative = path.join( + "exports", + `${stem}-${stamp.slice(0, 8)}-${stamp.slice(8)}-${nonce}.${extensions[parsed.format]}`, + ) + const target = path.join(root, relative) + if (parsed.format === "html") { + const renderer = new Renderer() + renderer.html = ({ text }) => escapeHtml(text) + renderer.link = ({ href, title, tokens }) => { + const content = renderer.parser.parseInline(tokens) + const target = safe(href, false) + if (!target) return content + const hint = title ? ` title="${escapeHtml(title)}"` : "" + return `${content}` + } + renderer.image = ({ href, title, text }) => { + const target = safe(href, true) + if (!target) return escapeHtml(text) + const hint = title ? ` title="${escapeHtml(title)}"` : "" + return `${escapeHtml(text)}` + } + const body = await marked.parse(markdown, { gfm: true, renderer }) + const base = `${path.relative(folder, path.dirname(source)).split(path.sep).join("/") || "."}/` + const title = path.basename(source, path.extname(source)) + const document = ` + + + + + + + ${escapeHtml(title)} + + + +${body} + + +` + await Bun.write(target, document) + const stat = await fs.stat(target) + return Result.parse({ + path: relative.split(path.sep).join("/"), + format: parsed.format, + size: stat.size, + created_at: new Date().toISOString(), + engine: "OpenScience Markdown", + readiness: parsed.readiness, + ...(review ? { review_id: review.id } : {}), + }) + } + const snapshotFile = path.join(folder, `.openscience-publication-${nonce}.md`) + await Bun.write(snapshotFile, snapshot) + const args = [ + "pandoc", + snapshotFile, + "--standalone", + `--resource-path=${path.dirname(source)}${path.delimiter}${root}`, + "--output", + target, + ...(parsed.format === "pdf" && support.pdf_engine ? [`--pdf-engine=${support.pdf_engine}`] : []), + ] + const proc = Bun.spawn(args, { + cwd: root, + env: await OpenScience.subprocessEnv(process.env), + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + }) + const [code, stdout, stderr] = await Promise.all([ + proc.exited, + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + ]).finally(() => fs.rm(snapshotFile, { force: true })) + if (code !== 0) { + await fs.rm(target, { force: true }) + throw new Error(stderr.trim() || stdout.trim() || `Pandoc exited with code ${code}`) + } + const stat = await fs.stat(target) + return Result.parse({ + path: relative.split(path.sep).join("/"), + format: parsed.format, + size: stat.size, + created_at: new Date().toISOString(), + engine: parsed.format === "pdf" ? `pandoc + ${support.pdf_engine}` : "pandoc", + readiness: parsed.readiness, + ...(review ? { review_id: review.id } : {}), + }) + } + + function resolve(root: string, file: string): string { + const target = path.resolve(root, file) + const relative = path.relative(root, target) + if (relative === ".." || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) { + throw new Error("Publication path escapes the project directory") + } + return target + } + + function safe(value: string, image: boolean): string | undefined { + const scheme = /^([a-z][a-z0-9+.-]*):/i.exec(value)?.[1]?.toLowerCase() + if (!scheme) return value + if (scheme === "http" || scheme === "https") return value + if (!image && scheme === "mailto") return value + if (image && scheme === "data" && /^data:image\//i.test(value)) return value + return undefined + } + + async function hash(value: ArrayBuffer) { + const digest = await crypto.subtle.digest("SHA-256", value) + return [...new Uint8Array(digest)].map((byte) => byte.toString(16).padStart(2, "0")).join("") + } +} diff --git a/backend/cli/src/file/review.ts b/backend/cli/src/file/review.ts new file mode 100644 index 00000000..9c64618a --- /dev/null +++ b/backend/cli/src/file/review.ts @@ -0,0 +1,602 @@ +import path from "node:path" +import { ulid } from "ulid" +import z from "zod" +import { Instance } from "../project/instance" +import { Provenance } from "../science/provenance/store" +import { Storage } from "../storage/storage" +import { ArtifactFile } from "./artifacts" + +export namespace PublicationReview { + export const Check = z.enum(["citation", "numeric", "figure", "provenance"]) + export type Check = z.infer + + export const Severity = z.enum(["blocking", "major", "minor", "info"]) + export type Severity = z.infer + + export const FindingStatus = z.enum(["open", "resolved", "overridden"]) + export type FindingStatus = z.infer + + export const Location = z.object({ + path: z.string(), + line: z.number().int().positive().optional(), + }) + export type Location = z.infer + + export const Resolution = z.object({ + kind: z.enum(["resolved", "overridden"]), + actor: z.string(), + reason: z.string(), + at: z.number(), + }) + export type Resolution = z.infer + + export const Finding = z.object({ + id: z.string(), + check: Check, + severity: Severity, + status: FindingStatus, + title: z.string(), + detail: z.string(), + evidence: z.string().array(), + location: Location, + resolution: Resolution.optional(), + }) + export type Finding = z.infer + + export const Event = z.object({ + version: z.number().int().positive(), + type: z.enum(["generated", "resolved", "overridden", "finalized"]), + actor: z.string(), + at: z.number(), + findingID: z.string().optional(), + reason: z.string().optional(), + }) + export type Event = z.infer + + export const Finalization = z.object({ + actor: z.string(), + at: z.number(), + artifactHash: z.string().regex(/^[a-f0-9]{64}$/), + }) + export type Finalization = z.infer + + export const Summary = z.object({ + total: z.number().int().nonnegative(), + open: z.number().int().nonnegative(), + blocking: z.number().int().nonnegative(), + major: z.number().int().nonnegative(), + minor: z.number().int().nonnegative(), + info: z.number().int().nonnegative(), + resolved: z.number().int().nonnegative(), + overridden: z.number().int().nonnegative(), + }) + export type Summary = z.infer + + export const Report = z.object({ + format: z.literal("openscience.publication-review.v1"), + id: z.string(), + projectID: z.string(), + path: z.string(), + artifactHash: z.string().regex(/^[a-f0-9]{64}$/), + version: z.number().int().positive(), + status: z.enum(["blocked", "warnings", "ready"]), + summary: Summary, + findings: Finding.array(), + events: Event.array(), + finalized: Finalization.optional(), + createdAt: z.number(), + updatedAt: z.number(), + }) + export type Report = z.infer + + export const State = Report.extend({ + stale: z.boolean(), + }) + export type State = z.infer + + export const RunInput = z.object({ + path: z.string().trim().min(1).max(10_000), + actor: z.string().trim().min(1).max(200).default("OpenScience"), + }) + export type RunInput = z.infer + + export const ResolveInput = z.object({ + status: z.enum(["resolved", "overridden"]), + actor: z.string().trim().min(1).max(200), + reason: z.string().trim().min(1).max(20_000), + }) + export type ResolveInput = z.infer + + export const FinalizeInput = z.object({ + actor: z.string().trim().min(1).max(200), + }) + export type FinalizeInput = z.infer + + const prefix = () => ["publication_review", Instance.project.id] + const key = (id: string) => [...prefix(), id] + + export async function run(input: RunInput): Promise { + const parsed = RunInput.parse(input) + const source = await target(parsed.path) + if (![".md", ".markdown"].includes(path.extname(source.absolute).toLowerCase())) { + throw new Error("Publication review currently requires a Markdown manuscript") + } + if (!(await Bun.file(source.absolute).exists())) { + throw new Error(`Publication manuscript not found: ${parsed.path}`) + } + const [text, artifactHash, graph, provenance, audit] = await Promise.all([ + Bun.file(source.absolute).text(), + digest(source.absolute), + Provenance.project(Instance.worktree), + ArtifactFile.provenance(Instance.worktree, source.relative), + ArtifactFile.audit(Instance.worktree), + ]) + const findings = [ + ...(await citations(text, source)), + ...(await numbers(text, source)), + ...(await figures(text, source, graph.nodes)), + ...(await reproducibility(source, provenance, audit)), + ] + const now = Date.now() + const report: Report = { + format: "openscience.publication-review.v1", + id: `review_${ulid()}`, + projectID: Instance.project.id, + path: source.relative, + artifactHash, + version: 1, + status: status(findings), + summary: summary(findings), + findings: findings.toSorted(compare), + events: [{ version: 1, type: "generated", actor: parsed.actor, at: now }], + createdAt: now, + updatedAt: now, + } + await Storage.write(key(report.id), report) + return Report.parse(report) + } + + export async function latest(filepath: string): Promise { + return (await history(filepath)).at(-1) + } + + export async function current(filepath: string): Promise { + const report = await latest(filepath) + if (!report) return + const source = await target(filepath) + const stale = + !(await Bun.file(source.absolute).exists()) || + (await digest(source.absolute).catch(() => "")) !== report.artifactHash + return State.parse({ ...report, stale }) + } + + export async function history(filepath: string): Promise { + const source = await target(filepath) + const keys = await Storage.list(prefix()) + const reports = await Promise.all( + keys.map((item) => Storage.read(item).then((value) => Report.parse(value))), + ) + return reports + .filter((report) => report.path === source.relative) + .toSorted((a, b) => a.createdAt - b.createdAt || a.id.localeCompare(b.id)) + } + + export async function get(id: string): Promise { + return Report.parse(await Storage.read(key(id))) + } + + export async function resolve(id: string, findingID: string, input: ResolveInput): Promise { + const parsed = ResolveInput.parse(input) + return Report.parse( + await Storage.update(key(id), (report) => { + if (report.finalized) throw new Error("A finalized publication review cannot be changed") + const finding = report.findings.find((item) => item.id === findingID) + if (!finding) throw new Error(`Review finding ${findingID} was not found`) + const now = Date.now() + finding.status = parsed.status + finding.resolution = { + kind: parsed.status, + actor: parsed.actor, + reason: parsed.reason, + at: now, + } + report.version += 1 + report.updatedAt = now + report.status = status(report.findings) + report.summary = summary(report.findings) + report.events.push({ + version: report.version, + type: parsed.status, + actor: parsed.actor, + at: now, + findingID, + reason: parsed.reason, + }) + }), + ) + } + + export async function finalize(id: string, input: FinalizeInput): Promise { + const parsed = FinalizeInput.parse(input) + const current = await get(id) + await assertCurrent(current) + if (current.findings.some((finding) => finding.severity === "blocking" && finding.status === "open")) { + throw new Error("Resolve or explicitly override all blocking findings before finalization") + } + return Report.parse( + await Storage.update(key(id), (report) => { + if (report.finalized) return + const now = Date.now() + report.version += 1 + report.updatedAt = now + report.finalized = { + actor: parsed.actor, + at: now, + artifactHash: report.artifactHash, + } + report.events.push({ + version: report.version, + type: "finalized", + actor: parsed.actor, + at: now, + }) + }), + ) + } + + export async function assertReady(filepath: string, id: string, artifactHash?: string): Promise { + const report = await get(id) + const source = await target(filepath) + if (report.path !== source.relative) throw new Error("The publication review belongs to a different manuscript") + await assertCurrent(report, artifactHash) + if (!report.finalized) throw new Error("The publication review has not been finalized") + if (report.findings.some((finding) => finding.severity === "blocking" && finding.status === "open")) { + throw new Error("The publication review still has open blocking findings") + } + return report + } + + async function citations(text: string, source: Awaited>): Promise { + const lines = text.split(/\r?\n/) + const definitions = new Set( + lines.map((line) => /^\s*\[\^([^\]]+)\]:/.exec(line)?.[1]).filter((value): value is string => Boolean(value)), + ) + const references = new Set() + for (const [index, line] of lines.entries()) { + for (const match of line.matchAll(/\[\^([^\]]+)\](?!:)/g)) { + if (definitions.has(match[1]!)) continue + references.add(`${match[1]}\0${index + 1}`) + } + } + const bib = await bibliography(text, source) + const keys = new Set() + for (const file of bib) { + const value = await Bun.file(file) + .text() + .catch(() => "") + for (const match of value.matchAll(/@\w+\s*\{\s*([^,\s]+)\s*,/g)) keys.add(match[1]!) + } + const cites = new Map() + for (const [index, line] of lines.entries()) { + for (const match of line.matchAll(/@([A-Za-z][A-Za-z0-9_.:+/-]*)/g)) { + if (!cites.has(match[1]!)) cites.set(match[1]!, index + 1) + } + } + const missing = await Promise.all( + [...cites] + .filter(([key]) => !keys.has(key)) + .map(([key, line]) => + finding({ + check: "citation", + severity: "blocking", + title: `Bibliography key @${key} is unresolved`, + detail: "The manuscript cites a key that is absent from its local bibliography files.", + evidence: bib.length + ? bib.map((file) => path.relative(Instance.worktree, file).replaceAll("\\", "/")) + : ["No local bibliography file was found."], + location: { path: source.relative, line }, + }), + ), + ) + const footnotes = await Promise.all( + [...references].map((value) => { + const [label, line] = value.split("\0") + return finding({ + check: "citation", + severity: "blocking", + title: `Footnote [^${label}] has no definition`, + detail: "Add a matching footnote definition or remove the unresolved reference.", + evidence: [`Referenced at ${source.relative}:${line}`], + location: { path: source.relative, line: Number(line) }, + }) + }), + ) + const placeholders = await Promise.all( + lines.flatMap((line, index) => { + if (!/\[(?:citation needed|cite|reference needed)\]|\bTODO\s*:?\s*cite\b/i.test(line)) return [] + return [ + finding({ + check: "citation", + severity: "blocking", + title: "Citation placeholder remains in the manuscript", + detail: "Replace the placeholder with a resolvable source before publication.", + evidence: [line.trim()], + location: { path: source.relative, line: index + 1 }, + }), + ] + }), + ) + return [...missing, ...footnotes, ...placeholders] + } + + async function numbers(text: string, source: Awaited>): Promise { + const lines = text.split(/\r?\n/) + return Promise.all( + lines.flatMap((line, index) => { + const claim = + /\b\d+(?:\.\d+)?\s*%/.test(line) || + /\bp\s*(?:<|>|=|≤|≥)\s*0?\.\d+/i.test(line) || + /\b(?:confidence interval|CI)\b/i.test(line) || + /\b\d+(?:\.\d+)?\s*(?:mg|µg|μg|ng|kg|mL|µL|μL|mm|cm|nm|µm|μm|Hz|kDa)\b/.test(line) + if (!claim) return [] + const traced = + /@[A-Za-z][A-Za-z0-9_.:+/-]*/.test(line) || + /\b(?:figure|fig\.?|table|supplement(?:ary)?)\s*[A-Za-z0-9]/i.test(line) || + /\[[^\]]+\]\([^)]*\.(?:csv|tsv|json|jsonl|parquet|arrow|xlsx?|ipynb)(?:[?#][^)]*)?\)/i.test(line) + if (traced) return [] + return [ + finding({ + check: "numeric", + severity: "major", + title: "Numeric claim has no inline evidence trace", + detail: + "Link this reported value to a citation, table, figure, notebook, or local data artifact so it can be independently checked.", + evidence: [line.trim()], + location: { path: source.relative, line: index + 1 }, + }), + ] + }), + ) + } + + async function figures( + text: string, + source: Awaited>, + nodes: Awaited>["nodes"], + ): Promise { + const lines = text.split(/\r?\n/) + const output: Finding[] = [] + for (const [index, line] of lines.entries()) { + for (const match of line.matchAll(/!\[([^\]]*)\]\(\s*(?:<([^>]+)>|([^\s)]+))(?:\s+["'][^"']*["'])?\s*\)/g)) { + const alt = match[1]!.trim() + const value = (match[2] ?? match[3]!).trim() + if (/^(?:https?:|data:)/i.test(value)) continue + const absolute = path.resolve(path.dirname(source.absolute), decodeURIComponent(value.split(/[?#]/)[0]!)) + const relative = path.relative(Instance.worktree, absolute).replaceAll("\\", "/") + const inside = await Instance.containsCanonicalPath(absolute) + const exists = inside && (await Bun.file(absolute).exists()) + if (!exists) { + output.push( + await finding({ + check: "figure", + severity: "blocking", + title: `Figure ${value} is missing`, + detail: inside + ? "The local figure referenced by this manuscript does not exist." + : "The figure reference resolves outside the opened project.", + evidence: [relative], + location: { path: source.relative, line: index + 1 }, + }), + ) + continue + } + if (!alt) { + output.push( + await finding({ + check: "figure", + severity: "minor", + title: `Figure ${value} has no alternative text`, + detail: "Add concise alternative text describing the scientific content of the figure.", + evidence: [relative], + location: { path: source.relative, line: index + 1 }, + }), + ) + } + const recorded = nodes.some((node) => { + if (node.kind !== "artifact" || !("path" in node) || !node.path) return false + const owner = typeof node.meta?.directory === "string" ? node.meta.directory : Instance.worktree + const nodePath = path.isAbsolute(node.path) ? node.path : path.resolve(owner, node.path) + return path.resolve(nodePath) === path.resolve(absolute) + }) + if (recorded) continue + output.push( + await finding({ + check: "figure", + severity: "major", + title: `Figure ${value} has no recorded provenance`, + detail: "Record the generating run, code, and source inputs for this local figure.", + evidence: [relative, "No matching artifact node exists in the project provenance graph."], + location: { path: source.relative, line: index + 1 }, + }), + ) + } + } + return output + } + + async function reproducibility( + source: Awaited>, + provenance: ArtifactFile.Provenance, + audit: ArtifactFile.Audit, + ): Promise { + const output: Finding[] = [] + if (!provenance.tracked || !provenance.commit) { + output.push( + await finding({ + check: "provenance", + severity: "blocking", + title: "Manuscript has no reachable Git snapshot", + detail: "Track and commit the manuscript so the reviewed source can be recovered.", + evidence: [`Git status: ${provenance.status}`], + location: { path: source.relative }, + }), + ) + } else if (provenance.dirty) { + output.push( + await finding({ + check: "provenance", + severity: "blocking", + title: "Manuscript differs from its recorded Git snapshot", + detail: "Commit the reviewed manuscript bytes before marking the publication ready.", + evidence: [`Git status: ${provenance.status}`, `Latest commit: ${provenance.commit.sha}`], + location: { path: source.relative }, + }), + ) + } + const failures = audit.checks.filter((check) => check.status === "fail") + for (const check of failures) { + if (check.id === "git-repository" || check.id === "git-commit") continue + output.push( + await finding({ + check: "provenance", + severity: "blocking", + title: check.label, + detail: check.detail, + evidence: [`Project reproducibility check: ${check.id}`], + location: { path: source.relative }, + }), + ) + } + const warnings = audit.checks.filter((check) => check.status === "warn") + for (const check of warnings) { + if (check.id === "git-clean" && provenance.dirty) continue + output.push( + await finding({ + check: "provenance", + severity: "minor", + title: check.label, + detail: check.detail, + evidence: [`Project reproducibility check: ${check.id}`], + location: { path: source.relative }, + }), + ) + } + return output + } + + async function bibliography(text: string, source: Awaited>): Promise { + const frontmatter = /^---\s*\n([\s\S]*?)\n---/m.exec(text)?.[1] ?? "" + const declared = frontmatter.split(/\r?\n/).flatMap((line) => { + const value = /^\s*bibliography\s*:\s*(.+)\s*$/i.exec(line)?.[1] + if (!value) return [] + return value + .replace(/^\[|\]$/g, "") + .split(",") + .map((item) => item.trim().replace(/^["']|["']$/g, "")) + .filter(Boolean) + }) + const candidates = [ + ...declared.map((file) => path.resolve(path.dirname(source.absolute), file)), + path.join(path.dirname(source.absolute), "references.bib"), + path.join(path.dirname(source.absolute), "bibliography.bib"), + path.join(path.dirname(source.absolute), "refs.bib"), + path.join(Instance.directory, "references.bib"), + path.join(Instance.directory, "bibliography.bib"), + path.join(Instance.directory, "refs.bib"), + path.join(Instance.worktree, "references.bib"), + path.join(Instance.worktree, "bibliography.bib"), + path.join(Instance.worktree, "refs.bib"), + ] + const unique = [...new Set(candidates)] + const safe = await Promise.all( + unique.map(async (file) => + (await Instance.containsCanonicalPath(file)) && (await Bun.file(file).exists()) ? file : undefined, + ), + ) + return safe.filter((file): file is string => Boolean(file)) + } + + async function target(value: string) { + const absolute = path.resolve(Instance.directory, value) + if (!(await Instance.containsCanonicalPath(absolute))) { + throw new Error(`Publication review target is outside the project: ${value}`) + } + return { + absolute, + relative: path.relative(Instance.worktree, absolute).replaceAll("\\", "/"), + } + } + + async function assertCurrent(report: Report, artifactHash?: string) { + const absolute = path.resolve(Instance.worktree, report.path) + if (!(await Instance.containsCanonicalPath(absolute))) { + throw new Error("The reviewed manuscript is outside the current project") + } + if (!(await Bun.file(absolute).exists())) throw new Error("The reviewed manuscript no longer exists") + if ((artifactHash ?? (await digest(absolute))) !== report.artifactHash) { + throw new Error("The manuscript changed after this publication review was generated") + } + } + + async function finding(input: Omit): Promise { + return { + ...input, + id: `finding_${(await hash(JSON.stringify(stable(input)))).slice(0, 20)}`, + status: "open", + } + } + + function summary(findings: Finding[]): Summary { + return { + total: findings.length, + open: findings.filter((finding) => finding.status === "open").length, + blocking: findings.filter((finding) => finding.severity === "blocking").length, + major: findings.filter((finding) => finding.severity === "major").length, + minor: findings.filter((finding) => finding.severity === "minor").length, + info: findings.filter((finding) => finding.severity === "info").length, + resolved: findings.filter((finding) => finding.status === "resolved").length, + overridden: findings.filter((finding) => finding.status === "overridden").length, + } + } + + function status(findings: Finding[]): Report["status"] { + if (findings.some((finding) => finding.severity === "blocking" && finding.status === "open")) return "blocked" + if (findings.some((finding) => finding.status === "open")) return "warnings" + return "ready" + } + + function compare(a: Finding, b: Finding) { + const rank: Record = { blocking: 0, major: 1, minor: 2, info: 3 } + return ( + rank[a.severity] - rank[b.severity] || (a.location.line ?? 0) - (b.location.line ?? 0) || a.id.localeCompare(b.id) + ) + } + + function stable(value: unknown): unknown { + if (Array.isArray(value)) return value.map(stable) + if (!value || typeof value !== "object") return value + return Object.fromEntries( + Object.entries(value as Record) + .toSorted(([a], [b]) => a.localeCompare(b)) + .map(([key, item]) => [key, stable(item)]), + ) + } + + async function digest(file: string) { + const hasher = new Bun.CryptoHasher("sha256") + const reader = Bun.file(file).stream().getReader() + const feed = async (): Promise => { + const chunk = await reader.read() + if (chunk.done) return + hasher.update(chunk.value) + return feed() + } + await feed() + return hasher.digest("hex") + } + + async function hash(value: string) { + const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(value)) + return [...new Uint8Array(digest)].map((byte) => byte.toString(16).padStart(2, "0")).join("") + } +} diff --git a/backend/cli/src/file/science.ts b/backend/cli/src/file/science.ts new file mode 100644 index 00000000..65dbbbcb --- /dev/null +++ b/backend/cli/src/file/science.ts @@ -0,0 +1,316 @@ +import path from "node:path" +import z from "zod" + +export namespace ScienceFile { + export const Format = z.enum(["bam", "cram", "h5ad", "loom"]) + export type Format = z.infer + + export const Inspection = z.object({ + format: Format, + name: z.string(), + size: z.number(), + modified: z.number(), + signature: z.boolean(), + index: z.string().optional(), + tool: z.object({ + name: z.string(), + available: z.boolean(), + detail: z.string().optional(), + }), + details: z.record(z.string(), z.unknown()), + }) + export type Inspection = z.infer + + const python = String.raw` +import json, sys +try: + import h5py +except Exception as exc: + print(json.dumps({"error": "h5py is not available", "detail": str(exc)})) + raise SystemExit(2) + +target = sys.argv[1] +result = {"groups": [], "datasets": [], "attributes": {}, "summary": {}} + +def clean(value): + if hasattr(value, "tolist"): + value = value.tolist() + if isinstance(value, bytes): + return value.decode("utf-8", "replace") + if isinstance(value, (str, int, float, bool)) or value is None: + return value + if isinstance(value, (list, tuple)): + return [clean(item) for item in value[:50]] + return str(value) + +def text(value): + value = clean(value) + return str(value) if value is not None else "" + +def labels(handle, key, indices): + if not key or "obs" not in handle: + return [] + value = handle["obs"].get(key) + if value is None: + return [] + try: + if isinstance(value, h5py.Group) and "codes" in value and "categories" in value: + codes = value["codes"][indices] + categories = value["categories"][:] + return [text(categories[int(code)]) if int(code) >= 0 and int(code) < len(categories) else "" for code in codes] + return [text(item) for item in value[indices]] + except Exception: + return [] + +with h5py.File(target, "r") as handle: + result["attributes"] = {str(key): clean(value) for key, value in list(handle.attrs.items())[:50]} + + def visit(name, value): + if len(result["groups"]) + len(result["datasets"]) >= 500: + return + if isinstance(value, h5py.Group): + result["groups"].append(name) + return + result["datasets"].append({ + "path": name, + "shape": list(value.shape), + "dtype": str(value.dtype), + "bytes": int(value.size * value.dtype.itemsize), + }) + + handle.visititems(visit) + matrix = handle.get("X") + if matrix is None: + matrix = handle.get("matrix") + if matrix is not None and hasattr(matrix, "shape"): + result["summary"]["matrix"] = list(matrix.shape) + if "obs" in handle: + obs_index = handle["obs"].get("_index") + if obs_index is not None: + result["summary"]["observations"] = len(obs_index) + if "var" in handle: + var_index = handle["var"].get("_index") + if var_index is not None: + result["summary"]["variables"] = len(var_index) + if matrix is not None and hasattr(matrix, "shape") and len(matrix.shape) >= 2: + result["summary"].setdefault("observations", int(matrix.shape[0])) + result["summary"].setdefault("variables", int(matrix.shape[1])) + if "obsm" in handle: + result["summary"]["embeddings"] = list(handle["obsm"].keys())[:100] + preferred = ["X_umap", "X_tsne", "X_pca", "spatial"] + names = list(handle["obsm"].keys()) + selected = next((name for name in preferred if name in names), names[0] if names else None) + value = handle["obsm"].get(selected) if selected else None + if value is not None and isinstance(value, h5py.Dataset) and len(value.shape) == 2 and value.shape[1] >= 2: + total = int(value.shape[0]) + count = min(total, 2500) + indices = [int(index * total / count) for index in range(count)] if count else [] + coords = value[indices, :2] if indices else [] + label_names = ["cell_type", "celltype", "leiden", "louvain", "cluster", "batch"] + label_key = next((name for name in label_names if "obs" in handle and name in handle["obs"]), None) + categories = labels(handle, label_key, indices) + result["embedding"] = { + "name": selected, + "label": label_key, + "total": total, + "points": [ + { + "x": float(point[0]), + "y": float(point[1]), + **({"label": categories[index]} if index < len(categories) and categories[index] else {}), + } + for index, point in enumerate(coords) + ], + } + if "layers" in handle: + result["summary"]["layers"] = list(handle["layers"].keys())[:100] + if "row_attrs" in handle: + result["summary"]["row_attributes"] = list(handle["row_attrs"].keys())[:100] + if "col_attrs" in handle: + result["summary"]["column_attributes"] = list(handle["col_attrs"].keys())[:100] + if "embedding" not in result: + candidates = [ + (name, handle["col_attrs"].get(name)) + for name in ["X_umap", "UMAP", "Embedding", "_Embedding", "TSNE"] + if name in handle["col_attrs"] + ] + selected = candidates[0] if candidates else None + if selected and isinstance(selected[1], h5py.Dataset) and len(selected[1].shape) == 2 and selected[1].shape[1] >= 2: + value = selected[1] + total = int(value.shape[0]) + count = min(total, 2500) + indices = [int(index * total / count) for index in range(count)] if count else [] + coords = value[indices, :2] if indices else [] + result["embedding"] = { + "name": selected[0], + "total": total, + "points": [{"x": float(point[0]), "y": float(point[1])} for point in coords], + } + +print(json.dumps(result)) +` + + export function format(file: string): Format | undefined { + const extension = path.extname(file).slice(1).toLowerCase() + return Format.options.find((value) => value === extension) + } + + export function binary(file: string): boolean { + return format(file) !== undefined + } + + export async function inspect(full: string, relative: string): Promise { + const kind = format(relative) + if (!kind) throw new Error(`Unsupported scientific binary format`) + const file = Bun.file(full) + if (!(await file.exists())) throw new Error(`File not found: ${relative}`) + const stat = await file.stat() + const bytes = new Uint8Array(await file.slice(0, 16).arrayBuffer()) + const base = { + format: kind, + name: path.basename(relative), + size: stat.size, + modified: stat.mtimeMs, + } + if (kind === "h5ad" || kind === "loom") return inspectHdf5(full, base, bytes) + return inspectAlignment(full, relative, base, bytes) + } + + async function inspectHdf5( + full: string, + base: Pick, + bytes: Uint8Array, + ): Promise { + const bin = Bun.which("python3") ?? Bun.which("python") + const signature = [0x89, 0x48, 0x44, 0x46, 0x0d, 0x0a, 0x1a, 0x0a].every((value, index) => bytes[index] === value) + if (!bin) { + return { + ...base, + signature, + tool: { name: "h5py", available: false, detail: "Python is not available on PATH" }, + details: {}, + } + } + const result = await command([bin, "-c", python, full], 20_000) + const data = result.code === 0 ? json(result.stdout) : undefined + return { + ...base, + signature, + tool: { + name: "h5py", + available: result.code === 0, + detail: + result.code === 0 + ? `inspected with ${path.basename(bin)}` + : detail(result.stdout, result.stderr) || + "Install h5py in the Python environment used to launch OpenScience", + }, + details: data ?? {}, + } + } + + async function inspectAlignment( + full: string, + relative: string, + base: Pick, + bytes: Uint8Array, + ): Promise { + const bin = Bun.which("samtools") + const cram = base.format === "cram" + const signature = cram + ? bytes[0] === 0x43 && bytes[1] === 0x52 && bytes[2] === 0x41 && bytes[3] === 0x4d + : bytes[0] === 0x1f && bytes[1] === 0x8b + const index = await findIndex(full, relative, cram) + const version = cram && signature ? `${bytes[4] ?? 0}.${bytes[5] ?? 0}` : undefined + if (!bin) { + return { + ...base, + signature, + index, + tool: { name: "samtools", available: false, detail: "Install samtools to inspect headers and references" }, + details: version ? { version } : {}, + } + } + const header = await command([bin, "view", "-H", full], 20_000) + const refs = header.stdout + .split(/\r?\n/) + .filter((line) => line.startsWith("@SQ")) + .map((line) => + Object.fromEntries( + line + .split("\t") + .slice(1) + .map((part) => part.split(":", 2)), + ), + ) + .map((record) => ({ name: record.SN ?? "", length: Number(record.LN) || 0 })) + const hd = header.stdout + .split(/\r?\n/) + .find((line) => line.startsWith("@HD")) + ?.split("\t") + .slice(1) + .map((part) => part.split(":", 2)) + const stats = index ? await command([bin, "idxstats", full], 20_000) : undefined + const chromosomes = + stats?.code === 0 + ? stats.stdout + .split(/\r?\n/) + .filter(Boolean) + .map((line) => line.split("\t")) + .filter((row) => row[0] !== "*") + .map((row) => ({ + name: row[0] ?? "", + length: Number(row[1]) || 0, + mapped: Number(row[2]) || 0, + unmapped: Number(row[3]) || 0, + })) + : [] + return { + ...base, + signature, + index, + tool: { + name: "samtools", + available: header.code === 0, + detail: header.code === 0 ? "header inspected locally" : detail(header.stdout, header.stderr), + }, + details: { + ...(version ? { version } : {}), + header: hd ? Object.fromEntries(hd) : {}, + references: refs, + chromosomes, + }, + } + } + + async function findIndex(full: string, relative: string, cram: boolean): Promise { + const extension = cram ? ".crai" : ".bai" + const candidates = [full + extension, full.replace(/\.[^.]+$/, extension)] + const found = await Promise.all( + candidates.map(async (candidate) => ((await Bun.file(candidate).exists()) ? candidate : undefined)), + ) + const value = found.find(Boolean) + if (!value) return + return path.join(path.dirname(relative), path.basename(value)).replace(/^\.\//, "") + } + + async function command(args: string[], timeout: number): Promise<{ code: number; stdout: string; stderr: string }> { + const process = Bun.spawn(args, { stdout: "pipe", stderr: "pipe" }) + const timer = setTimeout(() => process.kill(), timeout) + const code = await process.exited + clearTimeout(timer) + const [stdout, stderr] = await Promise.all([ + new Response(process.stdout).text(), + new Response(process.stderr).text(), + ]) + return { code, stdout, stderr } + } + + function json(value: string): Record | undefined { + return JSON.parse(value) as Record + } + + function detail(stdout: string, stderr: string): string { + return (stderr.trim() || stdout.trim()).slice(0, 500) + } +} diff --git a/backend/cli/src/file/starters.ts b/backend/cli/src/file/starters.ts new file mode 100644 index 00000000..ce32a937 --- /dev/null +++ b/backend/cli/src/file/starters.ts @@ -0,0 +1,138 @@ +import fs from "node:fs/promises" +import path from "node:path" +import z from "zod" + +export namespace StarterFile { + export const Template = z.enum(["single-cell", "dose-response", "protein-structure"]) + export type Template = z.infer + + export const Result = z.object({ + template: Template, + directory: z.string(), + files: z.string().array(), + notebook: z.string(), + readme: z.string(), + }) + export type Result = z.infer + + export async function create(root: string, template: Template): Promise { + const parsed = Template.parse(template) + const directory = path.join("openscience-starters", parsed) + const target = path.join(root, directory) + await fs.mkdir(path.dirname(target), { recursive: true }) + const created = await fs + .mkdir(target) + .then(() => true) + .catch((error) => { + if ((error as NodeJS.ErrnoException).code === "EEXIST") return false + throw error + }) + if (!created) throw new Error(`The ${parsed} starter already exists at ${directory}`) + const content = files(parsed) + const names = Object.keys(content).toSorted() + await Promise.all( + names.map(async (name) => { + const file = path.join(target, name) + await fs.mkdir(path.dirname(file), { recursive: true }) + await Bun.write(file, content[name]!) + }), + ) + return Result.parse({ + template: parsed, + directory, + files: names.map((name) => path.join(directory, name).split(path.sep).join("/")), + notebook: path.join(directory, "analysis.ipynb").split(path.sep).join("/"), + readme: path.join(directory, "README.md").split(path.sep).join("/"), + }) + } + + function files(template: Template): Record { + if (template === "single-cell") return singleCell() + if (template === "dose-response") return doseResponse() + return proteinStructure() + } + + function notebook(title: string, intro: string, cells: string[]): string { + return JSON.stringify( + { + cells: [ + { + cell_type: "markdown", + metadata: {}, + source: [`# ${title}\n`, `${intro}\n`], + }, + ...cells.map((source) => ({ + cell_type: "code", + execution_count: null, + metadata: {}, + outputs: [], + source: source.split("\n").map((line) => `${line}\n`), + })), + ], + metadata: { + kernelspec: { display_name: "Python 3", language: "python", name: "python3" }, + language_info: { name: "python", version: "3" }, + openscience: { starter: true }, + }, + nbformat: 4, + nbformat_minor: 5, + }, + null, + 2, + ) + } + + function singleCell(): Record { + return { + "README.md": + "# Single-cell starter\n\nA small, local-first expression matrix for exploring QC, normalization, clustering, and cell-type annotation. Open `analysis.ipynb` in OpenScience and run each cell. Replace `data/cells.csv` with your own matrix when ready.\n", + "data/cells.csv": + "cell,cell_type,CD3D,MS4A1,LYZ,NKG7,MKI67\ncell_001,T cell,9,0,1,6,0\ncell_002,T cell,8,0,0,7,1\ncell_003,B cell,0,10,1,0,0\ncell_004,B cell,0,8,0,1,0\ncell_005,Monocyte,1,0,11,2,0\ncell_006,Monocyte,0,0,9,1,1\ncell_007,NK cell,2,0,0,12,0\ncell_008,NK cell,1,0,1,10,0\ncell_009,Cycling,4,1,2,3,13\ncell_010,Cycling,3,2,1,4,11\n", + "analysis.ipynb": notebook( + "Single-cell expression starter", + "Inspect a tiny expression matrix, calculate per-cell QC, and visualize marker structure without external downloads.", + [ + "import csv\nfrom pathlib import Path\nrows = list(csv.DictReader(Path('data/cells.csv').open()))\ngenes = ['CD3D', 'MS4A1', 'LYZ', 'NKG7', 'MKI67']\nlen(rows), genes", + "qc = [{'cell': row['cell'], 'type': row['cell_type'], 'total': sum(int(row[g]) for g in genes), 'detected': sum(int(row[g]) > 0 for g in genes)} for row in rows]\nqc", + "from collections import defaultdict\nmeans = defaultdict(lambda: defaultdict(list))\nfor row in rows:\n for gene in genes:\n means[row['cell_type']][gene].append(int(row[gene]))\nsummary = {kind: {gene: round(sum(values[gene]) / len(values[gene]), 2) for gene in genes} for kind, values in means.items()}\nsummary", + ], + ), + } + } + + function doseResponse(): Record { + return { + "README.md": + "# Dose-response starter\n\nA compact plate-style dose-response dataset with vehicle controls and replicates. Use `analysis.ipynb` to aggregate response, estimate the half-maximal crossing, and review assay quality before replacing the sample CSV.\n", + "data/dose_response.csv": + "compound,dose_uM,replicate,response_pct\nVehicle,0,1,100\nVehicle,0,2,98\nCompound-A,0.001,1,96\nCompound-A,0.001,2,94\nCompound-A,0.01,1,87\nCompound-A,0.01,2,84\nCompound-A,0.1,1,62\nCompound-A,0.1,2,58\nCompound-A,1,1,31\nCompound-A,1,2,28\nCompound-A,10,1,9\nCompound-A,10,2,11\n", + "analysis.ipynb": notebook( + "Dose-response assay starter", + "Aggregate technical replicates and estimate the observed half-response crossing using only the Python standard library.", + [ + "import csv\nfrom pathlib import Path\nrows = list(csv.DictReader(Path('data/dose_response.csv').open()))\nrows[:3]", + "from collections import defaultdict\nseries = defaultdict(list)\nfor row in rows:\n if row['compound'] != 'Vehicle':\n series[float(row['dose_uM'])].append(float(row['response_pct']))\nmeans = {dose: round(sum(values) / len(values), 2) for dose, values in sorted(series.items())}\nmeans", + "crossing = min(means, key=lambda dose: abs(means[dose] - 50))\n{'nearest_half_max_dose_uM': crossing, 'response_pct': means[crossing], 'replicates_per_dose': {dose: len(values) for dose, values in series.items()}}", + ], + ), + } + } + + function proteinStructure(): Record { + return { + "README.md": + "# Protein-structure starter\n\nA tiny alanine peptide structure for learning the native PDB viewer, measuring geometry, and preparing downstream docking or molecular-dynamics work. Open `data/alanine.pdb` for the 3D view and `analysis.ipynb` for a dependency-free inspection.\n", + "data/alanine.pdb": + "HEADER OPENSCIENCE ALANINE STARTER\nATOM 1 N ALA A 1 -1.458 0.000 0.000 1.00 20.00 N\nATOM 2 CA ALA A 1 0.000 0.000 0.000 1.00 20.00 C\nATOM 3 C ALA A 1 0.540 1.430 0.000 1.00 20.00 C\nATOM 4 O ALA A 1 -0.160 2.390 0.000 1.00 20.00 O\nATOM 5 CB ALA A 1 0.510 -0.770 -1.220 1.00 20.00 C\nTER\nEND\n", + "analysis.ipynb": notebook( + "Protein structure starter", + "Parse atoms and calculate the structure centroid before moving to docking or molecular dynamics.", + [ + "from pathlib import Path\nlines = Path('data/alanine.pdb').read_text().splitlines()\natoms = [line for line in lines if line.startswith(('ATOM ', 'HETATM'))]\nlen(atoms)", + "coords = [(float(line[30:38]), float(line[38:46]), float(line[46:54])) for line in atoms]\ncentroid = tuple(round(sum(axis) / len(coords), 3) for axis in zip(*coords))\n{'atoms': len(atoms), 'centroid_angstrom': centroid}", + "elements = {}\nfor line in atoms:\n element = line[76:78].strip() or line[12:16].strip()[0]\n elements[element] = elements.get(element, 0) + 1\nelements", + ], + ), + } + } +} diff --git a/backend/cli/src/project/instance.ts b/backend/cli/src/project/instance.ts index 5ea2c2c8..390a45f5 100644 --- a/backend/cli/src/project/instance.ts +++ b/backend/cli/src/project/instance.ts @@ -68,6 +68,11 @@ export const Instance = { if (Instance.worktree === "/") return false return Filesystem.contains(Instance.worktree, filepath) }, + async containsCanonicalPath(filepath: string) { + if (await Filesystem.containsCanonical(Instance.directory, filepath)) return true + if (Instance.worktree === "/") return false + return Filesystem.containsCanonical(Instance.worktree, filepath) + }, state(init: () => S, dispose?: (state: Awaited) => Promise): () => S { return State.create(() => Instance.directory, init, dispose) }, diff --git a/backend/cli/src/provider/transform.ts b/backend/cli/src/provider/transform.ts index fcdc3abc..b1b8e083 100644 --- a/backend/cli/src/provider/transform.ts +++ b/backend/cli/src/provider/transform.ts @@ -1147,6 +1147,15 @@ export namespace ProviderTransform { export function error(providerID: string, error: APICallError) { let message = error.message + const body = error.responseBody?.toLowerCase() ?? "" + if ( + providerID === "openrouter" && + error.statusCode === 403 && + body.includes("this model is only available in the united states") && + body.includes('"provider_name":"meta"') + ) { + return "Muse Spark 1.1 is currently restricted by Meta to requests routed from the United States. Choose another model, or retry from a supported U.S. region." + } if (providerID.includes("github-copilot") && error.statusCode === 403) { return "Please reauthenticate with the copilot provider to ensure your credentials work properly with OpenScience." } diff --git a/backend/cli/src/science/provenance/review.ts b/backend/cli/src/science/provenance/review.ts index db278744..b1bd4c22 100644 --- a/backend/cli/src/science/provenance/review.ts +++ b/backend/cli/src/science/provenance/review.ts @@ -49,6 +49,7 @@ export namespace Review { /** Who recorded it (agent name). */ reviewer?: string sessionID?: string + directory?: string }): Promise { const relation = input.verdict ?? "refutes" const node = await Provenance.record({ @@ -64,6 +65,7 @@ export namespace Review { verdict: relation, reviewer: input.reviewer ?? "reviewer", sessionID: input.sessionID, + directory: input.directory, }, }) await Provenance.link({ from: node.id, to: input.target, relation }) diff --git a/backend/cli/src/science/provenance/store.ts b/backend/cli/src/science/provenance/store.ts index 106c49be..69980216 100644 --- a/backend/cli/src/science/provenance/store.ts +++ b/backend/cli/src/science/provenance/store.ts @@ -72,6 +72,7 @@ interface Graph { } const STORE_PATH = path.join(Global.Path.data, "provenance", "graph.json") +const lock = { current: Promise.resolve() as Promise } async function sha256(input: string): Promise { const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(input)) @@ -80,10 +81,20 @@ async function sha256(input: string): Promise { /** Deterministic content id from a node's identifying payload. */ export async function contentId(payload: unknown): Promise { - const canonical = JSON.stringify(payload, Object.keys(payload as object).sort()) + const canonical = JSON.stringify(stable(payload)) return (await sha256(canonical)).slice(0, 16) } +function stable(value: unknown): unknown { + if (Array.isArray(value)) return value.map(stable) + if (!value || typeof value !== "object") return value + return Object.fromEntries( + Object.entries(value as Record) + .toSorted(([a], [b]) => a.localeCompare(b)) + .map(([key, item]) => [key, stable(item)]), + ) +} + async function load(): Promise { const file = Bun.file(STORE_PATH) if (!(await file.exists())) return { version: 1, nodes: {}, edges: [] } @@ -98,25 +109,49 @@ async function save(graph: Graph): Promise { await Bun.write(STORE_PATH, JSON.stringify(graph, null, 2)) } +async function mutate(fn: (graph: Graph) => Promise | T): Promise { + const task = lock.current + .catch(() => undefined) + .then(async () => { + const graph = await load() + const result = await fn(graph) + await save(graph) + return result + }) + lock.current = task + return task +} + +function belongs(node: Node, directory: string): boolean { + const root = path.resolve(directory) + const nodeDirectory = node.meta?.directory + if (typeof nodeDirectory === "string" && path.resolve(nodeDirectory) === root) return true + if (node.kind !== "artifact" || !("path" in node) || !node.path || !path.isAbsolute(node.path)) return false + const relative = path.relative(root, path.resolve(node.path)) + return relative === "" || (!relative.startsWith(`..${path.sep}`) && relative !== ".." && !path.isAbsolute(relative)) +} + export namespace Provenance { /** Record a node. If `id` is omitted it is content-addressed from the node body. */ export async function record(node: Omit & { id?: string }): Promise { - const graph = await load() - const recordedAt = new Date().toISOString() - const id = node.id ?? (await contentId({ ...node })) - const full = { ...node, id, recordedAt } as Node - graph.nodes[id] = full - await save(graph) - return full + return mutate(async (graph) => { + const recordedAt = new Date().toISOString() + const id = node.id ?? (await contentId({ ...node })) + const full = { ...node, id, recordedAt } as Node + graph.nodes[id] = full + return full + }) } /** Link two existing nodes with a typed edge. */ export async function link(edge: Edge): Promise { - const graph = await load() - const exists = graph.edges.some((e) => e.from === edge.from && e.to === edge.to && e.relation === edge.relation) - if (!exists) graph.edges.push(edge) - await save(graph) - return edge + return mutate((graph) => { + if (!graph.nodes[edge.from]) throw new Error(`Provenance node ${edge.from} was not found`) + if (!graph.nodes[edge.to]) throw new Error(`Provenance node ${edge.to} was not found`) + const exists = graph.edges.some((e) => e.from === edge.from && e.to === edge.to && e.relation === edge.relation) + if (!exists) graph.edges.push(edge) + return edge + }) } /** Fetch a single node by id. */ @@ -160,5 +195,33 @@ export namespace Provenance { return Object.values(graph.nodes) } + /** Return only nodes belonging to a project plus their directly or transitively linked evidence. */ + export async function project(directory: string): Promise<{ nodes: Node[]; edges: Edge[] }> { + const graph = await load() + const seen = new Set( + Object.values(graph.nodes) + .filter((node) => belongs(node, directory)) + .map((node) => node.id), + ) + const queue = [...seen] + while (queue.length) { + const id = queue.shift()! + for (const edge of graph.edges) { + if (edge.from !== id && edge.to !== id) continue + const next = edge.from === id ? edge.to : edge.from + const node = graph.nodes[next] + if (seen.has(next) || !node) continue + const owner = node.meta?.directory + if (typeof owner === "string" && !belongs(node, directory)) continue + seen.add(next) + queue.push(next) + } + } + return { + nodes: [...seen].map((id) => graph.nodes[id]).filter((node): node is Node => !!node), + edges: graph.edges.filter((edge) => seen.has(edge.from) && seen.has(edge.to)), + } + } + export const path_ = STORE_PATH } diff --git a/backend/cli/src/server/routes/file.ts b/backend/cli/src/server/routes/file.ts index 961ebcdf..418b393d 100644 --- a/backend/cli/src/server/routes/file.ts +++ b/backend/cli/src/server/routes/file.ts @@ -6,6 +6,12 @@ import { Ripgrep } from "../../file/ripgrep" import { LSP } from "../../lsp" import { Instance } from "../../project/instance" import { lazy } from "../../util/lazy" +import { ScienceFile } from "../../file/science" +import { ArtifactFile } from "../../file/artifacts" +import { StarterFile } from "../../file/starters" +import { PublicationFile } from "../../file/publication" +import { ArtifactAnnotation } from "../../file/annotations" +import { PublicationReview } from "../../file/review" export const FileRoutes = lazy(() => new Hono() @@ -202,6 +208,388 @@ export const FileRoutes = lazy(() => return c.json(content) }, ) + .get( + "/file/inspect", + describeRoute({ + summary: "Inspect a scientific binary file", + description: "Inspect BAM, CRAM, H5AD, or LOOM metadata with locally available scientific tools.", + operationId: "file.inspect", + responses: { + 200: { + description: "Scientific file inspection", + content: { + "application/json": { + schema: resolver(ScienceFile.Inspection), + }, + }, + }, + }, + }), + validator( + "query", + z.object({ + path: z.string(), + }), + ), + async (c) => { + const result = await File.inspect(c.req.valid("query").path) + return c.json(result) + }, + ) + .get( + "/file/raw", + describeRoute({ + summary: "Download a file", + description: "Stream a project file without loading it into the JSON API as base64.", + operationId: "file.raw", + responses: { + 200: { + description: "Raw file contents", + }, + }, + }), + validator( + "query", + z.object({ + path: z.string(), + }), + ), + async (c) => { + const path = c.req.valid("query").path + const content = await File.raw(path) + return new Response(content, { + headers: { + "Content-Type": content.type || "application/octet-stream", + "Content-Length": String(content.size), + "Content-Disposition": `attachment; filename*=UTF-8''${encodeURIComponent(path.split("/").pop() || "download")}`, + }, + }) + }, + ) + .get( + "/file/artifacts", + describeRoute({ + summary: "List local research artifacts", + description: "Discover notebooks, datasets, figures, reports, models, and scientific files in the project.", + operationId: "file.artifacts", + responses: { + 200: { + description: "Research artifacts", + content: { + "application/json": { + schema: resolver(ArtifactFile.Info.array()), + }, + }, + }, + }, + }), + async (c) => c.json(await File.artifacts()), + ) + .get( + "/file/provenance", + describeRoute({ + summary: "Get local file provenance", + description: "Read Git branch, dirty state, and latest commit metadata for a project file.", + operationId: "file.provenance", + responses: { + 200: { + description: "Local provenance", + content: { + "application/json": { + schema: resolver(ArtifactFile.Provenance), + }, + }, + }, + }, + }), + validator( + "query", + z.object({ + path: z.string(), + }), + ), + async (c) => c.json(await File.provenance(c.req.valid("query").path)), + ) + .get( + "/file/reproducibility", + describeRoute({ + summary: "Audit project reproducibility", + description: + "Check Git state, locked dependencies, environment specifications, notebook structure, and research artifacts.", + operationId: "file.reproducibility", + responses: { + 200: { + description: "Project reproducibility audit", + content: { "application/json": { schema: resolver(ArtifactFile.Audit) } }, + }, + }, + }), + async (c) => c.json(await File.reproducibility()), + ) + .get( + "/file/annotations", + describeRoute({ + summary: "List artifact annotations", + description: "List durable review threads anchored to a project artifact.", + operationId: "file.annotations.list", + responses: { + 200: { + description: "Artifact annotations", + content: { "application/json": { schema: resolver(ArtifactAnnotation.Info.array()) } }, + }, + }, + }), + validator("query", z.object({ path: z.string() })), + async (c) => c.json(await ArtifactAnnotation.list(c.req.valid("query").path)), + ) + .post( + "/file/annotations", + describeRoute({ + summary: "Create an artifact annotation", + description: + "Create a durable review thread anchored to an artifact, text range, notebook cell, molecule, or locus.", + operationId: "file.annotations.create", + responses: { + 200: { + description: "Created annotation", + content: { "application/json": { schema: resolver(ArtifactAnnotation.Info) } }, + }, + }, + }), + validator("json", ArtifactAnnotation.Create), + async (c) => c.json(await ArtifactAnnotation.create(c.req.valid("json"))), + ) + .get( + "/file/annotations/:id/history", + describeRoute({ + summary: "Read artifact annotation history", + description: "Read every immutable revision of an artifact review thread, including a recoverable tombstone.", + operationId: "file.annotations.history", + responses: { + 200: { + description: "Versioned artifact annotation", + content: { "application/json": { schema: resolver(ArtifactAnnotation.Info) } }, + }, + }, + }), + validator("param", z.object({ id: z.string().startsWith("ann_") })), + async (c) => c.json(await ArtifactAnnotation.history(c.req.valid("param").id)), + ) + .patch( + "/file/annotations/:id", + describeRoute({ + summary: "Update an artifact annotation", + description: "Reply to, resolve, or reopen an artifact review thread.", + operationId: "file.annotations.update", + responses: { + 200: { + description: "Updated annotation", + content: { "application/json": { schema: resolver(ArtifactAnnotation.Info) } }, + }, + }, + }), + validator("param", z.object({ id: z.string().startsWith("ann_") })), + validator("json", ArtifactAnnotation.Update), + async (c) => c.json(await ArtifactAnnotation.update(c.req.valid("param").id, c.req.valid("json"))), + ) + .delete( + "/file/annotations/:id", + describeRoute({ + summary: "Tombstone an artifact annotation", + description: "Hide an artifact review thread while retaining its recoverable revision history.", + operationId: "file.annotations.delete", + responses: { + 200: { + description: "Tombstoned annotation", + content: { + "application/json": { + schema: resolver(z.object({ deleted: z.literal(true), version: z.number().int().positive() })), + }, + }, + }, + }, + }), + validator("param", z.object({ id: z.string().startsWith("ann_") })), + async (c) => c.json(await ArtifactAnnotation.remove(c.req.valid("param").id)), + ) + .get( + "/file/manifest", + describeRoute({ + summary: "Create an artifact integrity manifest", + description: "Hash every discovered research artifact and return a portable, deterministic manifest.", + operationId: "file.manifest", + responses: { + 200: { + description: "Artifact checksum manifest", + content: { "application/json": { schema: resolver(ArtifactFile.Manifest) } }, + }, + }, + }), + async (c) => { + c.header("Content-Disposition", 'attachment; filename="openscience-artifact-manifest.json"') + return c.json(await File.manifest()) + }, + ) + .post( + "/file/starters", + describeRoute({ + summary: "Create a local scientific starter project", + description: "Materialize a valid notebook, sample data, and README without external downloads.", + operationId: "file.starter", + responses: { + 200: { + description: "Created starter files", + content: { "application/json": { schema: resolver(StarterFile.Result) } }, + }, + }, + }), + validator("json", z.object({ template: StarterFile.Template })), + async (c) => c.json(await File.starter(c.req.valid("json").template)), + ) + .get( + "/file/publication/capabilities", + describeRoute({ + summary: "Inspect local publication export support", + description: "Detect Pandoc and a PDF engine before offering report export formats.", + operationId: "file.publicationCapabilities", + responses: { + 200: { + description: "Available local publication formats", + content: { "application/json": { schema: resolver(PublicationFile.Capabilities) } }, + }, + }, + }), + async (c) => c.json(await File.publicationCapabilities()), + ) + .post( + "/file/publication", + describeRoute({ + summary: "Export a Markdown research report", + description: "Create a timestamped HTML, PDF, DOCX, LaTeX, or PowerPoint publication artifact locally.", + operationId: "file.publication", + responses: { + 200: { + description: "Created publication artifact", + content: { "application/json": { schema: resolver(PublicationFile.Result) } }, + }, + }, + }), + validator("json", PublicationFile.Input), + async (c) => c.json(await File.publication(c.req.valid("json"))), + ) + .get( + "/file/reviews", + describeRoute({ + summary: "Read the current publication review", + description: + "Return the latest deterministic review report and whether it is stale for the current source bytes.", + operationId: "file.reviews.current", + responses: { + 200: { + description: "Current publication review", + content: { "application/json": { schema: resolver(PublicationReview.State) } }, + }, + 404: { description: "No publication review exists for this manuscript" }, + }, + }), + validator("query", z.object({ path: z.string().trim().min(1).max(10_000) })), + async (c) => { + const report = await File.reviewCurrent(c.req.valid("query").path) + if (!report) return c.json({ error: "No publication review exists for this manuscript" }, 404) + return c.json(report) + }, + ) + .get( + "/file/reviews/history", + describeRoute({ + summary: "List publication review history", + description: "List prior deterministic review reports for every reviewed version of a manuscript.", + operationId: "file.reviews.history", + responses: { + 200: { + description: "Publication review history", + content: { "application/json": { schema: resolver(PublicationReview.Report.array()) } }, + }, + }, + }), + validator("query", z.object({ path: z.string().trim().min(1).max(10_000) })), + async (c) => c.json(await File.reviewHistory(c.req.valid("query").path)), + ) + .post( + "/file/reviews", + describeRoute({ + summary: "Run deterministic publication checks", + description: + "Check citations, numeric traces, figures, and provenance for the exact Markdown manuscript bytes.", + operationId: "file.reviews.run", + responses: { + 200: { + description: "Generated publication review", + content: { "application/json": { schema: resolver(PublicationReview.Report) } }, + }, + }, + }), + validator("json", PublicationReview.RunInput), + async (c) => c.json(await File.review(c.req.valid("json"))), + ) + .patch( + "/file/reviews/:id/findings/:finding", + describeRoute({ + summary: "Resolve or override a publication finding", + description: "Record an attributed reason and close one deterministic review finding.", + operationId: "file.reviews.resolve", + responses: { + 200: { + description: "Updated publication review", + content: { "application/json": { schema: resolver(PublicationReview.Report) } }, + }, + 409: { description: "Finding cannot be updated" }, + }, + }), + validator( + "param", + z.object({ + id: z.string().startsWith("review_"), + finding: z.string().startsWith("finding_"), + }), + ), + validator("json", PublicationReview.ResolveInput), + async (c) => { + const params = c.req.valid("param") + const result = await File.reviewResolve(params.id, params.finding, c.req.valid("json")).then( + (value) => ({ value }), + (error) => ({ error: error instanceof Error ? error.message : String(error) }), + ) + if ("error" in result) return c.json({ error: result.error }, 409) + return c.json(result.value) + }, + ) + .post( + "/file/reviews/:id/finalize", + describeRoute({ + summary: "Finalize a publication review", + description: + "Bind publication-ready state to the exact reviewed source hash after all blocking findings close.", + operationId: "file.reviews.finalize", + responses: { + 200: { + description: "Finalized publication review", + content: { "application/json": { schema: resolver(PublicationReview.Report) } }, + }, + 409: { description: "Review is blocked, stale, or already invalid" }, + }, + }), + validator("param", z.object({ id: z.string().startsWith("review_") })), + validator("json", PublicationReview.FinalizeInput), + async (c) => { + const result = await File.reviewFinalize(c.req.valid("param").id, c.req.valid("json")).then( + (value) => ({ value }), + (error) => ({ error: error instanceof Error ? error.message : String(error) }), + ) + if ("error" in result) return c.json({ error: result.error }, 409) + return c.json(result.value) + }, + ) .get( "/file/status", describeRoute({ diff --git a/backend/cli/src/server/routes/notebook.ts b/backend/cli/src/server/routes/notebook.ts new file mode 100644 index 00000000..e95dedef --- /dev/null +++ b/backend/cli/src/server/routes/notebook.ts @@ -0,0 +1,137 @@ +import { Hono } from "hono" +import { describeRoute, validator } from "hono-openapi" +import z from "zod" +import { Instance } from "../../project/instance" +import { pythonKernels } from "../../tool/notebook" +import { rKernels } from "../../tool/rkernel" +import type { ExecuteResult, KernelOutput } from "../../science/kernel/types" +import { Provenance } from "../../science/provenance/store" +import { lazy } from "../../util/lazy" + +const Language = z.enum(["python", "r"]) +const Key = z.object({ + id: z.string().trim().min(1).max(1024), + language: Language, +}) +const Execute = Key.extend({ + code: z.string().max(2_000_000), + timeout: z.number().int().min(5_000).max(600_000).optional(), +}) + +type Language = z.infer + +const manager = (language: Language) => (language === "r" ? rKernels : pythonKernels) + +const key = (id: string) => `notebook-${Bun.hash(`${Instance.directory}\0${id}`).toString(36)}` + +function output(value: KernelOutput, execution: number | null) { + if (value.type === "stream") { + return { + output_type: "stream", + name: value.name ?? "stdout", + text: value.data?.["text/plain"] ?? "", + } + } + if (value.type === "error") { + return { + output_type: "error", + ename: value.error?.name ?? "Error", + evalue: value.error?.message ?? "Kernel execution failed", + traceback: value.error?.traceback ?? [], + } + } + return { + output_type: value.type === "result" ? "execute_result" : "display_data", + data: value.data ?? {}, + metadata: {}, + ...(value.type === "result" ? { execution_count: execution } : {}), + } +} + +function response(result: ExecuteResult) { + const execution = result.executionCount ?? null + return { + ok: result.ok, + execution_count: execution, + outputs: result.outputs.map((value) => output(value, execution)), + } +} + +export const NotebookRoutes = lazy(() => + new Hono() + .post( + "/execute", + describeRoute({ + summary: "Execute a notebook cell", + description: "Execute code in a persistent project-scoped Python or R kernel.", + operationId: "notebook.execute", + responses: { 200: { description: "Jupyter-compatible cell outputs" } }, + }), + validator("json", Execute), + async (c) => { + const body = c.req.valid("json") + const kernel = await manager(body.language).get(key(body.id), { cwd: Instance.directory }) + const result = await kernel.execute(body.code, { timeout: body.timeout }) + const node = await Provenance.record({ + kind: "run", + label: `${body.language} cell · ${body.id}`.slice(0, 140), + tool: "notebook", + sessionID: body.id, + inputs: { + path: body.id, + language: body.language, + code: body.code, + }, + status: result.ok ? "ok" : "error", + meta: { + directory: Instance.directory, + executionCount: result.executionCount ?? null, + outputTypes: result.outputs.map((value) => value.type), + }, + } as Parameters[0]) + return c.json({ ...response(result), provenance_id: node.id }) + }, + ) + .get( + "/status", + describeRoute({ + summary: "Get notebook kernel status", + operationId: "notebook.status", + responses: { 200: { description: "Kernel state" } }, + }), + validator("query", Key), + (c) => { + const query = c.req.valid("query") + return c.json({ active: manager(query.language).active(key(query.id)), language: query.language }) + }, + ) + .post( + "/restart", + describeRoute({ + summary: "Restart a notebook kernel", + operationId: "notebook.restart", + responses: { 200: { description: "Kernel state" } }, + }), + validator("json", Key), + async (c) => { + const body = c.req.valid("json") + await manager(body.language).release(key(body.id)) + return c.json({ active: false, language: body.language }) + }, + ) + .post( + "/interrupt", + describeRoute({ + summary: "Interrupt a notebook kernel", + description: "Stop the running cell and release its kernel. The next execution starts a fresh kernel.", + operationId: "notebook.interrupt", + responses: { 200: { description: "Kernel state" } }, + }), + validator("json", Key), + async (c) => { + const body = c.req.valid("json") + await manager(body.language).release(key(body.id)) + return c.json({ active: false, language: body.language }) + }, + ), +) diff --git a/backend/cli/src/server/routes/provenance.ts b/backend/cli/src/server/routes/provenance.ts new file mode 100644 index 00000000..45ac3ca3 --- /dev/null +++ b/backend/cli/src/server/routes/provenance.ts @@ -0,0 +1,186 @@ +import { Hono } from "hono" +import { describeRoute, validator } from "hono-openapi" +import z from "zod" +import { Instance } from "../../project/instance" +import { Provenance, type Edge, type Node } from "../../science/provenance/store" +import { Review } from "../../science/provenance/review" +import { lazy } from "../../util/lazy" + +const Kind = z.enum(["artifact", "run", "source", "claim"]) +const Relation = z.enum(["produced", "consumed", "derived-from", "supports", "refutes"]) +const NodeInput = z.object({ + kind: Kind, + label: z.string().trim().min(1).max(240), + artifact_type: z.string().trim().min(1).max(120).optional(), + path: z.string().max(4_000).optional(), + content_hash: z.string().max(240).optional(), + size: z.number().int().nonnegative().optional(), + tool: z.string().trim().min(1).max(240).optional(), + status: z.enum(["ok", "error"]).optional(), + meta: z.record(z.string(), z.unknown()).optional(), + derived_from: z.string().optional(), + relation: Relation.default("derived-from"), +}) +const ReviewInput = z.object({ + target: z.string(), + claim: z.string().trim().min(1).max(10_000), + issue: z.string().trim().min(1).max(10_000), + severity: z.enum(["blocking", "major", "minor", "info"]), + evidence: z.string().trim().min(1).max(20_000), + verdict: z.enum(["refutes", "supports"]).default("refutes"), +}) + +function summary(nodes: Node[], edges: Edge[]) { + const kinds = { artifact: 0, run: 0, source: 0, claim: 0 } + const reviews = { supports: 0, refutes: 0, blocking: 0, major: 0, minor: 0, info: 0 } + for (const node of nodes) { + kinds[node.kind] += 1 + if (node.meta?.review !== true) continue + const verdict = node.meta.verdict + const severity = node.meta.severity + if (verdict === "supports" || verdict === "refutes") reviews[verdict] += 1 + if (severity === "blocking" || severity === "major" || severity === "minor" || severity === "info") { + reviews[severity] += 1 + } + } + return { + total: nodes.length, + edges: edges.length, + kinds, + reviews, + orphan_edges: edges.filter( + (edge) => !nodes.some((node) => node.id === edge.from) || !nodes.some((node) => node.id === edge.to), + ).length, + } +} + +function lineage(id: string, nodes: Node[], edges: Edge[]) { + const ids = new Set([id]) + const queue = [id] + while (queue.length) { + const current = queue.shift()! + for (const edge of edges) { + if (edge.from !== current && edge.to !== current) continue + const next = edge.from === current ? edge.to : edge.from + if (ids.has(next)) continue + ids.add(next) + queue.push(next) + } + } + return { + nodes: nodes.filter((node) => ids.has(node.id)), + edges: edges.filter((edge) => ids.has(edge.from) && ids.has(edge.to)), + } +} + +export const ProvenanceRoutes = lazy(() => + new Hono() + .get( + "/", + describeRoute({ + summary: "List the project provenance graph", + description: "Returns project-scoped artifacts, runs, sources, claims, reviewer findings, and typed edges.", + operationId: "provenance.list", + responses: { 200: { description: "Project provenance graph" } }, + }), + async (c) => { + const graph = await Provenance.project(Instance.directory) + return c.json({ ...graph, summary: summary(graph.nodes, graph.edges) }) + }, + ) + .post( + "/nodes", + describeRoute({ + summary: "Record a project provenance node", + operationId: "provenance.record", + responses: { 200: { description: "Recorded node" }, 400: { description: "Invalid link target" } }, + }), + validator("json", NodeInput), + async (c) => { + const input = c.req.valid("json") + const graph = input.derived_from ? await Provenance.project(Instance.directory) : undefined + if (input.derived_from && !graph?.nodes.some((node) => node.id === input.derived_from)) { + return c.json({ error: "The provenance link target was not found" }, 400) + } + const node = await Provenance.record({ + kind: input.kind, + label: input.label, + ...(input.artifact_type ? { artifactType: input.artifact_type } : {}), + ...(input.path ? { path: input.path } : {}), + ...(input.content_hash ? { contentHash: input.content_hash } : {}), + ...(input.size !== undefined ? { size: input.size } : {}), + ...(input.tool ? { tool: input.tool } : {}), + ...(input.status ? { status: input.status } : {}), + meta: { ...input.meta, directory: Instance.directory }, + } as Parameters[0]) + if (input.derived_from) { + await Provenance.link({ from: node.id, to: input.derived_from, relation: input.relation }) + } + return c.json(node) + }, + ) + .post( + "/reviews", + describeRoute({ + summary: "Record a reviewer finding", + operationId: "provenance.review", + responses: { 200: { description: "Recorded finding" }, 400: { description: "Invalid target" } }, + }), + validator("json", ReviewInput), + async (c) => { + const input = c.req.valid("json") + const graph = await Provenance.project(Instance.directory) + if (!graph.nodes.some((node) => node.id === input.target)) { + return c.json({ error: "The review target was not found" }, 400) + } + const result = await Review.record({ + target: input.target, + finding: { + claim: input.claim, + issue: input.issue, + severity: input.severity, + evidence: input.evidence, + }, + verdict: input.verdict, + reviewer: "manual review", + directory: Instance.directory, + }) + return c.json(result) + }, + ) + .get( + "/export", + describeRoute({ + summary: "Export a project provenance audit", + operationId: "provenance.export", + responses: { 200: { description: "Portable JSON audit packet" } }, + }), + async (c) => { + const graph = await Provenance.project(Instance.directory) + c.header("Content-Disposition", 'attachment; filename="openscience-provenance-audit.json"') + return c.json({ + format: "openscience.provenance.audit.v1", + generated_at: new Date().toISOString(), + project: Instance.directory, + summary: summary(graph.nodes, graph.edges), + ...graph, + }) + }, + ) + .get( + "/:id", + describeRoute({ + summary: "Trace a provenance node", + operationId: "provenance.trace", + responses: { 200: { description: "Connected lineage" }, 404: { description: "Node not found" } }, + }), + validator("param", z.object({ id: z.string() })), + async (c) => { + const graph = await Provenance.project(Instance.directory) + const id = c.req.valid("param").id + if (!graph.nodes.some((node) => node.id === id)) return c.json({ error: "Provenance node not found" }, 404) + const connected = lineage(id, graph.nodes, graph.edges) + return c.json({ ...connected, summary: summary(connected.nodes, connected.edges) }) + }, + ), +) diff --git a/backend/cli/src/server/routes/settings/compute.ts b/backend/cli/src/server/routes/settings/compute.ts index 4269586d..ef5482bb 100644 --- a/backend/cli/src/server/routes/settings/compute.ts +++ b/backend/cli/src/server/routes/settings/compute.ts @@ -9,6 +9,7 @@ import { Env } from "../../../env" import { OpenScience } from "../../../openscience" import { errors } from "../../error" import { lazy } from "../../../util/lazy" +import { ComputeJobs } from "../../../compute/jobs" // ── Compute settings store ────────────────────────────────────────────────── // @@ -96,13 +97,7 @@ export namespace ComputeSettings { ] // ── Schemas ── - export const SshHost = z.object({ - id: z.string(), - label: z.string(), - host: z.string(), - user: z.string().optional(), - port: z.number().int().positive().optional(), - }) + export const SshHost = ComputeJobs.Host export type SshHost = z.infer export const Endpoint = z.object({ @@ -301,6 +296,10 @@ export namespace ComputeSettings { return view(stored) } + export async function findSshHost(target: string): Promise { + return (await read()).ssh_hosts.find((host) => host.id === target) + } + export async function addEndpoint(input: Omit): Promise { const stored = await read() stored.endpoints.push({ id: id(), ...input }) @@ -385,10 +384,32 @@ export const ComputeSettingsRoutes = lazy(() => host: z.string().min(1), user: z.string().optional(), port: z.number().int().positive().optional(), + scheduler: ComputeJobs.Scheduler.default("none"), + workdir: z.string().optional(), }), ), async (c) => c.json(await ComputeSettings.addSshHost(c.req.valid("json"))), ) + .post( + "/ssh/:id/test", + describeRoute({ + summary: "Test an SSH compute host", + operationId: "settings.compute.ssh.test", + responses: { + 200: { + description: "Connection result", + content: { "application/json": { schema: resolver(ComputeJobs.Probe) } }, + }, + ...errors(404), + }, + }), + validator("param", z.object({ id: z.string() })), + async (c) => { + const host = await ComputeSettings.findSshHost(c.req.valid("param").id) + if (!host) return c.json({ error: "SSH host not found" }, 404) + return c.json(await ComputeJobs.probe(host)) + }, + ) .delete( "/ssh/:id", describeRoute({ @@ -432,5 +453,94 @@ export const ComputeSettingsRoutes = lazy(() => }), validator("param", z.object({ id: z.string() })), async (c) => c.json(await ComputeSettings.removeEndpoint(c.req.valid("param").id)), + ) + .get( + "/jobs", + describeRoute({ + summary: "List local and remote compute jobs", + operationId: "settings.compute.jobs.list", + responses: { + 200: { + description: "Compute jobs", + content: { "application/json": { schema: resolver(ComputeJobs.Job.array()) } }, + }, + }, + }), + async (c) => c.json(await ComputeJobs.list()), + ) + .post( + "/jobs", + describeRoute({ + summary: "Start a local, SSH, Slurm, or PBS compute job", + operationId: "settings.compute.jobs.start", + responses: { + 200: { description: "Started job", content: { "application/json": { schema: resolver(ComputeJobs.Job) } } }, + ...errors(400), + }, + }), + validator("json", ComputeJobs.Input), + async (c) => { + const settings = await ComputeSettings.get() + const input = c.req.valid("json") + const hostId = input.target.kind === "ssh" ? input.target.host_id : undefined + if (hostId && !settings.ssh_hosts.some((host) => host.id === hostId)) { + return c.json({ error: "The selected SSH compute profile was not found" }, 400) + } + return c.json(await ComputeJobs.start(input, { hosts: settings.ssh_hosts })) + }, + ) + .delete( + "/jobs/completed", + describeRoute({ + summary: "Clear completed compute jobs", + operationId: "settings.compute.jobs.clear", + responses: { + 200: { + description: "Number cleared", + content: { + "application/json": { schema: resolver(z.object({ cleared: z.number().int().nonnegative() })) }, + }, + }, + }, + }), + async (c) => c.json({ cleared: await ComputeJobs.clear() }), + ) + .get( + "/jobs/:id/log", + describeRoute({ + summary: "Read a compute job log", + operationId: "settings.compute.jobs.log", + responses: { + 200: { + description: "Job output", + content: { "application/json": { schema: resolver(z.object({ log: z.string() })) } }, + }, + ...errors(404), + }, + }), + validator("param", z.object({ id: z.string() })), + async (c) => { + const job = await ComputeJobs.get(c.req.valid("param").id) + if (!job) return c.json({ error: "Compute job not found" }, 404) + return c.json({ log: await ComputeJobs.log(job.id) }) + }, + ) + .post( + "/jobs/:id/cancel", + describeRoute({ + summary: "Cancel a compute job", + operationId: "settings.compute.jobs.cancel", + responses: { + 200: { description: "Cancelled job", content: { "application/json": { schema: resolver(ComputeJobs.Job) } } }, + ...errors(404), + }, + }), + validator("param", z.object({ id: z.string() })), + async (c) => { + const settings = await ComputeSettings.get() + const job = await ComputeJobs.get(c.req.valid("param").id) + if (!job) return c.json({ error: "Compute job not found" }, 404) + return c.json(await ComputeJobs.cancel(job.id, { hosts: settings.ssh_hosts })) + }, ), ) diff --git a/backend/cli/src/server/server.ts b/backend/cli/src/server/server.ts index b95e192b..33aa5ca2 100644 --- a/backend/cli/src/server/server.ts +++ b/backend/cli/src/server/server.ts @@ -29,6 +29,8 @@ import { SessionRoutes } from "./routes/session" import { PtyRoutes } from "./routes/pty" import { McpRoutes } from "./routes/mcp" import { FileRoutes } from "./routes/file" +import { NotebookRoutes } from "./routes/notebook" +import { ProvenanceRoutes } from "./routes/provenance" import { ConfigRoutes } from "./routes/config" import { ExperimentalRoutes } from "./routes/experimental" import { ProviderRoutes } from "./routes/provider" @@ -278,6 +280,8 @@ export namespace Server { .route("/question", QuestionRoutes()) .route("/provider", ProviderRoutes()) .route("/", FileRoutes()) + .route("/notebook", NotebookRoutes()) + .route("/provenance", ProvenanceRoutes()) .route("/mcp", McpRoutes()) .route("/settings/skills", SettingsSkillsRoutes()) .route("/settings/memory", MemorySettingsRoutes()) diff --git a/backend/cli/src/tool/external-directory.ts b/backend/cli/src/tool/external-directory.ts index 1d3958fc..e17c0ee1 100644 --- a/backend/cli/src/tool/external-directory.ts +++ b/backend/cli/src/tool/external-directory.ts @@ -14,7 +14,7 @@ export async function assertExternalDirectory(ctx: Tool.Context, target?: string if (options?.bypass) return - if (Instance.containsPath(target)) return + if (await Instance.containsCanonicalPath(target)) return const kind = options?.kind ?? "file" const parentDir = kind === "directory" ? target : path.dirname(target) diff --git a/backend/cli/src/tool/notebook.ts b/backend/cli/src/tool/notebook.ts index 75c7b8b5..d111d7af 100644 --- a/backend/cli/src/tool/notebook.ts +++ b/backend/cli/src/tool/notebook.ts @@ -418,6 +418,10 @@ class PythonKernelManager implements KernelManager { this.kernels.delete(sessionID) } + active(sessionID: string): boolean { + return this.kernels.get(sessionID)?.ready ?? false + } + async shutdownAll(): Promise { for (const [id, k] of this.kernels) { await k.shutdown() diff --git a/backend/cli/src/tool/provenance.ts b/backend/cli/src/tool/provenance.ts index 15d90872..9fc3bd0a 100644 --- a/backend/cli/src/tool/provenance.ts +++ b/backend/cli/src/tool/provenance.ts @@ -2,6 +2,7 @@ import z from "zod" import { Tool } from "./tool" import { Provenance } from "../science/provenance/store" import { Review } from "../science/provenance/review" +import { Instance } from "../project/instance" /** * Agent-facing tools over the provenance DAG. Let the model record what it @@ -30,13 +31,19 @@ export const ProvenanceRecordTool = Tool.define("provenance_record", { .describe("Optional id of a parent node this was derived from (creates a 'derived-from' edge)"), }), async execute(params, ctx) { + if (params.derived_from) { + const graph = await Provenance.project(Instance.directory) + if (!graph.nodes.some((node) => node.id === params.derived_from)) { + throw new Error(`Provenance node ${params.derived_from} is not part of this project`) + } + } const node = await Provenance.record({ kind: params.kind, label: params.label, ...(params.artifact_type ? { artifactType: params.artifact_type } : {}), ...(params.path ? { path: params.path } : {}), ...(params.tool ? { tool: params.tool } : {}), - meta: { sessionID: ctx.sessionID, ...params.meta }, + meta: { sessionID: ctx.sessionID, directory: Instance.directory, ...params.meta }, } as Parameters[0]) if (params.derived_from) { @@ -68,8 +75,9 @@ export const ProvenanceQueryTool = Tool.define("provenance_query", { id: z.string().optional().describe("Node id to trace lineage for. Omit to list everything."), }), async execute(params, _ctx) { + const graph = await Provenance.project(Instance.directory) if (!params.id) { - const nodes = await Provenance.list() + const nodes = graph.nodes if (!nodes.length) { return { title: "Provenance", output: "No provenance nodes recorded yet.", metadata: { count: 0, edges: 0 } } } @@ -81,10 +89,23 @@ export const ProvenanceQueryTool = Tool.define("provenance_query", { } } - const { nodes, edges } = await Provenance.query(params.id) - if (!nodes.length) { + if (!graph.nodes.some((node) => node.id === params.id)) { return { title: "Provenance", output: `No node "${params.id}".`, metadata: { count: 0, edges: 0 } } } + const connected = new Set([params.id]) + const queue = [params.id] + while (queue.length) { + const current = queue.shift()! + for (const edge of graph.edges) { + if (edge.from !== current && edge.to !== current) continue + const next = edge.from === current ? edge.to : edge.from + if (connected.has(next)) continue + connected.add(next) + queue.push(next) + } + } + const nodes = graph.nodes.filter((node) => connected.has(node.id)) + const edges = graph.edges.filter((edge) => connected.has(edge.from) && connected.has(edge.to)) const nodeRows = nodes.map((n) => `- **${n.id}** [${n.kind}] ${n.label}`) const edgeRows = edges.map((e) => `- ${e.from} --${e.relation}--> ${e.to}`) return { @@ -127,6 +148,10 @@ export const ProvenanceReviewTool = Tool.define("provenance_review", { .describe("'refutes' flags a defect (default); 'supports' records a verified-sound check"), }), async execute(params, ctx) { + const graph = await Provenance.project(Instance.directory) + if (!graph.nodes.some((node) => node.id === params.target)) { + throw new Error(`Provenance node ${params.target} is not part of this project`) + } const { node, relation } = await Review.record({ target: params.target, finding: { @@ -138,6 +163,7 @@ export const ProvenanceReviewTool = Tool.define("provenance_review", { verdict: params.verdict, reviewer: ctx.agent, sessionID: ctx.sessionID, + directory: Instance.directory, }) return { title: `Review ${relation}: ${node.id}`, diff --git a/backend/cli/src/tool/rkernel.ts b/backend/cli/src/tool/rkernel.ts index 6774cffc..4d859f5a 100644 --- a/backend/cli/src/tool/rkernel.ts +++ b/backend/cli/src/tool/rkernel.ts @@ -382,6 +382,10 @@ class RKernelManager implements KernelManager { this.kernels.delete(sessionID) } + active(sessionID: string): boolean { + return this.kernels.get(sessionID)?.ready ?? false + } + async shutdownAll(): Promise { for (const [id, k] of this.kernels) { await k.shutdown() diff --git a/backend/cli/src/util/filesystem.ts b/backend/cli/src/util/filesystem.ts index aced6869..cda44c5f 100644 --- a/backend/cli/src/util/filesystem.ts +++ b/backend/cli/src/util/filesystem.ts @@ -1,5 +1,6 @@ import { realpathSync } from "fs" -import { dirname, isAbsolute, join, relative } from "path" +import { lstat, realpath, stat } from "fs/promises" +import { basename, dirname, isAbsolute, join, relative, resolve } from "path" export namespace Filesystem { export const exists = (p: string) => @@ -37,6 +38,41 @@ export namespace Filesystem { return rel === "" || (!rel.startsWith("..") && !isAbsolute(rel)) } + async function canonicalize(cursor: string, tail: string[]): Promise { + const result = await realpath(cursor).then( + (value) => ({ value }), + (error: NodeJS.ErrnoException) => ({ error }), + ) + if ("error" in result) { + if (result.error.code !== "ENOENT" && result.error.code !== "ENOTDIR") return + const info = await lstat(cursor).catch(() => undefined) + if (info?.isSymbolicLink()) return + const parent = dirname(cursor) + if (parent === cursor) return + return canonicalize(parent, [basename(cursor), ...tail]) + } + const base = result.value + if (!tail.length) return base + const info = await stat(base).catch(() => undefined) + if (!info?.isDirectory()) return + return resolve(base, ...tail) + } + + /** + * Resolve a path by filesystem identity, including a target that does not + * exist yet. Existing symlinks are followed; a broken symlink is rejected + * instead of being reconstructed as if it were an ordinary path segment. + */ + export function canonical(p: string): Promise { + return canonicalize(resolve(p), []) + } + + export async function containsCanonical(parent: string, child: string): Promise { + const [root, target] = await Promise.all([canonical(parent), canonical(child)]) + if (!root || !target) return false + return contains(root, target) + } + export async function findUp(target: string, start: string, stop?: string) { let current = start const result = [] diff --git a/backend/cli/test/compute/jobs.test.ts b/backend/cli/test/compute/jobs.test.ts new file mode 100644 index 00000000..584b7632 --- /dev/null +++ b/backend/cli/test/compute/jobs.test.ts @@ -0,0 +1,217 @@ +import { describe, expect, test } from "bun:test" +import fs from "node:fs/promises" +import path from "node:path" +import { ComputeJobs } from "../../src/compute/jobs" +import { tmpdir } from "../fixture/fixture" + +describe("ComputeJobs command adapters", () => { + const host = { + id: "cluster", + label: "Lab cluster", + host: "hpc.example.org", + user: "researcher", + port: 2222, + scheduler: "slurm" as const, + workdir: "/scratch/team project", + } + + test("builds a non-interactive SSH command for a Slurm job", () => { + const command = ComputeJobs.command( + { + id: "job-123", + name: "RNA benchmark", + command: "python train.py --label 'A B'", + cwd: "/scratch/team project", + resources: { + cpus: 8, + gpus: 2, + memory_gb: 48, + time_minutes: 95, + partition: "gpu-long", + }, + modules: ["cuda/12.4", "python/3.12"], + container: "/containers/research image.sif", + }, + host, + ) + + expect(command.argv.slice(0, 7)).toEqual(["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=8", "-p", "2222"]) + expect(command.argv).toContain("researcher@hpc.example.org") + expect(command.argv.at(-1)).toContain("sbatch --wait --parsable") + expect(command.argv.at(-1)).toContain("--cpus-per-task=8") + expect(command.argv.at(-1)).toContain("--gres=gpu:2") + expect(command.argv.at(-1)).toContain("--mem=48G") + expect(command.argv.at(-1)).toContain("--time=01:35:00") + expect(command.argv.at(-1)).toContain("--partition='gpu-long'") + expect(command.argv.at(-1)).toContain("module load") + expect(command.argv.at(-1)).toContain("cuda/12.4") + expect(command.argv.at(-1)).toContain("python/3.12") + expect(command.argv.at(-1)).toContain("apptainer exec") + expect(command.argv.at(-1)).toContain("/containers/research image.sif") + expect(command.argv.at(-1)).toContain("os-job-123") + expect(command.argv.at(-1)).toContain("python train.py") + }) + + test("builds PBS and direct SSH adapters from the same profile", () => { + const input = { + id: "job-9", + name: "Variant call", + command: "bash pipeline.sh", + cwd: "/work", + resources: { cpus: 4, gpus: 1, memory_gb: 16, time_minutes: 30 }, + } + const pbs = ComputeJobs.command(input, { ...host, scheduler: "pbs" }).argv.at(-1) + expect(pbs).toContain("qsub") + expect(pbs).toContain("select=1:ncpus=4:ngpus=1:mem=16gb") + expect(pbs).toContain("walltime=00:30:00") + expect(ComputeJobs.command(input, { ...host, scheduler: "none" }).argv.at(-1)).toContain("exec") + }) +}) + +describe("ComputeJobs local lifecycle", () => { + test("a missing working directory fails durably without crashing the server process", async () => { + await using tmp = await tmpdir() + const root = path.join(tmp.path, "state") + const cwd = path.join(tmp.path, "missing") + const cli = path.join(import.meta.dir, "../..") + const script = ` + import { ComputeJobs } from "./src/compute/jobs" + const job = await ComputeJobs.start( + { + name: "missing cwd", + command: "printf unreachable", + cwd: ${JSON.stringify(cwd)}, + target: { kind: "local" }, + }, + { root: ${JSON.stringify(root)} }, + ) + const result = await ComputeJobs.wait(job.id, { root: ${JSON.stringify(root)}, timeout: 5_000 }) + console.log(JSON.stringify(result)) + ` + const proc = Bun.spawn([process.execPath, "-e", script], { + cwd: cli, + stdout: "pipe", + stderr: "pipe", + }) + const [code, stdout, stderr] = await Promise.all([ + proc.exited, + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + ]) + + expect(code, stderr).toBe(0) + const job = ComputeJobs.Job.parse(JSON.parse(stdout.trim())) + expect(job.status).toBe("failed") + expect(job.exit_code).toBeNull() + expect(job.error).toMatch(/ENOENT|no such file or directory/i) + }) + + test("runs a real local job, persists status, and streams its log", async () => { + await using tmp = await tmpdir() + const root = path.join(tmp.path, "state") + const job = await ComputeJobs.start( + { + name: "deterministic smoke", + command: "printf 'alpha\\nbeta\\n'", + cwd: tmp.path, + target: { kind: "local" }, + }, + { root }, + ) + + const finished = await ComputeJobs.wait(job.id, { root, timeout: 5_000 }) + expect(finished.status).toBe("succeeded") + expect(finished.exit_code).toBe(0) + expect(await ComputeJobs.log(job.id, { root })).toContain("alpha\nbeta") + expect(finished.reproducibility).toMatchObject({ + platform: process.platform, + arch: process.arch, + command: "printf 'alpha\\nbeta\\n'", + }) + }) + + test("captures output artifacts, checksums, lockfiles, and checkpoints", async () => { + await using tmp = await tmpdir({ git: true }) + const root = path.join(tmp.path, "state") + await Bun.write(path.join(tmp.path, "requirements.txt"), "numpy==2.2.0\n") + const job = await ComputeJobs.start( + { + name: "artifact capture", + command: + "mkdir -p outputs checkpoints && printf 'metric,value\\nloss,0.1\\n' > outputs/results.csv && printf model > checkpoints/latest.ckpt", + cwd: tmp.path, + target: { kind: "local" }, + artifacts: ["outputs/**/*.csv"], + checkpoint: "checkpoints/latest.ckpt", + resources: { cpus: 2, memory_gb: 4 }, + }, + { root }, + ) + + const finished = await ComputeJobs.wait(job.id, { root, timeout: 5_000 }) + expect(finished.artifacts).toHaveLength(1) + expect(finished.artifacts?.[0]).toMatchObject({ + path: "outputs/results.csv", + size: 22, + }) + expect(finished.artifacts?.[0]?.sha256).toMatch(/^[a-f0-9]{64}$/) + expect(finished.checkpoint).toMatchObject({ + path: "checkpoints/latest.ckpt", + size: 5, + }) + expect(finished.reproducibility?.git?.dirty).toBe(true) + expect(finished.reproducibility?.lockfiles).toContainEqual( + expect.objectContaining({ + path: "requirements.txt", + sha256: expect.stringMatching(/^[a-f0-9]{64}$/), + }), + ) + }) + + test("cancels a running local process tree", async () => { + await using tmp = await tmpdir() + const root = path.join(tmp.path, "state") + const job = await ComputeJobs.start( + { + name: "cancel smoke", + command: "sleep 30", + cwd: tmp.path, + target: { kind: "local" }, + }, + { root }, + ) + + await ComputeJobs.cancel(job.id, { root }) + const cancelled = await ComputeJobs.wait(job.id, { root, timeout: 5_000 }) + expect(cancelled.status).toBe("cancelled") + }) + + test("recovers a completed detached job from its durable exit marker", async () => { + const root = await fs.mkdtemp(path.join(import.meta.dir, "jobs-recovery-")) + const id = "recovered-job" + await fs.mkdir(path.join(root, "jobs"), { recursive: true }) + await Bun.write( + path.join(root, "jobs.json"), + JSON.stringify([ + { + id, + name: "recovered", + command: "true", + target: { kind: "local" }, + target_label: "This computer", + scheduler: "none", + status: "running", + created_at: new Date(Date.now() - 10_000).toISOString(), + started_at: new Date(Date.now() - 9_000).toISOString(), + pid: 999_999, + }, + ]), + ) + await Bun.write(path.join(root, "jobs", `${id}.exit`), "0") + + const job = (await ComputeJobs.list({ root })).find((item) => item.id === id) + expect(job?.status).toBe("succeeded") + expect(job?.exit_code).toBe(0) + await fs.rm(root, { recursive: true, force: true }) + }) +}) diff --git a/backend/cli/test/file/artifacts.test.ts b/backend/cli/test/file/artifacts.test.ts new file mode 100644 index 00000000..938cffdb --- /dev/null +++ b/backend/cli/test/file/artifacts.test.ts @@ -0,0 +1,120 @@ +import { describe, expect, test } from "bun:test" +import { $ } from "bun" +import path from "node:path" +import { ArtifactFile } from "../../src/file/artifacts" +import { File } from "../../src/file" +import { Instance } from "../../src/project/instance" +import { tmpdir } from "../fixture/fixture" + +describe("ArtifactFile.classify", () => { + test.each([ + ["analysis.ipynb", "notebook"], + ["cells.h5ad", "dataset"], + ["counts.parquet", "dataset"], + ["figure.svg", "figure"], + ["manuscript.pdf", "report"], + ["protein.cif", "structure"], + ["reads.fastq", "sequence"], + ["cohort.vcf", "genomics"], + ["run.mzML", "spectrum"], + ["weights.safetensors", "model"], + ["bundle.zip", "archive"], + ])("classifies %s as %s", (file, kind) => { + expect(ArtifactFile.classify(file)?.kind).toBe(ArtifactFile.Kind.parse(kind)) + }) + + test("does not treat source code as a research artifact", () => { + expect(ArtifactFile.classify("pipeline.py")).toBeUndefined() + }) +}) + +describe("File.artifacts", () => { + test("discovers local artifacts recursively with metadata and skips dependency trees", async () => { + await using tmp = await tmpdir({ + init: async (directory) => { + await Bun.write(path.join(directory, "analysis.ipynb"), "{}") + await Bun.write(path.join(directory, "results", "figure.png"), Uint8Array.from([1, 2, 3])) + await Bun.write(path.join(directory, "results", "table.csv"), "x,y\n1,2\n") + await Bun.write(path.join(directory, "src", "pipeline.py"), "print('not an artifact')") + await Bun.write(path.join(directory, "node_modules", "package", "paper.pdf"), "skip") + }, + }) + await Instance.provide({ + directory: tmp.path, + fn: async () => { + const result = await File.artifacts() + expect(result.map((item) => item.path).toSorted()).toEqual([ + "analysis.ipynb", + "results/figure.png", + "results/table.csv", + ]) + expect(result.find((item) => item.path === "analysis.ipynb")).toMatchObject({ + kind: "notebook", + format: "ipynb", + size: 2, + }) + expect(result.find((item) => item.path === "results/figure.png")).toMatchObject({ + kind: "figure", + format: "png", + size: 3, + }) + }, + }) + }) +}) + +describe("File.provenance", () => { + test("reports branch, latest commit, and dirty state for a tracked artifact", async () => { + await using tmp = await tmpdir({ + git: true, + init: async (directory) => { + await Bun.write(path.join(directory, "results.csv"), "metric,value\naccuracy,0.9\n") + await $`git add results.csv`.cwd(directory).quiet() + await $`git -c user.name=OpenScience -c user.email=test@openscience.local commit -m "record baseline"` + .cwd(directory) + .quiet() + await Bun.write(path.join(directory, "results.csv"), "metric,value\naccuracy,0.95\n") + }, + }) + const branch = (await $`git branch --show-current`.cwd(tmp.path).quiet().text()).trim() + + await Instance.provide({ + directory: tmp.path, + fn: async () => { + const result = await File.provenance("results.csv") + expect(result).toMatchObject({ + path: "results.csv", + tracked: true, + dirty: true, + status: "modified", + branch, + commit: { + author: "OpenScience", + email: "test@openscience.local", + message: "record baseline", + }, + }) + expect(result.commit?.sha).toHaveLength(40) + }, + }) + }) + + test("reports a clean local-only state outside git", async () => { + await using tmp = await tmpdir({ + init: async (directory) => { + await Bun.write(path.join(directory, "report.pdf"), "pdf") + }, + }) + + await Instance.provide({ + directory: tmp.path, + fn: async () => { + expect(await File.provenance("report.pdf")).toMatchObject({ + tracked: false, + dirty: false, + status: "local", + }) + }, + }) + }) +}) diff --git a/backend/cli/test/file/path-traversal.test.ts b/backend/cli/test/file/path-traversal.test.ts index 380f8051..0f4f07b2 100644 --- a/backend/cli/test/file/path-traversal.test.ts +++ b/backend/cli/test/file/path-traversal.test.ts @@ -84,6 +84,64 @@ describe("File.read path traversal protection", () => { }, }) }) + + test("rejects reads through an internal symlink that resolves outside the project", async () => { + await using outside = await tmpdir({ + init: async (dir) => { + await Bun.write(path.join(dir, "secret.txt"), "external secret") + }, + }) + await using tmp = await tmpdir({ + init: async (dir) => { + await fs.symlink(outside.path, path.join(dir, "escape")) + }, + }) + + await Instance.provide({ + directory: tmp.path, + fn: async () => { + await expect(File.read("escape/secret.txt")).rejects.toThrow("Access denied: path escapes project directory") + await expect(File.raw("escape/secret.txt")).rejects.toThrow("Access denied: path escapes project directory") + await expect(File.inspect("escape/secret.txt")).rejects.toThrow("Access denied: path escapes project directory") + }, + }) + }) + + test("allows a new file below an internal directory", async () => { + await using tmp = await tmpdir({ + init: async (dir) => { + await fs.mkdir(path.join(dir, "results"), { recursive: true }) + }, + }) + + await Instance.provide({ + directory: tmp.path, + fn: async () => { + const result = await File.write("results/new.txt", "new result") + expect(result.content).toBe("new result") + expect(await Bun.file(path.join(tmp.path, "results", "new.txt")).text()).toBe("new result") + }, + }) + }) + + test("rejects a new write below an internal symlink to an external directory", async () => { + await using outside = await tmpdir() + await using tmp = await tmpdir({ + init: async (dir) => { + await fs.symlink(outside.path, path.join(dir, "escape")) + }, + }) + + await Instance.provide({ + directory: tmp.path, + fn: async () => { + await expect(File.write("escape/new.txt", "escaped")).rejects.toThrow( + "Access denied: path escapes project directory", + ) + expect(await Bun.file(path.join(outside.path, "new.txt")).exists()).toBe(false) + }, + }) + }) }) describe("File.list path traversal protection", () => { @@ -113,6 +171,26 @@ describe("File.list path traversal protection", () => { }, }) }) + + test("rejects listing through an internal symlink that resolves outside the project", async () => { + await using outside = await tmpdir({ + init: async (dir) => { + await Bun.write(path.join(dir, "secret.txt"), "external secret") + }, + }) + await using tmp = await tmpdir({ + init: async (dir) => { + await fs.symlink(outside.path, path.join(dir, "escape")) + }, + }) + + await Instance.provide({ + directory: tmp.path, + fn: async () => { + await expect(File.list("escape")).rejects.toThrow("Access denied: path escapes project directory") + }, + }) + }) }) describe("Instance.containsPath", () => { @@ -195,4 +273,23 @@ describe("Instance.containsPath", () => { }, }) }) + + test("canonical containment rejects a symlink escape but keeps an internal missing target", async () => { + await using outside = await tmpdir() + await using tmp = await tmpdir({ + git: true, + init: async (dir) => { + await fs.mkdir(path.join(dir, "results"), { recursive: true }) + await fs.symlink(outside.path, path.join(dir, "escape")) + }, + }) + + await Instance.provide({ + directory: tmp.path, + fn: async () => { + expect(await Instance.containsCanonicalPath(path.join(tmp.path, "escape", "secret.txt"))).toBe(false) + expect(await Instance.containsCanonicalPath(path.join(tmp.path, "results", "new.txt"))).toBe(true) + }, + }) + }) }) diff --git a/backend/cli/test/file/publication.test.ts b/backend/cli/test/file/publication.test.ts new file mode 100644 index 00000000..d2b6846b --- /dev/null +++ b/backend/cli/test/file/publication.test.ts @@ -0,0 +1,214 @@ +import { $ } from "bun" +import { describe, expect, test } from "bun:test" +import fs from "node:fs/promises" +import path from "node:path" +import { PublicationFile } from "../../src/file/publication" +import { PublicationReview } from "../../src/file/review" +import { Instance } from "../../src/project/instance" +import { tmpdir } from "../fixture/fixture" + +describe("PublicationFile", () => { + test("detects real local publication export capabilities", async () => { + const capabilities = await PublicationFile.capabilities() + expect(capabilities.formats.html).toBe(true) + expect(capabilities.formats.docx).toBe(capabilities.pandoc) + expect(capabilities.formats.pptx).toBe(capabilities.pandoc) + expect(capabilities.formats.pdf).toBe(capabilities.pandoc && Boolean(capabilities.pdf_engine)) + }) + + test("exports a secure standalone HTML publication without external tooling", async () => { + await using tmp = await tmpdir({ + init: async (directory) => { + await Bun.write( + path.join(directory, "report.md"), + "# Treatment response\n\nThe observed response was **42%**.\n\n\n", + ) + }, + }) + const result = await PublicationFile.render(tmp.path, { path: "report.md", format: "html" }) + expect(result.path).toMatch(/^exports\/report-\d{8}-\d{9}-[a-f0-9]{8}\.html$/) + expect(result.size).toBeGreaterThan(100) + expect(result.engine).toBe("OpenScience Markdown") + expect(result.readiness).toBe("draft") + const html = await Bun.file(path.join(tmp.path, result.path)).text() + expect(html).toContain("Treatment response") + expect(html).toContain("Content-Security-Policy") + expect(html).not.toContain("