diff --git a/electron/src/main/index.ts b/electron/src/main/index.ts index a1f953e3..67216c6d 100644 --- a/electron/src/main/index.ts +++ b/electron/src/main/index.ts @@ -513,6 +513,14 @@ function buildMenu(): void { // than the native picker — the user adds/reorders there, then confirms. click: () => win?.webContents.send('spyde:open_stack_dialog'), }, + { + label: 'Load In-Situ Data…', + // A potentiostat / holder log recorded alongside a movie. Auto- + // discovery already scans the movie's own folder on open; this is + // for a record that lives elsewhere, or one the duration scorer + // passed over. + click: () => pickInsituData(), + }, { type: 'separator' }, { label: 'Save Signal…', @@ -618,6 +626,26 @@ ipcMain.handle('spyde:open-file', async () => { } }) +/** Pick an instrument record (potentiostat / holder log) and attach it. + * `.txt` is offered because EC-Lab's ASCII export is routinely saved that way; + * Python dispatches on the file's own magic, not on the extension. */ +async function pickInsituData(): Promise { + const result = await dialog.showOpenDialog(win!, { + title: 'Load in-situ instrument data', + properties: ['openFile'], + filters: [ + { name: 'In-Situ Data', extensions: ['mpr', 'mpt', 'txt', 'mps'] }, + { name: 'BioLogic EC-Lab', extensions: ['mpr', 'mpt', 'mps'] }, + { name: 'All Files', extensions: ['*'] }, + ], + }) + if (!result.canceled) { + for (const p of result.filePaths) sendAction('load_insitu_data', { path: p }) + } +} + +ipcMain.handle('spyde:load-insitu-data', () => pickInsituData()) + /** Open a .zspy/.zarr DIRECTORY store (folder picker → load). */ ipcMain.handle('spyde:open-zarr-folder', async () => { const result = await dialog.showOpenDialog(win!, { diff --git a/electron/src/preload/index.ts b/electron/src/preload/index.ts index fd72034c..53ef2a56 100644 --- a/electron/src/preload/index.ts +++ b/electron/src/preload/index.ts @@ -110,6 +110,9 @@ contextBridge.exposeInMainWorld('electron', { /** Open a .zspy/.zarr DIRECTORY store (folder picker → load). */ openZarrFolder: (): Promise => ipcRenderer.invoke('spyde:open-zarr-folder'), + /** Pick an in-situ instrument record (.mpr/.mpt/.txt/.mps) and attach it. */ + loadInsituData: (): Promise => ipcRenderer.invoke('spyde:load-insitu-data'), + /** Quit the app (custom title-bar menu replaces native File→Quit). */ quit: (): Promise => ipcRenderer.invoke('spyde:quit'), diff --git a/electron/src/renderer/src/components/FloatingToolbar.tsx b/electron/src/renderer/src/components/FloatingToolbar.tsx index f1c5d2a2..2bab4d77 100644 --- a/electron/src/renderer/src/components/FloatingToolbar.tsx +++ b/electron/src/renderer/src/components/FloatingToolbar.tsx @@ -314,8 +314,12 @@ export function FloatingToolbar({ // clock runs and un-lights when playback auto-stops at the movie end. const pb = state.playback const playbackActive = a.name === 'Play' && pb.playing - const ffSpeed = a.name === 'Fast Forward' && pb.playing && pb.speed > 1 - ? pb.speed : 0 + // The "×N" badge shows whenever a speed above 1× is SELECTED, not only + // while the clock happens to be running: Fast Forward is a cycle, so + // pausing at ×8 and pressing Play must still read as ×8. Gating it on + // `pb.playing` made the speed vanish the moment you paused, which + // looked like the speed change had not been applied at all. + const ffSpeed = a.name === 'Fast Forward' && pb.speed > 1 ? pb.speed : 0 const active = openName === a.name || live.has(a.name) || (state.subItems.get(windowId)?.get(a.name)?.length ?? 0) > 0 || playbackActive @@ -714,12 +718,17 @@ const styles: Record = { border: 'none', cursor: 'pointer', width: 30, height: 30, borderRadius: 6, display: 'flex', alignItems: 'center', justifyContent: 'center', }, - // Small "×N" corner badge on the Fast Forward button while speed > 1. + // "×N" badge on the Fast Forward button while a speed above 1× is selected. + // Sized to stay legible at "32x" — the cycle reaches ×32, and the previous + // 8px corner chip was too small to read the speed off at a glance, which is + // the badge's entire job. speedBadge: { - position: 'absolute', bottom: -3, right: -3, + position: 'absolute', bottom: -5, right: -7, background: '#f38ba8', color: '#11111b', - fontSize: 8, fontWeight: 700, lineHeight: 1, - padding: '1px 3px', borderRadius: 6, pointerEvents: 'none', + fontSize: 10, fontWeight: 800, lineHeight: 1, + padding: '2px 4px', borderRadius: 7, pointerEvents: 'none', + letterSpacing: '-0.2px', + boxShadow: '0 0 0 1.5px #11111b', }, subBar: { ...subBase, top: '100%', marginTop: 10, animation: 'spyde-pop 130ms ease-out' }, subBarUp: { ...subBase, bottom: '100%', marginBottom: 10, animation: 'spyde-pop-up 130ms ease-out' }, diff --git a/electron/src/renderer/src/components/MenuBar.tsx b/electron/src/renderer/src/components/MenuBar.tsx index 903ab911..4cbf2850 100644 --- a/electron/src/renderer/src/components/MenuBar.tsx +++ b/electron/src/renderer/src/components/MenuBar.tsx @@ -187,6 +187,7 @@ export function MenuBar({ onStartGuide, onShowInfo }: { { label: 'Open…', onClick: () => window.electron.openFile() }, { label: 'Open Zarr Folder (.zspy)…', onClick: () => window.electron.openZarrFolder() }, { label: 'Load Stack…', onClick: () => openStackDialog() }, + { label: 'Load In-Situ Data…', onClick: () => window.electron.loadInsituData() }, { separator: true }, { label: 'Save Signal…', onClick: () => window.electron.saveDialog() }, { separator: true }, diff --git a/electron/src/renderer/src/components/MovieEditor.tsx b/electron/src/renderer/src/components/MovieEditor.tsx index 6b36f6d9..00acc05e 100644 --- a/electron/src/renderer/src/components/MovieEditor.tsx +++ b/electron/src/renderer/src/components/MovieEditor.tsx @@ -24,7 +24,7 @@ import React from 'react' import { useSpyDE } from '../kernel/SpyDEContext' import { SeamlessFigureFrame } from './ReportFigureCell' import { WINDOW_DRAG_MIME } from '../kernel/dnd' -import type { MovieStateMessage, MovieParams, MovieAnnotation, MovieSpeedSegment } from '../kernel/protocol' +import type { MovieStateMessage, MovieParams, MovieAnnotation, MovieSpeedSegment, MovieTextOverlay } from '../kernel/protocol' interface Props { cellId: string @@ -35,7 +35,17 @@ interface Props { const CMAPS = ['gray', 'viridis', 'magma', 'inferno', 'plasma', 'cividis', 'hot', 'jet'] const DOWNSAMPLES = [1, 2, 4, 8] // Speed presets (a segment's multiplier). 0 = hold (freeze); <1 slow-mo; >1 ff. -const SPEEDS = [0, 0.25, 0.5, 1, 2, 4, 8] +// Segment speed multipliers. Reaches 32x for the same reason the playback +// cycle does: an in-situ acquisition is thousands of frames of slow change, +// and a long uneventful stretch wants skipping hard, not merely fast. +const SPEEDS = [0, 0.25, 0.5, 1, 2, 4, 8, 16, 32] + +/** A burn-in's clip label: its literal text, or the quantity it tracks. */ +function burninLabel(o: MovieTextOverlay): string { + if (o.builtin === 'label') return o.text || 'Text' + if (o.builtin === 'time') return 'Timestamp' + return o.label || o.insitu_channel || 'value' +} const SPEED_LABEL = (s: number) => (s === 0 ? 'hold' : `${s}×`) // A selected timeline clip (which lane + index), so the inspector edits it. @@ -185,6 +195,11 @@ export function MovieEditor({ cellId, sendAction, onClose }: Props) { // ── overlay + speed lists ───────────────────────────────────────────────── const anns = st?.annotations ?? [] const textOverlays = st?.text_overlays ?? [] + // Everything addable as burnt-in text (static label, clock, instrument + // channels) — the backend owns the list so the rail has one source of truth. + const burninSources = st?.burnin_sources + ?? [{ source: 'label', label: 'Text', units: '' }, + { source: 'time', label: 'Timestamp', units: 's' }] const speedSegs = st?.speed_segments ?? [] const setAnnotations = (list: MovieAnnotation[]) => { setSt(s => (s ? { ...s, annotations: list } : s)); tune({ annotations: list }) @@ -254,11 +269,28 @@ export function MovieEditor({ cellId, sendAction, onClose }: Props) {
{/* Left rail — just the ADD buttons (compact). */}
-
Add overlay
- + {/* Burnt-in TEXT — one section, one button per source. Static text, + the clock and each attached instrument channel are the same kind + of object (they differ only in where the string comes from), so + they share this list, the Burn-in timeline lane and one + inspector. */} +
Burn-in text
+ {burninSources.map(s => ( + + ))} +
+
Shapes
+
+
Timing
-
+
Crop
) } @@ -509,6 +549,10 @@ function RenderControls({ params, patchParams }: { onChange={(b) => patchParams({ axes: b })} /> patchParams({ scalebar: b })} /> + {/* Timestamp is an ordinary burn-in overlay (`builtin: "time"`), so the + checkbox only adds/removes it — position, size and colour are edited + on the Signal lane like a voltage's, not with a second set of + controls here. */} patchParams({ timestamp: b })} />
@@ -644,6 +688,8 @@ const styles: Record = { body: { flex: 1, display: 'flex', minHeight: 0 }, rail: { display: 'flex', flexDirection: 'column', gap: 5, padding: 12, borderRight: '1px solid #313244', flexShrink: 0, width: 132, overflowY: 'auto' }, railHead: { fontSize: 10, fontWeight: 700, color: '#89b4fa', textTransform: 'uppercase', letterSpacing: 0.4 }, + // Hairline between the rail's groups (burn-in text / shapes / timing). + railRule: { height: 1, background: 'rgba(255,255,255,0.10)', margin: '10px 0 8px' }, toolBtn: { background: '#1e1e2e', color: '#cdd6f4', border: '1px solid #313244', borderRadius: 5, padding: '5px 8px', fontSize: 11.5, cursor: 'pointer', textAlign: 'left' }, toolBtnActive: { background: '#89b4fa', color: '#11111b', borderColor: '#89b4fa', fontWeight: 700 }, center: { flex: 1, display: 'flex', flexDirection: 'column', minWidth: 0, padding: 14, gap: 10 }, diff --git a/electron/src/renderer/src/electron.d.ts b/electron/src/renderer/src/electron.d.ts index 784cf466..7762a95f 100644 --- a/electron/src/renderer/src/electron.d.ts +++ b/electron/src/renderer/src/electron.d.ts @@ -17,6 +17,7 @@ declare global { action: (action: string, payload?: Record, windowId?: number) => void openFile: () => Promise openZarrFolder: () => Promise + loadInsituData: () => Promise quit: () => Promise saveDialog: () => Promise pickFile: (opts: { name?: string; extensions?: string[] }) => Promise diff --git a/electron/src/renderer/src/kernel/protocol.ts b/electron/src/renderer/src/kernel/protocol.ts index e2795a75..ed914db6 100644 --- a/electron/src/renderer/src/kernel/protocol.ts +++ b/electron/src/renderer/src/kernel/protocol.ts @@ -300,7 +300,7 @@ export interface PlaybackStateMessage extends MsgBase { type: 'playback_state' /** True while the movie clock is running. */ playing: boolean - /** Current speed multiplier (1/2/4/8) — drives the Fast Forward "×N" badge. */ + /** Current speed multiplier (1/2/4/8/16/32) — drives the Fast Forward "×N" badge. */ speed?: number loop?: boolean } @@ -561,6 +561,14 @@ export interface ReportCell { * at render time; `fmt` is a Python-style format string over {label,value,units}. */ export interface MovieTextOverlay { source?: Record // a SignalRef dict (opaque to the renderer) + /** An attached instrument channel (spyde/insitu), e.g. "Ewe/V". */ + insitu_channel?: string + /** A built-in value needing no source: "time" is the elapsed-time burn-in, + * which is an ordinary overlay so it gets the same widget, timeline clip + * and inspector controls as any other. */ + builtin?: 'time' | 'label' + /** Literal string for a builtin:'label' overlay. */ + text?: string label?: string units?: string fmt?: string @@ -623,6 +631,8 @@ export interface MovieParams { cmap?: string clim?: [number, number] | null timestamp?: boolean + /** Burnt-in timestamp colour ("#rrggbb"); white when unset. */ + timestamp_color?: string scalebar?: boolean axes?: boolean // draw calibrated axis ticks (default true) t_start?: number @@ -659,6 +669,26 @@ export interface MovieStateMessage extends MsgBase { signal_window_id: number | null // the MDI window holding the signal figure nav_fig_id: string | null // the 1-D navigator figure (shown beside, opt) current_index: number // the navigator's current time index + /** Instrument channels aligned to this movie (electrochemistry potential / + * current, holder temperature…). Already per-frame on the tree, so the + * editor offers each as a one-click burn-in overlay; empty when none is + * attached. See spyde/insitu and movie_add_insitu_overlay. */ + insitu_channels?: InsituChannelOption[] + /** Everything addable as burnt-in text: static label, clock, channels. */ + burnin_sources?: BurninSource[] +} + +/** One attached instrument channel offered as a burn-in overlay. */ +export interface BurninSource { + source: string // "label" | "time" | an instrument channel key + label: string + units: string +} + +export interface InsituChannelOption { + channel: string // the backend key, e.g. "Ewe/V" + label: string // display name, e.g. "Ewe" + units: string // e.g. "V" } /** Export finished (spyde:movie_done). */ diff --git a/electron/tests/insitu_echem.spec.ts b/electron/tests/insitu_echem.spec.ts new file mode 100644 index 00000000..a3925d0b --- /dev/null +++ b/electron/tests/insitu_echem.spec.ts @@ -0,0 +1,285 @@ +/** + * insitu_echem.spec.ts — an in-situ movie opens with its REAL frame timing and + * with the electrochemistry recorded beside it attached as navigator lanes. + * + * This is the part a green pytest run cannot see. The unit tests prove the + * readers and the alignment maths; only the app can show that the chips + * actually appear on the navigator, that shift-clicking them stacks E and I + * against the movie's own time cursor, and that dragging that cursor still + * moves the movie. + * + * DEV-BOX ONLY: it drives a real 132 GB DE acquisition (with its + * `.spyde-nav.npz` sidecar, so no navigator recompute) plus the BioLogic + * records beside it. CI has neither, so the whole file skips when the folder + * is absent rather than shipping a fake that would pass without proving + * anything. + */ +import { test, expect } from '@playwright/test' +import { existsSync, mkdirSync } from 'fs' +import { join } from 'path' + +const { + launchApp, backendAction, waitForSubwindowCount, navWindow, backendErrorLines, +} = require('./_harness.cjs') + +const DATA_DIR = + 'D:\\InsituElectroChemistry\\directelectron_good-electrochemistry-movie_2026-07-30_0335' +const MOVIE = join(DATA_DIR, '20251117_88071_movie.mrc') +const EC_RUN = join(DATA_DIR, 'New TEM NZH-001-03-floating_02_CV_C01.mpr') +const SHOTS = join(__dirname, '..', 'insitu_echem_shots') + +const POTENTIAL_CHIP = 'Ewe (V)' +const CURRENT_CHIP = 'I (µA)' + +test.skip(!existsSync(MOVIE), `in-situ electrochemistry dataset not present (${MOVIE})`) +test.describe.configure({ mode: 'serial' }) + +let ctx: any + +test.beforeAll(async () => { + mkdirSync(SHOTS, { recursive: true }) + ctx = await launchApp({ env: { SPYDE_LOG_LEVEL: 'INFO' } }) +}) + +test.afterAll(async () => { + await ctx?.app?.close() +}) + +/** The most recent backend log line containing `needle`. + * `waitForLog` resolves with the line only for a FUTURE match — a line already + * in the buffer resolves undefined — so always read the buffer afterwards. */ +function lastLine(needle: string): string { + const hit = [...ctx.backend.logBuffer].reverse().find((l: string) => l.includes(needle)) + return hit ?? '' +} + +/** The nav window's id, taken from its chip strip's testid. */ +async function navChipsId(nav: any): Promise { + const testid = await nav.locator('[data-testid^="nav-chips-"]').first().getAttribute('data-testid') + return String(testid).replace('nav-chips-', '') +} + +test('a DE movie opens with its timestamps as the time base', async () => { + const { page, backend } = ctx + + await backendAction(page, 'open_file', { path: MOVIE }) + await waitForSubwindowCount(page, 2, 180_000) + await page.screenshot({ path: join(SHOTS, '01-movie-open.png') }) + + // The reader derives 1/(fps*sum) = 8.19 ms; the timestamps say 32.76 ms. + await backend.waitForLog('Frame timing from', 120_000) + const timing = lastLine('Frame timing from') + expect(timing).toContain('movie_timestamps.csv') + expect(timing).toContain('32.760 ms/frame') + expect(timing).toContain('30.53 fps') + // and the reader's wrong value is reported alongside it, 4x too fast + expect(timing).toContain('reader said 8.19') + + // The SIGNAL axes too: this is a TEM image, so nm — but rsciio reads the + // imaging exposure's camera length of 0 as "has a camera length" and + // calibrates it as diffraction at the unset -1 nm^-1. + await backend.waitForLog('frame pixel size from', 30_000) + const px = lastLine('frame pixel size from') + expect(px).toContain('1.14786 nm/px') + expect(px).toContain('reader said -1 nm^-1') +}) + +test('electrochemistry beside the movie attaches as navigator chips', async () => { + const { page, backend } = ctx + + await backend.waitForLog('Attached Cyclic Voltammetry', 120_000) + const attached = lastLine('Attached Cyclic Voltammetry') + expect(attached).toContain('floating_02_CV_C01') + expect(attached).toContain('aligned by span') + expect(attached).toContain('100% of frames covered') + + const nav = navWindow(page) + const navId = await navChipsId(nav) + await expect(nav.getByTestId(`nav-chip-${POTENTIAL_CHIP}-${navId}`)).toBeVisible() + await expect(nav.getByTestId(`nav-chip-${CURRENT_CHIP}-${navId}`)).toBeVisible() + await page.screenshot({ path: join(SHOTS, '02-nav-chips.png') }) +}) + +test('shift-clicking the chips stacks E and I on the movie time cursor', async () => { + const { page } = ctx + const nav = navWindow(page) + const navId = await navChipsId(nav) + + await nav.getByTestId(`nav-chip-${POTENTIAL_CHIP}-${navId}`).click() + await nav.getByTestId(`nav-chip-${CURRENT_CHIP}-${navId}`).click({ modifiers: ['Shift'] }) + + // The stacked view is a distinct figure the backend emits with + // view_kind:'stacked'; wait for the frame to swap rather than sleeping. + await expect + .poll(async () => nav.locator('iframe').count(), { timeout: 60_000 }) + .toBeGreaterThan(0) + await page.waitForTimeout(1_500) // let the figure paint before capturing + await nav.screenshot({ path: join(SHOTS, '03-stacked-lanes.png') }) + await page.screenshot({ path: join(SHOTS, '04-full-window.png') }) +}) + +/** Dominant x (CSS px) of the ORANGE stacked-lane cursor, or null. + * `crosshairAt` in the harness finds the GREEN navigator crosshair; the + * stacked lanes' linked cursor is #ff9100. */ +async function laneCursorX(nav: any): Promise { + // Iterate EVERY iframe in the window, not just the first: the stacked figure + // mounts as its own iframe alongside the ones already there, so `.first()` + // reads a figure with no cursor in it at all. + const count = await nav.locator('iframe').count() + for (let i = 0; i < count; i++) { + const ifel = await nav.locator('iframe').nth(i).elementHandle() + const frame = ifel && (await ifel.contentFrame()) + if (!frame) continue + const x = await frameCursorX(frame) + if (x !== null) return x + } + return null +} + +async function frameCursorX(frame: any): Promise { + return frame.evaluate(() => { + let best: { x: number; n: number } | null = null + for (const c of Array.from(document.querySelectorAll('canvas')) as HTMLCanvasElement[]) { + const g = c.getContext('2d') + if (!g || !c.width || !c.height) continue + const d = g.getImageData(0, 0, c.width, c.height).data + const cols = new Int32Array(c.width) + let total = 0 + for (let y = 0; y < c.height; y++) { + for (let x = 0; x < c.width; x++) { + const p = (y * c.width + x) * 4 + const r = d[p], gr = d[p + 1], b = d[p + 2], a = d[p + 3] + // #ff9100 = (255, 145, 0), the vline widget orange. + if (a > 40 && r > 200 && gr > 90 && gr < 200 && b < 60) { cols[x]++; total++ } + } + } + if (!total || (best && total <= best.n)) continue + let bx = 0 + for (let x = 1; x < c.width; x++) if (cols[x] > cols[bx]) bx = x + const rect = c.getBoundingClientRect() + best = { x: rect.left + (bx + 0.5) * (rect.width / c.width), n: total } + } + return best ? best.x : null + }) +} + +test('dragging a lane cursor tracks the pointer without jumping back', async () => { + const { page } = ctx + const nav = navWindow(page) + const navId = await navChipsId(nav) + + // Back to the stacked view (the previous test switched to a single lane). + await nav.getByTestId(`nav-chip-${POTENTIAL_CHIP}-${navId}`).click() + await nav.getByTestId(`nav-chip-${CURRENT_CHIP}-${navId}`).click({ modifiers: ['Shift'] }) + await page.waitForTimeout(2_000) + + // Find the iframe the cursor actually lives in, and use ITS box for the + // page-coordinate maths — the stacked figure is not necessarily iframe 0. + let box: any = null + let startX: number | null = null + const nFrames = await nav.locator('iframe').count() + for (let i = 0; i < nFrames; i++) { + const ifel = await nav.locator('iframe').nth(i).elementHandle() + const frame = ifel && (await ifel.contentFrame()) + if (!frame) continue + const x = await frameCursorX(frame) + if (x !== null) { + startX = x + box = await nav.locator('iframe').nth(i).boundingBox() + break + } + } + expect(startX, 'no orange lane cursor found in any iframe').not.toBeNull() + + // Walk the cursor right and sample after each step. The bug this guards is + // the ASYNC write-back landing on the line still under the pointer: the + // cursor snaps back to the last committed frame, the next move drags it + // forward again, and the samples oscillate instead of advancing. + const gy = box!.y + box!.height * 0.25 + await page.mouse.move(box!.x + startX!, gy) + await page.mouse.down() + const samples: number[] = [] + for (let i = 1; i <= 6; i++) { + await page.mouse.move(box!.x + startX! + i * 22, gy, { steps: 3 }) + await page.waitForTimeout(350) + const x = await laneCursorX(nav) + if (x !== null) samples.push(x) + } + await page.mouse.up() + await page.waitForTimeout(600) + await nav.screenshot({ path: join(SHOTS, '06-after-drag.png') }) + + console.log('lane cursor drag samples:', samples.map(s => s.toFixed(1)).join(', ')) + expect(samples.length).toBeGreaterThanOrEqual(4) + expect(samples[samples.length - 1]).toBeGreaterThan(samples[0] + 20) + // Monotonic within a frame-quantisation tolerance — a snap-back to the last + // committed position is tens of px, far outside this. + for (let i = 1; i < samples.length; i++) { + expect(samples[i], `sample ${i} went backwards: ${samples.join(', ')}`) + .toBeGreaterThan(samples[i - 1] - 6) + } +}) + +test('selecting one lane shows the WHOLE experiment, not just its start', async () => { + const { page } = ctx + const nav = navWindow(page) + const navId = await navChipsId(nav) + + // Plain click = switch the live navigator in place to just this lane. + await nav.getByTestId(`nav-chip-${POTENTIAL_CHIP}-${navId}`).click() + await page.waitForTimeout(1_500) + await nav.screenshot({ path: join(SHOTS, '05-single-lane.png') }) + + // The lane must carry the movie's TIME calibration. Uncalibrated it plots + // over frame index (0…7913) while the selector works in seconds (0…259), so + // the cursor only ever reaches the first ~3% — "I can only see the beginning + // of the experiment". Read the axis the figure actually rendered. + const xMax = await nav.locator('iframe').first().contentFrame() + .locator('text=/^\\d+$/').last().innerText().catch(() => '') + // 259 s of movie: the last x tick belongs to the seconds axis, not to a + // frame count in the thousands. + if (xMax) expect(Number(xMax)).toBeLessThan(1_000) +}) + +test('loading a record explicitly works too', async () => { + const { page, backend } = ctx + // The File-menu picker returns a path; drive the same backend action it sends + // (a native dialog can't be driven from Playwright). + const before = ctx.backend.logBuffer.length + await backendAction(page, 'load_insitu_data', { path: EC_RUN }) + await expect + .poll(() => ctx.backend.logBuffer.slice(before) + .some((l: string) => /insitu: (Attached|could not attach)/.test(l)), + { timeout: 60_000 }) + .toBe(true) + const fresh = ctx.backend.logBuffer.slice(before).join('\n') + expect(fresh).toContain('Attached Cyclic Voltammetry') + expect(fresh).not.toContain('could not attach') +}) + +test('fast forward shows the speed it is running at, up to x32', async () => { + const { page } = ctx + const nav = navWindow(page) + const ff = nav.getByTitle('Fast Forward').or(nav.locator('[data-testid="toolbar-btn-Fast Forward"]')) + const badge = page.getByTestId('playback-speed-badge') + + // 1x -> no badge. Then 2, 4, 8, 16, 32 as the cycle advances. + await expect(badge).toHaveCount(0) + for (const expected of ['2x', '4x', '8x', '16x', '32x']) { + await ff.first().click() + await expect(badge).toHaveText(expected, { timeout: 10_000 }) + } + await page.screenshot({ path: join(SHOTS, '07-speed-32x.png') }) + + // Once more wraps to 1x and the badge goes away. + await ff.first().click() + await expect(badge).toHaveCount(0, { timeout: 10_000 }) + await backendAction(page, 'playback', { command: 'stop' }) +}) + +test('no renderer JS errors and no backend errors', async () => { + ctx.assertNoJsErrors() + const errors = backendErrorLines(ctx.backend) + .filter((l: string) => !/DeprecationWarning|VisibleDeprecation/.test(l)) + expect(errors, `backend errors:\n${errors.join('\n')}`).toEqual([]) +}) diff --git a/spyde/actions/base.py b/spyde/actions/base.py index c36103d3..60c68240 100644 --- a/spyde/actions/base.py +++ b/spyde/actions/base.py @@ -99,7 +99,7 @@ def play_pause(toolbar: "ActionContext", toggled=None, *args, **kwargs): def fast_forward(toolbar: "ActionContext", toggled=None, *args, **kwargs): - """Fast-forward = speed multiplier. Cycles 2x → 4x → 8x → back to 1x. Pressed + """Fast-forward = speed multiplier. Cycles 2x → 4x → 8x → 16x → 32x → back to 1x. Pressed while stopped, starts playback at 2x; while playing, bumps the speed one notch (staying at 1x after 8x).""" session = _session_of(toolbar) diff --git a/spyde/actions/movie_export/encoder.py b/spyde/actions/movie_export/encoder.py index 718799c0..dec57453 100644 --- a/spyde/actions/movie_export/encoder.py +++ b/spyde/actions/movie_export/encoder.py @@ -32,9 +32,23 @@ log = logging.getLogger(__name__) -# H.264 quality (imageio scale 0..10, higher = better). 7 is a good -# size/quality balance for scientific movies. -_H264_QUALITY = 7 +# H.264 quality (imageio scale 0..10, higher = better). +# +# 9, not 7. A fast-forward segment SUBSAMPLES the source (one output frame per +# N source frames — see pipeline.frame_indices_with_speed), so at 16×/32× two +# consecutive output frames can be ~80 source frames apart. Every P-frame then +# carries a near-total rewrite of the picture, and at quality 7 x264 spends its +# budget on the low-frequency bulk and smears the high-frequency detail — +# visibly, the burnt-in timestamp and scale bar TEAR and ghost while the image +# itself looks fine. The content is noisy electron-microscope data, which is +# expensive to encode at the best of times. +_H264_QUALITY = 9 + +# Cap the keyframe interval. Without it x264 picks a GOP of ~250 frames and, +# on a heavily subsampled sequence, a scene-cut-every-frame stream never gets a +# clean reference to recover against. One keyframe per ~1 s of output bounds +# how far any artifact can propagate. +_H264_GOP = 24 class Writer(Protocol): @@ -57,11 +71,21 @@ def __init__(self, path: str, fps: float, size: tuple[int, int]): # size is (W, H); imageio infers it from the first appended frame, but we # keep it for the GIF path and for validation. self._w, self._h = int(size[0]), int(size[1]) - self._writer = imageio.get_writer( - path, fps=float(fps), codec="libx264", + kwargs = dict( + fps=float(fps), codec="libx264", quality=_H264_QUALITY, macro_block_size=1, pixelformat="yuv420p", ffmpeg_log_level="error", ) + # `output_params` is imageio-ffmpeg's passthrough to the CLI. It is not + # in every imageio version's signature, so a TypeError falls back to + # the plain writer rather than failing the export — the GOP cap is a + # quality improvement, never a requirement. + try: + self._writer = imageio.get_writer( + path, output_params=["-g", str(_H264_GOP)], **kwargs) + except TypeError: + log.debug("imageio has no output_params; encoding without a GOP cap") + self._writer = imageio.get_writer(path, **kwargs) def append(self, rgb: np.ndarray) -> None: self._writer.append_data(np.ascontiguousarray(rgb, dtype=np.uint8)) diff --git a/spyde/actions/movie_export/pipeline.py b/spyde/actions/movie_export/pipeline.py index ba8ca305..dadf9135 100644 --- a/spyde/actions/movie_export/pipeline.py +++ b/spyde/actions/movie_export/pipeline.py @@ -171,11 +171,69 @@ def _text_with_shadow(draw, xy, text, font, fill, shadow): draw.text((x, y), text, font=font, fill=fill) -def _draw_timestamp(img, t_sec: float, font): +def label_overlay(frame_h: int = 512, *, text: str = "Label", + color: str = "#ffffff") -> dict: + """A STATIC text burn-in, as an ordinary text-overlay dict. + + Free text used to live in ``annotations`` (``kind: "text"``) while a live + value lived in ``text_overlays`` — two representations, two add paths, two + timeline lanes and two inspectors for what is, on the frame, the same + thing: some text at a position. Both are text overlays now; only the SOURCE + of the string differs (``builtin: "label"`` carries it literally, + ``builtin: "time"`` derives it from the frame, an instrument channel reads + it per frame). + """ + return { + "builtin": "label", + "text": str(text), + "xy": [8, max(4, int(frame_h * 0.08))], + "size": 18, + "color": color, + } + + +def time_overlay(frame_h: int = 512, *, color: str = "#ffffff") -> dict: + """The built-in elapsed-time overlay, as an ordinary text-overlay dict. + + The timestamp used to be a boolean param drawn at a fixed ``(6, 4)`` with + its own font rule and no presence in the editor — so toggling it changed + nothing on screen, and it could not be moved, recoloured or time-gated like + every other burnt-in value. Expressing it as a normal overlay makes one set + of controls cover all of them. + """ + return { + "builtin": "time", + "label": "t", + "units": "s", + "xy": [8, max(4, int(frame_h * 0.02))], + "size": 18, + "color": color, + } + + +def has_time_overlay(overlays) -> bool: + return any(isinstance(o, dict) and o.get("builtin") == "time" + for o in (overlays or ())) + + +def _hex_to_rgb(value, default=_TS_COLOR) -> tuple: + """``"#ff9100"`` → ``(255, 145, 0)``; anything unparseable → *default*.""" + text = str(value or "").strip().lstrip("#") + if len(text) == 3: + text = "".join(c * 2 for c in text) + if len(text) != 6: + return default + try: + return tuple(int(text[i:i + 2], 16) for i in (0, 2, 4)) + except ValueError: + return default + + +def _draw_timestamp(img, t_sec: float, font, color=None): from PIL import ImageDraw draw = ImageDraw.Draw(img) _text_with_shadow(draw, (6, 4), f"t = {t_sec:.2f} s", font, - _TS_COLOR, _TS_SHADOW) + _hex_to_rgb(color), _TS_SHADOW) def _draw_scalebar(img, scale_x: float, units: str, font): @@ -286,6 +344,48 @@ def sc(v, o=0.0): log.debug("annotation %r draw failed: %s", kind, e) +# A burnt-in text size is quoted at this output height and scaled from it. +# 512 px matches the timestamp's own rule (out_h // 28) closely enough that a +# default-sized overlay and the timestamp read as the same size. +_TEXT_REF_H = 512 + + +def _overlay_font_px(size, out_h: int) -> int: + """A burnt-in text size in OUTPUT pixels. + + The size on an overlay is quoted against a reference frame, not in absolute + pixels, because the same movie exports at wildly different resolutions — + the timestamp already scales itself (``out_h // 28``) while text overlays + did not, so on a 4096 px source an 18 pt overlay came out as an unreadable + speck beside a 36 px timestamp. That is what "the voltage doesn't show on + export" was: drawn, but hair-thin. + """ + try: + pt = float(size) + except (TypeError, ValueError): + pt = 18.0 + return max(10, int(round(pt * max(1, int(out_h)) / _TEXT_REF_H))) + + +def _overlay_number(val) -> str: + """Format a live overlay value legibly across the ranges these carry. + + A fixed ``.2f`` is right for a potential (0.56 V) and useless for the + current beside it (2.1e-04 mA renders as "0.00"). Small magnitudes get + significant figures instead. + """ + try: + v = float(val) + except (TypeError, ValueError): + return "—" + if v != v: # NaN — outside the instrument record + return "—" + mag = abs(v) + if mag and (mag < 0.01 or mag >= 1e5): + return f"{v:.3g}" + return f"{v:.2f}" + + def _draw_text_overlays(img, overlays, values_at_t, t_sec: float, k: int, crop_origin=(0, 0)): """Draw each 1-D-signal-as-text overlay: a live value formatted as text (e.g. @@ -304,12 +404,27 @@ def _draw_text_overlays(img, overlays, values_at_t, t_sec: float, k: int, def sc(v, o=0.0): return (float(v) - o) / max(1, k) + out_h = img.size[1] for ov, val in zip(overlays, values_at_t): if not isinstance(ov, dict) or not _in_time_range(t_sec, ov): continue + # The elapsed-time overlay is a BUILT-IN: its value is the frame's own + # time, so it needs no captured trace. Everything else about it — the + # position, size, colour, time gate, the editor's live widget — is the + # ordinary overlay machinery, which is the point of routing it here + # rather than drawing it from its own special case. + if ov.get("builtin") == "time": + val = t_sec try: xy = ov.get("xy", [8, 8]) - font = _load_font(int(ov.get("size", 18))) + font = _load_font(_overlay_font_px(ov.get("size", 18), out_h)) + if ov.get("builtin") == "label": + # A static label: the string IS the content, no value to format. + _text_with_shadow( + draw, (sc(xy[0], ox), sc(xy[1], oy)), + str(ov.get("text", "") or ""), font, + _color(ov.get("color", "#ffffff")), (0, 0, 0, 220)) + continue color = _color(ov.get("color", "#ffffff")) label = str(ov.get("label", "") or "") units = str(ov.get("units", "") or "") @@ -321,9 +436,9 @@ def sc(v, o=0.0): try: text = fmt.format(label=label, value=float(val), units=units) except (KeyError, IndexError, ValueError): - text = f"{label} = {float(val):.2f} {units}".strip() + text = f"{label} = {_overlay_number(val)} {units}".strip() else: - text = f"{label} = {float(val):.2f} {units}".strip() + text = f"{label} = {_overlay_number(val)} {units}".strip() _text_with_shadow(draw, (sc(xy[0], ox), sc(xy[1], oy)), text, font, color, (0, 0, 0, 220)) except Exception as e: @@ -581,10 +696,77 @@ def _segment_end_frame(segments, t_sec, sc, t1) -> float: return t_sec / sc -def _base_frame(raw, t: int, crop, k: int) -> np.ndarray: - """Read source frame *t*, apply the CROP (source px, before downsample), then - the spatial downsample. Memory-safe: one :func:`read_frame` slice.""" - return downsample(apply_crop(read_frame(raw, t), crop), k) +def speed_at_frame(speed_segments, src_frame: float, scale_s: float) -> float: + """The speed multiplier in force at source frame *src_frame* (1× outside + every segment).""" + t_sec = float(src_frame) * (float(scale_s) or 1.0) + for seg in (speed_segments or ()): + try: + tr = seg.get("time_range") + if tr and float(tr[0]) <= t_sec <= float(tr[1]): + return max(0.0, float(seg.get("speed", 1.0))) + except (TypeError, ValueError, AttributeError): + continue + return 1.0 + + +def integration_window(fps: float, scale_s: float, speed: float = 1.0) -> int: + """How many SOURCE frames to average into one output frame. + + Dropping from a 30.5 frame/s acquisition to a 12 frame/s movie means each + output frame stands for ~2.5 source frames. Picking one and discarding the + rest throws away real signal — on noisy electron-microscope data the + integrated frame is visibly cleaner — so the fps reduction INTEGRATES, the + same way the spatial ``downsample`` box-means rather than decimating. + + Speed never increases the window. A 32× segment advances the cursor ~81 + source frames per output frame, but integrating 81 would smear a second of + real change into one picture; a fast-forward is a SUB-SELECTION, so it + keeps the same short window and simply jumps further between them. + + Speed does reduce it: in slow motion the output advances a fraction of a + frame, and integrating more than it advances would double-count the same + source frames into consecutive output frames and blur the result. Hence + ``min(1, speed)`` — the window is what fps asks for, capped by what the + output frame actually spans. + """ + if fps <= 0 or scale_s <= 0: + return 1 + span = (1.0 / float(fps)) / float(scale_s) + return max(1, int(round(span * min(1.0, max(0.0, float(speed)))))) + + +def read_frame_integrated(raw, t: int, n: int, n_frames: int) -> np.ndarray: + """Mean of source frames ``[t, t+n)``, clamped to the dataset. + + Reads ONE FRAME AT A TIME into a running accumulator — never a stacked + slice — so the memory-safety contract holds exactly as it does for the + single-frame path, whatever ``n`` is. + """ + n = max(1, int(n)) + if n == 1: + return read_frame(raw, t) + stop = min(int(n_frames), int(t) + n) + start = max(0, min(int(t), stop - 1)) + acc = None + count = 0 + for i in range(start, stop): + frame = np.asarray(read_frame(raw, i), dtype=np.float32) + acc = frame if acc is None else acc + frame + count += 1 + if acc is None or count == 0: + return read_frame(raw, t) + return acc / float(count) + + +def _base_frame(raw, t: int, crop, k: int, n_int: int = 1, + n_frames: int = 0) -> np.ndarray: + """Read source frame *t* (optionally integrating *n_int* frames), apply the + CROP (source px, before downsample), then the spatial downsample. + Memory-safe: one :func:`read_frame` slice at a time.""" + frame = (read_frame(raw, t) if n_int <= 1 + else read_frame_integrated(raw, t, n_int, n_frames)) + return downsample(apply_crop(frame, crop), k) def _resolve_clim(first: np.ndarray, clim): @@ -605,6 +787,7 @@ def _resolve_clim(first: np.ndarray, clim): def _compose_frame(frame, lut, lo, hi, out_h, out_w, *, t_sec, k, anns, text_overlays, text_values, timestamp, scalebar, + ts_color=None, sb_scale, sig_units, ts_font, sb_font, inset=None, inset_i=0, overlay=None, raw_over=None, src_t=0, crop_origin=(0, 0)): """LUT + fit + the whole overlay stack for ONE already-read (cropped + @@ -627,8 +810,11 @@ def _compose_frame(frame, lut, lo, hi, out_h, out_w, *, t_sec, k, _draw_annotations(img, anns, t_sec, k, crop_origin) if text_overlays: _draw_text_overlays(img, text_overlays, text_values, t_sec, k, crop_origin) - if timestamp: - _draw_timestamp(img, t_sec, ts_font) + # Only the LEGACY path draws it here: a spec that has been migrated carries + # a `builtin: "time"` entry in text_overlays and gets it drawn there, with + # the position/size/colour/time-gate every other overlay has. + if timestamp and not has_time_overlay(text_overlays): + _draw_timestamp(img, t_sec, ts_font, ts_color) if scalebar: _draw_scalebar(img, sb_scale, sig_units, sb_font) if inset is not None: @@ -649,10 +835,16 @@ def render_single_frame(raw, t: int, *, params: dict, n_frames: int, crop = p.get("crop") cmap = str(p.get("cmap", "gray") or "gray") timestamp = bool(p.get("timestamp", True)) + ts_color = p.get("timestamp_color") scalebar = bool(p.get("scalebar", True)) and sig_scale_x > 0 anns = p.get("annotations") or [] lut = build_lut(cmap) - frame = _base_frame(raw, int(t), crop, k) + frame = _base_frame( + raw, int(t), crop, k, + integration_window(float(p.get("fps", 10.0) or 10.0), scale_s, + speed_at_frame(p.get("speed_segments") or [], + int(t), scale_s)), + n_frames) lo, hi = _resolve_clim(frame, p.get("clim")) rgb0 = even_crop(apply_lut(frame, lut, lo, hi)) out_h, out_w = rgb0.shape[:2] @@ -665,7 +857,8 @@ def render_single_frame(raw, t: int, *, params: dict, n_frames: int, return _compose_frame( frame, lut, lo, hi, out_h, out_w, t_sec=t_sec, k=k, anns=anns, text_overlays=(text_overlays or []), - text_values=(text_values or []), timestamp=timestamp, scalebar=scalebar, + text_values=(text_values or []), timestamp=timestamp, ts_color=ts_color, + scalebar=scalebar, sb_scale=sb_scale, sig_units=sig_units, ts_font=ts_font, sb_font=sb_font, crop_origin=crop_origin, overlay=overlay, raw_over=raw_over, src_t=int(t)) @@ -741,6 +934,7 @@ def export_movie(raw, *, path: str, params: dict, n_frames: int, clim = p.get("clim") crop = p.get("crop") timestamp = bool(p.get("timestamp", True)) + ts_color = p.get("timestamp_color") scalebar = bool(p.get("scalebar", True)) and sig_scale_x > 0 anns = p.get("annotations") or [] text_overlays = list(text_overlays or []) @@ -767,7 +961,13 @@ def export_movie(raw, *, path: str, params: dict, n_frames: int, raise _Cancelled() # Auto contrast from the FIRST rendered frame when clim is unset (robust 2-98%). - first = _base_frame(raw, idxs[0], crop, k) + # Frames to integrate per output frame. Sized by the FPS reduction and + # capped by the local speed (see integration_window) — a fast-forward + # sub-selects with the same short window, it does not average more. + segs = p.get("speed_segments") or [] + n_ints = [integration_window(fps, scale_s, + speed_at_frame(segs, i, scale_s)) for i in idxs] + first = _base_frame(raw, idxs[0], crop, k, n_ints[0], n_frames) # Cancel immediately AFTER the probe read too (a cancel that arrived while the # read was in flight stops us before we open the writer / encode any frame). @@ -820,12 +1020,13 @@ def export_movie(raw, *, path: str, params: dict, n_frames: int, for fi, t in enumerate(idxs): if should_cancel is not None and should_cancel(): raise _Cancelled() - frame = first if fi == 0 else _base_frame(raw, t, crop, k) + frame = (first if fi == 0 else + _base_frame(raw, t, crop, k, n_ints[fi], n_frames)) text_values = [(r[fi] if r is not None else None) for r in text_resampled] img = _compose_frame( frame, lut, lo, hi, out_h, out_w, t_sec=times[fi], k=k, anns=anns, text_overlays=text_overlays, text_values=text_values, - timestamp=timestamp, scalebar=scalebar, sb_scale=sb_scale, + timestamp=timestamp, ts_color=ts_color, scalebar=scalebar, sb_scale=sb_scale, sig_units=sig_units, ts_font=ts_font, sb_font=sb_font, inset=inset, inset_i=fi, overlay=overlay, raw_over=raw_over, src_t=t, diff --git a/spyde/actions/movie_export/traces.py b/spyde/actions/movie_export/traces.py index 6658ef8f..cf9d3ef2 100644 --- a/spyde/actions/movie_export/traces.py +++ b/spyde/actions/movie_export/traces.py @@ -122,12 +122,83 @@ def capture_from_plot(plot, *, color: str | None = None) -> "TraceSpec | None": x=x, y=y) -def from_metadata(signal, key: str): # pragma: no cover - documented seam only - """SEAM (NOT IMPLEMENTED): build a TraceSpec from per-frame values baked into - a movie's ``original_metadata`` (e.g. a temperature/pressure column from the - DE-MRC reader). The movie time base would be the trace's own x; this is the - obvious growth point for CSV import too. Kept as a stub so the source can be - added without reshaping :class:`TraceSpec`.""" - raise NotImplementedError( - "trace-from-original_metadata is a post-v1 seam; only 1-D plot capture " - "is implemented (see capture_from_plot).") +def from_insitu_channel(tree, channel: str, *, color: str | None = None): + """Capture a :class:`TraceSpec` from a movie's ATTACHED instrument channel. + + This is the second trace source (``capture_from_plot`` is the first): a + per-frame column that :mod:`spyde.insitu` already aligned and resampled + onto this movie's frame times, sitting on the tree as ``insitu_channels``. + It needs no 1-D plot window to exist — an electrochemistry potential lives + on the tree from the moment the movie opens, so it can be burnt into the + frame directly. + + The x base is the movie's OWN time axis in seconds (index × scale), not the + instrument clock: an overlay is resampled against ``movie_times`` at render + time, and the instrument clock has a lag offset that would shift every + value by that lag. The values are already per-frame, so this is an identity + resample — but going through the same path keeps one code path for both + sources. + + Frames outside the instrument record are NaN in ``insitu_channels`` and + stay NaN here; :func:`~spyde.actions.movie_export.pipeline._draw_text_overlays` + paints a dash for those rather than a made-up number. + + Returns None when the tree has no such channel. + """ + channels = getattr(tree, "insitu_channels", None) or {} + values = channels.get(channel) + if values is None: + log.debug("traces: no in-situ channel %r on this tree", channel) + return None + y = np.asarray(values, dtype=float) + if y.size == 0: + return None + try: + ax = tree.root.axes_manager.navigation_axes[0] + scale, offset = float(ax.scale), float(ax.offset) + except Exception: + scale, offset = 1.0, 0.0 + x = np.arange(y.size, dtype=float) * scale + offset + + label, units = _split_channel_name(channel) + return TraceSpec(id=new_trace_id(), label=label, + color=str(color or _TRACE_COLORS[0]), units=units, x=x, y=y) + + +def _split_channel_name(channel: str) -> tuple[str, str]: + """``"Ewe/V"`` → ``("Ewe", "V")``; ``"/mA"`` → ``("I", "mA")``. + + EC-Lab names a column ``quantity/unit`` and brackets an averaged one, which + reads badly burnt into a frame ("/mA = 0.0002"). Split it so the overlay + can render "I = 0.0002 mA" like every other trace. + """ + name, _, unit = str(channel).rpartition("/") + if not name: + name, unit = str(channel), "" + return name.strip().strip("<>") or str(channel), unit.strip() + + +def insitu_channel_options(tree) -> list[dict]: + """The instrument channels on *tree* worth offering as an overlay. + + Flags and bookkeeping columns (``mode``, ``error``, ``counter inc.``) are + excluded — they are booleans that read as "0"/"1" burnt into a frame. The + result is renderer-facing: ``[{"channel", "label", "units"}]``. + """ + channels = getattr(tree, "insitu_channels", None) or {} + out: list[dict] = [] + for name in channels: + if name == "time/s" or name in _SKIP_CHANNELS: + continue + values = np.asarray(channels[name], dtype=float) + if values.size == 0 or not np.isfinite(values).any(): + continue + label, units = _split_channel_name(name) + out.append({"channel": name, "label": label, "units": units}) + return out + + +# Per-frame columns that carry no meaning as a burnt-in number. +_SKIP_CHANNELS = frozenset({ + "mode", "error", "ox/red", "control changes", "Ns changes", "counter inc.", +}) diff --git a/spyde/actions/navigator_views.py b/spyde/actions/navigator_views.py index 4391181f..ed6bedea 100644 --- a/spyde/actions/navigator_views.py +++ b/spyde/actions/navigator_views.py @@ -249,11 +249,26 @@ class _StackedNavCursor: selector commits a new index and fires this cursor's ``index_hook``, which sets every row's line ``x`` to match. - The guard (``_busy``) stops the ``set → pointer_move → handler → set`` echo - (widget ``.set`` fires ``pointer_move``) AND the drive→hook→drive loop. - Position writes to widgets are UI updates, so when the sync originates on the - ``_NavDispatcher`` thread (the index_hook) they are marshalled onto the - asyncio main thread via ``session._dispatch_to_main``.""" + Exactly ONE line is the master at a time. Whichever the user is dragging + holds that role for the length of the drag; every other line is a pure + follower and is never written back to by its own handler. + + That asymmetry is load-bearing, not tidiness. There are two guards here + because there are two different races: + + * ``_busy`` stops the SYNCHRONOUS echo — a widget's ``.set`` itself fires + ``pointer_move``, so mirroring would re-enter the handler immediately. + * ``_master`` stops the ASYNCHRONOUS write-back. The index hook runs on the + ``_NavDispatcher`` thread and is marshalled onto the main thread, so it + lands SOME TIME AFTER the drag handler has returned and cleared + ``_busy``. Without ``_master`` it then writes the last committed + (index-quantised) position onto the line the user is still holding; the + pointer has moved on, the next ``pointer_move`` yanks it back, and the + cursor visibly jumps to and fro for the whole drag. ``_busy`` cannot + cover this — it is long since false by the time the write-back arrives. + + ``pointer_up`` releases the master so the authoritative committed position + can then settle every line, including the one just let go.""" def __init__(self, session, window_id: int, widgets, sel): self.session = session @@ -261,6 +276,9 @@ def __init__(self, session, window_id: int, widgets, sel): self.widgets = list(widgets) self.sel = sel self._busy = False + # The widget currently under the pointer — the master. None when no + # drag is in progress, i.e. every line follows. + self._master = None self._handlers: list = [] # keep wrapper refs alive (weak registration) self._closed = False self._wire_drag() @@ -269,18 +287,34 @@ def __init__(self, session, window_id: int, widgets, sel): # ── row line dragged → drive the real selector + mirror the other rows ── def _wire_drag(self) -> None: for w in self.widgets: - h = self._make_drag_handler(w) - self._handlers.append(h) - for et in ("pointer_move", "pointer_up"): + move = self._make_drag_handler(w) + release = self._make_release_handler(w) + self._handlers.extend((move, release)) + for et, h in (("pointer_move", move), ("pointer_up", release)): try: w.add_event_handler(h, et) except Exception as e: log.debug("wiring stacked cursor %s handler failed: %s", et, e) + def _make_release_handler(self, src): + """``pointer_up``: drive the final position, then hand back the master + role so the committed index can settle every line.""" + drive = self._make_drag_handler(src) + + def handler(_ev=None): + try: + drive() + finally: + if self._master is src: + self._master = None + return handler + def _make_drag_handler(self, src): def handler(_ev=None): if self._busy: return + # Claim the master role for as long as this line is being dragged. + self._master = src self._busy = True try: x = float(src.get("x")) @@ -335,11 +369,18 @@ def _on_selector_index(self, indices) -> None: self.session._dispatch_to_main(lambda: self._set_all_lines(x)) def _set_all_lines(self, x: float) -> None: + """Write the committed position onto every FOLLOWER line. + + The master (the line under the pointer, if any) is deliberately skipped + — see the class docstring. Writing to it is what made the cursor jump + to and fro mid-drag.""" if self._closed or self._busy: return self._busy = True try: for w in self.widgets: + if w is self._master: + continue try: w.set(x=float(x)) except Exception as e: @@ -352,6 +393,7 @@ def close(self) -> None: keep syncing (and can be GC'd). Called on chip switch-back / window close.""" self._closed = True + self._master = None try: if self._index_hook in self.sel.index_hooks: self.sel.index_hooks.remove(self._index_hook) diff --git a/spyde/actions/playback.py b/spyde/actions/playback.py index a6db21f7..0a094221 100644 --- a/spyde/actions/playback.py +++ b/spyde/actions/playback.py @@ -27,7 +27,7 @@ Fast-forward = speed multiplier ------------------------------- -``fast_forward()`` cycles the speed 1×→2×→4×→8×→1× (starting playback at 2× if +``fast_forward()`` cycles the speed 1×→2×→4×→8×→16×→32×→1× (starting playback at 2× if stopped). ``play()`` is a plain toggle at 1× (or the current speed). Both step the same selector via ``translate_pixels`` + ``delayed_update_data(force=True)`` — the same path a manual drag uses — so no new frame-read machinery is introduced. At the @@ -46,8 +46,12 @@ logger = logging.getLogger(__name__) -# Speed multipliers cycled by Fast Forward. -SPEED_CYCLE = (1, 2, 4, 8) +# Speed multipliers cycled by Fast Forward. Reaches ×32 because an in-situ +# movie is often thousands of frames of slow change — a 7914-frame acquisition +# is 4.3 minutes at ×1 and still 32 s at ×8. The clock is wall-clock-paced with +# a frame-skip (see `_tick`), so a high multiplier drops frames rather than +# demanding an impossible paint rate; ×32 costs no more per second than ×1. +SPEED_CYCLE = (1, 2, 4, 8, 16, 32) # Legacy fallback frame rate when the time axis carries no usable scale. DEFAULT_FPS = 10.0 @@ -87,10 +91,11 @@ class MoviePlaybackController: State machine (all guarded by ``self._lock``): • ``_playing`` — a clock thread is running. - • ``speed`` — current multiplier (1/2/4/8); the "×N" the UI shows. + • ``speed`` — current multiplier (1/2/4/8/16/32); the "×N" the UI shows. • ``loop`` — wrap to the start at the end instead of stopping. ``play()`` is a plain toggle at the current (or requested) speed; - ``fast_forward()`` starts at 2× if stopped, else bumps 1→2→4→8→1 while playing. + ``fast_forward()`` starts at 2× if stopped, else bumps 1→2→4→8→16→32→1 while + playing. """ def __init__(self, session) -> None: @@ -268,7 +273,7 @@ def fast_forward(self, loop: "bool | None" = None) -> bool: """Fast-forward = speed multiplier cycle. Stopped → start playing at 2×. - Playing → bump the speed 1→2→4→8→1 (stays playing at 1× after 8×). + Playing → bump the speed 1→2→4→8→16→32→1 (back to 1× after 32×). Returns True when playback is running afterwards.""" if loop is not None: self.loop = bool(loop) diff --git a/spyde/actions/registry.py b/spyde/actions/registry.py index 0aa04db2..f1b62fa5 100644 --- a/spyde/actions/registry.py +++ b/spyde/actions/registry.py @@ -225,6 +225,8 @@ "movie_tune": "spyde.actions.report.movie.movie_tune", "movie_crop_mode": "spyde.actions.report.movie.movie_crop_mode", "movie_add_text_overlay": "spyde.actions.report.movie.movie_add_text_overlay", + "movie_add_burnin": "spyde.actions.report.movie.movie_add_burnin", + "movie_add_insitu_overlay": "spyde.actions.report.movie.movie_add_insitu_overlay", "movie_add_overlay_image": "spyde.actions.report.movie.movie_add_overlay_image", "movie_drop_window": "spyde.actions.report.movie.movie_drop_window", "movie_export": "spyde.actions.report.movie.movie_export", diff --git a/spyde/actions/report/movie.py b/spyde/actions/report/movie.py index ff877dd7..b6854bf0 100644 --- a/spyde/actions/report/movie.py +++ b/spyde/actions/report/movie.py @@ -57,6 +57,7 @@ from spyde.actions.lifecycle import bump_generation, is_current, run_on_worker from spyde.actions.playback import _units_to_seconds from spyde.actions.movie_export import traces as _traces +from spyde.actions.movie_export import pipeline as _pipeline from spyde.actions.movie_export.pipeline import ( export_movie, render_single_frame, _Cancelled, ) @@ -75,7 +76,7 @@ # (seeded properly from the signal's time axis + the plot's cmap on movie_open). _DEFAULT_PARAMS = dict( fps=12, downsample=1, stride=1, cmap="gray", clim=None, - timestamp=True, scalebar=True, t_start=0, t_end=0, + timestamp=True, timestamp_color="#ffffff", scalebar=True, t_start=0, t_end=0, ) # fps auto-seed clamp band (real-time can be absurd) — mirrors the old wizard. @@ -235,12 +236,18 @@ def sync_overlay_widgets(self) -> None: if p2 is None: return # Drop the previous widgets (a rebuild supersedes them wholesale). + # BOTH sets: the text-overlay labels used to have their dict cleared + # without ever being removed from the plot, so every resync stacked + # another copy of every label on the figure — one add read as two. try: for w in self._ann_widgets.values(): p2._widgets.pop(getattr(w, "id", None), None) + for lw, _ov in getattr(self, "_text_overlay_widgets", {}).values(): + p2._widgets.pop(getattr(lw, "id", None), None) except Exception as e: log.debug("movie clear overlay widgets failed: %s", e) self._ann_widgets = {} + self._text_overlay_widgets = {} self._widget_handlers = [] cur_sec = self.current_index() * self.scale_seconds() anns = (self.cell.movie.annotations if self.cell.movie else None) or [] @@ -299,6 +306,13 @@ def sync_overlay_widgets(self) -> None: color=str(ov.get("color", "#ffffff") or "#ffffff"), show_handles=False) self._text_overlay_widgets[i] = (lw, ov) + # PERSIST a drag. Without this the label moves on screen and the + # spec keeps its original xy, so the export draws it somewhere + # else entirely — the editor and the movie disagreeing about + # where the timestamp and the voltage sit. + handler = _make_burnin_widget_handler(self, i) + lw.add_event_handler(handler, "pointer_up") + self._widget_handlers.append(handler) except Exception as e: log.debug("movie add text-overlay widget failed: %s", e) try: @@ -677,6 +691,125 @@ def _plot_clim(self): # ── text-overlay trace captures ────────────────────────────────────────────── + def _overlays_with_time(self, spec) -> list: + """The spec's burn-in overlays, MIGRATING the two legacy shapes into it. + + Everything that draws text on a frame is one kind of object now, so a + spec written before that has to be upgraded in place the first time it + is read: + + * ``params["timestamp"]`` — a bool with a hardcoded position — becomes + a ``builtin: "time"`` overlay; + * ``annotations`` entries with ``kind: "text"`` — free labels that had + their own add button, lane and inspector — become ``builtin: + "label"`` overlays. + + After this there is ONE list, ONE add path and ONE set of controls for + burnt-in text, which is the whole point; the shapes (rect/circle/arrow) + stay in ``annotations`` because they are not text. + """ + overlays = list(spec.text_overlays or []) + params = spec.params or {} + # Migrate ONCE. Re-deriving on every read means the legacy flags stay + # authoritative forever: deleting the timestamp clip from the timeline + # left `params["timestamp"]` true, so the very next read put it back and + # the removal never stuck. After this flag is set the overlay LIST is + # the only truth. + if params.get("burnin_migrated"): + return overlays + changed = False + + # Legacy free-text annotations → static label overlays. + anns = list(spec.annotations or []) + legacy_text = [a for a in anns + if isinstance(a, dict) and a.get("kind") == "text"] + if legacy_text: + for ann in legacy_text: + ov = _pipeline.label_overlay( + self.frame_size()[1], + text=str(ann.get("text", "") or "Label"), + color=str(ann.get("color", "#ffffff") or "#ffffff")) + ov["xy"] = [int(v) for v in (ann.get("xy") or ov["xy"])] + ov["size"] = int(ann.get("size", ov["size"]) or ov["size"]) + if ann.get("time_range"): + ov["time_range"] = list(ann["time_range"]) + overlays.append(ov) + spec.annotations = [a for a in anns + if not (isinstance(a, dict) + and a.get("kind") == "text")] + changed = True + + if bool(params.get("timestamp", True)) and not _pipeline.has_time_overlay(overlays): + overlays.insert(0, _pipeline.time_overlay( + self.frame_size()[1], + color=str(params.get("timestamp_color") or "#ffffff"))) + changed = True + + if changed: + spec.text_overlays = overlays + spec.params = {**params, "burnin_migrated": True} + return overlays + + # The burn-in sources the editor can add. `label` and `time` are always + # available; the rest are whatever instrument channels are attached. + def burnin_sources(self) -> list: + """``[{source, label, units}]`` — everything addable as burnt-in text.""" + out = [{"source": "label", "label": "Text", "units": ""}, + {"source": "time", "label": "Timestamp", "units": "s"}] + for opt in self.insitu_channel_options(): + out.append({"source": opt["channel"], "label": opt["label"], + "units": opt["units"]}) + return out + + def add_burnin(self, source: str, xy=None) -> bool: + """Add ONE burnt-in text overlay of any kind. The single add path. + + *source* is ``"label"``, ``"time"``, or an instrument channel key. All + three produce an entry in ``text_overlays`` differing only in where the + string comes from. + """ + spec = self.cell.movie + if spec is None: + return False + existing = self._overlays_with_time(spec) + fw, fh = self.frame_size() + if source == "label": + ov = _pipeline.label_overlay(fh) + elif source == "time": + if _pipeline.has_time_overlay(existing): + return False # only one clock makes sense + ov = _pipeline.time_overlay(fh) + else: + option = {o["channel"]: o + for o in self.insitu_channel_options()}.get(str(source)) + if option is None: + return False + ov = {"insitu_channel": option["channel"], "label": option["label"], + "units": option["units"], "size": 18} + ov["color"] = _traces.color_for_index(len(existing)) + ov["xy"] = ([int(xy[0]), int(xy[1])] if xy is not None + else [int(fw * 0.06), + int(fh * 0.9) - _overlay_row_step(fh) * len(existing)]) + spec.text_overlays = list(existing) + [ov] + return True + + def set_timestamp_enabled(self, on: bool) -> None: + """Add or remove the built-in time overlay (the Timestamp checkbox).""" + spec = self.cell.movie + if spec is None: + return + self._overlays_with_time(spec) # migrate first, then edit + overlays = [o for o in (spec.text_overlays or []) + if not (isinstance(o, dict) and o.get("builtin") == "time")] + if on: + params = spec.params or {} + overlays.insert(0, _pipeline.time_overlay( + self.frame_size()[1], + color=str(params.get("timestamp_color") or "#ffffff"))) + spec.text_overlays = overlays + # Keep the legacy flag honest for anything still reading it. + spec.params = {**(spec.params or {}), "timestamp": bool(on)} + def rebuild_text_traces(self) -> list: """For every 1-D-signal-as-text overlay on the spec, attach a live ``_trace`` (a captured :class:`TraceSpec`) so export/preview can resample @@ -689,10 +822,27 @@ def rebuild_text_traces(self) -> list: spec = self.cell.movie cache = getattr(self, "_overlay_traces", None) or {} out = [] - for ov in (spec.text_overlays or []): + for ov in self._overlays_with_time(spec): ov = dict(ov) + # The elapsed-time overlay's value is the frame's own time, so it + # carries no trace — the drawing/format paths read `builtin`. + if ov.get("builtin") == "time": + out.append(ov) + continue ref = ov.get("source") tr = None + # An INSTRUMENT channel (electrochemistry potential/current, holder + # temperature) is already per-frame on the tree — no source window + # has to exist, and it is re-read every rebuild rather than cached, + # so re-attaching a different record updates the overlay too. + channel = ov.get("insitu_channel") + if channel: + tr = _traces.from_insitu_channel( + self.tree, str(channel), color=ov.get("color")) + if tr is not None: + ov["_trace"] = tr + out.append(ov) + continue key = _overlay_key(ref) if key is not None and key in cache: tr = cache[key] @@ -881,12 +1031,48 @@ def state(self) -> dict: "signal_window_id": self.signal_window_id(), "nav_fig_id": self.nav_fig_id(), "current_index": self.current_index(), + # Instrument channels attached to this movie (electrochemistry + # potential/current, holder temperature…). Already per-frame, so + # the editor can offer them as burn-in overlays directly — no 1-D + # source window has to be open and dragged in. + "insitu_channels": self.insitu_channel_options(), + # Everything addable as burnt-in text, in one list — the editor + # renders one button per entry instead of a text button here, a + # checkbox there and a channel list somewhere else. + "burnin_sources": self.burnin_sources(), } + def insitu_channel_options(self) -> list: + """Instrument channels this movie can burn in (see + :func:`spyde.actions.movie_export.traces.insitu_channel_options`).""" + try: + return _traces.insitu_channel_options(self.tree) + except Exception as e: + log.debug("listing in-situ overlay channels failed: %s", e) + return [] + + def add_insitu_overlay(self, channel: str, xy=None, **kw) -> bool: + """Add a burn-in for one attached instrument channel. + + Kept as a name; :meth:`add_burnin` is the one implementation. + """ + return self.add_burnin(str(channel), xy=xy) + def emit(self) -> None: ipc.emit(self.state()) +def _overlay_row_step(frame_h: int) -> int: + """Vertical gap between stacked burn-in overlays, in SOURCE pixels. + + It has to scale with the frame. A fixed 30 px is a readable row on a 512 px + frame and 0.7 % of a 4096 px one — on a real DE movie two overlays landed + 30 px apart on a 4096 px frame, which on screen is the same line twice and + reads as a duplicated text box rather than as two overlays. + """ + return max(30, int(round(frame_h * 0.045))) + + def _public_overlay(ov: dict) -> dict: """A text-overlay dict WITHOUT the ephemeral ``_trace`` (for the wire / serialization).""" @@ -937,7 +1123,12 @@ def _format_overlay_value(ov: dict, frame: int, scale_s: float, label = str(ov.get("label", "") or "") units = str(ov.get("units", "") or "") fmt = str(ov.get("fmt", "") or "") - val = _overlay_value_at(ov.get("_trace"), frame, scale_s, n_frames) + if ov.get("builtin") == "time": + # Its value IS the frame's time — no trace to resolve, which is why it + # would otherwise render as a dash in the editor. + val = float(frame) * float(scale_s or 0.0) + else: + val = _overlay_value_at(ov.get("_trace"), frame, scale_s, n_frames) if val is None: return f"{label} = —" if label else "—" if fmt: @@ -945,7 +1136,9 @@ def _format_overlay_value(ov: dict, frame: int, scale_s: float, return fmt.format(label=label, value=val, units=units) except (KeyError, IndexError, ValueError): pass - return f"{label} = {val:.2f} {units}".strip() + # Same formatter the burnt-in frames use, so the editor preview and the + # export never disagree about how a value reads. + return f"{label} = {_pipeline._overlay_number(val)} {units}".strip() def _make_movie_crop_handler(st): @@ -977,6 +1170,41 @@ def _on_drag_end(event): return _on_drag_end +def _make_burnin_widget_handler(st, index: int): + """``pointer_up`` handler persisting a dragged BURN-IN label's position. + + The sibling of :func:`_make_movie_widget_handler` for ``text_overlays``. + Geometry is in IMAGE pixels, which for a signal frame ARE source pixels — + the same units the export's ``xy`` is in — so it maps straight across with + no conversion. + + ``rebuild_text_traces`` returns copies IN SPEC ORDER, so the widget index is + the spec index; the bounds check below is what keeps that assumption honest + if an overlay is removed between the build and the drag. + """ + def _on_drag_end(event): + try: + widget = getattr(event, "source", None) + g = getattr(widget, "_data", None) if widget is not None else None + if not isinstance(g, dict): + return + spec = st.cell.movie + overlays = (spec.text_overlays if spec else None) or [] + if not (0 <= index < len(overlays)): + return + nx = int(round(float(g.get("x", 0)))) + ny = int(round(float(g.get("y", 0)))) + if overlays[index].get("xy") != [nx, ny]: + overlays[index]["xy"] = [nx, ny] + spec.text_overlays = overlays + st.mgr.dirty = True + st.emit() + except Exception as e: + log.debug("burn-in drag persist failed (idx %s): %s", index, e) + + return _on_drag_end + + def _make_movie_widget_handler(st, ann_index: int, kind: str): """A module-level closure (NOT a bound method — anyplotlib sets ``fn._event_types`` on the handler) returning a ``pointer_up`` handler that @@ -1290,6 +1518,7 @@ def movie_tune(session, plot, payload) -> None: if cell is None or cell.cell_type != "movie" or cell.movie is None: return spec = cell.movie + want_timestamp = None if "params" in payload and isinstance(payload["params"], dict): p = dict(spec.params or {}) incoming = payload["params"] @@ -1306,9 +1535,20 @@ def movie_tune(session, plot, payload) -> None: p["clim"] = ([float(cl[0]), float(cl[1])] if cl and len(cl) == 2 and cl[0] is not None and cl[1] is not None else None) - for key in ("timestamp", "scalebar", "axes"): + for key in ("scalebar", "axes"): if key in incoming: p[key] = bool(incoming[key]) + if incoming.get("timestamp_color"): + p["timestamp_color"] = str(incoming["timestamp_color"]) + # The Timestamp checkbox adds/removes the built-in time OVERLAY, so the + # editor shows the change immediately (it used to flip a render-time + # param that nothing on screen reflected) and the timestamp is then + # movable, recolourable and time-gateable like every other overlay. + if "timestamp" in incoming: + p["timestamp"] = bool(incoming["timestamp"]) + # Applied AFTER the text_overlays replacement below — a payload + # carrying both would otherwise clobber the overlay we just added. + want_timestamp = p["timestamp"] # Clamp the time range to the dataset (n_frames known only with a source). n = st.n_frames() if st is not None and st.has_source else None if n: @@ -1331,10 +1571,16 @@ def movie_tune(session, plot, payload) -> None: if "out_size" in payload: os_ = payload["out_size"] spec.out_size = ([int(v) for v in os_] if os_ and len(os_) == 2 else None) + if want_timestamp is not None and st is not None: + st.set_timestamp_enabled(want_timestamp) mgr.dirty = True - # An annotation change → rebuild the draggable widgets on the live figure so a - # newly-added / removed / retimed overlay is immediately editable on the figure. - if st is not None and "annotations" in payload: + # An annotation / overlay change → rebuild the draggable widgets on the live + # figure so a newly-added, removed or retimed overlay is immediately visible + # and editable there. `text_overlays` is in this list because toggling the + # timestamp now adds/removes one, and without a resync the editor showed no + # change at all — the original "toggling Timestamp does nothing" report. + if st is not None and ("annotations" in payload or "text_overlays" in payload + or want_timestamp is not None): st.sync_overlay_widgets() # A render-param change (cmap / clim / axes) → push it to the live figure. if st is not None and "params" in payload: @@ -1449,7 +1695,10 @@ def movie_add_text_overlay(session, plot, payload) -> None: if tr is None: ipc.emit_error("Add text overlay: source window is not a 1-D plot.") return - xy = payload.get("xy") or [8, 8 + 26 * len(cell.movie.text_overlays)] + st_for_size = _sessions(mgr).get(cell.id) + fh = st_for_size.frame_size()[1] if st_for_size is not None else 512 + xy = payload.get("xy") or [ + 8, 8 + _overlay_row_step(fh) * len(cell.movie.text_overlays)] ov = { "source": SignalRef.from_plot(src).to_dict(), "label": str(payload.get("label") or tr.label or "value"), @@ -1469,6 +1718,52 @@ def movie_add_text_overlay(session, plot, payload) -> None: mgr.emit_state() +def movie_add_burnin(session, plot, payload) -> None: + """Add ONE burnt-in text overlay (``{cell_id, source, xy?}``). + + The single add path for everything that draws text on a frame: ``source`` + is ``"label"`` (static text), ``"time"`` (the elapsed-time clock), or an + attached instrument channel key such as ``"Ewe/V"``. They differ only in + where the string comes from — position, size, colour, time gate, the + draggable widget and the timeline clip are identical, which is why they are + one object with one action rather than three. Emits ``movie_state``. + """ + mgr = _manager(session) + if not mgr.open: + return + cell = mgr.doc.cell_by_id(payload.get("cell_id")) + if cell is None or cell.cell_type != "movie" or cell.movie is None: + return + st = _sessions(mgr).get(cell.id) + if st is None: + ipc.emit_error("Add burn-in: the movie editor is not open.") + return + source = str(payload.get("source") or payload.get("channel") or "label") + if not st.add_burnin(source, xy=payload.get("xy")): + if source == "time": + ipc.emit_error("The timestamp is already on this movie.") + else: + ipc.emit_error( + f"Add burn-in: no in-situ channel {source!r} on this movie. " + "Load its instrument data first (File ▸ Load In-Situ Data…)." + ) + return + if source == "time": + # Keep the legacy flag in step so a reopened spec doesn't re-add it. + st.cell.movie.params = {**(st.cell.movie.params or {}), "timestamp": True} + mgr.dirty = True + st.sync_overlay_widgets() # show it on the figure immediately + st.emit() + mgr.emit_state() + + +# Back-compat alias: the previous, channel-only entry point. +def movie_add_insitu_overlay(session, plot, payload) -> None: + payload = dict(payload or {}) + payload.setdefault("source", payload.get("channel")) + movie_add_burnin(session, plot, payload) + + def movie_export(session, plot, payload) -> None: """Render the movie to ``{cell_id, path}`` on a worker thread — memory-safe, generation-guarded, per-frame cancellable, with partial-file cleanup and a @@ -1631,7 +1926,11 @@ def _bake_poster(raw, params, n_frames, scale_s, sig_scale_x, sig_units, tvals = None overlays = list(text_overlays or []) if overlays: - from spyde.actions.movie_export.pipeline import _overlay_value_at + # `_overlay_value_at` is defined in THIS module — the old import + # pulled it from `pipeline`, where it has never existed, so this + # raised and the poster bake was silently skipped. It only ran when + # a movie had text overlays, which used to be the uncommon case; + # the timestamp being an overlay now makes it every movie. tvals = [_overlay_value_at(o.get("_trace"), t0, scale_s, n_frames) for o in overlays] img = render_single_frame(raw, t0, params=pp, n_frames=n_frames, diff --git a/spyde/backend/_session_actions.py b/spyde/backend/_session_actions.py index b09eb999..bb62af48 100644 --- a/spyde/backend/_session_actions.py +++ b/spyde/backend/_session_actions.py @@ -218,6 +218,8 @@ def dispatch_action(self, msg: dict) -> None: self.open_file(payload["path"]) elif action == "open_stack": self.open_stack(payload.get("paths") or []) + elif action == "load_insitu_data": + self.load_insitu_data(payload["path"], window_id) elif action == "confirm_nav_shape": self._confirm_nav_shape(payload) elif action == "playback": @@ -497,9 +499,9 @@ def console(self): def _handle_playback(self, payload: dict) -> None: """Play / pause / fast-forward the movie time navigator. Commands: ``play`` / ``pause`` / ``toggle`` (real-time on/off) / ``fast_forward`` - (speed cycle 2→4→8→1) / ``step`` (single frame) / ``set_speed`` / + (speed cycle 2→4→8→16→32→1) / ``step`` (single frame) / ``set_speed`` / ``set_loop``. Playback is real-time (paced from the time axis), so there is - no ``fps``/``step`` speed control any more — ``speed`` is a 1/2/4/8x + no ``fps``/``step`` speed control any more — ``speed`` is a 1/2/4/8/16/32x multiplier and ``loop`` wraps at the end.""" cmd = payload.get("command", "toggle") pb = self.playback diff --git a/spyde/backend/_session_files.py b/spyde/backend/_session_files.py index 5cd73b9b..36912d4d 100644 --- a/spyde/backend/_session_files.py +++ b/spyde/backend/_session_files.py @@ -189,6 +189,79 @@ def open_file(self, path: str) -> None: name=f"load-{name}", ).start() + # Instrument-record extensions the File ▸ Load In-Situ Data… picker offers. + # `.txt` is here because EC-Lab's ASCII export is routinely saved that way; + # the reader dispatches on the file's own magic, not on the extension. + INSITU_EXTS = (".mpr", ".mpt", ".txt", ".mps") + + def load_insitu_data(self, path: str, window_id: int | None = None) -> None: + """Attach an explicitly chosen instrument record to a movie tree. + + Targets the tree owning *window_id* when the renderer supplies one + (the user's focused window), else the most recent tree that has a + frame time base. Attaching needs per-frame timestamps, so a tree + without them is reported rather than silently skipped — that is the + single most likely reason for this to do nothing. + """ + if not os.path.isfile(path): + emit_error(f"File not found: {path}") + return + if _path_ext(path) == ".mps": + self._describe_mps(path) + return + tree = self._insitu_target_tree(window_id) + if tree is None: + emit_error("Open an in-situ movie first, then load its instrument data.") + return + try: + from spyde.insitu.attach import attach_ec_file + result = attach_ec_file(tree, path) + except Exception as e: + emit_error(f"Failed to load {os.path.basename(path)}: {e}") + return + if result: + log.info("insitu: %s", result.describe()) + emit_status(result.describe()) + else: + log.info("insitu: could not attach %s — %s", + os.path.basename(path), result.reason) + emit_error(f"Could not attach {os.path.basename(path)} — {result.reason}") + + def _insitu_target_tree(self, window_id: int | None): + """The tree an attach should land on: the focused window's, else the + most recently opened one that has a per-frame time base.""" + if window_id is not None: + plot = self._plot_by_window_id(window_id) + tree = getattr(getattr(plot, "plot_state", None), "signal_tree", None) + if tree is None: + tree = getattr(plot, "signal_tree", None) + if tree is not None: + return tree + from spyde.insitu.attach import movie_clock_for + for tree in reversed(self.signal_trees): + try: + if movie_clock_for(tree) is not None: + return tree + except Exception as e: + log.debug("probing tree for a frame time base failed: %s", e) + return self.signal_trees[-1] if self.signal_trees else None + + def _describe_mps(self, path: str) -> None: + """An ``.mps`` is the recipe, not the data — say so, and say what it + planned, so picking one is a useful mistake rather than a silent one.""" + try: + from spyde.insitu.eclab import read_mps + settings = read_mps(path) + except Exception as e: + emit_error(f"Failed to read {os.path.basename(path)}: {e}") + return + names = [t.get("name", "?") for t in settings.get("techniques", [])] + emit_status( + f"{os.path.basename(path)} is an EC-Lab settings file (no samples). " + f"It runs {len(names)} technique(s): {', '.join(names) or 'none'}. " + "Load the matching .mpr for the data." + ) + def _open_if_dense_vectors(self, sig, path: str) -> bool: """If *sig* is a saved dense-vectors carrier, reconstruct the vectors and open a Find-Vectors result tree. Returns True if it handled the file.""" @@ -368,14 +441,204 @@ def _load_file_thread(self, path: str) -> None: return for sig in signal: self._maybe_set_insitu_signal_type(sig) - self._add_signal(sig, source_path=path, - navigator_override=_reader_navigator(sig)) + clock = self._apply_frame_timestamps(sig, path) + self._apply_de_pixel_size(sig, path, clock) + tree = self._add_signal(sig, source_path=path, + navigator_override=_reader_navigator(sig)) + self._maybe_attach_insitu(tree, path, clock) self._add_recent(path) ipc.emit({"type": "recent_files", "paths": self._recent_files[:20]}) except Exception as e: ipc.emit({"type": "loading", "busy": False, "text": ""}) emit_error(f"Failed to load {os.path.basename(path)}: {e}") + @staticmethod + def _apply_frame_timestamps(sig: BaseSignal, path: str): + """Calibrate a movie's time axis from its REAL per-frame timestamps. + + A Direct Electron acquisition writes ``*_movie_timestamps.csv`` beside + the movie with one row per saved frame. That file is ground truth, and + the reader's calibration is not: RosettaSciIO derives the period as + ``1 / (fps * Autosave Movie Sum Count)`` when a summed movie's saved + period is ``sum_count / fps`` — wrong by ``sum_count**2``, so a + routine 2-frame sum lands the axis 4× too fast. Everything downstream + inherits that: playback wall-clock pacing, the movie exporter's time + base, the metadata panel's FPS. + + The axis stays UNIFORM (scale = the median measured period) rather + than becoming a per-frame non-uniform axis, because the 1-D selector + resolves a widget position to an index with ``(x - offset) / scale`` + — a non-uniform axis would break navigation for a jitter that is + microseconds wide in practice. The exact times are kept on the + returned clock (and on the tree) for anything that needs them, and a + real gap is reported rather than smoothed over. + + Returns the :class:`~spyde.insitu.de_movie.MovieClock` when it applied, + else None. Best-effort throughout — a movie with no sidecar, or one + whose sidecar disagrees with the data, loads exactly as before. + """ + try: + from spyde.insitu.de_movie import read_movie_clock + except Exception as e: # pragma: no cover - import guard + log.debug("insitu timestamps unavailable: %s", e) + return None + try: + if not FileLoaderMixin._is_movie_time_axis(sig): + return None + clock = read_movie_clock(path) + if clock.n_frames < 2 or clock.frame_period <= 0: + return None + nav = sig.axes_manager.navigation_axes[0] + n_nav = int(sig.axes_manager.navigation_shape[0]) + if clock.n_frames != n_nav: + # One info file per acquisition but one movie per autosave + # session, so a stale/foreign sidecar is a real possibility. + # The measured period is still right (it is a property of the + # camera, not of which frames landed in this file), so take it + # and refuse only the per-frame mapping. + log.warning( + "insitu: %s lists %d frames but the movie has %d — using " + "its frame period, not its per-frame times", + os.path.basename(clock.timestamps_path or path), + clock.n_frames, n_nav, + ) + nav.scale = clock.frame_period + nav.units = "s" + return None + before = float(nav.scale) + nav.scale = clock.frame_period + nav.units = "s" + if not str(getattr(nav, "name", "") or "").strip(): + nav.name = "time" + # Correct the recorded fps too, not just the axis. The reader takes + # `frames_per_second` straight from the DE info file, where it is + # the CAMERA's rate — with `Autosave Movie Sum Count = N` the SAVED + # frames arrive N times slower. Leaving it means the metadata panel + # (which prefers the explicit key — see metadata_extract) reports + # 61 fps beside a 32.76 ms/frame axis: two answers to one question. + # Fix the value at the source rather than teaching every reader of + # it to distrust it. + try: + # Rounded: the metadata chip renders the stored number + # verbatim, and "30.525030441931396 fps" is noise. The AXIS + # keeps the full-precision period; this is the display value. + sig.metadata.set_item( + "Acquisition_instrument.TEM.frames_per_second", + round(1.0 / clock.frame_period, 4), + ) + except Exception as e: + log.debug("stamping corrected fps failed: %s", e) + dropped = clock.dropped_frames() + if dropped.size: + log.info("insitu: %d frame gap(s) in %s", dropped.size, + os.path.basename(clock.timestamps_path or path)) + if abs(before - clock.frame_period) > 1e-9: + msg = ( + f"Frame timing from {os.path.basename(clock.timestamps_path)}: " + f"{clock.frame_period * 1e3:.3f} ms/frame " + f"({1 / clock.frame_period:.2f} fps)" + ) + # Logged as well as emitted: emit_status is the PLOTAPP line + # protocol, consumed by the Electron main process, so it never + # reaches stderr where the e2e harness can see it. + log.info("insitu: %s (reader said %.5f ms)", msg, before * 1e3) + emit_status(msg) + return clock + except Exception as e: + log.debug("applying frame timestamps failed: %s", e) + return None + + @staticmethod + def _apply_de_pixel_size(sig: BaseSignal, path: str, clock=None) -> bool: + """Calibrate a DE frame's SIGNAL axes from its ``*_info.txt``. + + Companion to :meth:`_apply_frame_timestamps` — same principle, the + other axis. RosettaSciIO decides imaging-vs-diffraction with + ``camera_length != -1``, but an imaging exposure records ``0``, so a + plain TEM image is calibrated as diffraction: ``nm^-1`` units and the + unset ``-1`` diffraction pixel size, while the correct specimen pixel + size sits in the same file (see + :func:`spyde.insitu.de_movie.spatial_calibration`). + + Applied only when the info file gives a positive scale, and only to a + 2-D signal. Returns True when it changed anything. + """ + try: + from spyde.insitu.de_movie import ( + find_movie_sidecars, read_info, spatial_calibration, + ) + except Exception as e: # pragma: no cover - import guard + log.debug("insitu pixel size unavailable: %s", e) + return False + try: + if sig.axes_manager.signal_dimension != 2: + return False + info = getattr(clock, "info", None) + info_path = getattr(clock, "info_path", None) + if not info: + _ts, info_path = find_movie_sidecars(path) + if not info_path: + return False + info = read_info(info_path) + calibration = spatial_calibration(info) + if calibration is None: + return False + scale_y, scale_x, units = calibration + axes = sig.axes_manager.signal_axes + # HyperSpy orders signal_axes x-first. + before = (float(axes[0].scale), str(axes[0].units or "")) + if (abs(before[0] - scale_x) < 1e-12 and before[1] == units): + return False + axes[0].scale, axes[0].units = scale_x, units + axes[1].scale, axes[1].units = scale_y, units + # The reader also NAMED them for the branch it took: an image + # mis-read as diffraction comes out as "kx"/"ky". Rename to match + # the space the units now say they are in, but only when they + # carry the reader's own default — never clobber a user's name. + wanted = ("x", "y") if units == "nm" else ("kx", "ky") + unwanted = ("kx", "ky") if units == "nm" else ("x", "y") + for axis, name, stale in zip(axes, wanted, unwanted): + if str(getattr(axis, "name", "") or "").strip().lower() == stale: + axis.name = name + log.info( + "insitu: frame pixel size from %s: %.6g %s/px (reader said " + "%.6g %s)", os.path.basename(info_path or path), scale_x, units, + before[0], before[1] or "-", + ) + emit_status( + f"Pixel size from {os.path.basename(info_path or path)}: " + f"{scale_x:.5g} {units}/px" + ) + return True + except Exception as e: + log.debug("applying DE pixel size failed: %s", e) + return False + + def _maybe_attach_insitu(self, tree, path: str, clock=None) -> None: + """Look beside a freshly-opened movie for instrument data to attach. + + Auto-discovery is silent unless it finds a record whose duration + actually matches the movie's — see + :func:`spyde.insitu.attach.discover_and_attach`. Best-effort: a folder + of unrelated records, an unreadable file or a missing sidecar all just + mean the movie opens as it always did. + """ + if tree is None: + return + try: + from spyde.insitu.attach import discover_and_attach + if clock is not None: + tree.insitu_clock = clock + result = discover_and_attach(tree, path) + except Exception as e: + log.debug("in-situ auto-discovery failed: %s", e) + return + if result: + log.info("insitu: %s", result.describe()) + emit_status(result.describe()) + else: + log.info("insitu: nothing attached — %s", result.reason) + @staticmethod def _is_self_describing(path: str) -> bool: """True for HyperSpy-native formats that store full axes (shape + diff --git a/spyde/insitu/__init__.py b/spyde/insitu/__init__.py new file mode 100644 index 00000000..0603d0e8 --- /dev/null +++ b/spyde/insitu/__init__.py @@ -0,0 +1,51 @@ +"""In-situ auxiliary channels — the non-image time series recorded *alongside* a +movie, and the clock arithmetic that puts them on the same axis as its frames. + +An in-situ experiment produces two independent recordings of the same event: +the camera's frame stack, and whatever the stimulus instrument logged (a +potentiostat's E/I, a heating holder's temperature, a gas cell's pressure). +They come off two machines with two unsynchronised clocks and two sampling +rates, so "what was the potential in frame 4210?" is not a lookup — it is an +alignment problem. This package answers it in three pieces: + +* :mod:`~spyde.insitu.de_movie` reads the Direct Electron sidecars that give a + movie its REAL per-frame time base — ``*_movie_timestamps.csv`` (one row per + saved frame) and ``*_info.txt``. Without them a movie only has the reader's + uniform ``1/fps`` guess. +* :mod:`~spyde.insitu.eclab` reads BioLogic EC-Lab potentiostat records — + ``.mpr`` (binary), ``.mpt``/``.txt`` (ASCII export) and ``.mps`` (settings). +* :mod:`~spyde.insitu.align` finds the offset between the two clocks and + resamples the instrument channels onto the frame times. + +The alignment is deliberately kept separate from both readers: it takes plain +time vectors, so a temperature log or any other per-time channel can reuse it +without either reader being involved. +""" +from __future__ import annotations + +from spyde.insitu.align import ( + Alignment, + align_clocks, + ec_time_for_frames, + frame_for_ec_sample, + match_runs, + resample_to_frames, +) +from spyde.insitu.de_movie import MovieClock, find_movie_sidecars, read_movie_clock +from spyde.insitu.eclab import EcRun, find_ec_runs, read_ec_file, read_mps + +__all__ = [ + "Alignment", + "EcRun", + "MovieClock", + "align_clocks", + "ec_time_for_frames", + "find_ec_runs", + "find_movie_sidecars", + "frame_for_ec_sample", + "match_runs", + "read_ec_file", + "read_movie_clock", + "read_mps", + "resample_to_frames", +] diff --git a/spyde/insitu/align.py b/spyde/insitu/align.py new file mode 100644 index 00000000..404c6f0b --- /dev/null +++ b/spyde/insitu/align.py @@ -0,0 +1,327 @@ +"""Put a movie's frames and an instrument's samples on one time axis. + +The two recordings share an experiment but not a clock. The camera stamps +frames with a free-running monotonic counter; the potentiostat stamps samples +with seconds since its own acquisition start, anchored to the naive local wall +clock of a different PC. Neither knows about the other, and the sampling rates +differ (here: 30.5 frame/s against 12.5 sample/s). Aligning them means finding +one number — the instrument-clock time of movie frame 0 — after which +everything else is interpolation. + +Three ways to find that number, in descending order of trustworthiness: + +``span`` + The two records have the same duration, so they were started and stopped + together; align first sample to first frame. This needs no clock agreement + at all, which is exactly why it is the default when it applies: it is + immune to unsynchronised PCs, timezones and DST. A duration agreement to + well under a second across several minutes is not a coincidence. + +``absolute`` + Convert both to UTC and subtract. Requires the caller to say what timezone + the instrument PC was in (nothing in an EC-Lab file records it) and leans + on the camera's epoch stamp, which has 1 s resolution and is written at + acquisition stop rather than start. Good for picking WHICH run pairs with + which movie; too coarse to trust for the final offset when ``span`` is + available. + +``manual`` + The caller knows better — a trigger wire, a lab notebook, a feature both + records saw. Always available as an override. + +When both ``span`` and ``absolute`` are possible, the result reports the UTC +offset the span solution *implies* (:attr:`Alignment.implied_utc_offset_hours`). +That is the honest way to surface clock skew: if it lands near a whole number of +hours the two PCs simply disagreed about the timezone; if it does not, one of +their clocks was genuinely wrong, and you can see by how much. +""" +from __future__ import annotations + +import datetime as dt +import logging +from dataclasses import dataclass, field + +import numpy as np + +from spyde.insitu.de_movie import MovieClock, info_matches_movie +from spyde.insitu.eclab import DISCRETE_CHANNELS, EcRun + +log = logging.getLogger(__name__) + +# Durations this close (relative, or absolute seconds — whichever is looser) +# count as "started and stopped together". +SPAN_REL_TOL = 0.02 +SPAN_ABS_TOL = 1.0 + + +@dataclass +class Alignment: + """The solved offset between a movie's clock and an instrument's clock. + + :attr:`lag_s` is the whole answer: the instrument-clock time (seconds since + the instrument's acquisition start, the same origin as ``EcRun.time_s``) at + which movie frame 0 was exposed. Everything else on this object is + diagnostics for judging whether to believe it. + """ + + method: str + lag_s: float + duration_mismatch_s: float = 0.0 + overlap_s: float = 0.0 + covered_fraction: float = 0.0 + implied_utc_offset_hours: float | None = None + notes: list[str] = field(default_factory=list) + + @property + def trustworthy(self) -> bool: + """True when the instrument covers essentially the whole movie.""" + return self.covered_fraction > 0.99 + + def describe(self) -> str: + lines = [ + f"method={self.method} lag={self.lag_s:.3f} s " + f"duration mismatch={self.duration_mismatch_s:+.3f} s " + f"overlap={self.overlap_s:.1f} s " + f"frames covered={self.covered_fraction * 100:.1f}%" + ] + if self.implied_utc_offset_hours is not None: + off = self.implied_utc_offset_hours + nearest = round(off) + lines.append( + f"implied instrument-PC UTC offset={off:+.4f} h " + f"({(off - nearest) * 3600:+.1f} s from UTC{nearest:+d})" + ) + lines.extend(f" note: {n}" for n in self.notes) + return "\n".join(lines) + + +def _span_lag(clock: MovieClock, run: EcRun) -> float: + """Instrument-clock time of frame 0 under the co-started assumption.""" + return float(run.time_s[0]) + + +def _absolute_lag(clock: MovieClock, run: EcRun, utc_offset_hours: float, + notes: list[str]) -> float | None: + """Instrument-clock time of frame 0 from the two absolute stamps.""" + epoch = clock.epoch_utc + if epoch is None or run.start is None: + return None + if not info_matches_movie(clock): + notes.append( + "the info file's frame count does not match this movie, so its " + "epoch stamp belongs to a different autosave session" + ) + # The epoch stamp is written at acquisition STOP, so it dates the LAST + # frame; walk back over the movie to reach frame 0. + frame0_utc = epoch - clock.duration + ec_start_utc = run.start_utc(utc_offset_hours) + assert ec_start_utc is not None + return float(frame0_utc - ec_start_utc.timestamp()) + + +def _implied_utc_offset(clock: MovieClock, run: EcRun, lag_s: float) -> float | None: + """The instrument PC's UTC offset implied by *lag_s*, in hours.""" + epoch = clock.epoch_utc + if epoch is None or run.start is None: + return None + frame0_utc = epoch - clock.duration + # Frame 0 in instrument-local wall clock: + frame0_local = run.start + dt.timedelta(seconds=lag_s) + naive_utc = frame0_local.replace(tzinfo=dt.timezone.utc).timestamp() + return (naive_utc - frame0_utc) / 3600.0 + + +def _coverage(clock: MovieClock, run: EcRun, lag_s: float) -> tuple[float, float]: + """``(overlap_seconds, fraction_of_frames_with_instrument_data)``.""" + if clock.n_frames == 0 or run.n_points == 0: + return 0.0, 0.0 + t = clock.t + lag_s + lo, hi = float(run.time_s[0]), float(run.time_s[-1]) + inside = (t >= lo) & (t <= hi) + overlap = max(0.0, min(t[-1], hi) - max(t[0], lo)) + return float(overlap), float(np.count_nonzero(inside) / t.size) + + +def align_clocks( + clock: MovieClock, + run: EcRun, + *, + method: str = "auto", + lag_s: float | None = None, + ec_utc_offset_hours: float | None = None, +) -> Alignment: + """Solve the movie↔instrument clock offset. + + *method* is ``"auto"`` (span if the durations agree, else absolute), + ``"span"``, ``"absolute"`` or ``"manual"``. ``"manual"`` requires *lag_s*; + ``"absolute"`` requires *ec_utc_offset_hours* (the instrument PC's offset + from UTC — no EC-Lab file records it). + """ + if clock.n_frames == 0: + raise ValueError("movie clock has no frames") + if run.n_points == 0: + raise ValueError(f"{run.name}: instrument run has no samples") + + notes: list[str] = [] + mismatch = run.duration - clock.duration + span_tol = max(SPAN_ABS_TOL, SPAN_REL_TOL * max(run.duration, clock.duration)) + spans_agree = abs(mismatch) <= span_tol + + if method == "manual": + if lag_s is None: + raise ValueError("method='manual' requires lag_s") + chosen, chosen_lag = "manual", float(lag_s) + elif method == "span": + chosen, chosen_lag = "span", _span_lag(clock, run) + if not spans_agree: + notes.append( + f"durations differ by {mismatch:+.2f} s (tolerance {span_tol:.2f} s) " + "— the records may not be co-extensive" + ) + elif method == "absolute": + if ec_utc_offset_hours is None: + raise ValueError("method='absolute' requires ec_utc_offset_hours") + got = _absolute_lag(clock, run, ec_utc_offset_hours, notes) + if got is None: + raise ValueError( + "cannot align by absolute time: need both the movie's epoch " + "stamp (from *_info.txt) and the run's acquisition start" + ) + chosen, chosen_lag = "absolute", got + elif method == "auto": + if spans_agree: + chosen, chosen_lag = "span", _span_lag(clock, run) + notes.append( + f"durations agree to {abs(mismatch):.3f} s over {run.duration:.1f} s " + "— treating the two records as co-started" + ) + elif ec_utc_offset_hours is not None: + got = _absolute_lag(clock, run, ec_utc_offset_hours, notes) + if got is None: + raise ValueError("no usable absolute stamps for method='auto'") + chosen, chosen_lag = "absolute", got + notes.append( + f"durations differ by {mismatch:+.2f} s, so the records are not " + "co-extensive; fell back to the absolute clocks" + ) + else: + raise ValueError( + f"durations differ by {mismatch:+.2f} s (tolerance {span_tol:.2f} s), " + "so the records are not co-extensive. Pass ec_utc_offset_hours to " + "align by absolute time, or lag_s with method='manual'." + ) + else: + raise ValueError(f"unknown alignment method {method!r}") + + overlap, covered = _coverage(clock, run, chosen_lag) + if covered < 1.0: + notes.append( + f"{(1 - covered) * 100:.1f}% of frames fall outside the instrument " + "record and will resample to NaN" + ) + return Alignment( + method=chosen, + lag_s=chosen_lag, + duration_mismatch_s=float(mismatch), + overlap_s=overlap, + covered_fraction=covered, + implied_utc_offset_hours=_implied_utc_offset(clock, run, chosen_lag), + notes=notes, + ) + + +def ec_time_for_frames(clock: MovieClock, alignment: Alignment) -> np.ndarray: + """Each movie frame's time on the instrument clock (seconds since the + instrument's acquisition start).""" + return clock.t + alignment.lag_s + + +def resample_to_frames( + clock: MovieClock, + run: EcRun, + alignment: Alignment, + channels: list[str] | None = None, +) -> dict[str, np.ndarray]: + """Resample the instrument's channels onto the movie's frame times. + + Returns one float64 array per channel, each ``clock.n_frames`` long, plus + ``"time/s"`` (the instrument-clock time of every frame). Frames outside the + instrument record are **NaN**, never clamped — a movie that ran on past the + end of the sweep must not show the last potential held flat, because that + is a measurement that was never made. + + Continuous channels are linearly interpolated. Channels that step rather + than vary (:data:`~spyde.insitu.eclab.DISCRETE_CHANNELS` — cycle number, + I Range, the mode/ox-red flags) take the nearest sample instead, since an + interpolated cycle 1.5 or a current range between two ranges never existed. + """ + t_frames = ec_time_for_frames(clock, alignment) + t_ec = run.time_s + order = np.argsort(t_ec) + t_sorted = t_ec[order] + inside = (t_frames >= t_sorted[0]) & (t_frames <= t_sorted[-1]) + + wanted = channels if channels is not None else list(run.channels) + out: dict[str, np.ndarray] = {"time/s": t_frames} + for name in wanted: + values = run.channels.get(name) + if values is None: + log.warning("align: no channel %r in %s", name, run.name) + continue + y = np.asarray(values, dtype=float)[order] + if name in DISCRETE_CHANNELS or np.asarray(values).dtype.kind in "biu": + idx = np.searchsorted(t_sorted, t_frames) + idx = np.clip(idx, 1, t_sorted.size - 1) + left = t_frames - t_sorted[idx - 1] <= t_sorted[idx] - t_frames + resampled = y[np.where(left, idx - 1, idx)] + else: + resampled = np.interp(t_frames, t_sorted, y) + out[name] = np.where(inside, resampled, np.nan) + return out + + +def frame_for_ec_sample( + clock: MovieClock, run: EcRun, alignment: Alignment +) -> np.ndarray: + """The nearest movie frame index for each instrument sample. + + The inverse mapping of :func:`resample_to_frames`, for going the other way: + "show me the frame where the current peaked". Samples falling outside the + movie get ``-1``. + """ + t_frames = ec_time_for_frames(clock, alignment) + idx = np.searchsorted(t_frames, run.time_s) + idx = np.clip(idx, 1, max(1, t_frames.size - 1)) + left = run.time_s - t_frames[idx - 1] <= t_frames[idx] - run.time_s + nearest = np.where(left, idx - 1, idx) + outside = (run.time_s < t_frames[0]) | (run.time_s > t_frames[-1]) + return np.where(outside, -1, nearest).astype(np.int64) + + +def match_runs( + clock: MovieClock, + runs: list[EcRun], + *, + ec_utc_offset_hours: float | None = None, +) -> list[tuple[EcRun, Alignment]]: + """Rank instrument runs by how well each explains this movie. + + The scoring is duration agreement: of all the techniques in a session, the + one whose record is the same length as the movie is the one that ran + *during* it. Runs that cannot be aligned at all are dropped. Best first. + """ + scored: list[tuple[float, EcRun, Alignment]] = [] + for run in runs: + if run.n_points < 2 or clock.duration <= 0: + continue + try: + alignment = align_clocks( + clock, run, method="auto", ec_utc_offset_hours=ec_utc_offset_hours + ) + except ValueError as exc: + log.debug("align: %s does not match this movie (%s)", run.name, exc) + continue + penalty = abs(alignment.duration_mismatch_s) / max(clock.duration, 1e-9) + scored.append((penalty - alignment.covered_fraction, run, alignment)) + scored.sort(key=lambda item: item[0]) + return [(run, alignment) for _, run, alignment in scored] diff --git a/spyde/insitu/attach.py b/spyde/insitu/attach.py new file mode 100644 index 00000000..ae73881c --- /dev/null +++ b/spyde/insitu/attach.py @@ -0,0 +1,294 @@ +"""Attach aligned instrument channels to a loaded movie's signal tree. + +This is the glue between the readers in this package and the app: it finds the +instrument record that belongs to a movie, solves the clock offset, resamples +the channels onto the frame times, and registers the interesting ones as named +navigator signals so they appear as chips beside the movie's own navigator. + +Registering them as navigators rather than inventing a display is deliberate. +A named navigator already gets a chip, already stacks onto one shared time +cursor when several are selected (``navigator_views.select_navigator``), and +already drives the movie when dragged — all of which is written and tested. An +E/t trace is an ordinary line, so the generic stacked builder draws it +correctly; nothing here needs its own figure code. + +Two entry points, one path underneath: + +* :func:`discover_and_attach` — the automatic one, run when a movie loads. It + scans the movie's own folder, and stays silent unless it finds a record that + really does explain the movie (:func:`~spyde.insitu.align.match_runs` + scores on duration agreement). +* :func:`attach_ec_file` — the manual one, behind File ▸ Load In-Situ Data…, + for a record that lives somewhere else or that the scorer passed over. + +Only the potential and current become chips. The full resampled table is left +on the tree as ``insitu_channels``, because a chip strip holding eleven +entries — most of them flags — makes the two that matter harder to find, not +easier. +""" +from __future__ import annotations + +import logging +import os + +import numpy as np + +from spyde.insitu.align import align_clocks, match_runs, resample_to_frames +from spyde.insitu.de_movie import MovieClock, read_movie_clock +from spyde.insitu.eclab import EcRun, find_ec_runs, read_ec_file + +log = logging.getLogger(__name__) + +POTENTIAL_LANE = "Ewe (V)" +CURRENT_LANE = "I" + +# Below this the current reads better in µA than in mA — a 0.0001 axis is +# unreadable. The lane NAME carries whichever unit was chosen, so the scaling +# is never silent. +_MICROAMP_CUTOFF_MA = 1.0 + + +class AttachResult: + """What an attach attempt did, for the caller's status line.""" + + def __init__(self, run=None, alignment=None, lanes=(), reason=""): + self.run = run + self.alignment = alignment + self.lanes = tuple(lanes) + self.reason = reason + + def __bool__(self) -> bool: + return bool(self.lanes) + + def describe(self) -> str: + if not self: + return self.reason or "no in-situ data attached" + al = self.alignment + return ( + f"Attached {self.run.technique} from {self.run.name} — " + f"{self.run.n_points} samples aligned by {al.method}, " + f"{abs(al.duration_mismatch_s):.2f} s over {self.run.duration:.1f} s, " + f"{al.covered_fraction * 100:.0f}% of frames covered" + ) + + +def movie_clock_for(tree, path: str | None = None) -> MovieClock | None: + """The tree's frame time base — cached on the tree by the loader, else read.""" + clock = getattr(tree, "insitu_clock", None) + if clock is not None: + return clock + source = path or getattr(tree, "source_path", None) + if not source: + return None + try: + clock = read_movie_clock(source) + except (FileNotFoundError, ValueError, OSError) as exc: + log.debug("insitu: no frame time base for %s (%s)", source, exc) + return None + tree.insitu_clock = clock + return clock + + +def _current_lane(values: np.ndarray) -> tuple[str, np.ndarray]: + """Name and scale the current lane so its axis is readable.""" + finite = values[np.isfinite(values)] + peak = float(np.max(np.abs(finite))) if finite.size else 0.0 + if 0 < peak < _MICROAMP_CUTOFF_MA: + return f"{CURRENT_LANE} (µA)", values * 1e3 + return f"{CURRENT_LANE} (mA)", values + + +def _register_lanes(tree, columns: dict[str, np.ndarray]) -> list[str]: + """Ensure the potential and current exist as named navigator signals. + + A navigator signal must be exactly as long as the movie's navigation axis + (``_preprocess_navigator`` enforces it), and NaN would poison the display + levels — so frames outside the instrument record are filled with the + nearest in-record value here. The authoritative NaN-bearing arrays stay on + ``tree.insitu_channels``; this is a display copy. + + Attaching twice is an ordinary thing to do — auto-discovery runs on open + and the user may then pick a record by hand — so a lane that already + exists has its VALUES replaced in place rather than being skipped or + registered again. Re-registering would add a second plot state for the + same name; skipping would leave the old run's trace on screen while the + tree claims the new one. Mutating the existing signal's array keeps one + plot state and one truth. + + Returns every lane now present, whether it was created or updated — so the + caller can tell "this run has no E/I channel" (empty) from "nothing left + to do" (non-empty), which is not the same outcome. + """ + # The movie's OWN navigation calibration has to land on each lane's signal + # axis. A 1-D selector turns a widget position into a frame index with + # ``(x - offset) / scale`` read from the navigator plot's own signal axis, + # so an uncalibrated lane is indexed in seconds against a scale of 1 — the + # stacked view's x-axis reads 0…7913 instead of 0…259 s, and dragging its + # cursor resolves to the wrong frame. ``calibrated_nav_signal`` is the + # existing helper for exactly this; imported lazily so this package stays + # usable without the backend. + from spyde.backend._session_files import calibrated_nav_signal + + lanes: list[tuple[str, np.ndarray]] = [] + potential = next( + (columns[k] for k in ("Ewe/V", "/V", "|E|/V") if k in columns), None + ) + if potential is not None: + lanes.append((POTENTIAL_LANE, potential)) + current = next( + (columns[k] for k in ("/mA", "I/mA", "|I|/mA") if k in columns), None + ) + if current is not None: + lanes.append(_current_lane(current)) + + present: list[str] = [] + existing = getattr(tree, "navigator_signals", {}) + for name, values in lanes: + display = _fill_edges(values).astype(np.float32) + try: + if name in existing: + _replace_lane_data(existing[name], display) + else: + lane = calibrated_nav_signal(display, tree.root) + lane.metadata.General.title = name + tree.add_navigator_signal(name, lane) + present.append(name) + except Exception as exc: + # Warning, not debug: a lane that silently fails to register looks + # from the outside exactly like "this run has no E/I channel", and + # chasing that took a round trip through the real app. + log.warning("insitu: registering navigator %r failed: %s", name, exc) + return present + + +def _replace_lane_data(entry, values: np.ndarray) -> None: + """Overwrite an already-registered lane's samples in place. + + ``add_navigator_signal`` stores whatever ``_preprocess_navigator`` returned, + and that is ALWAYS a list — ``[signal]`` for a plain trace, or + ``[navigator, signal]`` for a navigated one — never the bare signal handed + in. So unwrap before touching ``.data``, and write through the entries + whose shape matches rather than rebinding the list. + """ + targets = entry if isinstance(entry, (list, tuple)) else [entry] + updated = 0 + for signal in targets: + data = getattr(signal, "data", None) + if data is None: + continue + shape = np.asarray(data).shape + if shape != values.shape: + continue + signal.data = values.reshape(shape) + updated += 1 + if not updated: + shapes = [getattr(getattr(s, "data", None), "shape", None) for s in targets] + raise ValueError( + f"no registered lane array matches {values.shape} (found {shapes})" + ) + + +def _fill_edges(values: np.ndarray) -> np.ndarray: + """Replace NaN with the nearest finite value (display copy only).""" + out = np.asarray(values, dtype=float).copy() + finite = np.isfinite(out) + if not finite.any(): + return np.zeros_like(out) + idx = np.arange(out.size) + out[~finite] = np.interp(idx[~finite], idx[finite], out[finite]) + return out + + +def _store(tree, clock, run, alignment, columns) -> None: + tree.insitu_clock = clock + tree.insitu_run = run + tree.insitu_alignment = alignment + tree.insitu_channels = columns + + +def attach_run(tree, clock: MovieClock, run: EcRun, alignment) -> AttachResult: + """Resample *run* onto *clock*'s frames and register the display lanes.""" + columns = resample_to_frames(clock, run, alignment) + nav = _nav_size(tree) + if nav is not None and nav != clock.n_frames: + return AttachResult( + reason=( + f"the movie has {nav} frames but " + f"{os.path.basename(clock.timestamps_path or '?')} lists " + f"{clock.n_frames} — cannot map instrument samples to frames" + ) + ) + _store(tree, clock, run, alignment, columns) + lanes = _register_lanes(tree, columns) + if not lanes: + return AttachResult( + run, alignment, (), + f"{run.name} has no potential or current channel — it records " + f"{', '.join(sorted(run.channels)) or 'nothing'}", + ) + return AttachResult(run, alignment, lanes) + + +def _nav_size(tree) -> int | None: + try: + shape = tree.root.axes_manager.navigation_shape + return int(shape[0]) if len(shape) == 1 else None + except Exception: + return None + + +def discover_and_attach(tree, path: str) -> AttachResult: + """Look beside *path* for an instrument record that explains this movie. + + Silent by design when nothing matches: an in-situ movie sitting in a folder + of unrelated records is the common case, and a false attach is worse than + none. :func:`~spyde.insitu.align.match_runs` only returns runs whose + duration agrees with the movie's, so "found nothing" here means "found + nothing that could plausibly have been recorded during this movie". + """ + clock = movie_clock_for(tree, path) + if clock is None: + return AttachResult(reason="no frame timestamps beside this movie") + directory = os.path.dirname(os.path.abspath(path)) or "." + try: + runs = find_ec_runs(directory) + except OSError as exc: + return AttachResult(reason=f"could not scan {directory}: {exc}") + if not runs: + return AttachResult(reason="no EC-Lab records in this folder") + ranked = match_runs(clock, runs) + if not ranked: + return AttachResult( + reason=( + f"{len(runs)} EC-Lab record(s) beside this movie, none matching " + f"its {clock.duration:.1f} s duration" + ) + ) + run, alignment = ranked[0] + return attach_run(tree, clock, run, alignment) + + +def attach_ec_file(tree, ec_path: str, *, movie_path: str | None = None, + method: str = "auto", lag_s: float | None = None, + ec_utc_offset_hours: float | None = None) -> AttachResult: + """Attach one explicitly chosen instrument record to *tree*.""" + clock = movie_clock_for(tree, movie_path) + if clock is None: + return AttachResult( + reason=("this dataset has no per-frame time base — a DE movie needs " + "its *_movie_timestamps.csv beside it") + ) + try: + run = read_ec_file(ec_path) + except (ValueError, OSError) as exc: + return AttachResult(reason=f"could not read {os.path.basename(ec_path)}: {exc}") + if run.n_points < 2: + return AttachResult(reason=f"{run.name} has no samples") + try: + alignment = align_clocks( + clock, run, method=method, lag_s=lag_s, + ec_utc_offset_hours=ec_utc_offset_hours, + ) + except ValueError as exc: + return AttachResult(reason=f"could not align {run.name}: {exc}") + return attach_run(tree, clock, run, alignment) diff --git a/spyde/insitu/de_movie.py b/spyde/insitu/de_movie.py new file mode 100644 index 00000000..26714576 --- /dev/null +++ b/spyde/insitu/de_movie.py @@ -0,0 +1,369 @@ +"""Direct Electron movie sidecars — the REAL per-frame time base. + +A DE acquisition writes, beside ``_movie.mrc``: + +* ``_movie_timestamps.csv`` — ``Frame Index, Timestamp (s), Electrons``, + one row per SAVED frame. ``Timestamp (s)`` is a free-running monotonic camera + clock (values in the ~10^6 s range: uptime, not an epoch), so it says exactly + *when each frame happened relative to every other frame* and nothing about + where that sits in wall-clock time. +* ``_info.txt`` — ``key = value`` acquisition metadata, including + ``Timestamp (seconds since Epoch)``, the single absolute anchor available. + +Two reasons this matters more than it looks: + +**The uniform axis is a guess, and here it is a wrong one.** RosettaSciIO's MRC +reader derives the time scale from the metadata as +``1 / (Frames Per Second * Autosave Movie Sum Count)`` (``rsciio/mrc/_api.py``). +Summing ``N`` camera frames into one saved frame makes the saved period +``N / fps``, not ``1 / (fps*N)`` — so a 2-frame sum at 61.05 fps is calibrated +0.00819 s/frame when the timestamps say 0.03276 s/frame, a factor of ``N**2`` +too fast. The CSV is ground truth and does not need the formula to be right. + +**The epoch anchor is coarse and LATE.** ``Timestamp (seconds since Epoch)`` has +1 s resolution and is written when the info file is written — after the +acquisition stops (the same file reports ``Acquisition Status = Stopped`` and an +``Acquisition Counter`` already incremented past this dataset's). So it dates +the END of the movie, give or take the flush, and it is a UTC epoch while the +instrument log it must be matched against is normally naive local time. Treat it +as a coarse hint, never as a precise sync — which is why +:mod:`spyde.insitu.align` prefers to match on span and keeps the absolute route +as a cross-check. +""" +from __future__ import annotations + +import csv +import datetime as dt +import glob +import logging +import os +import re +from dataclasses import dataclass, field + +import numpy as np + +log = logging.getLogger(__name__) + +TIMESTAMPS_SUFFIX = "_movie_timestamps.csv" +INFO_SUFFIX = "_info.txt" + +# The CSV column header, normalised (lowercased, stripped). Kept loose because +# the units suffix has changed between DE server versions. +_FRAME_COL = "frame index" +_TIME_COL = "timestamp" +_ELECTRONS_COL = "electrons" + + +@dataclass +class MovieClock: + """The per-frame time base of one DE movie, plus its acquisition metadata. + + ``t_camera`` is the raw monotonic camera clock in seconds — only DIFFERENCES + of it are meaningful. ``t`` is the same thing rebased to zero at the first + saved frame, which is what you want as the movie's own time axis. + """ + + frame_index: np.ndarray + t_camera: np.ndarray + electrons: np.ndarray | None = None + info: dict[str, str] = field(default_factory=dict) + timestamps_path: str | None = None + info_path: str | None = None + + @property + def n_frames(self) -> int: + return int(self.t_camera.size) + + @property + def t(self) -> np.ndarray: + """Frame times in seconds since the first saved frame.""" + if self.t_camera.size == 0: + return self.t_camera + return self.t_camera - self.t_camera[0] + + @property + def duration(self) -> float: + """First-frame-to-last-frame span in seconds (one period short of the + total exposed time — see :attr:`frame_period`).""" + if self.t_camera.size < 2: + return 0.0 + return float(self.t_camera[-1] - self.t_camera[0]) + + @property + def frame_period(self) -> float: + """Median saved-frame period in seconds (robust to a dropped frame).""" + if self.t_camera.size < 2: + return 0.0 + return float(np.median(np.diff(self.t_camera))) + + @property + def epoch_utc(self) -> float | None: + """``Timestamp (seconds since Epoch)`` from the info file, or None. + + Coarse (1 s) and stamped at acquisition STOP — see the module docstring. + """ + raw = self.info.get("Timestamp (seconds since Epoch)") + if raw is None: + return None + try: + return float(raw) + except ValueError: + log.warning("de_movie: uninterpretable epoch timestamp %r", raw) + return None + + def epoch_datetime(self, tz: dt.tzinfo | None = dt.timezone.utc) -> dt.datetime | None: + """:attr:`epoch_utc` as an aware datetime, or None.""" + ep = self.epoch_utc + if ep is None: + return None + return dt.datetime.fromtimestamp(ep, dt.timezone.utc).astimezone(tz) + + def dropped_frames(self, tol: float = 0.5) -> np.ndarray: + """Indices ``i`` where the gap to frame ``i+1`` exceeds ``1 + tol`` + periods — i.e. where the camera lost frames.""" + if self.t_camera.size < 3: + return np.empty(0, dtype=int) + gaps = np.diff(self.t_camera) + period = self.frame_period + if period <= 0: + return np.empty(0, dtype=int) + return np.flatnonzero(gaps > period * (1.0 + tol)) + + @property + def reader_frame_period(self) -> float | None: + """The period RosettaSciIO's MRC reader would derive from the metadata. + + Exposed so a caller can SEE the discrepancy rather than guess at it; + compare against :attr:`frame_period`. + """ + try: + fps = float(self.info["Frames Per Second"]) + n_sum = float(self.info.get("Autosave Movie Sum Count", 1) or 1) + except (KeyError, TypeError, ValueError): + return None + if fps <= 0 or n_sum <= 0: + return None + return 1.0 / (fps * n_sum) + + +def _strip_suffix(path: str) -> str: + """``…_movie.mrc`` / ``…_movie_timestamps.csv`` / ``…_info.txt`` → ``…``.""" + base = path + for suffix in (TIMESTAMPS_SUFFIX, INFO_SUFFIX): + if base.endswith(suffix): + return base[: -len(suffix)] + base = os.path.splitext(base)[0] + if base.endswith("_movie"): + base = base[: -len("_movie")] + return base + + +def find_movie_sidecars(path: str) -> tuple[str | None, str | None]: + """Locate ``(timestamps_csv, info_txt)`` for *path* (the .mrc, or either + sidecar). Returns None for whichever is absent. + + DE names the info file after the ACQUISITION while the movie file also + carries a per-autosave-session number (``…_88071_run1_2616_movie.mrc`` vs + ``…_88071_info.txt``), so an exact stem match is not enough — we widen to a + glob in the same directory and take the longest common prefix. + """ + stem = _strip_suffix(path) + ts = stem + TIMESTAMPS_SUFFIX + info = stem + INFO_SUFFIX + found_ts = ts if os.path.exists(ts) else None + found_info = info if os.path.exists(info) else None + + directory = os.path.dirname(os.path.abspath(path)) or "." + name = os.path.basename(stem) + for suffix, current in ((TIMESTAMPS_SUFFIX, found_ts), (INFO_SUFFIX, found_info)): + if current is not None: + continue + best, best_len = None, 0 + for cand in glob.glob(os.path.join(glob.escape(directory), "*" + suffix)): + cand_stem = os.path.basename(cand)[: -len(suffix)] + shared = len(os.path.commonprefix([name, cand_stem])) + # Require a real shared prefix, not just a leading date. + if shared > best_len and shared >= min(8, len(cand_stem)): + best, best_len = cand, shared + if suffix == TIMESTAMPS_SUFFIX: + found_ts = best + else: + found_info = best + return found_ts, found_info + + +def read_info(path: str) -> dict[str, str]: + """Parse a DE ``*_info.txt`` into ``{key: value}`` (both stripped).""" + out: dict[str, str] = {} + with open(path, encoding="utf-8", errors="replace") as fh: + for line in fh: + key, sep, value = line.partition("=") + if not sep: + continue + out[key.strip()] = value.strip() + return out + + +def read_timestamps(path: str) -> tuple[np.ndarray, np.ndarray, np.ndarray | None]: + """Parse a DE ``*_movie_timestamps.csv`` → ``(frame_index, t_camera, electrons)``. + + Column order is taken from the header rather than assumed, and a trailing + torn row (acquisition killed mid-write) is dropped rather than raising. + """ + frames: list[int] = [] + times: list[float] = [] + electrons: list[float] = [] + with open(path, newline="", encoding="utf-8", errors="replace") as fh: + reader = csv.reader(fh) + try: + header = next(reader) + except StopIteration: + raise ValueError(f"{path}: empty timestamps file") from None + norm = [c.strip().lower() for c in header] + + def _col(want: str) -> int | None: + for i, c in enumerate(norm): + if c.startswith(want): + return i + return None + + i_frame, i_time, i_el = _col(_FRAME_COL), _col(_TIME_COL), _col(_ELECTRONS_COL) + if i_time is None: + raise ValueError(f"{path}: no 'Timestamp' column in header {header!r}") + for row_no, row in enumerate(reader, start=2): + if not row or not row[0].strip(): + continue + try: + t = float(row[i_time]) + f = int(float(row[i_frame])) if i_frame is not None else len(times) + e = float(row[i_el]) if i_el is not None and i_el < len(row) else np.nan + except (IndexError, ValueError): + log.warning("de_movie: dropping unparseable row %d of %s", row_no, path) + continue + frames.append(f) + times.append(t) + electrons.append(e) + + t = np.asarray(times, dtype=float) + idx = np.asarray(frames, dtype=np.int64) + el = np.asarray(electrons, dtype=float) + return idx, t, (None if el.size == 0 or np.all(np.isnan(el)) else el) + + +def read_movie_clock(path: str) -> MovieClock: + """Read the frame time base for the movie at *path*. + + *path* may be the ``.mrc``, the timestamps ``.csv`` or the ``_info.txt`` — + the siblings are located automatically. Raises ``FileNotFoundError`` if no + timestamps file can be found, since without it there is no real time base. + """ + ts_path, info_path = find_movie_sidecars(path) + if ts_path is None: + raise FileNotFoundError( + f"no '*{TIMESTAMPS_SUFFIX}' sidecar found beside {path!r}; " + "the movie has no per-frame time base without it" + ) + idx, t, electrons = read_timestamps(ts_path) + info = read_info(info_path) if info_path else {} + + if t.size and np.any(np.diff(t) <= 0): + log.warning("de_movie: %s is not strictly increasing", ts_path) + + clock = MovieClock( + frame_index=idx, + t_camera=t, + electrons=electrons, + info=info, + timestamps_path=ts_path, + info_path=info_path, + ) + reader_period = clock.reader_frame_period + if reader_period and clock.frame_period > 0: + ratio = clock.frame_period / reader_period + if abs(ratio - 1.0) > 0.01: + log.info( + "de_movie: %s measured frame period %.6f s vs reader-derived " + "%.6f s (x%.3g) — using the timestamps", + os.path.basename(ts_path), clock.frame_period, reader_period, ratio, + ) + return clock + + +_FRAMES_WRITTEN_KEYS = ("Autosave Movie Frames Written", "Number of Frames Processed") + + +def info_matches_movie(clock: MovieClock) -> bool: + """True when the info file's frame count agrees with the timestamps file. + + DE writes ONE info file per acquisition but a new movie file per autosave + session, so a mismatched count means the info file (and therefore its epoch + stamp) belongs to a DIFFERENT session and must not be used as this movie's + absolute anchor. + """ + for key in _FRAMES_WRITTEN_KEYS: + raw = clock.info.get(key) + if raw is None: + continue + try: + return int(float(raw)) == clock.n_frames + except ValueError: + continue + return False + + +def _info_float(info: dict, key: str) -> float | None: + raw = info.get(key) + if raw is None: + return None + try: + return float(raw) + except (TypeError, ValueError): + return None + + +def spatial_calibration(info: dict) -> tuple[float, float, str] | None: + """``(scale_y, scale_x, units)`` for a DE frame, or None if undeterminable. + + DE records the real-space pixel size as ``Specimen Pixel Size X/Y + (nanometers)`` and the reciprocal one as ``Diffraction Pixel Size X/Y``, + with ``-1`` meaning "not applicable". Which pair applies is decided by + ``Instrument Project Camera Length (centimeters)``: a genuine diffraction + exposure has a POSITIVE camera length. + + RosettaSciIO gets that test wrong — it asks ``camera_length != -1``, so the + ``0`` an imaging exposure records reads as "has a camera length" and the + frame is calibrated as diffraction. The reciprocal pixel size is then the + unset ``-1``, which survives its own ``== -1`` guard (the value is a string + at that point), and a TEM image ends up at ``-1.00 nm^-1`` per pixel with + the correct ``1.14786 nm`` sitting unused in the same file. + + So the sentinel test here is ``> 0``, applied to both candidates: a scale + is only used if it is a real positive number. + """ + camera_length = _info_float(info, "Instrument Project Camera Length (centimeters)") + diff_x = _info_float(info, "Diffraction Pixel Size X") + diff_y = _info_float(info, "Diffraction Pixel Size Y") + spec_x = _info_float(info, "Specimen Pixel Size X (nanometers)") + spec_y = _info_float(info, "Specimen Pixel Size Y (nanometers)") + + diffracting = bool(camera_length and camera_length > 0) + if diffracting and diff_x and diff_x > 0 and diff_y and diff_y > 0: + return diff_y, diff_x, "nm^-1" + if spec_x and spec_x > 0 and spec_y and spec_y > 0: + return spec_y, spec_x, "nm" + # Imaging exposure with no specimen pixel size, or a diffraction one with + # no diffraction pixel size — nothing trustworthy to say. + if diff_x and diff_x > 0 and diff_y and diff_y > 0: + return diff_y, diff_x, "nm^-1" + return None + + +def parse_de_datetime(text: str) -> dt.datetime | None: + """Parse the ``YYYYMMDD`` prefix DE puts on dataset names, for a sanity date.""" + m = re.match(r"(\d{4})(\d{2})(\d{2})", text.strip()) + if not m: + return None + try: + return dt.datetime(int(m[1]), int(m[2]), int(m[3])) + except ValueError: + return None diff --git a/spyde/insitu/eclab.py b/spyde/insitu/eclab.py new file mode 100644 index 00000000..e9c2c5e8 --- /dev/null +++ b/spyde/insitu/eclab.py @@ -0,0 +1,621 @@ +"""BioLogic EC-Lab potentiostat records — ``.mpr``, ``.mpt``/``.txt``, ``.mps``. + +An EC-Lab experiment is one *settings* file (``.mps``, the recipe) plus one +*record* per linked technique, written twice over: a binary ``.mpr`` and, +whenever the operator remembers to export it, an ASCII ``.mpt`` (often saved as +``.txt``). Both carry the same samples; only the ``.mpr`` is guaranteed to +exist, so it is the primary path here and the ASCII reader is the cross-check. + +Time in an EC-Lab record is ``time/s``, seconds since the ACQUISITION started — +which is *before* the technique started, because a linked sequence keeps one +clock across all its techniques. A second technique therefore begins at a +``time/s`` of tens or hundreds of seconds, not at zero. The absolute origin for +that clock lives in the ``VMP LOG`` module as an OLE date (and in the ASCII +header as ``Acquisition started on``); it is the EC-Lab PC's **naive local wall +clock**, with no timezone recorded anywhere in the file. + +Format notes (reverse-engineered; cross-checked against an ASCII export of the +same run, see ``spyde/tests/migrated/test_eclab.py``): + +* A module header is ``b"MODULE"`` + 10-byte short name + 25-byte long name, + then either a 32-bit length, or — when that field reads ``0xFFFFFFFF`` — a + 64-bit length following it. Both layouts appear in the wild. +* The ``VMP data`` body is ``npts:u4, ncols:u2, ids:u2[ncols]`` and then a + fixed block of padding before the records. Rather than hard-code that padding + (it is version-dependent), the record start is DERIVED as + ``len(body) - npts * record_size``, which self-checks the column layout: if + the assumed widths were wrong the subtraction would not land on a plausible + offset over a run of zero bytes. +* Several column IDs are *flags* packed into a single leading ``u1`` rather than + columns of their own. + +Coverage: the column table below holds the IDs seen in real CV/OCV records. An +unrecognised ID is not guessed at — a wrong width silently shifts every later +column and corrupts the whole record — except in the one case where exactly one +ID is unknown and the byte arithmetic pins its width unambiguously. Anything +else raises :class:`UnsupportedColumns`, which names the IDs and points at the +ASCII export as the way through. +""" +from __future__ import annotations + +import datetime as dt +import glob +import logging +import os +import re +import struct +from dataclasses import dataclass, field + +import numpy as np + +log = logging.getLogger(__name__) + +MPR_MAGIC = b"BIO-LOGIC MODULAR FILE\x1a" +MPT_MAGIC = "EC-Lab ASCII FILE" + +_OLE_EPOCH = dt.datetime(1899, 12, 30) +# Candidate byte offsets of the acquisition-start OLE date inside VMP LOG. +# Which one is populated varies between EC-Lab versions; the first that decodes +# to a sane calendar date wins. +_LOG_DATE_OFFSETS = (585, 465, 469, 473, 993) +_OLE_MIN, _OLE_MAX = 36525.0, 73050.0 # 2000-01-01 .. 2100-01-01 + +# Column IDs that are BIT FLAGS sharing one leading u1 byte, not columns. +_FLAG_IDS: dict[int, tuple[str, int]] = { + 1: ("mode", 0x03), + 2: ("ox/red", 0x04), + 3: ("error", 0x08), + 21: ("control changes", 0x10), + 31: ("Ns changes", 0x20), + 65: ("counter inc.", 0x80), +} + +# Column ID -> (name, numpy dtype). Verified against an ASCII export. +_COLUMN_IDS: dict[int, tuple[str, str]] = { + 4: ("time/s", "/mA", "/V", " None: + self.time_s = np.asarray(self.time_s, dtype=float) + + @property + def n_points(self) -> int: + return int(self.time_s.size) + + @property + def duration(self) -> float: + """First-to-last sample span in seconds.""" + if self.time_s.size < 2: + return 0.0 + return float(self.time_s[-1] - self.time_s[0]) + + @property + def sample_period(self) -> float: + """Median sampling period in seconds.""" + if self.time_s.size < 2: + return 0.0 + return float(np.median(np.diff(self.time_s))) + + @property + def name(self) -> str: + return os.path.splitext(os.path.basename(self.path))[0] + + def start_utc(self, utc_offset_hours: float) -> dt.datetime | None: + """:attr:`start` reinterpreted as being *utc_offset_hours* ahead of UTC. + + The file records no timezone, so the offset has to come from the caller + (or from :func:`spyde.insitu.align.align`'s span match, which sidesteps + the question entirely). + """ + if self.start is None: + return None + return (self.start - dt.timedelta(hours=utc_offset_hours)).replace( + tzinfo=dt.timezone.utc + ) + + def potential(self) -> np.ndarray | None: + """The working-electrode potential, whichever column carries it.""" + for key in ("Ewe/V", "/V", "Ewe-Ece/V", "|E|/V"): + if key in self.channels: + return self.channels[key] + return None + + def current(self) -> np.ndarray | None: + """The current in mA, whichever column carries it.""" + for key in ("/mA", "I/mA", "|I|/mA"): + if key in self.channels: + return self.channels[key] + return None + + def describe(self) -> str: + when = self.start.strftime("%Y-%m-%d %H:%M:%S.%f")[:-3] if self.start else "?" + return ( + f"{self.name}: {self.technique}, {self.n_points} pts, " + f"{self.time_s[0]:.2f}–{self.time_s[-1]:.2f} s " + f"({self.duration:.2f} s @ {self.sample_period * 1e3:.1f} ms), " + f"acq start {when}" + ) if self.n_points else f"{self.name}: {self.technique}, empty" + + +# -------------------------------------------------------------------------- +# .mpr (binary) +# -------------------------------------------------------------------------- + +@dataclass +class _Module: + offset: int + short: str + long: str + version: int + date: str + body: bytes + + +def _read_modules(buf: bytes) -> list[_Module]: + mods: list[_Module] = [] + i = buf.find(b"MODULE") + while i >= 0: + short = buf[i + 6 : i + 16].decode("latin1").strip() + long = buf[i + 16 : i + 41].decode("latin1").strip() + (first,) = struct.unpack_from(" dt.datetime | None: + if not (_OLE_MIN < value < _OLE_MAX): + return None + try: + return _OLE_EPOCH + dt.timedelta(days=float(value)) + except (OverflowError, ValueError): + return None + + +def _log_start_date(mod: _Module | None) -> dt.datetime | None: + if mod is None: + return None + for off in _LOG_DATE_OFFSETS: + if off + 8 > len(mod.body): + continue + (raw,) = struct.unpack_from(" str: + """Technique name from the settings module's embedded strings, else the + filename's EC-Lab suffix (``…_02_CV_C01.mpr`` → ``CV``).""" + if settings is not None: + text = settings.body.decode("latin1", errors="replace") + for name in _TECHNIQUES: + if name in text: + return name + m = re.search(r"_\d+_([A-Za-z]+)_C\d+", os.path.basename(path)) + if not m: + return "unknown" + code = m.group(1).upper() + return _TECHNIQUE_CODES.get(code, code) + + +def _layout(ids: tuple[int, ...]) -> tuple[list[tuple[str, str]], dict[str, int], list[int]]: + """``ids`` → (numpy dtype fields, flag masks, unknown ids). + + Flag IDs collapse into a single leading ``flags`` field; everything else + becomes one field, in the order the IDs appear. + """ + fields: list[tuple[str, str]] = [] + flags: dict[str, int] = {} + unknown: list[int] = [] + for cid in ids: + if cid in _FLAG_IDS: + name, mask = _FLAG_IDS[cid] + if not flags: + fields.append(("flags", " bool: + """True when ``body[first:last]`` is the all-zero run that precedes the + records (bar the lone marker byte EC-Lab sets in some versions).""" + padding = bytearray(body[first:last]) + marker = _PADDING_MARKER - first + if 0 <= marker < len(padding): + padding[marker] = 0 + return not any(padding) + + +def _resolve_unknown(fields: list[tuple[str, str]], body: bytes, npts: int, + header_end: int, unknown: list[int], + path: str) -> list[tuple[str, str]]: + """Pin the width of a single unknown column from the record size. + + The records sit flush against the END of the data module behind a run of + zero padding, so a candidate width is right only if it puts the first + record where the padding stops. That is the whole test: try each plausible + width and keep the one whose implied start lands on clean padding. + + With two or more unknowns there is nothing to solve — several width + combinations would satisfy the same total — and a wrong width silently + shifts every column after it, so we refuse rather than guess. + """ + if not unknown: + return fields + if len(unknown) > 1: + raise UnsupportedColumns( + f"{os.path.basename(path)}: unknown EC-Lab column IDs {unknown} — " + "cannot determine their byte widths. Export the run as ASCII from " + "EC-Lab (File > Export as text) and load the .mpt/.txt instead." + ) + known = sum(np.dtype(d).itemsize for _, d in fields if d) + for width, dtype in _WIDTH_TO_DTYPE.items(): + start = len(body) - npts * (known + width) + if start >= header_end and _padding_is_clean(body, header_end, start): + log.warning( + "eclab: %s has unrecognised column ID %d; reading it as a " + "%d-byte 'unknown_%d' pinned by the record size", + os.path.basename(path), unknown[0], width, unknown[0], + ) + return [(n, d or dtype) for n, d in fields] + raise UnsupportedColumns( + f"{os.path.basename(path)}: unknown EC-Lab column ID {unknown[0]} — no " + f"byte width places {npts} records inside a {len(body)} B data module. " + "Export the run as ASCII from EC-Lab and load the .mpt/.txt instead." + ) + + +def read_mpr(path: str) -> EcRun: + """Read a BioLogic ``.mpr`` binary record.""" + with open(path, "rb") as fh: + buf = fh.read() + if not buf.startswith(MPR_MAGIC): + raise ValueError(f"{path}: not a BioLogic .mpr file") + + mods = _read_modules(buf) + by_short = {m.short: m for m in mods} + data_mod = next((m for m in mods if m.short.startswith("VMP data")), None) + if data_mod is None: + raise ValueError(f"{path}: no 'VMP data' module") + + body = data_mod.body + (npts,) = struct.unpack_from(" dict[str, str]: + """Best-effort ``key : value`` pairs from the settings module's ASCII.""" + if mod is None: + return {} + text = mod.body.decode("latin1", errors="replace") + out: dict[str, str] = {} + for line in re.split(r"[\r\n\x00]+", text): + key, sep, value = line.partition(" : ") + if sep and 0 < len(key.strip()) < 60 and key.strip().isprintable(): + out[key.strip()] = value.strip() + return out + + +# -------------------------------------------------------------------------- +# .mpt / .txt (ASCII export) +# -------------------------------------------------------------------------- + +_ACQ_START_RE = re.compile( + r"Acquisition started on\s*:\s*(\d{1,2}/\d{1,2}/\d{4}\s+[\d:.]+)" +) +_ABS_TIME_RE = re.compile(r"^\d{1,2}/\d{1,2}/\d{4}\s") + + +def _parse_ec_datetime(text: str) -> dt.datetime | None: + for fmt in ("%m/%d/%Y %H:%M:%S.%f", "%m/%d/%Y %H:%M:%S", + "%d/%m/%Y %H:%M:%S.%f", "%d/%m/%Y %H:%M:%S"): + try: + return dt.datetime.strptime(text.strip(), fmt) + except ValueError: + continue + return None + + +def _to_float(token: str) -> float: + """EC-Lab writes in the exporting PC's locale — decimal comma is common.""" + token = token.strip() + if not token: + return np.nan + try: + return float(token) + except ValueError: + try: + return float(token.replace(",", ".")) + except ValueError: + return np.nan + + +def read_mpt(path: str) -> EcRun: + """Read an EC-Lab ASCII export (``.mpt``, often saved as ``.txt``). + + Handles both time conventions EC-Lab can export: ``time/s`` as elapsed + seconds, or as an absolute ``MM/DD/YYYY HH:MM:SS.ffff`` stamp (which is + converted back to elapsed seconds so both readers agree). + """ + with open(path, encoding="latin1") as fh: + lines = fh.read().splitlines() + if not lines or MPT_MAGIC not in lines[0]: + raise ValueError(f"{path}: not an EC-Lab ASCII file") + + n_header = 3 + for line in lines[:10]: + m = re.search(r"Nb header lines\s*:\s*(\d+)", line) + if m: + n_header = int(m.group(1)) + break + header = lines[: n_header - 1] + technique = next( + (t for line in header for t in _TECHNIQUES if line.strip() == t), "unknown" + ) + start = None + for line in header: + m = _ACQ_START_RE.search(line) + if m: + start = _parse_ec_datetime(m.group(1)) + break + + columns = [c.strip() for c in lines[n_header - 1].split("\t") if c.strip()] + rows = [line.split("\t") for line in lines[n_header:] if line.strip()] + if not rows: + raise ValueError(f"{path}: no data rows after {n_header} header lines") + + time_idx = next((i for i, c in enumerate(columns) if c.startswith("time/s")), None) + absolute_time = ( + time_idx is not None + and len(rows[0]) > time_idx + and bool(_ABS_TIME_RE.match(rows[0][time_idx].strip())) + ) + + values: dict[str, list[float]] = {c: [] for c in columns} + stamps: list[dt.datetime] = [] + for row in rows: + for i, name in enumerate(columns): + token = row[i] if i < len(row) else "" + if i == time_idx and absolute_time: + when = _parse_ec_datetime(token) + stamps.append(when or dt.datetime.min) + values[name].append(np.nan) + else: + values[name].append(_to_float(token)) + + channels = {c: np.asarray(v, dtype=float) for c, v in values.items()} + if absolute_time: + origin = start or (stamps[0] if stamps else None) + time_s = np.asarray([(s - origin).total_seconds() for s in stamps], dtype=float) + channels.pop(columns[time_idx], None) + if start is None: + start = origin + else: + time_s = channels.pop(columns[time_idx]) if time_idx is not None else np.arange( + len(rows), dtype=float + ) + + settings = {} + for line in header: + key, sep, value = line.partition(" : ") + if sep: + settings[key.strip()] = value.strip() + + return EcRun(path=path, technique=technique, start=start, time_s=time_s, + channels=channels, settings=settings) + + +# -------------------------------------------------------------------------- +# .mps (settings) + discovery +# -------------------------------------------------------------------------- + +def read_mps(path: str) -> dict: + """Parse an EC-Lab ``.mps`` settings file into ``{header, techniques}``. + + The ``.mps`` holds no samples — it is the recipe. Useful for knowing which + techniques a sequence was *meant* to run, and in what order. + """ + with open(path, encoding="latin1") as fh: + lines = fh.read().splitlines() + header: dict[str, str] = {} + techniques: list[dict[str, str]] = [] + current: dict[str, str] | None = None + for line in lines: + stripped = line.strip() + m = re.match(r"Technique\s*:\s*(\d+)", stripped) + if m: + current = {"index": m.group(1), "name": ""} + techniques.append(current) + continue + if current is not None and not current["name"] and stripped: + current["name"] = stripped + continue + key, sep, value = stripped.partition(" : ") + if sep: + (current if current is not None else header)[key.strip()] = value.strip() + return {"path": path, "header": header, "techniques": techniques} + + +def read_ec_file(path: str) -> EcRun: + """Read any EC-Lab record, dispatching on content rather than extension + (the ASCII export is routinely saved as ``.txt``).""" + with open(path, "rb") as fh: + head = fh.read(len(MPR_MAGIC)) + if head.startswith(MPR_MAGIC): + return read_mpr(path) + return read_mpt(path) + + +def find_ec_runs(directory: str) -> list[EcRun]: + """Read every EC-Lab record in *directory*, oldest acquisition first. + + Prefers the ``.mpr`` when a run has both a binary and an ASCII export, so + the same technique is not returned twice. Identity is the run's *sample + window* — acquisition start plus point count plus first/last timestamp — + rather than the filename, because EC-Lab exports pick up suffixes + (``-et``, ``(2)``) and the same run can be exported more than once. + Unreadable files are logged and skipped rather than failing the whole scan. + """ + seen: dict[tuple, EcRun] = {} + candidates = sorted(glob.glob(os.path.join(glob.escape(directory), "*"))) + binaries = [p for p in candidates if p.lower().endswith(".mpr")] + ascii_files = [p for p in candidates if p.lower().endswith((".mpt", ".txt"))] + for path in binaries + ascii_files: + try: + run = read_ec_file(path) + except (ValueError, OSError, struct.error) as exc: + log.debug("eclab: skipping %s (%s)", os.path.basename(path), exc) + continue + key = ( + run.start, + run.n_points, + round(float(run.time_s[0]), 3) if run.n_points else 0.0, + round(float(run.time_s[-1]), 3) if run.n_points else 0.0, + ) + seen.setdefault(key, run) + runs = list(seen.values()) + runs.sort(key=lambda r: (r.start or dt.datetime.min, r.time_s[0] if r.n_points else 0)) + return runs diff --git a/spyde/metadata_extract.py b/spyde/metadata_extract.py index 18ecd0c9..ab2fba78 100644 --- a/spyde/metadata_extract.py +++ b/spyde/metadata_extract.py @@ -245,6 +245,12 @@ def build_metadata_dict(signal_tree: "BaseSignalTree") -> dict[str, dict[str, st # numbers instead of "--". We require a recognised time UNIT (not just a # "time"-ish name) and convert it to seconds — deriving from an unconvertible # unit (or a bare uncalibrated name) would show a wrong fps, worse than "--". + # + # The key stays authoritative on purpose. When a DE movie's per-frame + # timestamps disagree with it (a summed movie's key holds the CAMERA rate, + # not the saved-frame rate), the loader corrects the KEY at load time + # (`_session_files._apply_frame_timestamps`) rather than this precedence — + # fixing the recorded value beats teaching every reader of it to distrust it. try: movie = subsections.get("Movie / In-Situ") if movie is not None: diff --git a/spyde/tests/migrated/test_insitu_align.py b/spyde/tests/migrated/test_insitu_align.py new file mode 100644 index 00000000..47cc8a11 --- /dev/null +++ b/spyde/tests/migrated/test_insitu_align.py @@ -0,0 +1,514 @@ +"""In-situ auxiliary channels: DE frame timestamps, EC-Lab records, clock align. + +Everything here is synthesised into ``tmp_path`` — no instrument files on disk — +but the synthesis follows the real byte layout, so the ``.mpr`` builder below +doubles as the format's specification. The numbers in +:class:`TestRealWorldShape` are taken from a real paired acquisition (a DE +Apollo movie and a BioLogic SP-200 cyclic voltammogram recorded together) and +pin the two behaviours that make the alignment work at all: matching on span +rather than on the two disagreeing wall clocks, and refusing to invent +instrument data outside the record. +""" +from __future__ import annotations + +import datetime as dt +import struct + +import numpy as np +import pytest + +from spyde.insitu import align as align_mod +from spyde.insitu import de_movie, eclab + +# -------------------------------------------------------------------------- +# builders +# -------------------------------------------------------------------------- + +MPR_HEADER = eclab.MPR_MAGIC + b" " * 25 + b"\x00" * 4 # 52 bytes, as EC-Lab writes +_DATA_START = 1007 # padding before the records, EC-Lab v11 + + +def _module(short: str, long: str, version: int, body: bytes) -> bytes: + """One ``MODULE`` block in the 64-bit-length layout.""" + return ( + b"MODULE" + + short.encode().ljust(10) + + long.encode().ljust(25) + + struct.pack(" bytes: + """A ``VMP LOG`` carrying the acquisition start as an OLE date.""" + body = bytearray(8010) + days = (start - dt.datetime(1899, 12, 30)).total_seconds() / 86400.0 + struct.pack_into(" bytes: + """Assemble a minimal but structurally faithful ``.mpr``.""" + npts = len(records) + header = struct.pack("/mA", " np.ndarray: + rec = np.zeros(n, dtype=CV_DTYPE) + t = t0 + dt_s * np.arange(n) + rec["time/s"] = t + rec["Ewe/V"] = 0.02 * (t - t0) # a 20 mV/s ramp + rec["control/V"] = rec["Ewe/V"] + rec["/mA"] = 1e-4 * np.sin(t) + rec["cycle number"] = np.where(np.arange(n) < n // 2, 1.0, 2.0) + rec["I Range"] = np.where(np.arange(n) < n // 2, 52, 53) + rec["flags"] = np.where(np.arange(n) % 2, 0x04, 0x00) | 0x02 # ox/red + mode + return rec + + +def write_movie(tmp_path, *, stem="20251117_88071", n=200, period=0.03276, + t0=1133858.28439, epoch=None, info_extra=None, frames_written=None): + """Write a DE timestamps CSV (+ info.txt) and return the .mrc path.""" + csv_path = tmp_path / f"{stem}{de_movie.TIMESTAMPS_SUFFIX}" + lines = ["Frame Index, Timestamp (s), Electrons"] + for i in range(n): + lines.append(f"{i}, {t0 + i * period:.6f}, {31000000 + i}") + csv_path.write_text("\n".join(lines) + "\n") + + info = { + "Frames Per Second": "61.05006", + "Autosave Movie Sum Count": "2", + "Autosave Movie Frames Written": str(n if frames_written is None else frames_written), + "Acquisition Status": "Stopped", + } + if epoch is not None: + info["Timestamp (seconds since Epoch)"] = str(epoch) + info.update(info_extra or {}) + info_path = tmp_path / f"{stem}{de_movie.INFO_SUFFIX}" + info_path.write_text( + "\n".join(f"{k:<60} = {v}" for k, v in info.items()) + "\n" + ) + mrc = tmp_path / f"{stem}_movie.mrc" + mrc.write_bytes(b"") + return str(mrc) + + +EC_START = dt.datetime(2025, 11, 17, 14, 50, 59, 687000) + + +def _eu(value: float, fmt: str) -> str: + """A number as EC-Lab writes it on a decimal-comma locale.""" + return format(value, fmt).replace(".", ",") + + +def write_mpt(tmp_path, name="run.txt", *, absolute=False, n=50): + """An EC-Lab ASCII export. + + ``Nb header lines`` counts every line up to AND INCLUDING the column + header, so it is derived from the header list rather than hard-coded — + getting that off by one is the classic way to misread these files. + """ + header = [ + "EC-Lab ASCII FILE", + "Nb header lines : {n}", + "", + "Cyclic Voltammetry", + "", + "Acquisition started on : 11/17/2025 14:50:59.687", + "Device : SP-200 (SN 1593)", + "", + "ox/red\ttime/s\tcontrol/V\tEwe/V\t/mA\t", + ] + header[1] = header[1].format(n=len(header)) + + rows = [] + for i in range(n): + t = 31.56 + 0.08 * i + ramp = 0.02 * (t - 31.56) + # A datetime keeps its dot; only the numbers take a decimal comma. + when = (EC_START + dt.timedelta(seconds=t)).strftime("%m/%d/%Y %H:%M:%S.%f")[:-2] + t_col = when if absolute else _eu(t, ".15E") + rows.append("\t".join([ + str(i % 2), t_col, _eu(ramp, ".7E"), _eu(ramp, ".7E"), + _eu(1e-4 * np.sin(t), ".15E"), + ])) + + path = tmp_path / name + path.write_text("\n".join(header + rows) + "\n", encoding="latin1") + return str(path) + + +# -------------------------------------------------------------------------- +# DE movie sidecars +# -------------------------------------------------------------------------- + +class TestMovieClock: + def test_reads_frame_times(self, tmp_path): + mrc = write_movie(tmp_path, n=200) + clock = de_movie.read_movie_clock(mrc) + assert clock.n_frames == 200 + assert clock.frame_period == pytest.approx(0.03276, abs=1e-9) + assert clock.duration == pytest.approx(199 * 0.03276, abs=1e-6) + assert clock.t[0] == 0.0 + + def test_finds_sidecars_from_any_member(self, tmp_path): + mrc = write_movie(tmp_path) + for probe in (mrc, mrc.replace("_movie.mrc", de_movie.TIMESTAMPS_SUFFIX), + mrc.replace("_movie.mrc", de_movie.INFO_SUFFIX)): + ts, info = de_movie.find_movie_sidecars(probe) + assert ts is not None and info is not None + + def test_finds_sidecar_when_movie_carries_a_session_number(self, tmp_path): + """DE names the info file per ACQUISITION but the movie per autosave + session, so the stems do not match exactly.""" + write_movie(tmp_path, stem="20251117_88071") + movie = tmp_path / "20251117_88071_run1_2616_movie.mrc" + movie.write_bytes(b"") + ts, info = de_movie.find_movie_sidecars(str(movie)) + assert ts is not None and info is not None + + def test_reader_derived_period_disagrees_with_the_timestamps(self, tmp_path): + """RosettaSciIO derives 1/(fps*sum) where the true saved period is + sum/fps — a factor of sum**2. The CSV is ground truth; this pins that + the discrepancy is visible rather than silent.""" + clock = de_movie.read_movie_clock(write_movie(tmp_path)) + assert clock.reader_frame_period == pytest.approx(1 / (61.05006 * 2)) + assert clock.frame_period / clock.reader_frame_period == pytest.approx(4.0, rel=1e-3) + + def test_epoch_is_read_as_utc(self, tmp_path): + clock = de_movie.read_movie_clock(write_movie(tmp_path, epoch=1763387769)) + assert clock.epoch_utc == 1763387769.0 + assert clock.epoch_datetime().strftime("%Y-%m-%d %H:%M:%S") == "2025-11-17 13:56:09" + + def test_info_frame_count_gates_the_epoch_anchor(self, tmp_path): + """A DE acquisition writes ONE info file but a movie per autosave + session; a count mismatch means the epoch stamp is another session's.""" + good = de_movie.read_movie_clock(write_movie(tmp_path, n=50)) + assert de_movie.info_matches_movie(good) + other = tmp_path / "b" + other.mkdir() + bad = de_movie.read_movie_clock( + write_movie(other, n=50, frames_written=3005) + ) + assert not de_movie.info_matches_movie(bad) + + def test_detects_dropped_frames(self, tmp_path): + mrc = write_movie(tmp_path, n=10) + csv = mrc.replace("_movie.mrc", de_movie.TIMESTAMPS_SUFFIX) + lines = open(csv).read().splitlines() + # Blow a hole in the middle: frame 5 arrives three periods late. + rows = [lines[0]] + for i, line in enumerate(lines[1:]): + idx, t, e = line.split(",") + rows.append(f"{idx},{float(t) + (0.0655 if i >= 5 else 0):.6f},{e}") + open(csv, "w").write("\n".join(rows) + "\n") + clock = de_movie.read_movie_clock(mrc) + assert clock.dropped_frames().tolist() == [4] + + def test_missing_timestamps_raises(self, tmp_path): + (tmp_path / "x_movie.mrc").write_bytes(b"") + with pytest.raises(FileNotFoundError): + de_movie.read_movie_clock(str(tmp_path / "x_movie.mrc")) + + def test_torn_row_is_dropped_not_fatal(self, tmp_path): + mrc = write_movie(tmp_path, n=10) + csv = mrc.replace("_movie.mrc", de_movie.TIMESTAMPS_SUFFIX) + with open(csv, "a") as fh: + fh.write("10, \n") # acquisition killed mid-write + assert de_movie.read_movie_clock(mrc).n_frames == 10 + + +# -------------------------------------------------------------------------- +# EC-Lab records +# -------------------------------------------------------------------------- + +class TestMpr: + def test_round_trips_columns_and_flags(self, tmp_path): + rec = cv_records(n=100) + path = tmp_path / "r_02_CV_C01.mpr" + path.write_bytes(build_mpr(ids=CV_IDS, records=rec)) + run = eclab.read_mpr(str(path)) + + assert run.n_points == 100 + assert run.time_s == pytest.approx(rec["time/s"]) + assert run.channels["Ewe/V"] == pytest.approx(rec["Ewe/V"]) + assert run.channels["/mA"] == pytest.approx(rec["/mA"]) + assert run.channels["I Range"].tolist() == rec["I Range"].tolist() + # flags share one u1: ox/red alternates, mode is the low 2 bits + assert run.channels["ox/red"].tolist() == [bool(i % 2) for i in range(100)] + assert set(np.unique(run.channels["mode"])) == {2} + assert run.unknown_columns == () + + def test_reads_the_acquisition_start(self, tmp_path): + path = tmp_path / "r_02_CV_C01.mpr" + path.write_bytes(build_mpr(ids=CV_IDS, records=cv_records())) + run = eclab.read_mpr(str(path)) + assert run.start == dt.datetime(2025, 11, 17, 14, 50, 59, 687000) + + def test_technique_falls_back_to_the_filename_code(self, tmp_path): + path = tmp_path / "r_02_CV_C01.mpr" + path.write_bytes(build_mpr(ids=CV_IDS, records=cv_records(), + settings_text=b"\x00")) + assert eclab.read_mpr(str(path)).technique == "Cyclic Voltammetry" + + def test_one_unknown_column_is_pinned_by_the_record_size(self, tmp_path): + """An unrecognised ID is readable when the byte arithmetic leaves only + one possible width.""" + dtype = np.dtype(CV_DTYPE.descr + [("mystery", "= 0 + # every mapped frame is within half a frame period of its sample + t_frames = align_mod.ec_time_for_frames(clock, al) + err = np.abs(t_frames[frames[inside]] - run.time_s[inside]) + assert err.max() <= clock.frame_period / 2 + 1e-9 + + +class TestMatchRuns: + def test_picks_the_run_whose_duration_matches(self, tmp_path): + """Of several techniques in a session, the one that ran DURING the + movie is the one with the movie's duration.""" + mrc = write_movie(tmp_path, n=200, epoch=1763387769) + clock = de_movie.read_movie_clock(mrc) + n_match = int(np.ceil(clock.duration / 0.08)) + 1 + for name, n_ec in (("a_02_CV_C01.mpr", 30), + ("b_02_CV_C01.mpr", n_match), + ("c_02_CV_C01.mpr", 12)): + (tmp_path / name).write_bytes( + build_mpr(ids=CV_IDS, records=cv_records(n=n_ec)) + ) + runs = eclab.find_ec_runs(str(tmp_path)) + ranked = align_mod.match_runs(clock, runs) + assert ranked, "no run matched the movie" + best, al = ranked[0] + assert best.path.endswith("b_02_CV_C01.mpr") + assert al.method == "span" + assert al.trustworthy + + def test_no_candidates_returns_empty(self, tmp_path): + clock, run = _paired(tmp_path, n_ec=30) + assert align_mod.match_runs(clock, [run]) == [] + + +class TestRealWorldShape: + """Numbers from a real paired DE Apollo + BioLogic SP-200 acquisition. + + The point is the *relationship*: 7914 frames at 30.525 fps and 3256 samples + at 12.5 Hz cover the same 259.2 s, while the two PCs' wall clocks disagree + by very nearly an hour. Span matching gets the right answer without ever + consulting those clocks. + """ + + def test_span_beats_a_one_hour_clock_skew(self, tmp_path): + mrc = write_movie(tmp_path, n=7914, period=0.03276, epoch=1763387769) + clock = de_movie.read_movie_clock(mrc) + assert clock.duration == pytest.approx(259.23, abs=0.01) + + path = tmp_path / "floating_02_CV_C01.mpr" + path.write_bytes( + build_mpr(ids=CV_IDS, records=cv_records(n=3243, t0=31.56, dt_s=0.08)) + ) + run = eclab.read_ec_file(str(path)) + + al = align_mod.align_clocks(clock, run) + assert al.method == "span" + assert abs(al.duration_mismatch_s) < 1.0 + assert al.covered_fraction == pytest.approx(1.0, abs=1e-3) + # The camera's own epoch stamp puts the movie an hour off the EC clock; + # span matching is immune to that, and reports the skew instead. + assert round(al.implied_utc_offset_hours) == 1 + + cols = align_mod.resample_to_frames(clock, run, al) + assert np.isfinite(cols["Ewe/V"]).all() + assert cols["Ewe/V"].shape == (7914,) diff --git a/spyde/tests/migrated/test_insitu_attach.py b/spyde/tests/migrated/test_insitu_attach.py new file mode 100644 index 00000000..2a3e13fc --- /dev/null +++ b/spyde/tests/migrated/test_insitu_attach.py @@ -0,0 +1,444 @@ +"""Wiring: a movie's timestamps calibrate its time axis, and instrument data +recorded beside it attaches as navigator lanes. + +The two behaviours pinned here are the ones a green reader suite cannot see: +that the loader PREFERS the timestamps sidecar over the reader's derived +period (which is wrong by ``sum_count**2`` for a summed DE movie), and that +auto-discovery stays quiet unless a record really does explain the movie. +""" +from __future__ import annotations + +import numpy as np +import pytest +import hyperspy.api as hs + +from spyde.backend._session_files import FileLoaderMixin +from spyde.insitu import attach as attach_mod +from spyde.insitu import de_movie +from spyde.tests.migrated.test_insitu_align import ( + CV_IDS, build_mpr, cv_records, write_movie, write_mpt, +) + +N_FRAMES = 120 +PERIOD = 0.03276 + + +def movie_signal(n=N_FRAMES, frame=8): # noqa: D401 + """A minimal in-situ movie signal: nav-dim 1, 2-D frames, time axis.""" + sig = hs.signals.Signal2D(np.zeros((n, frame, frame), np.uint16)) + ax = sig.axes_manager.navigation_axes[0] + ax.name, ax.units = "time", "sec" + ax.scale = 1.0 / (61.05006 * 2) # what the MRC reader would have set + return sig + + +def paired(tmp_path, *, n_ec=None, ec_dt=0.08): + mrc = write_movie(tmp_path, n=N_FRAMES, period=PERIOD, epoch=1763387769) + duration = (N_FRAMES - 1) * PERIOD + if n_ec is None: + n_ec = int(np.ceil(duration / ec_dt)) + 1 + (tmp_path / "r_02_CV_C01.mpr").write_bytes( + build_mpr(ids=CV_IDS, records=cv_records(n=n_ec, dt_s=ec_dt)) + ) + return mrc + + +class _FakeTree: + """Enough tree surface for attach: nav shape + navigator registration. + + ``navigator_signals`` stores a LIST, because that is what the real + ``BaseSignalTree.add_navigator_signal`` stores — ``_preprocess_navigator`` + returns ``[signal]`` (or ``[navigator, signal]``), never the bare signal it + was handed. A fake that stored the signal directly let a re-attach bug + through this suite and all the way into the running app. + """ + + def __init__(self, n=N_FRAMES, source_path=None): + self.root = movie_signal(n) + self.navigator_signals: dict = {} + self.source_path = source_path + self.registered: list[tuple[str, np.ndarray]] = [] + + def add_navigator_signal(self, name, signal): + self.navigator_signals[name] = [signal] + self.registered.append((name, np.asarray(signal.data))) + + def lane_data(self, name) -> np.ndarray: + return np.asarray(self.navigator_signals[name][0].data) + + +class TestTimeAxisFromTimestamps: + def test_timestamps_override_the_readers_period(self, tmp_path): + """The reader's 1/(fps*sum) is 4× too fast for a 2-frame sum; the CSV + is ground truth and must win.""" + mrc = write_movie(tmp_path, n=N_FRAMES, period=PERIOD) + sig = movie_signal() + before = float(sig.axes_manager.navigation_axes[0].scale) + + clock = FileLoaderMixin._apply_frame_timestamps(sig, mrc) + + assert clock is not None + ax = sig.axes_manager.navigation_axes[0] + assert ax.scale == pytest.approx(PERIOD, abs=1e-9) + assert ax.units == "s" + assert before / ax.scale == pytest.approx(0.25, rel=1e-3) # was 4× fast + + def test_the_recorded_fps_is_corrected_too(self, tmp_path): + """The metadata panel prefers the explicit fps KEY over the axis, so + leaving the reader's camera rate there would report 61 fps beside a + 32.76 ms/frame axis. Fix the value, not the precedence.""" + from spyde.metadata_extract import build_metadata_dict + + mrc = write_movie(tmp_path, n=N_FRAMES, period=PERIOD) + sig = movie_signal() + sig.metadata.set_item("Acquisition_instrument.TEM.frames_per_second", 61.05006) + + FileLoaderMixin._apply_frame_timestamps(sig, mrc) + + recorded = sig.metadata.get_item( + "Acquisition_instrument.TEM.frames_per_second") + # Rounded for display — the metadata chip prints this number verbatim, + # so "30.525030441931396 fps" is noise. The AXIS keeps full precision. + assert recorded == pytest.approx(1 / PERIOD, rel=1e-4) + assert len(str(recorded)) <= 8, f"unrounded fps on the chip: {recorded}" + + from spyde.tests.migrated.test_movie_metadata import _Tree + panel = build_metadata_dict(_Tree(sig))["Movie / In-Situ"] + assert "30.5" in panel["FPS"], panel["FPS"] + assert "61" not in panel["FPS"], panel["FPS"] + + def test_frame_count_mismatch_keeps_the_period_but_not_the_mapping(self, tmp_path): + """A stale sidecar from another autosave session still knows the + camera's frame period — that is a camera property. What it cannot do is + map ITS frames onto THIS movie's, so no clock is returned.""" + mrc = write_movie(tmp_path, n=N_FRAMES + 40, period=PERIOD) + sig = movie_signal(n=N_FRAMES) + + clock = FileLoaderMixin._apply_frame_timestamps(sig, mrc) + + assert clock is None + assert sig.axes_manager.navigation_axes[0].scale == pytest.approx(PERIOD) + + def test_no_sidecar_leaves_the_signal_untouched(self, tmp_path): + (tmp_path / "bare_movie.mrc").write_bytes(b"") + sig = movie_signal() + before = float(sig.axes_manager.navigation_axes[0].scale) + assert FileLoaderMixin._apply_frame_timestamps( + sig, str(tmp_path / "bare_movie.mrc")) is None + assert sig.axes_manager.navigation_axes[0].scale == before + + def test_non_movie_signals_are_skipped(self, tmp_path): + """A 4D-STEM scan has no frame time base to fix.""" + mrc = write_movie(tmp_path, n=N_FRAMES, period=PERIOD) + scan = hs.signals.Signal2D(np.zeros((4, 5, 8, 8), np.uint16)) + assert FileLoaderMixin._apply_frame_timestamps(scan, mrc) is None + + +class TestPixelSize: + """RosettaSciIO decides imaging-vs-diffraction with ``camera_length != -1``, + but an IMAGING exposure records 0 — so a TEM image is calibrated as + diffraction (nm^-1) at the unset -1 pixel size.""" + + def _sig(self): + s = movie_signal(n=4, frame=8) + # What the reader leaves on a mis-detected imaging exposure. + for ax, name in zip(s.axes_manager.signal_axes, ("kx", "ky")): + ax.scale, ax.units, ax.name = -1.0, "nm^-1", name + return s + + def test_imaging_axes_are_renamed_out_of_reciprocal_space(self, tmp_path): + """The reader names them kx/ky for the diffraction branch it wrongly + took; nm axes called "kx" are just as wrong as nm^-1 ones.""" + mrc = write_movie(tmp_path, n=4, info_extra={ + "Instrument Project Camera Length (centimeters)": "0", + "Specimen Pixel Size X (nanometers)": "1.14786", + "Specimen Pixel Size Y (nanometers)": "1.14786", + }) + sig = self._sig() + FileLoaderMixin._apply_de_pixel_size(sig, mrc) + assert [ax.name for ax in sig.axes_manager.signal_axes] == ["x", "y"] + + def test_a_user_named_axis_is_not_clobbered(self, tmp_path): + mrc = write_movie(tmp_path, n=4, info_extra={ + "Instrument Project Camera Length (centimeters)": "0", + "Specimen Pixel Size X (nanometers)": "1.14786", + "Specimen Pixel Size Y (nanometers)": "1.14786", + }) + sig = self._sig() + for ax in sig.axes_manager.signal_axes: + ax.name = "my axis" + FileLoaderMixin._apply_de_pixel_size(sig, mrc) + assert [ax.name for ax in sig.axes_manager.signal_axes] == ["my axis"] * 2 + + def test_imaging_exposure_gets_nm_from_the_specimen_pixel_size(self, tmp_path): + mrc = write_movie(tmp_path, n=4, info_extra={ + "Instrument Project Camera Length (centimeters)": "0", + "Diffraction Pixel Size X": "-1", + "Diffraction Pixel Size Y": "-1", + "Specimen Pixel Size X (nanometers)": "1.14786", + "Specimen Pixel Size Y (nanometers)": "1.14786", + }) + sig = self._sig() + assert FileLoaderMixin._apply_de_pixel_size(sig, mrc) + for ax in sig.axes_manager.signal_axes: + assert ax.scale == pytest.approx(1.14786) + assert ax.units == "nm" + + def test_real_diffraction_keeps_reciprocal_units(self, tmp_path): + mrc = write_movie(tmp_path, n=4, info_extra={ + "Instrument Project Camera Length (centimeters)": "80", + "Diffraction Pixel Size X": "0.0031", + "Diffraction Pixel Size Y": "0.0031", + "Specimen Pixel Size X (nanometers)": "-1", + "Specimen Pixel Size Y (nanometers)": "-1", + }) + sig = self._sig() + assert FileLoaderMixin._apply_de_pixel_size(sig, mrc) + for ax in sig.axes_manager.signal_axes: + assert ax.scale == pytest.approx(0.0031) + assert ax.units == "nm^-1" + + def test_no_usable_pixel_size_changes_nothing(self, tmp_path): + mrc = write_movie(tmp_path, n=4, info_extra={ + "Instrument Project Camera Length (centimeters)": "0", + "Diffraction Pixel Size X": "-1", + "Diffraction Pixel Size Y": "-1", + "Specimen Pixel Size X (nanometers)": "-1", + "Specimen Pixel Size Y (nanometers)": "-1", + }) + sig = self._sig() + assert not FileLoaderMixin._apply_de_pixel_size(sig, mrc) + assert sig.axes_manager.signal_axes[0].units == "nm^-1" + + def test_a_1d_signal_is_left_alone(self, tmp_path): + import hyperspy.api as hs + mrc = write_movie(tmp_path, n=4, info_extra={ + "Specimen Pixel Size X (nanometers)": "1.1", + "Specimen Pixel Size Y (nanometers)": "1.1", + }) + line = hs.signals.Signal1D(np.zeros((4, 8), np.uint16)) + assert not FileLoaderMixin._apply_de_pixel_size(line, mrc) + + +class TestAutoDiscovery: + def test_attaches_a_matching_record_as_navigator_lanes(self, tmp_path): + mrc = paired(tmp_path) + tree = _FakeTree(source_path=mrc) + + result = attach_mod.discover_and_attach(tree, mrc) + + assert result, result.reason + names = [n for n, _ in tree.registered] + assert attach_mod.POTENTIAL_LANE in names + assert any(n.startswith(attach_mod.CURRENT_LANE) for n in names) + for _, values in tree.registered: + assert values.shape == (N_FRAMES,) + assert np.isfinite(values).all(), "a lane must not carry NaN" + assert result.alignment.method == "span" + + def test_lanes_carry_the_movies_time_calibration(self, tmp_path): + """A lane's signal axis MUST be the movie's time axis. A 1-D selector + maps a widget position to a frame with ``(x - offset) / scale`` read + off the shown navigator's own axis, so an uncalibrated lane plots in + frame index and its cursor resolves to the wrong frame.""" + mrc = paired(tmp_path) + tree = _FakeTree(source_path=mrc) + # The loader calibrates the root's time axis first; the lanes copy it. + FileLoaderMixin._apply_frame_timestamps(tree.root, mrc) + attach_mod.discover_and_attach(tree, mrc) + + root_ax = tree.root.axes_manager.navigation_axes[0] + assert root_ax.scale == pytest.approx(PERIOD, abs=1e-9) + assert tree.navigator_signals, "no lanes registered" + for name in tree.navigator_signals: + lane_ax = tree.navigator_signals[name][0].axes_manager.signal_axes[0] + assert lane_ax.scale == pytest.approx(root_ax.scale), name + assert lane_ax.offset == pytest.approx(root_ax.offset), name + assert lane_ax.units == root_ax.units, name + # …so the lane's x runs over the movie's DURATION in seconds, not + # over its frame COUNT. + span = lane_ax.scale * (N_FRAMES - 1) + assert span == pytest.approx((N_FRAMES - 1) * PERIOD, rel=1e-6), name + + def test_current_lane_is_named_for_the_unit_it_uses(self, tmp_path): + """A µA-scale current plotted as mA is an unreadable 0.0001 axis; the + rescale is fine but the NAME has to say so.""" + mrc = paired(tmp_path) + tree = _FakeTree(source_path=mrc) + attach_mod.discover_and_attach(tree, mrc) + current = [n for n, _ in tree.registered if n.startswith(attach_mod.CURRENT_LANE)] + assert current == ["I (µA)"] + + def test_silent_when_nothing_matches_the_duration(self, tmp_path): + """A folder of unrelated records must not produce a false attach.""" + mrc = paired(tmp_path, n_ec=12) # far too short for the movie + tree = _FakeTree(source_path=mrc) + + result = attach_mod.discover_and_attach(tree, mrc) + + assert not result + assert tree.registered == [] + assert "none matching" in result.reason + + def test_silent_with_no_records_at_all(self, tmp_path): + mrc = write_movie(tmp_path, n=N_FRAMES, period=PERIOD) + tree = _FakeTree(source_path=mrc) + result = attach_mod.discover_and_attach(tree, mrc) + assert not result + assert "no EC-Lab records" in result.reason + + def test_refuses_when_the_movie_has_no_time_base(self, tmp_path): + (tmp_path / "bare_movie.mrc").write_bytes(b"") + tree = _FakeTree() + result = attach_mod.discover_and_attach(tree, str(tmp_path / "bare_movie.mrc")) + assert not result + assert "timestamps" in result.reason + + def test_stores_the_full_table_on_the_tree(self, tmp_path): + """Only two channels become chips; everything else stays reachable.""" + mrc = paired(tmp_path) + tree = _FakeTree(source_path=mrc) + attach_mod.discover_and_attach(tree, mrc) + assert set(tree.insitu_channels) >= {"time/s", "Ewe/V", "/mA", + "cycle number", "I Range"} + assert tree.insitu_run is not None + assert tree.insitu_alignment.covered_fraction == pytest.approx(1.0, abs=0.02) + + def test_a_frame_count_mismatch_refuses_to_map(self, tmp_path): + mrc = paired(tmp_path) + tree = _FakeTree(n=N_FRAMES - 5, source_path=mrc) # movie shorter than CSV + result = attach_mod.discover_and_attach(tree, mrc) + assert not result + assert "cannot map" in result.reason + + +class TestManualAttach: + def test_attaches_an_explicitly_chosen_file(self, tmp_path): + mrc = write_movie(tmp_path, n=N_FRAMES, period=PERIOD) + elsewhere = tmp_path / "elsewhere" + elsewhere.mkdir() + duration = (N_FRAMES - 1) * PERIOD + n_ec = int(np.ceil(duration / 0.08)) + 1 + ec = elsewhere / "r_02_CV_C01.mpr" + ec.write_bytes(build_mpr(ids=CV_IDS, records=cv_records(n=n_ec))) + tree = _FakeTree(source_path=mrc) + + result = attach_mod.attach_ec_file(tree, str(ec), movie_path=mrc) + + assert result, result.reason + assert attach_mod.POTENTIAL_LANE in tree.navigator_signals + + def test_reads_the_ascii_export_too(self, tmp_path): + mrc = write_movie(tmp_path, n=N_FRAMES, period=PERIOD) + duration = (N_FRAMES - 1) * PERIOD + n_ec = int(np.ceil(duration / 0.08)) + 1 + ec = write_mpt(tmp_path, "export.txt", n=n_ec) + tree = _FakeTree(source_path=mrc) + assert attach_mod.attach_ec_file(tree, ec, movie_path=mrc) + + def test_manual_override_of_the_lag(self, tmp_path): + mrc = paired(tmp_path, n_ec=12) # too short to auto-align + tree = _FakeTree(source_path=mrc) + ec = str(tmp_path / "r_02_CV_C01.mpr") + + assert not attach_mod.attach_ec_file(tree, ec, movie_path=mrc) + result = attach_mod.attach_ec_file( + tree, ec, movie_path=mrc, method="manual", lag_s=31.56 + ) + assert result, result.reason + assert result.alignment.lag_s == pytest.approx(31.56) + + def test_reattaching_the_same_run_succeeds(self, tmp_path): + """Auto-discovery runs on open, so picking the same record by hand + afterwards is the NORMAL second call — it must read as success, not as + 'this run has no potential or current channel'.""" + mrc = paired(tmp_path) + tree = _FakeTree(source_path=mrc) + ec = str(tmp_path / "r_02_CV_C01.mpr") + + first = attach_mod.discover_and_attach(tree, mrc) + second = attach_mod.attach_ec_file(tree, ec, movie_path=mrc) + + assert first and second, second.reason + assert set(second.lanes) == set(first.lanes) + # ONE plot state per lane, not one per attach. + names = [n for n, _ in tree.registered] + assert len(names) == len(set(names)) + + def test_reattach_replaces_the_lane_values(self, tmp_path): + """A different record under the same lane name must REPLACE the trace, + not leave the previous run's on screen.""" + mrc = paired(tmp_path) + tree = _FakeTree(source_path=mrc) + attach_mod.discover_and_attach(tree, mrc) + before = tree.lane_data(attach_mod.POTENTIAL_LANE).copy() + + duration = (N_FRAMES - 1) * PERIOD + n_ec = int(np.ceil(duration / 0.08)) + 1 + other = tmp_path / "other_02_CV_C01.mpr" + records = cv_records(n=n_ec) + records["Ewe/V"] = records["Ewe/V"] * -1.0 - 0.25 # a distinguishable sweep + other.write_bytes(build_mpr(ids=CV_IDS, records=records)) + + assert attach_mod.attach_ec_file(tree, str(other), movie_path=mrc) + after = tree.lane_data(attach_mod.POTENTIAL_LANE) + assert not np.allclose(before, after), "the lane still shows the old run" + assert after.shape == before.shape + + def test_a_run_with_no_e_or_i_says_what_it_does_have(self, tmp_path): + mrc = write_movie(tmp_path, n=N_FRAMES, period=PERIOD) + clock = de_movie.read_movie_clock(mrc) + tree = _FakeTree(source_path=mrc) + run = type("R", (), {})() + # An OCV record carries /V, so strip to something that carries + # neither a potential nor a current. + from spyde.insitu.eclab import EcRun + run = EcRun(path="x_01_OCV_C01.mpr", technique="Open Circuit Voltage", + start=None, time_s=np.linspace(0, clock.duration, 40), + channels={"cycle number": np.zeros(40)}) + from spyde.insitu.align import align_clocks + result = attach_mod.attach_run(tree, clock, run, align_clocks(clock, run)) + assert not result + assert "cycle number" in result.reason + + def test_unreadable_file_reports_rather_than_raises(self, tmp_path): + mrc = write_movie(tmp_path, n=N_FRAMES, period=PERIOD) + junk = tmp_path / "junk.mpr" + junk.write_bytes(b"not an instrument record") + tree = _FakeTree(source_path=mrc) + result = attach_mod.attach_ec_file(tree, str(junk), movie_path=mrc) + assert not result + assert "could not read" in result.reason + + +class TestSessionEntryPoint: + def test_mps_is_reported_as_a_recipe_not_data(self, window, tmp_path): + """Picking the .mps is an easy mistake — it holds no samples. Say what + it planned instead of failing blankly.""" + session = window["window"] + mps = tmp_path / "seq.mps" + mps.write_text( + "EC-LAB SETTING FILE\n\nNumber of linked techniques : 2\n\n" + "Technique : 1\nOpen Circuit Voltage\ntR (h:m:s) 0:00:30\n\n" + "Technique : 2\nCyclic Voltammetry\nEi (V) 0,000\n", + encoding="latin1", + ) + session.load_insitu_data(str(mps)) + text = " ".join(str(m) for m in window["messages"]) + assert "settings file" in text + assert "Cyclic Voltammetry" in text + + def test_missing_file_errors(self, window, tmp_path): + session = window["window"] + session.load_insitu_data(str(tmp_path / "nope.mpr")) + text = " ".join(str(m) for m in window["messages"]) + assert "not found" in text.lower() + + def test_no_movie_open_is_reported(self, window, tmp_path): + session = window["window"] + (tmp_path / "r_02_CV_C01.mpr").write_bytes( + build_mpr(ids=CV_IDS, records=cv_records(n=50)) + ) + session.load_insitu_data(str(tmp_path / "r_02_CV_C01.mpr")) + text = " ".join(str(m) for m in window["messages"]) + assert "in-situ movie" in text.lower() diff --git a/spyde/tests/migrated/test_movie_block.py b/spyde/tests/migrated/test_movie_block.py index 0a85888c..85b630fa 100644 --- a/spyde/tests/migrated/test_movie_block.py +++ b/spyde/tests/migrated/test_movie_block.py @@ -568,13 +568,17 @@ def test_add_text_overlay_from_1d_window(self, movie_dataset): "cell_id": cell_id, "source_window_id": line_plot.window_id, "label": "T"}) cell = session._report.doc.cell_by_id(cell_id) - assert len(cell.movie.text_overlays) == 1 - ov = cell.movie.text_overlays[0] + # The timestamp is a burn-in overlay now, so a movie always has one + # before anything is added; count only the dragged-in signal. + added = [o for o in cell.movie.text_overlays if not o.get('builtin')] + assert len(added) == 1 + ov = added[0] assert ov["label"] == "T" assert isinstance(ov.get("source"), dict) # a SignalRef dict # The editor state ships the overlay WITHOUT the ephemeral _trace. st = _latest(messages, "movie_state") - assert len(st["text_overlays"]) == 1 + emitted = [o for o in st["text_overlays"] if not o.get("builtin")] + assert len(emitted) == 1 assert "_trace" not in st["text_overlays"][0] def test_text_overlay_live_value_on_figure(self, movie_dataset): @@ -606,8 +610,10 @@ def _is_temp(p): M.movie_add_text_overlay(session, None, { "cell_id": cell_id, "source_window_id": tline.window_id, "label": "T"}) sess = M._sessions(session._report)[cell_id] - assert len(sess._text_overlay_widgets) == 1, "no live text-overlay widget" - lw = sess._text_overlay_widgets[0][0] + widgets = [w for w in sess._text_overlay_widgets.values() + if not w[1].get("builtin")] + assert len(widgets) == 1, "no live text-overlay widget" + lw = widgets[0][0] assert "100" in lw.text, lw.text # frame 0 → 100 M.movie_scrub(session, None, {"cell_id": cell_id, "t": 5}) _wait_until(lambda: sess.current_index() == 5) diff --git a/spyde/tests/migrated/test_movie_insitu_overlay.py b/spyde/tests/migrated/test_movie_insitu_overlay.py new file mode 100644 index 00000000..d6afbfad --- /dev/null +++ b/spyde/tests/migrated/test_movie_insitu_overlay.py @@ -0,0 +1,504 @@ +"""Burn an attached instrument channel into the movie's frames. + +The movie editor's original text overlay needed a live 1-D plot window to drag +in. An instrument channel — an electrochemistry potential, a holder temperature +— is already per-frame on the tree once :mod:`spyde.insitu` has aligned it, so +it needs no window at all. This covers that second trace source (the +``from_metadata`` seam ``traces.py`` documented) and the editor plumbing on top. +""" +from __future__ import annotations + +import numpy as np +import pytest +import hyperspy.api as hs + +from spyde.actions.movie_export import traces as _traces + + +class _Tree: + def __init__(self, n=50, scale=0.03276, channels=None): + self.root = hs.signals.Signal2D(np.zeros((n, 4, 4), np.uint16)) + ax = self.root.axes_manager.navigation_axes[0] + ax.name, ax.units, ax.scale = "time", "s", scale + self.insitu_channels = channels if channels is not None else { + "time/s": np.arange(n) * scale + 31.56, + "Ewe/V": np.linspace(0.0, 0.8, n), + "/mA": np.linspace(-1e-3, 1e-3, n), + "cycle number": np.ones(n), + "I Range": np.full(n, 52.0), + "mode": np.full(n, 2.0), + "ox/red": np.zeros(n), + "error": np.zeros(n), + } + + +class TestFromInsituChannel: + def test_captures_a_channel_on_the_movie_time_base(self): + tree = _Tree(n=50, scale=0.03276) + tr = _traces.from_insitu_channel(tree, "Ewe/V") + assert tr is not None + assert tr.label == "Ewe" + assert tr.units == "V" + assert tr.y.shape == (50,) + # x is the MOVIE's own time axis, NOT the instrument clock — an overlay + # is resampled against movie_times, and the instrument clock carries the + # alignment lag (here 31.56 s) that would shift every value. + assert tr.x[0] == pytest.approx(0.0) + assert tr.x[-1] == pytest.approx(49 * 0.03276) + + def test_bracketed_average_names_are_cleaned(self): + """EC-Lab writes "/mA"; burnt into a frame that should read "I".""" + tr = _traces.from_insitu_channel(_Tree(), "/mA") + assert (tr.label, tr.units) == ("I", "mA") + + def test_missing_channel_returns_none(self): + assert _traces.from_insitu_channel(_Tree(), "Nope/V") is None + + def test_no_channels_at_all_returns_none(self): + tree = _Tree(channels={}) + assert _traces.from_insitu_channel(tree, "Ewe/V") is None + + def test_nan_outside_the_record_is_preserved(self): + """Frames the instrument never covered must stay NaN so the overlay + paints a dash, not a fabricated value.""" + n = 20 + values = np.linspace(0, 1, n) + values[-5:] = np.nan + tree = _Tree(n=n, channels={"Ewe/V": values}) + tr = _traces.from_insitu_channel(tree, "Ewe/V") + assert np.isnan(tr.y[-5:]).all() + assert np.isfinite(tr.y[:-5]).all() + + def test_resample_onto_movie_times_is_the_identity(self): + """The values are already per-frame, so resampling at the frame times + must return them unchanged — the overlay path is shared with dragged + 1-D traces and must not shift an in-situ channel.""" + tree = _Tree(n=40, scale=0.05) + tr = _traces.from_insitu_channel(tree, "Ewe/V") + got = tr.resample(tr.x) + assert got == pytest.approx(tr.y) + + +class TestChannelOptions: + def test_lists_the_meaningful_channels(self): + options = _traces.insitu_channel_options(_Tree()) + names = [o["channel"] for o in options] + assert "Ewe/V" in names and "/mA" in names + # time/s is the axis, not a reading; flags are booleans that read as + # "0"/"1" burnt into a frame. + for skipped in ("time/s", "mode", "error", "ox/red"): + assert skipped not in names + + def test_labels_and_units_are_split(self): + options = {o["channel"]: o for o in _traces.insitu_channel_options(_Tree())} + assert options["Ewe/V"]["label"] == "Ewe" + assert options["Ewe/V"]["units"] == "V" + + def test_all_nan_channel_is_not_offered(self): + tree = _Tree(n=10, channels={"Ewe/V": np.full(10, np.nan)}) + assert _traces.insitu_channel_options(tree) == [] + + def test_no_insitu_data_offers_nothing(self): + tree = _Tree() + del tree.insitu_channels + assert _traces.insitu_channel_options(tree) == [] + + +class _FakeSpec: + def __init__(self): + self.text_overlays: list = [] + self.annotations: list = [] + self.params: dict = {"timestamp": False} + + +class _FakeEditSession: + """The slice of MovieEditSession the burn-in add path touches.""" + from spyde.actions.report.movie import MovieEditSession + add_burnin = MovieEditSession.add_burnin + add_insitu_overlay = MovieEditSession.add_insitu_overlay + burnin_sources = MovieEditSession.burnin_sources + insitu_channel_options = MovieEditSession.insitu_channel_options + _overlays_with_time = MovieEditSession._overlays_with_time + set_timestamp_enabled = MovieEditSession.set_timestamp_enabled + + def __init__(self, tree): + self.tree = tree + + class _Cell: + pass + self.cell = _Cell() + self.cell.movie = _FakeSpec() + + def frame_size(self): + return (512, 512) + + +class TestAddOverlay: + def test_adds_an_overlay_bound_to_the_channel(self): + st = _FakeEditSession(_Tree()) + assert st.add_insitu_overlay("Ewe/V") + (ov,) = st.cell.movie.text_overlays + assert ov["insitu_channel"] == "Ewe/V" + assert ov["label"] == "Ewe" and ov["units"] == "V" + # No SignalRef — the whole point is that no source window is needed. + assert "source" not in ov + + def test_unknown_channel_is_refused(self): + st = _FakeEditSession(_Tree()) + assert not st.add_insitu_overlay("Nope/V") + assert st.cell.movie.text_overlays == [] + + def test_a_flag_channel_is_refused(self): + """Only what `insitu_channel_options` offers can be added.""" + st = _FakeEditSession(_Tree()) + assert not st.add_insitu_overlay("ox/red") + + def test_successive_overlays_stack_down_the_frame(self): + st = _FakeEditSession(_Tree()) + st.add_insitu_overlay("Ewe/V") + st.add_insitu_overlay("/mA") + ys = [o["xy"][1] for o in st.cell.movie.text_overlays] + assert ys[1] < ys[0], "the second overlay should not land on the first" + colors = {o["color"] for o in st.cell.movie.text_overlays} + assert len(colors) == 2, "each overlay should take its own colour" + + def test_the_row_gap_scales_with_the_frame(self): + """A fixed 30 px gap is a readable row on a 512 px frame and 0.7% of a + 4096 px one — two overlays then land on the same line and read as one + duplicated text box.""" + from spyde.actions.report.movie import _overlay_row_step + + class _Big(_FakeEditSession): + def frame_size(self): + return (4096, 4096) + + st = _Big(_Tree()) + st.add_insitu_overlay("Ewe/V") + st.add_insitu_overlay("/mA") + ys = [o["xy"][1] for o in st.cell.movie.text_overlays] + assert ys[0] - ys[1] == _overlay_row_step(4096) + assert ys[0] - ys[1] >= 180, "rows too close to tell apart on a 4k frame" + # …and a small frame keeps a sane minimum rather than collapsing. + assert _overlay_row_step(256) == 30 + + +class TestBurnInLegibility: + """What "the voltage doesn't show on export" actually was: drawn, but at an + absolute 18 px beside a frame-relative timestamp, so a speck on a 4k movie — + and formatted `.2f`, which renders a µA current as "0.00".""" + + def test_font_size_scales_with_the_output_frame(self): + from spyde.actions.movie_export.pipeline import _overlay_font_px + # The timestamp's own rule is out_h // 28; a default overlay should + # land in the same ballpark rather than staying at a fixed 18 px. + assert _overlay_font_px(18, 512) == 18 + assert _overlay_font_px(18, 1024) == 36 + assert _overlay_font_px(18, 1024) == pytest.approx(1024 // 28, abs=4) + # …and never collapses to nothing on a tiny export. + assert _overlay_font_px(18, 64) >= 10 + + def test_font_size_survives_junk(self): + from spyde.actions.movie_export.pipeline import _overlay_font_px + assert _overlay_font_px(None, 512) == 18 + assert _overlay_font_px("big", 512) == 18 + + def test_small_magnitudes_keep_their_significant_figures(self): + from spyde.actions.movie_export.pipeline import _overlay_number + assert _overlay_number(0.5603) == "0.56" + # A µA-scale current in mA — ".2f" would render this as "0.00". + assert _overlay_number(2.67e-4) == "0.000267" + assert _overlay_number(-6.1e-3) == "-0.0061" + assert _overlay_number(1.5e6) == "1.5e+06" + + def test_nan_and_junk_paint_a_dash(self): + from spyde.actions.movie_export.pipeline import _overlay_number + assert _overlay_number(float("nan")) == "—" + assert _overlay_number(None) == "—" + assert _overlay_number([1, 2]) == "—" + + def test_the_overlay_actually_lights_pixels_on_a_rendered_frame(self): + """End-to-end through the real compose path — the check a green unit + suite could not make, and the one that would have caught this.""" + from spyde.actions.movie_export import pipeline + + n, edge = 12, 256 + raw = np.full((n, edge, edge), 12000, np.uint16) + tree = _Tree(n=n) + tr = _traces.from_insitu_channel(tree, "Ewe/V") + overlay = { + "insitu_channel": "Ewe/V", "label": "Ewe", "units": "V", + "xy": [12, int(edge * 0.85)], "size": 18, "color": "#ffcc00", + "_trace": tr, + } + values = pipeline._resample_text_overlays( + [overlay], np.arange(n) * 0.03276, src_indices=np.arange(n)) + img = pipeline.render_single_frame( + raw, 5, params=dict(fps=12, downsample=1, stride=1, cmap="gray", + clim=None, timestamp=False, scalebar=False, + t_start=0, t_end=n - 1), + n_frames=n, scale_s=0.03276, sig_scale_x=1.0, sig_units="nm", + text_overlays=[overlay], + text_values=[None if v is None else v[5] for v in values], + ) + a = np.asarray(img) + lit = np.count_nonzero((a[..., 0] > 180) & (a[..., 1] > 140) & (a[..., 2] < 90)) + assert lit > 60, f"the burnt-in overlay is invisible ({lit} px lit)" + + +class TestOneAddPath: + """Static text, the clock and an instrument channel are the same object and + go through ONE action. They used to have three add paths landing in two + different lists across two timeline lanes.""" + + def test_every_source_produces_a_text_overlay(self): + st = _FakeEditSession(_Tree()) + for source in ("label", "time", "Ewe/V"): + assert st.add_burnin(source), source + kinds = [o.get("builtin") or o.get("insitu_channel") + for o in st.cell.movie.text_overlays] + assert kinds == ["time", "label", "Ewe/V"] or set(kinds) == { + "time", "label", "Ewe/V"} + # …and nothing landed in `annotations`, which is for shapes now. + assert st.cell.movie.annotations == [] + + def test_they_all_carry_the_same_editable_fields(self): + st = _FakeEditSession(_Tree()) + for source in ("label", "time", "Ewe/V"): + st.add_burnin(source) + for ov in st.cell.movie.text_overlays: + for key in ("xy", "size", "color"): + assert key in ov, f"{ov.get('builtin') or ov.get('insitu_channel')} lacks {key}" + + def test_the_source_list_covers_label_time_and_channels(self): + sources = [s["source"] for s in _FakeEditSession(_Tree()).burnin_sources()] + assert sources[:2] == ["label", "time"] + assert "Ewe/V" in sources and "/mA" in sources + + def test_only_one_clock(self): + st = _FakeEditSession(_Tree()) + assert st.add_burnin("time") + assert not st.add_burnin("time"), "a second timestamp makes no sense" + + def test_adding_twice_adds_exactly_two(self): + """The reported bug was one add showing as two.""" + st = _FakeEditSession(_Tree()) + st.add_burnin("Ewe/V") + st.add_burnin("Ewe/V") + assert len(st.cell.movie.text_overlays) == 2 + + def test_legacy_text_annotations_migrate_to_overlays(self): + st = _FakeEditSession(_Tree()) + st.cell.movie.annotations = [ + {"kind": "text", "text": "Before", "xy": [10, 20], "size": 24, + "color": "#ff0000", "time_range": [0.0, 1.0]}, + {"kind": "rect", "xy": [0, 0], "wh": [10, 10]}, + ] + overlays = st._overlays_with_time(st.cell.movie) + labels = [o for o in overlays if o.get("builtin") == "label"] + assert len(labels) == 1 + assert labels[0]["text"] == "Before" + assert labels[0]["xy"] == [10, 20] + assert labels[0]["size"] == 24 + assert labels[0]["time_range"] == [0.0, 1.0] + # The SHAPE stays an annotation — only text moved. + assert [a["kind"] for a in st.cell.movie.annotations] == ["rect"] + + def test_a_static_label_draws_its_literal_text(self): + from spyde.actions.movie_export import pipeline + n, edge = 4, 96 + raw = np.full((n, edge, edge), 9000, np.uint16) + ov = pipeline.label_overlay(edge, text="Hello", color="#ffcc00") + img = pipeline.render_single_frame( + raw, 1, params=dict(fps=12, downsample=1, stride=1, cmap="gray", + clim=None, timestamp=False, scalebar=False, + t_start=0, t_end=n - 1), + n_frames=n, scale_s=0.5, sig_scale_x=1.0, sig_units="nm", + text_overlays=[ov], text_values=[None]) + a = np.asarray(img) + lit = np.count_nonzero((a[..., 0] > 180) & (a[..., 1] > 140) & (a[..., 2] < 90)) + assert lit > 20, "a static label overlay drew nothing" + + +class TestDragPersists: + """Dragging a burn-in in the editor moved the WIDGET and left the spec's + `xy` untouched, so the export drew it somewhere else — the editor and the + movie disagreeing about where the timestamp and the voltage sit.""" + + def _dragged(self, index, x, y): + from spyde.actions.report.movie import _make_burnin_widget_handler + st = _FakeEditSession(_Tree()) + st.add_burnin("time") + st.add_burnin("Ewe/V") + st.mgr = type("M", (), {"dirty": False})() + st.emit = lambda: None + + class _Ev: + source = type("W", (), {"_data": {"x": x, "y": y}})() + _make_burnin_widget_handler(st, index)(_Ev()) + return st + + def test_a_drag_writes_the_new_position_to_the_spec(self): + st = self._dragged(0, 3100.4, 120.6) + assert st.cell.movie.text_overlays[0]["xy"] == [3100, 121] + + def test_it_moves_only_the_dragged_one(self): + st = self._dragged(1, 900, 40) + assert st.cell.movie.text_overlays[1]["xy"] == [900, 40] + assert st.cell.movie.text_overlays[0]["xy"] != [900, 40] + + def test_it_marks_the_report_dirty(self): + assert self._dragged(0, 10, 10).mgr.dirty is True + + def test_an_out_of_range_index_is_ignored(self): + from spyde.actions.report.movie import _make_burnin_widget_handler + st = _FakeEditSession(_Tree()) + st.add_burnin("time") + st.mgr = type("M", (), {"dirty": False})() + st.emit = lambda: None + + class _Ev: + source = type("W", (), {"_data": {"x": 5, "y": 5}})() + _make_burnin_widget_handler(st, 7)(_Ev()) # removed since the build + assert st.cell.movie.text_overlays[0]["xy"] != [5, 5] + + def test_every_burn_in_label_gets_a_handler_wired(self): + import inspect + from spyde.actions.report.movie import MovieEditSession + src = inspect.getsource(MovieEditSession.sync_overlay_widgets) + tail = src.split("_text_overlay_widgets[i]", 1)[1] + assert "_make_burnin_widget_handler" in tail + assert 'add_event_handler(handler, "pointer_up")' in tail + + +class TestWidgetLeak: + """`sync_overlay_widgets` cleared the text-overlay DICT but never removed + the label widgets from the plot, so every resync stacked another copy of + every label — one add read as two.""" + + def test_resync_removes_the_previous_labels(self): + import inspect + from spyde.actions.report.movie import MovieEditSession + src = inspect.getsource(MovieEditSession.sync_overlay_widgets) + head = src.split("cur_sec", 1)[0] + assert "_text_overlay_widgets" in head, ( + "text-overlay widgets are not popped off the plot before a rebuild" + ) + assert head.count("p2._widgets.pop") >= 2 + + +class TestTimestampIsAnOrdinaryOverlay: + """The timestamp used to be a bool param drawn at a fixed spot with its own + font rule and NO presence in the editor — so toggling it changed nothing on + screen and it could not be moved, recoloured or time-gated like the values + burnt in beside it. It is now a `builtin: "time"` text overlay. + """ + + def _session(self): + st = _FakeEditSession(_Tree()) + st.cell.movie.params = {"timestamp": True} + return st + + def test_a_legacy_param_migrates_to_a_real_overlay(self): + from spyde.actions.movie_export.pipeline import has_time_overlay + st = self._session() + overlays = st._overlays_with_time(st.cell.movie) + assert has_time_overlay(overlays) + # …and it is persisted, so it is only synthesised once. + assert has_time_overlay(st.cell.movie.text_overlays) + + def test_it_carries_the_same_fields_as_any_other_overlay(self): + st = self._session() + (ov,) = st._overlays_with_time(st.cell.movie) + for key in ("label", "units", "xy", "size", "color"): + assert key in ov, f"timestamp overlay is missing {key!r}" + + def test_toggling_off_removes_it_and_leaves_the_others(self): + st = self._session() + st._overlays_with_time(st.cell.movie) + st.add_insitu_overlay("Ewe/V") + st.set_timestamp_enabled(False) + kinds = [o.get("builtin") or o.get("insitu_channel") + for o in st.cell.movie.text_overlays] + assert kinds == ["Ewe/V"] + st.set_timestamp_enabled(True) + assert "time" in [o.get("builtin") for o in st.cell.movie.text_overlays] + + def test_removing_the_clip_STICKS(self): + """The migration must run ONCE. Re-deriving it on every read left the + legacy `params["timestamp"]` authoritative forever, so deleting the + timestamp clip from the timeline was undone by the next read.""" + st = self._session() + st._overlays_with_time(st.cell.movie) + assert st.cell.movie.text_overlays + st.cell.movie.text_overlays = [] # the timeline's remove + again = st._overlays_with_time(st.cell.movie) + assert [o.get("builtin") for o in again] == [] + + def test_migration_does_not_re_add_after_a_toggle_off(self): + st = self._session() + st.set_timestamp_enabled(False) + assert st._overlays_with_time(st.cell.movie) == [] + assert st.cell.movie.params["timestamp"] is False + + def test_legacy_text_annotations_migrate_only_once(self): + st = _FakeEditSession(_Tree()) + st.cell.movie.annotations = [{"kind": "text", "text": "A", "xy": [1, 2]}] + st._overlays_with_time(st.cell.movie) + assert len(st.cell.movie.text_overlays) == 1 + st.cell.movie.text_overlays = [] # user deletes it + assert st._overlays_with_time(st.cell.movie) == [] + + def test_its_value_is_the_frames_own_time(self): + """No trace to resolve — the editor label rendered a dash before.""" + from spyde.actions.report.movie import _format_overlay_value + ov = {"builtin": "time", "label": "t", "units": "s"} + assert _format_overlay_value(ov, 100, 0.03276, 200) == "t = 3.28 s" + + def test_the_burnt_in_frame_draws_it_from_the_overlay_path(self): + from spyde.actions.movie_export import pipeline + + n, edge = 6, 128 + raw = np.full((n, edge, edge), 9000, np.uint16) + ov = pipeline.time_overlay(edge, color="#ffcc00") + params = dict(fps=12, downsample=1, stride=1, cmap="gray", clim=None, + timestamp=True, scalebar=False, t_start=0, t_end=n - 1) + img = pipeline.render_single_frame( + raw, 3, params=params, n_frames=n, scale_s=0.5, + sig_scale_x=1.0, sig_units="nm", + text_overlays=[ov], text_values=[None]) + a = np.asarray(img) + lit = np.count_nonzero((a[..., 0] > 180) & (a[..., 1] > 140) & (a[..., 2] < 90)) + assert lit > 20, "the timestamp overlay drew nothing" + + def test_the_legacy_path_does_not_double_draw(self): + """A migrated spec still has `timestamp: True`; the old fixed-position + draw must stand down once the overlay exists, or the frame carries two + timestamps.""" + import inspect + from spyde.actions.movie_export import pipeline + src = inspect.getsource(pipeline._compose_frame) + assert "not has_time_overlay(text_overlays)" in src + + +class TestTimestampColour: + def test_hex_is_parsed(self): + from spyde.actions.movie_export.pipeline import _hex_to_rgb + assert _hex_to_rgb("#ff9100") == (255, 145, 0) + assert _hex_to_rgb("f90") == (255, 153, 0) + + def test_unset_or_junk_falls_back_to_white(self): + from spyde.actions.movie_export.pipeline import _TS_COLOR, _hex_to_rgb + for bad in (None, "", "not-a-colour", "#12345"): + assert _hex_to_rgb(bad) == _TS_COLOR + + def test_the_default_params_carry_a_timestamp_colour(self): + from spyde.actions.report.movie import _DEFAULT_PARAMS + assert _DEFAULT_PARAMS["timestamp_color"] == "#ffffff" + + def test_the_pipeline_threads_the_colour_through(self): + """The param must reach `_draw_timestamp`, not just sit in the dict.""" + import inspect + from spyde.actions.movie_export import pipeline + src = inspect.getsource(pipeline) + assert 'p.get("timestamp_color")' in src + assert "_draw_timestamp(img, t_sec, ts_font, ts_color)" in src diff --git a/spyde/tests/migrated/test_movie_integration.py b/spyde/tests/migrated/test_movie_integration.py new file mode 100644 index 00000000..3e7c8e8d --- /dev/null +++ b/spyde/tests/migrated/test_movie_integration.py @@ -0,0 +1,149 @@ +"""Export resampling: what bins, what sub-selects. + +Three controls reduce the data on the way out, and they must not all behave the +same way: + +* **downsample** (spatial) box-MEANS k×k blocks — it already did. +* **fps** (temporal) INTEGRATES the source frames each output frame stands for. + Dropping 30.5 frame/s to 12 frame/s means ~2.5 source frames per output one; + keeping one and binning the rest throws away real signal, which on noisy + in-situ data is visible as noise the integrated frame does not have. +* **speed segments** SUB-SELECT. A 32x segment jumps ~81 source frames per + output frame; integrating 81 would smear a second of real change into one + picture, so the window stays what fps asked for and the cursor simply jumps + further. +""" +from __future__ import annotations + +import numpy as np +import pytest + +from spyde.actions.movie_export import pipeline + +SCALE_S = 0.03276 # a real DE movie: 30.525 frame/s + + +class TestSpatialBinning: + def test_downsample_is_a_box_mean_not_a_decimation(self): + frame = np.arange(16, dtype=np.float32).reshape(4, 4) + got = pipeline.downsample(frame, 2) + # Top-left 2x2 block is 0,1,4,5 -> mean 2.5 (a decimation would give 0). + assert got[0, 0] == pytest.approx(2.5) + assert got.shape == (2, 2) + + def test_k_of_one_is_a_no_op(self): + frame = np.arange(9, dtype=np.float32).reshape(3, 3) + assert pipeline.downsample(frame, 1) is frame + + +class TestIntegrationWindow: + def test_fps_reduction_sets_the_window(self): + # 30.525 source frame/s -> 12 frame/s output = 2.54 source frames each. + assert pipeline.integration_window(12.0, SCALE_S) == 3 + assert pipeline.integration_window(30.525, SCALE_S) == 1 + + def test_speed_up_does_NOT_widen_the_window(self): + """The whole point: a fast-forward sub-selects, it does not average + more. 32x advances ~81 frames but still integrates the fps window.""" + base = pipeline.integration_window(12.0, SCALE_S, speed=1.0) + for speed in (2, 4, 8, 16, 32): + assert pipeline.integration_window(12.0, SCALE_S, speed=speed) == base + + def test_slow_motion_narrows_it(self): + """In slow-mo the output advances a fraction of a frame; integrating + more than it advances would double-count frames into consecutive output + frames and blur it.""" + assert pipeline.integration_window(12.0, SCALE_S, speed=0.25) == 1 + assert pipeline.integration_window(12.0, SCALE_S, speed=0.0) == 1 + + def test_degenerate_inputs_fall_back_to_one_frame(self): + assert pipeline.integration_window(0, SCALE_S) == 1 + assert pipeline.integration_window(12.0, 0) == 1 + assert pipeline.integration_window(12.0, SCALE_S, speed=-3) == 1 + + def test_never_below_one(self): + # A faster output than the source cannot integrate a fraction of a frame. + assert pipeline.integration_window(120.0, SCALE_S) == 1 + + +class TestSpeedAtFrame: + def test_inside_a_segment(self): + segs = [{"time_range": [1.0, 2.0], "speed": 8}] + assert pipeline.speed_at_frame(segs, 45, SCALE_S) == 8 # 1.47 s + assert pipeline.speed_at_frame(segs, 5, SCALE_S) == 1.0 # 0.16 s + + def test_no_segments_is_one(self): + assert pipeline.speed_at_frame([], 10, SCALE_S) == 1.0 + + def test_malformed_segments_are_skipped(self): + assert pipeline.speed_at_frame([{"speed": 4}, None], 10, SCALE_S) == 1.0 + + +class TestIntegratedRead: + def _raw(self, n=10, edge=4): + # Frame i is filled with i, so a mean over a window is exactly its + # arithmetic mean — an integration bug shows up as a wrong number. + return np.stack([np.full((edge, edge), i, np.uint16) for i in range(n)]) + + def test_it_averages_the_window(self): + got = pipeline.read_frame_integrated(self._raw(), 2, 4, 10) + assert got.mean() == pytest.approx(3.5) # (2+3+4+5)/4 + + def test_window_of_one_is_the_plain_read(self): + got = pipeline.read_frame_integrated(self._raw(), 7, 1, 10) + assert got.mean() == pytest.approx(7.0) + + def test_it_clamps_at_the_end(self): + got = pipeline.read_frame_integrated(self._raw(n=10), 8, 5, 10) + assert got.mean() == pytest.approx(8.5) # (8+9)/2, not out of range + + def test_it_reads_one_frame_at_a_time(self): + """The memory-safety contract: never a stacked slice, whatever n is.""" + seen = [] + + class _Probe: + def __init__(self, data): + self.data = data + + def __getitem__(self, key): + seen.append(key) + return self.data[key] + + raw = _Probe(self._raw(n=10)) + pipeline.read_frame_integrated(raw, 1, 5, 10) + assert len(seen) == 5 + for key in seen: + # Each read is a scalar frame index, never a slice object. + assert isinstance(key[0], (int, np.integer)), key + + def test_integrating_reduces_noise(self): + """The reason this exists, on data shaped like the real thing.""" + rng = np.random.default_rng(0) + raw = (rng.normal(1000, 50, (16, 32, 32))).astype(np.float32) + single = pipeline.read_frame_integrated(raw, 0, 1, 16) + integrated = pipeline.read_frame_integrated(raw, 0, 8, 16) + assert integrated.std() < single.std() * 0.6 + + +class TestExportUsesIt: + def test_a_fast_segment_sub_selects_rather_than_smears(self): + """End-to-end: the window used inside a 32x segment is the fps window, + not the ~81 frames the cursor advances.""" + n = 400 + idxs = pipeline.frame_indices_with_speed( + n, 0, n - 1, 1, [{"time_range": [0.0, n * SCALE_S], "speed": 32}], + fps=12, scale_s=SCALE_S) + assert len(idxs) >= 2 + step = idxs[1] - idxs[0] + assert step > 40, f"a 32x segment should jump far, stepped {step}" + window = pipeline.integration_window( + 12.0, SCALE_S, pipeline.speed_at_frame( + [{"time_range": [0.0, n * SCALE_S], "speed": 32}], idxs[0], SCALE_S)) + assert window == 3 + assert window < step, "integration must not span the whole jump" + + def test_the_preview_integrates_like_the_export(self): + """Editor preview and export must not disagree about the picture.""" + import inspect + src = inspect.getsource(pipeline.render_single_frame) + assert "integration_window" in src diff --git a/spyde/tests/migrated/test_playback.py b/spyde/tests/migrated/test_playback.py index 99c4d084..1c6ccbee 100644 --- a/spyde/tests/migrated/test_playback.py +++ b/spyde/tests/migrated/test_playback.py @@ -215,14 +215,29 @@ def test_fast_forward_cycles_speed(self): assert pb.fast_forward() is True assert pb.is_playing is True assert pb.speed == 2 - # Then 2→4→8→1 while playing. + # Then 2→4→8→16→32→1 while playing. The cycle reaches ×32 because an + # in-situ acquisition is often thousands of frames: 7914 frames of real + # data is 4.3 minutes at ×1 and still 32 s at ×8. pb.fast_forward(); assert pb.speed == 4 pb.fast_forward(); assert pb.speed == 8 + pb.fast_forward(); assert pb.speed == 16 + pb.fast_forward(); assert pb.speed == 32 pb.fast_forward(); assert pb.speed == 1 # wraps, still playing assert pb.is_playing is True pb.fast_forward(); assert pb.speed == 2 pb.pause() + def test_speed_cycle_is_the_declared_one(self): + from spyde.actions.playback import SPEED_CYCLE + pb, _ = _controller(n=100000, scale=1.0, units="s") + seen = [] + pb.fast_forward() + for _ in range(len(SPEED_CYCLE)): + seen.append(pb.speed) + pb.fast_forward() + pb.pause() + assert sorted(set(seen)) == sorted(SPEED_CYCLE) + def test_fast_forward_emits_speed_in_state(self): # The playback_state emit carries the current speed. emitted = [] diff --git a/spyde/tests/migrated/test_stacked_cursor_master.py b/spyde/tests/migrated/test_stacked_cursor_master.py new file mode 100644 index 00000000..1c79a759 --- /dev/null +++ b/spyde/tests/migrated/test_stacked_cursor_master.py @@ -0,0 +1,195 @@ +"""Stacked 1-D navigator lanes: one master, the rest followers. + +The failure this pins is an interaction bug you cannot see in a screenshot and +cannot see in a synchronous unit test either — it needs the write-back to +arrive AFTER the drag handler has returned, which is exactly what +``_dispatch_to_main`` does in the app. + +While a lane is being dragged its line is the MASTER. The selector's index hook +fires on the ``_NavDispatcher`` thread with the committed (index-quantised) +position and is marshalled onto the main thread, landing some milliseconds +later — by which time the synchronous ``_busy`` guard is long since false. If +that write-back touches the held line, it snaps back to the last committed +frame while the pointer is somewhere else, the next ``pointer_move`` drags it +forward again, and the cursor oscillates for the whole drag. +""" +from __future__ import annotations + +import numpy as np +import pytest + +from spyde.actions.navigator_views import _StackedNavCursor + + +class _FakeWidget: + """A VLine stand-in. ``set`` fires ``pointer_move`` like the real one.""" + + def __init__(self, x=0.0): + self._x = float(x) + self._handlers: dict[str, list] = {} + + def add_event_handler(self, fn, event_type): + self._handlers.setdefault(event_type, []).append(fn) + + def get(self, key): + assert key == "x" + return self._x + + @property + def x(self): + return self._x + + @x.setter + def x(self, value): + self._x = float(value) + + def set(self, x=None, **_kw): + if x is None: + return + self._x = float(x) + # The real widget echoes a pointer_move on a programmatic set — the + # whole reason _busy exists. + self._fire("pointer_move") + + def _fire(self, event_type): + for fn in list(self._handlers.get(event_type, ())): + fn() + + def drag_to(self, x: float): + self._x = float(x) + self._fire("pointer_move") + + def release(self): + self._fire("pointer_up") + + +class _FakeSelector: + def __init__(self, scale=0.03276): + self.index_hooks: list = [] + self._widget = _FakeWidget() + self.current_indices = np.array([0]) + self.scale = scale + self.updates = 0 + self.current_plot = self + + # `_selector_axis` reads current_plot.plot_state.current_signal… + @property + def plot_state(self): + return self + + @property + def current_signal(self): + scale = self.scale + + class _Axis: + pass + + ax = _Axis() + ax.scale, ax.offset = scale, 0.0 + + class _AM: + signal_axes = [ax] + + class _Sig: + axes_manager = _AM() + + return _Sig() + + def delayed_update_data(self, force=False): + self.updates += 1 + + +class _FakeSession: + """Defers marshalled work, so a test can land it at a chosen moment — + the app's ``_dispatch_to_main`` is likewise asynchronous.""" + + def __init__(self): + self.pending: list = [] + + def _dispatch_to_main(self, fn): + self.pending.append(fn) + + def flush(self): + pending, self.pending = self.pending, [] + for fn in pending: + fn() + + +@pytest.fixture +def stacked(): + session = _FakeSession() + widgets = [_FakeWidget(), _FakeWidget(), _FakeWidget()] + sel = _FakeSelector() + cursor = _StackedNavCursor(session, 1, widgets, sel) + return session, widgets, sel, cursor + + +class TestMasterFollower: + def test_drag_mirrors_to_the_other_lanes(self, stacked): + _session, widgets, sel, _cursor = stacked + widgets[0].drag_to(4.2) + assert [w.x for w in widgets] == [4.2, 4.2, 4.2] + assert sel._widget.x == pytest.approx(4.2) + assert sel.updates == 1 + + def test_late_writeback_does_not_move_the_held_line(self, stacked): + """THE regression: the index hook lands after the handler returned.""" + session, widgets, sel, cursor = stacked + + widgets[0].drag_to(4.2) # user drags lane 0 to 4.2 + # The dispatcher commits frame 128 (128 * 0.03276 = 4.19328) and fires + # the hook on its own thread; the write-back is marshalled. + for hook in sel.index_hooks: + hook(np.array([128])) + widgets[0].drag_to(4.9) # user keeps dragging BEFORE it lands + session.flush() # …and now it lands + + assert widgets[0].x == pytest.approx(4.9), ( + "the held line was yanked back to the committed position" + ) + # The followers do take the committed position — that is their job. + assert widgets[1].x == pytest.approx(4.19328) + assert widgets[2].x == pytest.approx(4.19328) + + def test_release_hands_the_master_role_back(self, stacked): + """After pointer_up every line settles on the committed position.""" + session, widgets, sel, cursor = stacked + + widgets[0].drag_to(4.2) + widgets[0].release() + assert cursor._master is None + + for hook in sel.index_hooks: + hook(np.array([128])) + session.flush() + + assert [pytest.approx(w.x) for w in widgets] == [pytest.approx(4.19328)] * 3 + + def test_only_one_master_at_a_time(self, stacked): + """Mirroring fires the followers' own pointer_move handlers; they must + not steal the master role from the line actually under the pointer.""" + _session, widgets, _sel, cursor = stacked + widgets[1].drag_to(2.0) + assert cursor._master is widgets[1] + assert [w.x for w in widgets] == [2.0, 2.0, 2.0] + + def test_programmatic_move_with_no_drag_syncs_every_line(self, stacked): + """Playback moves the selector with nobody dragging — all lanes follow.""" + session, widgets, sel, _cursor = stacked + for hook in sel.index_hooks: + hook(np.array([64])) + session.flush() + assert [pytest.approx(w.x) for w in widgets] == [pytest.approx(64 * 0.03276)] * 3 + + def test_drag_does_not_re_enter_via_the_mirror_echo(self, stacked): + """`set` echoes a pointer_move; one drag must drive the selector once.""" + _session, widgets, sel, _cursor = stacked + widgets[2].drag_to(1.5) + assert sel.updates == 1 + + def test_close_releases_the_master_and_stops_syncing(self, stacked): + session, widgets, sel, cursor = stacked + widgets[0].drag_to(4.2) + cursor.close() + assert cursor._master is None + assert sel.index_hooks == [] diff --git a/spyde/toolbars.yaml b/spyde/toolbars.yaml index ca4db289..cf1ad628 100644 --- a/spyde/toolbars.yaml +++ b/spyde/toolbars.yaml @@ -34,7 +34,7 @@ functions: toggle: True toolbar_side: right Fast Forward: - description: Fast-forward — cycle the playback speed 2x → 4x → 8x → 1x. Starts playback if stopped. + description: Fast-forward — cycle the playback speed 2x → 4x → 8x → 16x → 32x → 1x. Starts playback if stopped. icon: drawing/toolbars/icons/fastforward.svg function: spyde.actions.base.fast_forward plot_dim: [ 1 ]