From 3b1a8d9d12570159fd35c9ff5eb5bcbeb8006247 Mon Sep 17 00:00:00 2001 From: Carter Francis Date: Thu, 6 Aug 2026 10:02:35 -0500 Subject: [PATCH 1/3] feat(insitu): align electrochemistry with an in-situ movie's real frame times An in-situ experiment records the same event twice -- the camera's frames and the potentiostat's E/I -- on two machines with two unsynchronised clocks and two sampling rates. `spyde/insitu/` reads both and puts them on one axis: * `de_movie.py` -- the DE `*_movie_timestamps.csv` / `*_info.txt` sidecars. * `eclab.py` -- BioLogic `.mpr` (binary), `.mpt`/`.txt` (ASCII), `.mps`. * `align.py` -- the clock solve + resample onto the frame times. * `attach.py` -- registers the channels as navigator chips on the tree. Alignment matches on SPAN, not on the wall clocks. On the reference dataset the two PCs disagreed by 59 min 41 s (the EC box was on UTC+1, the camera stamps UTC); duration agreement got 0.104 s over 259.3 s and needs no clock agreement at all. The absolute route stays as a fallback and the implied UTC offset is reported rather than hidden. Opening a `.mrc` now also fixes two RosettaSciIO calibration bugs from the info file: * the time axis is derived `1/(fps*sum_count)` where a summed movie's saved period is `sum_count/fps` -- wrong by `sum_count**2`, so a routine 2-frame sum lands 4x too fast (122.1 -> 30.525 fps here). The timestamps are ground truth and the recorded fps is corrected to match, rather than flipping metadata_extract's deliberate key-beats-axis precedence. * imaging-vs-diffraction is decided with `camera_length != -1`, but an imaging exposure records 0 -- so a TEM image was calibrated as diffraction at the unset -1 nm^-1 with kx/ky axes, while `Specimen Pixel Size X = 1.14786 nm` sat in the same file. Auto-discovery stays silent unless a record's duration actually matches the movie; File > Load In-Situ Data... loads one from elsewhere. Our own .mpr parser rather than galvani (GPLv3+, so licence-compatible): it mis-reads extended column IDs -- its v0 path takes every other byte, so ID 438 `step time/s` reads as 182 -- and failed on 1 of the 6 real files. Validated against an ASCII export of the same run to 1e-8. --- electron/src/main/index.ts | 28 + electron/src/preload/index.ts | 3 + .../src/renderer/src/components/MenuBar.tsx | 1 + electron/src/renderer/src/electron.d.ts | 1 + electron/tests/insitu_echem.spec.ts | 285 ++++++++ spyde/backend/_session_actions.py | 6 +- spyde/backend/_session_files.py | 267 +++++++- spyde/insitu/__init__.py | 51 ++ spyde/insitu/align.py | 327 +++++++++ spyde/insitu/attach.py | 294 +++++++++ spyde/insitu/de_movie.py | 369 +++++++++++ spyde/insitu/eclab.py | 621 ++++++++++++++++++ spyde/metadata_extract.py | 6 + spyde/tests/migrated/test_insitu_align.py | 514 +++++++++++++++ spyde/tests/migrated/test_insitu_attach.py | 444 +++++++++++++ 15 files changed, 3213 insertions(+), 4 deletions(-) create mode 100644 electron/tests/insitu_echem.spec.ts create mode 100644 spyde/insitu/__init__.py create mode 100644 spyde/insitu/align.py create mode 100644 spyde/insitu/attach.py create mode 100644 spyde/insitu/de_movie.py create mode 100644 spyde/insitu/eclab.py create mode 100644 spyde/tests/migrated/test_insitu_align.py create mode 100644 spyde/tests/migrated/test_insitu_attach.py diff --git a/electron/src/main/index.ts b/electron/src/main/index.ts index a1f953e3..67216c6d 100644 --- a/electron/src/main/index.ts +++ b/electron/src/main/index.ts @@ -513,6 +513,14 @@ function buildMenu(): void { // than the native picker — the user adds/reorders there, then confirms. click: () => win?.webContents.send('spyde:open_stack_dialog'), }, + { + label: 'Load In-Situ Data…', + // A potentiostat / holder log recorded alongside a movie. Auto- + // discovery already scans the movie's own folder on open; this is + // for a record that lives elsewhere, or one the duration scorer + // passed over. + click: () => pickInsituData(), + }, { type: 'separator' }, { label: 'Save Signal…', @@ -618,6 +626,26 @@ ipcMain.handle('spyde:open-file', async () => { } }) +/** Pick an instrument record (potentiostat / holder log) and attach it. + * `.txt` is offered because EC-Lab's ASCII export is routinely saved that way; + * Python dispatches on the file's own magic, not on the extension. */ +async function pickInsituData(): Promise { + const result = await dialog.showOpenDialog(win!, { + title: 'Load in-situ instrument data', + properties: ['openFile'], + filters: [ + { name: 'In-Situ Data', extensions: ['mpr', 'mpt', 'txt', 'mps'] }, + { name: 'BioLogic EC-Lab', extensions: ['mpr', 'mpt', 'mps'] }, + { name: 'All Files', extensions: ['*'] }, + ], + }) + if (!result.canceled) { + for (const p of result.filePaths) sendAction('load_insitu_data', { path: p }) + } +} + +ipcMain.handle('spyde:load-insitu-data', () => pickInsituData()) + /** Open a .zspy/.zarr DIRECTORY store (folder picker → load). */ ipcMain.handle('spyde:open-zarr-folder', async () => { const result = await dialog.showOpenDialog(win!, { diff --git a/electron/src/preload/index.ts b/electron/src/preload/index.ts index fd72034c..53ef2a56 100644 --- a/electron/src/preload/index.ts +++ b/electron/src/preload/index.ts @@ -110,6 +110,9 @@ contextBridge.exposeInMainWorld('electron', { /** Open a .zspy/.zarr DIRECTORY store (folder picker → load). */ openZarrFolder: (): Promise => ipcRenderer.invoke('spyde:open-zarr-folder'), + /** Pick an in-situ instrument record (.mpr/.mpt/.txt/.mps) and attach it. */ + loadInsituData: (): Promise => ipcRenderer.invoke('spyde:load-insitu-data'), + /** Quit the app (custom title-bar menu replaces native File→Quit). */ quit: (): Promise => ipcRenderer.invoke('spyde:quit'), diff --git a/electron/src/renderer/src/components/MenuBar.tsx b/electron/src/renderer/src/components/MenuBar.tsx index 903ab911..4cbf2850 100644 --- a/electron/src/renderer/src/components/MenuBar.tsx +++ b/electron/src/renderer/src/components/MenuBar.tsx @@ -187,6 +187,7 @@ export function MenuBar({ onStartGuide, onShowInfo }: { { label: 'Open…', onClick: () => window.electron.openFile() }, { label: 'Open Zarr Folder (.zspy)…', onClick: () => window.electron.openZarrFolder() }, { label: 'Load Stack…', onClick: () => openStackDialog() }, + { label: 'Load In-Situ Data…', onClick: () => window.electron.loadInsituData() }, { separator: true }, { label: 'Save Signal…', onClick: () => window.electron.saveDialog() }, { separator: true }, diff --git a/electron/src/renderer/src/electron.d.ts b/electron/src/renderer/src/electron.d.ts index 784cf466..7762a95f 100644 --- a/electron/src/renderer/src/electron.d.ts +++ b/electron/src/renderer/src/electron.d.ts @@ -17,6 +17,7 @@ declare global { action: (action: string, payload?: Record, windowId?: number) => void openFile: () => Promise openZarrFolder: () => Promise + loadInsituData: () => Promise quit: () => Promise saveDialog: () => Promise pickFile: (opts: { name?: string; extensions?: string[] }) => Promise diff --git a/electron/tests/insitu_echem.spec.ts b/electron/tests/insitu_echem.spec.ts new file mode 100644 index 00000000..a3925d0b --- /dev/null +++ b/electron/tests/insitu_echem.spec.ts @@ -0,0 +1,285 @@ +/** + * insitu_echem.spec.ts — an in-situ movie opens with its REAL frame timing and + * with the electrochemistry recorded beside it attached as navigator lanes. + * + * This is the part a green pytest run cannot see. The unit tests prove the + * readers and the alignment maths; only the app can show that the chips + * actually appear on the navigator, that shift-clicking them stacks E and I + * against the movie's own time cursor, and that dragging that cursor still + * moves the movie. + * + * DEV-BOX ONLY: it drives a real 132 GB DE acquisition (with its + * `.spyde-nav.npz` sidecar, so no navigator recompute) plus the BioLogic + * records beside it. CI has neither, so the whole file skips when the folder + * is absent rather than shipping a fake that would pass without proving + * anything. + */ +import { test, expect } from '@playwright/test' +import { existsSync, mkdirSync } from 'fs' +import { join } from 'path' + +const { + launchApp, backendAction, waitForSubwindowCount, navWindow, backendErrorLines, +} = require('./_harness.cjs') + +const DATA_DIR = + 'D:\\InsituElectroChemistry\\directelectron_good-electrochemistry-movie_2026-07-30_0335' +const MOVIE = join(DATA_DIR, '20251117_88071_movie.mrc') +const EC_RUN = join(DATA_DIR, 'New TEM NZH-001-03-floating_02_CV_C01.mpr') +const SHOTS = join(__dirname, '..', 'insitu_echem_shots') + +const POTENTIAL_CHIP = 'Ewe (V)' +const CURRENT_CHIP = 'I (µA)' + +test.skip(!existsSync(MOVIE), `in-situ electrochemistry dataset not present (${MOVIE})`) +test.describe.configure({ mode: 'serial' }) + +let ctx: any + +test.beforeAll(async () => { + mkdirSync(SHOTS, { recursive: true }) + ctx = await launchApp({ env: { SPYDE_LOG_LEVEL: 'INFO' } }) +}) + +test.afterAll(async () => { + await ctx?.app?.close() +}) + +/** The most recent backend log line containing `needle`. + * `waitForLog` resolves with the line only for a FUTURE match — a line already + * in the buffer resolves undefined — so always read the buffer afterwards. */ +function lastLine(needle: string): string { + const hit = [...ctx.backend.logBuffer].reverse().find((l: string) => l.includes(needle)) + return hit ?? '' +} + +/** The nav window's id, taken from its chip strip's testid. */ +async function navChipsId(nav: any): Promise { + const testid = await nav.locator('[data-testid^="nav-chips-"]').first().getAttribute('data-testid') + return String(testid).replace('nav-chips-', '') +} + +test('a DE movie opens with its timestamps as the time base', async () => { + const { page, backend } = ctx + + await backendAction(page, 'open_file', { path: MOVIE }) + await waitForSubwindowCount(page, 2, 180_000) + await page.screenshot({ path: join(SHOTS, '01-movie-open.png') }) + + // The reader derives 1/(fps*sum) = 8.19 ms; the timestamps say 32.76 ms. + await backend.waitForLog('Frame timing from', 120_000) + const timing = lastLine('Frame timing from') + expect(timing).toContain('movie_timestamps.csv') + expect(timing).toContain('32.760 ms/frame') + expect(timing).toContain('30.53 fps') + // and the reader's wrong value is reported alongside it, 4x too fast + expect(timing).toContain('reader said 8.19') + + // The SIGNAL axes too: this is a TEM image, so nm — but rsciio reads the + // imaging exposure's camera length of 0 as "has a camera length" and + // calibrates it as diffraction at the unset -1 nm^-1. + await backend.waitForLog('frame pixel size from', 30_000) + const px = lastLine('frame pixel size from') + expect(px).toContain('1.14786 nm/px') + expect(px).toContain('reader said -1 nm^-1') +}) + +test('electrochemistry beside the movie attaches as navigator chips', async () => { + const { page, backend } = ctx + + await backend.waitForLog('Attached Cyclic Voltammetry', 120_000) + const attached = lastLine('Attached Cyclic Voltammetry') + expect(attached).toContain('floating_02_CV_C01') + expect(attached).toContain('aligned by span') + expect(attached).toContain('100% of frames covered') + + const nav = navWindow(page) + const navId = await navChipsId(nav) + await expect(nav.getByTestId(`nav-chip-${POTENTIAL_CHIP}-${navId}`)).toBeVisible() + await expect(nav.getByTestId(`nav-chip-${CURRENT_CHIP}-${navId}`)).toBeVisible() + await page.screenshot({ path: join(SHOTS, '02-nav-chips.png') }) +}) + +test('shift-clicking the chips stacks E and I on the movie time cursor', async () => { + const { page } = ctx + const nav = navWindow(page) + const navId = await navChipsId(nav) + + await nav.getByTestId(`nav-chip-${POTENTIAL_CHIP}-${navId}`).click() + await nav.getByTestId(`nav-chip-${CURRENT_CHIP}-${navId}`).click({ modifiers: ['Shift'] }) + + // The stacked view is a distinct figure the backend emits with + // view_kind:'stacked'; wait for the frame to swap rather than sleeping. + await expect + .poll(async () => nav.locator('iframe').count(), { timeout: 60_000 }) + .toBeGreaterThan(0) + await page.waitForTimeout(1_500) // let the figure paint before capturing + await nav.screenshot({ path: join(SHOTS, '03-stacked-lanes.png') }) + await page.screenshot({ path: join(SHOTS, '04-full-window.png') }) +}) + +/** Dominant x (CSS px) of the ORANGE stacked-lane cursor, or null. + * `crosshairAt` in the harness finds the GREEN navigator crosshair; the + * stacked lanes' linked cursor is #ff9100. */ +async function laneCursorX(nav: any): Promise { + // Iterate EVERY iframe in the window, not just the first: the stacked figure + // mounts as its own iframe alongside the ones already there, so `.first()` + // reads a figure with no cursor in it at all. + const count = await nav.locator('iframe').count() + for (let i = 0; i < count; i++) { + const ifel = await nav.locator('iframe').nth(i).elementHandle() + const frame = ifel && (await ifel.contentFrame()) + if (!frame) continue + const x = await frameCursorX(frame) + if (x !== null) return x + } + return null +} + +async function frameCursorX(frame: any): Promise { + return frame.evaluate(() => { + let best: { x: number; n: number } | null = null + for (const c of Array.from(document.querySelectorAll('canvas')) as HTMLCanvasElement[]) { + const g = c.getContext('2d') + if (!g || !c.width || !c.height) continue + const d = g.getImageData(0, 0, c.width, c.height).data + const cols = new Int32Array(c.width) + let total = 0 + for (let y = 0; y < c.height; y++) { + for (let x = 0; x < c.width; x++) { + const p = (y * c.width + x) * 4 + const r = d[p], gr = d[p + 1], b = d[p + 2], a = d[p + 3] + // #ff9100 = (255, 145, 0), the vline widget orange. + if (a > 40 && r > 200 && gr > 90 && gr < 200 && b < 60) { cols[x]++; total++ } + } + } + if (!total || (best && total <= best.n)) continue + let bx = 0 + for (let x = 1; x < c.width; x++) if (cols[x] > cols[bx]) bx = x + const rect = c.getBoundingClientRect() + best = { x: rect.left + (bx + 0.5) * (rect.width / c.width), n: total } + } + return best ? best.x : null + }) +} + +test('dragging a lane cursor tracks the pointer without jumping back', async () => { + const { page } = ctx + const nav = navWindow(page) + const navId = await navChipsId(nav) + + // Back to the stacked view (the previous test switched to a single lane). + await nav.getByTestId(`nav-chip-${POTENTIAL_CHIP}-${navId}`).click() + await nav.getByTestId(`nav-chip-${CURRENT_CHIP}-${navId}`).click({ modifiers: ['Shift'] }) + await page.waitForTimeout(2_000) + + // Find the iframe the cursor actually lives in, and use ITS box for the + // page-coordinate maths — the stacked figure is not necessarily iframe 0. + let box: any = null + let startX: number | null = null + const nFrames = await nav.locator('iframe').count() + for (let i = 0; i < nFrames; i++) { + const ifel = await nav.locator('iframe').nth(i).elementHandle() + const frame = ifel && (await ifel.contentFrame()) + if (!frame) continue + const x = await frameCursorX(frame) + if (x !== null) { + startX = x + box = await nav.locator('iframe').nth(i).boundingBox() + break + } + } + expect(startX, 'no orange lane cursor found in any iframe').not.toBeNull() + + // Walk the cursor right and sample after each step. The bug this guards is + // the ASYNC write-back landing on the line still under the pointer: the + // cursor snaps back to the last committed frame, the next move drags it + // forward again, and the samples oscillate instead of advancing. + const gy = box!.y + box!.height * 0.25 + await page.mouse.move(box!.x + startX!, gy) + await page.mouse.down() + const samples: number[] = [] + for (let i = 1; i <= 6; i++) { + await page.mouse.move(box!.x + startX! + i * 22, gy, { steps: 3 }) + await page.waitForTimeout(350) + const x = await laneCursorX(nav) + if (x !== null) samples.push(x) + } + await page.mouse.up() + await page.waitForTimeout(600) + await nav.screenshot({ path: join(SHOTS, '06-after-drag.png') }) + + console.log('lane cursor drag samples:', samples.map(s => s.toFixed(1)).join(', ')) + expect(samples.length).toBeGreaterThanOrEqual(4) + expect(samples[samples.length - 1]).toBeGreaterThan(samples[0] + 20) + // Monotonic within a frame-quantisation tolerance — a snap-back to the last + // committed position is tens of px, far outside this. + for (let i = 1; i < samples.length; i++) { + expect(samples[i], `sample ${i} went backwards: ${samples.join(', ')}`) + .toBeGreaterThan(samples[i - 1] - 6) + } +}) + +test('selecting one lane shows the WHOLE experiment, not just its start', async () => { + const { page } = ctx + const nav = navWindow(page) + const navId = await navChipsId(nav) + + // Plain click = switch the live navigator in place to just this lane. + await nav.getByTestId(`nav-chip-${POTENTIAL_CHIP}-${navId}`).click() + await page.waitForTimeout(1_500) + await nav.screenshot({ path: join(SHOTS, '05-single-lane.png') }) + + // The lane must carry the movie's TIME calibration. Uncalibrated it plots + // over frame index (0…7913) while the selector works in seconds (0…259), so + // the cursor only ever reaches the first ~3% — "I can only see the beginning + // of the experiment". Read the axis the figure actually rendered. + const xMax = await nav.locator('iframe').first().contentFrame() + .locator('text=/^\\d+$/').last().innerText().catch(() => '') + // 259 s of movie: the last x tick belongs to the seconds axis, not to a + // frame count in the thousands. + if (xMax) expect(Number(xMax)).toBeLessThan(1_000) +}) + +test('loading a record explicitly works too', async () => { + const { page, backend } = ctx + // The File-menu picker returns a path; drive the same backend action it sends + // (a native dialog can't be driven from Playwright). + const before = ctx.backend.logBuffer.length + await backendAction(page, 'load_insitu_data', { path: EC_RUN }) + await expect + .poll(() => ctx.backend.logBuffer.slice(before) + .some((l: string) => /insitu: (Attached|could not attach)/.test(l)), + { timeout: 60_000 }) + .toBe(true) + const fresh = ctx.backend.logBuffer.slice(before).join('\n') + expect(fresh).toContain('Attached Cyclic Voltammetry') + expect(fresh).not.toContain('could not attach') +}) + +test('fast forward shows the speed it is running at, up to x32', async () => { + const { page } = ctx + const nav = navWindow(page) + const ff = nav.getByTitle('Fast Forward').or(nav.locator('[data-testid="toolbar-btn-Fast Forward"]')) + const badge = page.getByTestId('playback-speed-badge') + + // 1x -> no badge. Then 2, 4, 8, 16, 32 as the cycle advances. + await expect(badge).toHaveCount(0) + for (const expected of ['2x', '4x', '8x', '16x', '32x']) { + await ff.first().click() + await expect(badge).toHaveText(expected, { timeout: 10_000 }) + } + await page.screenshot({ path: join(SHOTS, '07-speed-32x.png') }) + + // Once more wraps to 1x and the badge goes away. + await ff.first().click() + await expect(badge).toHaveCount(0, { timeout: 10_000 }) + await backendAction(page, 'playback', { command: 'stop' }) +}) + +test('no renderer JS errors and no backend errors', async () => { + ctx.assertNoJsErrors() + const errors = backendErrorLines(ctx.backend) + .filter((l: string) => !/DeprecationWarning|VisibleDeprecation/.test(l)) + expect(errors, `backend errors:\n${errors.join('\n')}`).toEqual([]) +}) diff --git a/spyde/backend/_session_actions.py b/spyde/backend/_session_actions.py index b09eb999..bb62af48 100644 --- a/spyde/backend/_session_actions.py +++ b/spyde/backend/_session_actions.py @@ -218,6 +218,8 @@ def dispatch_action(self, msg: dict) -> None: self.open_file(payload["path"]) elif action == "open_stack": self.open_stack(payload.get("paths") or []) + elif action == "load_insitu_data": + self.load_insitu_data(payload["path"], window_id) elif action == "confirm_nav_shape": self._confirm_nav_shape(payload) elif action == "playback": @@ -497,9 +499,9 @@ def console(self): def _handle_playback(self, payload: dict) -> None: """Play / pause / fast-forward the movie time navigator. Commands: ``play`` / ``pause`` / ``toggle`` (real-time on/off) / ``fast_forward`` - (speed cycle 2→4→8→1) / ``step`` (single frame) / ``set_speed`` / + (speed cycle 2→4→8→16→32→1) / ``step`` (single frame) / ``set_speed`` / ``set_loop``. Playback is real-time (paced from the time axis), so there is - no ``fps``/``step`` speed control any more — ``speed`` is a 1/2/4/8x + no ``fps``/``step`` speed control any more — ``speed`` is a 1/2/4/8/16/32x multiplier and ``loop`` wraps at the end.""" cmd = payload.get("command", "toggle") pb = self.playback diff --git a/spyde/backend/_session_files.py b/spyde/backend/_session_files.py index 5cd73b9b..36912d4d 100644 --- a/spyde/backend/_session_files.py +++ b/spyde/backend/_session_files.py @@ -189,6 +189,79 @@ def open_file(self, path: str) -> None: name=f"load-{name}", ).start() + # Instrument-record extensions the File ▸ Load In-Situ Data… picker offers. + # `.txt` is here because EC-Lab's ASCII export is routinely saved that way; + # the reader dispatches on the file's own magic, not on the extension. + INSITU_EXTS = (".mpr", ".mpt", ".txt", ".mps") + + def load_insitu_data(self, path: str, window_id: int | None = None) -> None: + """Attach an explicitly chosen instrument record to a movie tree. + + Targets the tree owning *window_id* when the renderer supplies one + (the user's focused window), else the most recent tree that has a + frame time base. Attaching needs per-frame timestamps, so a tree + without them is reported rather than silently skipped — that is the + single most likely reason for this to do nothing. + """ + if not os.path.isfile(path): + emit_error(f"File not found: {path}") + return + if _path_ext(path) == ".mps": + self._describe_mps(path) + return + tree = self._insitu_target_tree(window_id) + if tree is None: + emit_error("Open an in-situ movie first, then load its instrument data.") + return + try: + from spyde.insitu.attach import attach_ec_file + result = attach_ec_file(tree, path) + except Exception as e: + emit_error(f"Failed to load {os.path.basename(path)}: {e}") + return + if result: + log.info("insitu: %s", result.describe()) + emit_status(result.describe()) + else: + log.info("insitu: could not attach %s — %s", + os.path.basename(path), result.reason) + emit_error(f"Could not attach {os.path.basename(path)} — {result.reason}") + + def _insitu_target_tree(self, window_id: int | None): + """The tree an attach should land on: the focused window's, else the + most recently opened one that has a per-frame time base.""" + if window_id is not None: + plot = self._plot_by_window_id(window_id) + tree = getattr(getattr(plot, "plot_state", None), "signal_tree", None) + if tree is None: + tree = getattr(plot, "signal_tree", None) + if tree is not None: + return tree + from spyde.insitu.attach import movie_clock_for + for tree in reversed(self.signal_trees): + try: + if movie_clock_for(tree) is not None: + return tree + except Exception as e: + log.debug("probing tree for a frame time base failed: %s", e) + return self.signal_trees[-1] if self.signal_trees else None + + def _describe_mps(self, path: str) -> None: + """An ``.mps`` is the recipe, not the data — say so, and say what it + planned, so picking one is a useful mistake rather than a silent one.""" + try: + from spyde.insitu.eclab import read_mps + settings = read_mps(path) + except Exception as e: + emit_error(f"Failed to read {os.path.basename(path)}: {e}") + return + names = [t.get("name", "?") for t in settings.get("techniques", [])] + emit_status( + f"{os.path.basename(path)} is an EC-Lab settings file (no samples). " + f"It runs {len(names)} technique(s): {', '.join(names) or 'none'}. " + "Load the matching .mpr for the data." + ) + def _open_if_dense_vectors(self, sig, path: str) -> bool: """If *sig* is a saved dense-vectors carrier, reconstruct the vectors and open a Find-Vectors result tree. Returns True if it handled the file.""" @@ -368,14 +441,204 @@ def _load_file_thread(self, path: str) -> None: return for sig in signal: self._maybe_set_insitu_signal_type(sig) - self._add_signal(sig, source_path=path, - navigator_override=_reader_navigator(sig)) + clock = self._apply_frame_timestamps(sig, path) + self._apply_de_pixel_size(sig, path, clock) + tree = self._add_signal(sig, source_path=path, + navigator_override=_reader_navigator(sig)) + self._maybe_attach_insitu(tree, path, clock) self._add_recent(path) ipc.emit({"type": "recent_files", "paths": self._recent_files[:20]}) except Exception as e: ipc.emit({"type": "loading", "busy": False, "text": ""}) emit_error(f"Failed to load {os.path.basename(path)}: {e}") + @staticmethod + def _apply_frame_timestamps(sig: BaseSignal, path: str): + """Calibrate a movie's time axis from its REAL per-frame timestamps. + + A Direct Electron acquisition writes ``*_movie_timestamps.csv`` beside + the movie with one row per saved frame. That file is ground truth, and + the reader's calibration is not: RosettaSciIO derives the period as + ``1 / (fps * Autosave Movie Sum Count)`` when a summed movie's saved + period is ``sum_count / fps`` — wrong by ``sum_count**2``, so a + routine 2-frame sum lands the axis 4× too fast. Everything downstream + inherits that: playback wall-clock pacing, the movie exporter's time + base, the metadata panel's FPS. + + The axis stays UNIFORM (scale = the median measured period) rather + than becoming a per-frame non-uniform axis, because the 1-D selector + resolves a widget position to an index with ``(x - offset) / scale`` + — a non-uniform axis would break navigation for a jitter that is + microseconds wide in practice. The exact times are kept on the + returned clock (and on the tree) for anything that needs them, and a + real gap is reported rather than smoothed over. + + Returns the :class:`~spyde.insitu.de_movie.MovieClock` when it applied, + else None. Best-effort throughout — a movie with no sidecar, or one + whose sidecar disagrees with the data, loads exactly as before. + """ + try: + from spyde.insitu.de_movie import read_movie_clock + except Exception as e: # pragma: no cover - import guard + log.debug("insitu timestamps unavailable: %s", e) + return None + try: + if not FileLoaderMixin._is_movie_time_axis(sig): + return None + clock = read_movie_clock(path) + if clock.n_frames < 2 or clock.frame_period <= 0: + return None + nav = sig.axes_manager.navigation_axes[0] + n_nav = int(sig.axes_manager.navigation_shape[0]) + if clock.n_frames != n_nav: + # One info file per acquisition but one movie per autosave + # session, so a stale/foreign sidecar is a real possibility. + # The measured period is still right (it is a property of the + # camera, not of which frames landed in this file), so take it + # and refuse only the per-frame mapping. + log.warning( + "insitu: %s lists %d frames but the movie has %d — using " + "its frame period, not its per-frame times", + os.path.basename(clock.timestamps_path or path), + clock.n_frames, n_nav, + ) + nav.scale = clock.frame_period + nav.units = "s" + return None + before = float(nav.scale) + nav.scale = clock.frame_period + nav.units = "s" + if not str(getattr(nav, "name", "") or "").strip(): + nav.name = "time" + # Correct the recorded fps too, not just the axis. The reader takes + # `frames_per_second` straight from the DE info file, where it is + # the CAMERA's rate — with `Autosave Movie Sum Count = N` the SAVED + # frames arrive N times slower. Leaving it means the metadata panel + # (which prefers the explicit key — see metadata_extract) reports + # 61 fps beside a 32.76 ms/frame axis: two answers to one question. + # Fix the value at the source rather than teaching every reader of + # it to distrust it. + try: + # Rounded: the metadata chip renders the stored number + # verbatim, and "30.525030441931396 fps" is noise. The AXIS + # keeps the full-precision period; this is the display value. + sig.metadata.set_item( + "Acquisition_instrument.TEM.frames_per_second", + round(1.0 / clock.frame_period, 4), + ) + except Exception as e: + log.debug("stamping corrected fps failed: %s", e) + dropped = clock.dropped_frames() + if dropped.size: + log.info("insitu: %d frame gap(s) in %s", dropped.size, + os.path.basename(clock.timestamps_path or path)) + if abs(before - clock.frame_period) > 1e-9: + msg = ( + f"Frame timing from {os.path.basename(clock.timestamps_path)}: " + f"{clock.frame_period * 1e3:.3f} ms/frame " + f"({1 / clock.frame_period:.2f} fps)" + ) + # Logged as well as emitted: emit_status is the PLOTAPP line + # protocol, consumed by the Electron main process, so it never + # reaches stderr where the e2e harness can see it. + log.info("insitu: %s (reader said %.5f ms)", msg, before * 1e3) + emit_status(msg) + return clock + except Exception as e: + log.debug("applying frame timestamps failed: %s", e) + return None + + @staticmethod + def _apply_de_pixel_size(sig: BaseSignal, path: str, clock=None) -> bool: + """Calibrate a DE frame's SIGNAL axes from its ``*_info.txt``. + + Companion to :meth:`_apply_frame_timestamps` — same principle, the + other axis. RosettaSciIO decides imaging-vs-diffraction with + ``camera_length != -1``, but an imaging exposure records ``0``, so a + plain TEM image is calibrated as diffraction: ``nm^-1`` units and the + unset ``-1`` diffraction pixel size, while the correct specimen pixel + size sits in the same file (see + :func:`spyde.insitu.de_movie.spatial_calibration`). + + Applied only when the info file gives a positive scale, and only to a + 2-D signal. Returns True when it changed anything. + """ + try: + from spyde.insitu.de_movie import ( + find_movie_sidecars, read_info, spatial_calibration, + ) + except Exception as e: # pragma: no cover - import guard + log.debug("insitu pixel size unavailable: %s", e) + return False + try: + if sig.axes_manager.signal_dimension != 2: + return False + info = getattr(clock, "info", None) + info_path = getattr(clock, "info_path", None) + if not info: + _ts, info_path = find_movie_sidecars(path) + if not info_path: + return False + info = read_info(info_path) + calibration = spatial_calibration(info) + if calibration is None: + return False + scale_y, scale_x, units = calibration + axes = sig.axes_manager.signal_axes + # HyperSpy orders signal_axes x-first. + before = (float(axes[0].scale), str(axes[0].units or "")) + if (abs(before[0] - scale_x) < 1e-12 and before[1] == units): + return False + axes[0].scale, axes[0].units = scale_x, units + axes[1].scale, axes[1].units = scale_y, units + # The reader also NAMED them for the branch it took: an image + # mis-read as diffraction comes out as "kx"/"ky". Rename to match + # the space the units now say they are in, but only when they + # carry the reader's own default — never clobber a user's name. + wanted = ("x", "y") if units == "nm" else ("kx", "ky") + unwanted = ("kx", "ky") if units == "nm" else ("x", "y") + for axis, name, stale in zip(axes, wanted, unwanted): + if str(getattr(axis, "name", "") or "").strip().lower() == stale: + axis.name = name + log.info( + "insitu: frame pixel size from %s: %.6g %s/px (reader said " + "%.6g %s)", os.path.basename(info_path or path), scale_x, units, + before[0], before[1] or "-", + ) + emit_status( + f"Pixel size from {os.path.basename(info_path or path)}: " + f"{scale_x:.5g} {units}/px" + ) + return True + except Exception as e: + log.debug("applying DE pixel size failed: %s", e) + return False + + def _maybe_attach_insitu(self, tree, path: str, clock=None) -> None: + """Look beside a freshly-opened movie for instrument data to attach. + + Auto-discovery is silent unless it finds a record whose duration + actually matches the movie's — see + :func:`spyde.insitu.attach.discover_and_attach`. Best-effort: a folder + of unrelated records, an unreadable file or a missing sidecar all just + mean the movie opens as it always did. + """ + if tree is None: + return + try: + from spyde.insitu.attach import discover_and_attach + if clock is not None: + tree.insitu_clock = clock + result = discover_and_attach(tree, path) + except Exception as e: + log.debug("in-situ auto-discovery failed: %s", e) + return + if result: + log.info("insitu: %s", result.describe()) + emit_status(result.describe()) + else: + log.info("insitu: nothing attached — %s", result.reason) + @staticmethod def _is_self_describing(path: str) -> bool: """True for HyperSpy-native formats that store full axes (shape + diff --git a/spyde/insitu/__init__.py b/spyde/insitu/__init__.py new file mode 100644 index 00000000..0603d0e8 --- /dev/null +++ b/spyde/insitu/__init__.py @@ -0,0 +1,51 @@ +"""In-situ auxiliary channels — the non-image time series recorded *alongside* a +movie, and the clock arithmetic that puts them on the same axis as its frames. + +An in-situ experiment produces two independent recordings of the same event: +the camera's frame stack, and whatever the stimulus instrument logged (a +potentiostat's E/I, a heating holder's temperature, a gas cell's pressure). +They come off two machines with two unsynchronised clocks and two sampling +rates, so "what was the potential in frame 4210?" is not a lookup — it is an +alignment problem. This package answers it in three pieces: + +* :mod:`~spyde.insitu.de_movie` reads the Direct Electron sidecars that give a + movie its REAL per-frame time base — ``*_movie_timestamps.csv`` (one row per + saved frame) and ``*_info.txt``. Without them a movie only has the reader's + uniform ``1/fps`` guess. +* :mod:`~spyde.insitu.eclab` reads BioLogic EC-Lab potentiostat records — + ``.mpr`` (binary), ``.mpt``/``.txt`` (ASCII export) and ``.mps`` (settings). +* :mod:`~spyde.insitu.align` finds the offset between the two clocks and + resamples the instrument channels onto the frame times. + +The alignment is deliberately kept separate from both readers: it takes plain +time vectors, so a temperature log or any other per-time channel can reuse it +without either reader being involved. +""" +from __future__ import annotations + +from spyde.insitu.align import ( + Alignment, + align_clocks, + ec_time_for_frames, + frame_for_ec_sample, + match_runs, + resample_to_frames, +) +from spyde.insitu.de_movie import MovieClock, find_movie_sidecars, read_movie_clock +from spyde.insitu.eclab import EcRun, find_ec_runs, read_ec_file, read_mps + +__all__ = [ + "Alignment", + "EcRun", + "MovieClock", + "align_clocks", + "ec_time_for_frames", + "find_ec_runs", + "find_movie_sidecars", + "frame_for_ec_sample", + "match_runs", + "read_ec_file", + "read_movie_clock", + "read_mps", + "resample_to_frames", +] diff --git a/spyde/insitu/align.py b/spyde/insitu/align.py new file mode 100644 index 00000000..404c6f0b --- /dev/null +++ b/spyde/insitu/align.py @@ -0,0 +1,327 @@ +"""Put a movie's frames and an instrument's samples on one time axis. + +The two recordings share an experiment but not a clock. The camera stamps +frames with a free-running monotonic counter; the potentiostat stamps samples +with seconds since its own acquisition start, anchored to the naive local wall +clock of a different PC. Neither knows about the other, and the sampling rates +differ (here: 30.5 frame/s against 12.5 sample/s). Aligning them means finding +one number — the instrument-clock time of movie frame 0 — after which +everything else is interpolation. + +Three ways to find that number, in descending order of trustworthiness: + +``span`` + The two records have the same duration, so they were started and stopped + together; align first sample to first frame. This needs no clock agreement + at all, which is exactly why it is the default when it applies: it is + immune to unsynchronised PCs, timezones and DST. A duration agreement to + well under a second across several minutes is not a coincidence. + +``absolute`` + Convert both to UTC and subtract. Requires the caller to say what timezone + the instrument PC was in (nothing in an EC-Lab file records it) and leans + on the camera's epoch stamp, which has 1 s resolution and is written at + acquisition stop rather than start. Good for picking WHICH run pairs with + which movie; too coarse to trust for the final offset when ``span`` is + available. + +``manual`` + The caller knows better — a trigger wire, a lab notebook, a feature both + records saw. Always available as an override. + +When both ``span`` and ``absolute`` are possible, the result reports the UTC +offset the span solution *implies* (:attr:`Alignment.implied_utc_offset_hours`). +That is the honest way to surface clock skew: if it lands near a whole number of +hours the two PCs simply disagreed about the timezone; if it does not, one of +their clocks was genuinely wrong, and you can see by how much. +""" +from __future__ import annotations + +import datetime as dt +import logging +from dataclasses import dataclass, field + +import numpy as np + +from spyde.insitu.de_movie import MovieClock, info_matches_movie +from spyde.insitu.eclab import DISCRETE_CHANNELS, EcRun + +log = logging.getLogger(__name__) + +# Durations this close (relative, or absolute seconds — whichever is looser) +# count as "started and stopped together". +SPAN_REL_TOL = 0.02 +SPAN_ABS_TOL = 1.0 + + +@dataclass +class Alignment: + """The solved offset between a movie's clock and an instrument's clock. + + :attr:`lag_s` is the whole answer: the instrument-clock time (seconds since + the instrument's acquisition start, the same origin as ``EcRun.time_s``) at + which movie frame 0 was exposed. Everything else on this object is + diagnostics for judging whether to believe it. + """ + + method: str + lag_s: float + duration_mismatch_s: float = 0.0 + overlap_s: float = 0.0 + covered_fraction: float = 0.0 + implied_utc_offset_hours: float | None = None + notes: list[str] = field(default_factory=list) + + @property + def trustworthy(self) -> bool: + """True when the instrument covers essentially the whole movie.""" + return self.covered_fraction > 0.99 + + def describe(self) -> str: + lines = [ + f"method={self.method} lag={self.lag_s:.3f} s " + f"duration mismatch={self.duration_mismatch_s:+.3f} s " + f"overlap={self.overlap_s:.1f} s " + f"frames covered={self.covered_fraction * 100:.1f}%" + ] + if self.implied_utc_offset_hours is not None: + off = self.implied_utc_offset_hours + nearest = round(off) + lines.append( + f"implied instrument-PC UTC offset={off:+.4f} h " + f"({(off - nearest) * 3600:+.1f} s from UTC{nearest:+d})" + ) + lines.extend(f" note: {n}" for n in self.notes) + return "\n".join(lines) + + +def _span_lag(clock: MovieClock, run: EcRun) -> float: + """Instrument-clock time of frame 0 under the co-started assumption.""" + return float(run.time_s[0]) + + +def _absolute_lag(clock: MovieClock, run: EcRun, utc_offset_hours: float, + notes: list[str]) -> float | None: + """Instrument-clock time of frame 0 from the two absolute stamps.""" + epoch = clock.epoch_utc + if epoch is None or run.start is None: + return None + if not info_matches_movie(clock): + notes.append( + "the info file's frame count does not match this movie, so its " + "epoch stamp belongs to a different autosave session" + ) + # The epoch stamp is written at acquisition STOP, so it dates the LAST + # frame; walk back over the movie to reach frame 0. + frame0_utc = epoch - clock.duration + ec_start_utc = run.start_utc(utc_offset_hours) + assert ec_start_utc is not None + return float(frame0_utc - ec_start_utc.timestamp()) + + +def _implied_utc_offset(clock: MovieClock, run: EcRun, lag_s: float) -> float | None: + """The instrument PC's UTC offset implied by *lag_s*, in hours.""" + epoch = clock.epoch_utc + if epoch is None or run.start is None: + return None + frame0_utc = epoch - clock.duration + # Frame 0 in instrument-local wall clock: + frame0_local = run.start + dt.timedelta(seconds=lag_s) + naive_utc = frame0_local.replace(tzinfo=dt.timezone.utc).timestamp() + return (naive_utc - frame0_utc) / 3600.0 + + +def _coverage(clock: MovieClock, run: EcRun, lag_s: float) -> tuple[float, float]: + """``(overlap_seconds, fraction_of_frames_with_instrument_data)``.""" + if clock.n_frames == 0 or run.n_points == 0: + return 0.0, 0.0 + t = clock.t + lag_s + lo, hi = float(run.time_s[0]), float(run.time_s[-1]) + inside = (t >= lo) & (t <= hi) + overlap = max(0.0, min(t[-1], hi) - max(t[0], lo)) + return float(overlap), float(np.count_nonzero(inside) / t.size) + + +def align_clocks( + clock: MovieClock, + run: EcRun, + *, + method: str = "auto", + lag_s: float | None = None, + ec_utc_offset_hours: float | None = None, +) -> Alignment: + """Solve the movie↔instrument clock offset. + + *method* is ``"auto"`` (span if the durations agree, else absolute), + ``"span"``, ``"absolute"`` or ``"manual"``. ``"manual"`` requires *lag_s*; + ``"absolute"`` requires *ec_utc_offset_hours* (the instrument PC's offset + from UTC — no EC-Lab file records it). + """ + if clock.n_frames == 0: + raise ValueError("movie clock has no frames") + if run.n_points == 0: + raise ValueError(f"{run.name}: instrument run has no samples") + + notes: list[str] = [] + mismatch = run.duration - clock.duration + span_tol = max(SPAN_ABS_TOL, SPAN_REL_TOL * max(run.duration, clock.duration)) + spans_agree = abs(mismatch) <= span_tol + + if method == "manual": + if lag_s is None: + raise ValueError("method='manual' requires lag_s") + chosen, chosen_lag = "manual", float(lag_s) + elif method == "span": + chosen, chosen_lag = "span", _span_lag(clock, run) + if not spans_agree: + notes.append( + f"durations differ by {mismatch:+.2f} s (tolerance {span_tol:.2f} s) " + "— the records may not be co-extensive" + ) + elif method == "absolute": + if ec_utc_offset_hours is None: + raise ValueError("method='absolute' requires ec_utc_offset_hours") + got = _absolute_lag(clock, run, ec_utc_offset_hours, notes) + if got is None: + raise ValueError( + "cannot align by absolute time: need both the movie's epoch " + "stamp (from *_info.txt) and the run's acquisition start" + ) + chosen, chosen_lag = "absolute", got + elif method == "auto": + if spans_agree: + chosen, chosen_lag = "span", _span_lag(clock, run) + notes.append( + f"durations agree to {abs(mismatch):.3f} s over {run.duration:.1f} s " + "— treating the two records as co-started" + ) + elif ec_utc_offset_hours is not None: + got = _absolute_lag(clock, run, ec_utc_offset_hours, notes) + if got is None: + raise ValueError("no usable absolute stamps for method='auto'") + chosen, chosen_lag = "absolute", got + notes.append( + f"durations differ by {mismatch:+.2f} s, so the records are not " + "co-extensive; fell back to the absolute clocks" + ) + else: + raise ValueError( + f"durations differ by {mismatch:+.2f} s (tolerance {span_tol:.2f} s), " + "so the records are not co-extensive. Pass ec_utc_offset_hours to " + "align by absolute time, or lag_s with method='manual'." + ) + else: + raise ValueError(f"unknown alignment method {method!r}") + + overlap, covered = _coverage(clock, run, chosen_lag) + if covered < 1.0: + notes.append( + f"{(1 - covered) * 100:.1f}% of frames fall outside the instrument " + "record and will resample to NaN" + ) + return Alignment( + method=chosen, + lag_s=chosen_lag, + duration_mismatch_s=float(mismatch), + overlap_s=overlap, + covered_fraction=covered, + implied_utc_offset_hours=_implied_utc_offset(clock, run, chosen_lag), + notes=notes, + ) + + +def ec_time_for_frames(clock: MovieClock, alignment: Alignment) -> np.ndarray: + """Each movie frame's time on the instrument clock (seconds since the + instrument's acquisition start).""" + return clock.t + alignment.lag_s + + +def resample_to_frames( + clock: MovieClock, + run: EcRun, + alignment: Alignment, + channels: list[str] | None = None, +) -> dict[str, np.ndarray]: + """Resample the instrument's channels onto the movie's frame times. + + Returns one float64 array per channel, each ``clock.n_frames`` long, plus + ``"time/s"`` (the instrument-clock time of every frame). Frames outside the + instrument record are **NaN**, never clamped — a movie that ran on past the + end of the sweep must not show the last potential held flat, because that + is a measurement that was never made. + + Continuous channels are linearly interpolated. Channels that step rather + than vary (:data:`~spyde.insitu.eclab.DISCRETE_CHANNELS` — cycle number, + I Range, the mode/ox-red flags) take the nearest sample instead, since an + interpolated cycle 1.5 or a current range between two ranges never existed. + """ + t_frames = ec_time_for_frames(clock, alignment) + t_ec = run.time_s + order = np.argsort(t_ec) + t_sorted = t_ec[order] + inside = (t_frames >= t_sorted[0]) & (t_frames <= t_sorted[-1]) + + wanted = channels if channels is not None else list(run.channels) + out: dict[str, np.ndarray] = {"time/s": t_frames} + for name in wanted: + values = run.channels.get(name) + if values is None: + log.warning("align: no channel %r in %s", name, run.name) + continue + y = np.asarray(values, dtype=float)[order] + if name in DISCRETE_CHANNELS or np.asarray(values).dtype.kind in "biu": + idx = np.searchsorted(t_sorted, t_frames) + idx = np.clip(idx, 1, t_sorted.size - 1) + left = t_frames - t_sorted[idx - 1] <= t_sorted[idx] - t_frames + resampled = y[np.where(left, idx - 1, idx)] + else: + resampled = np.interp(t_frames, t_sorted, y) + out[name] = np.where(inside, resampled, np.nan) + return out + + +def frame_for_ec_sample( + clock: MovieClock, run: EcRun, alignment: Alignment +) -> np.ndarray: + """The nearest movie frame index for each instrument sample. + + The inverse mapping of :func:`resample_to_frames`, for going the other way: + "show me the frame where the current peaked". Samples falling outside the + movie get ``-1``. + """ + t_frames = ec_time_for_frames(clock, alignment) + idx = np.searchsorted(t_frames, run.time_s) + idx = np.clip(idx, 1, max(1, t_frames.size - 1)) + left = run.time_s - t_frames[idx - 1] <= t_frames[idx] - run.time_s + nearest = np.where(left, idx - 1, idx) + outside = (run.time_s < t_frames[0]) | (run.time_s > t_frames[-1]) + return np.where(outside, -1, nearest).astype(np.int64) + + +def match_runs( + clock: MovieClock, + runs: list[EcRun], + *, + ec_utc_offset_hours: float | None = None, +) -> list[tuple[EcRun, Alignment]]: + """Rank instrument runs by how well each explains this movie. + + The scoring is duration agreement: of all the techniques in a session, the + one whose record is the same length as the movie is the one that ran + *during* it. Runs that cannot be aligned at all are dropped. Best first. + """ + scored: list[tuple[float, EcRun, Alignment]] = [] + for run in runs: + if run.n_points < 2 or clock.duration <= 0: + continue + try: + alignment = align_clocks( + clock, run, method="auto", ec_utc_offset_hours=ec_utc_offset_hours + ) + except ValueError as exc: + log.debug("align: %s does not match this movie (%s)", run.name, exc) + continue + penalty = abs(alignment.duration_mismatch_s) / max(clock.duration, 1e-9) + scored.append((penalty - alignment.covered_fraction, run, alignment)) + scored.sort(key=lambda item: item[0]) + return [(run, alignment) for _, run, alignment in scored] diff --git a/spyde/insitu/attach.py b/spyde/insitu/attach.py new file mode 100644 index 00000000..ae73881c --- /dev/null +++ b/spyde/insitu/attach.py @@ -0,0 +1,294 @@ +"""Attach aligned instrument channels to a loaded movie's signal tree. + +This is the glue between the readers in this package and the app: it finds the +instrument record that belongs to a movie, solves the clock offset, resamples +the channels onto the frame times, and registers the interesting ones as named +navigator signals so they appear as chips beside the movie's own navigator. + +Registering them as navigators rather than inventing a display is deliberate. +A named navigator already gets a chip, already stacks onto one shared time +cursor when several are selected (``navigator_views.select_navigator``), and +already drives the movie when dragged — all of which is written and tested. An +E/t trace is an ordinary line, so the generic stacked builder draws it +correctly; nothing here needs its own figure code. + +Two entry points, one path underneath: + +* :func:`discover_and_attach` — the automatic one, run when a movie loads. It + scans the movie's own folder, and stays silent unless it finds a record that + really does explain the movie (:func:`~spyde.insitu.align.match_runs` + scores on duration agreement). +* :func:`attach_ec_file` — the manual one, behind File ▸ Load In-Situ Data…, + for a record that lives somewhere else or that the scorer passed over. + +Only the potential and current become chips. The full resampled table is left +on the tree as ``insitu_channels``, because a chip strip holding eleven +entries — most of them flags — makes the two that matter harder to find, not +easier. +""" +from __future__ import annotations + +import logging +import os + +import numpy as np + +from spyde.insitu.align import align_clocks, match_runs, resample_to_frames +from spyde.insitu.de_movie import MovieClock, read_movie_clock +from spyde.insitu.eclab import EcRun, find_ec_runs, read_ec_file + +log = logging.getLogger(__name__) + +POTENTIAL_LANE = "Ewe (V)" +CURRENT_LANE = "I" + +# Below this the current reads better in µA than in mA — a 0.0001 axis is +# unreadable. The lane NAME carries whichever unit was chosen, so the scaling +# is never silent. +_MICROAMP_CUTOFF_MA = 1.0 + + +class AttachResult: + """What an attach attempt did, for the caller's status line.""" + + def __init__(self, run=None, alignment=None, lanes=(), reason=""): + self.run = run + self.alignment = alignment + self.lanes = tuple(lanes) + self.reason = reason + + def __bool__(self) -> bool: + return bool(self.lanes) + + def describe(self) -> str: + if not self: + return self.reason or "no in-situ data attached" + al = self.alignment + return ( + f"Attached {self.run.technique} from {self.run.name} — " + f"{self.run.n_points} samples aligned by {al.method}, " + f"{abs(al.duration_mismatch_s):.2f} s over {self.run.duration:.1f} s, " + f"{al.covered_fraction * 100:.0f}% of frames covered" + ) + + +def movie_clock_for(tree, path: str | None = None) -> MovieClock | None: + """The tree's frame time base — cached on the tree by the loader, else read.""" + clock = getattr(tree, "insitu_clock", None) + if clock is not None: + return clock + source = path or getattr(tree, "source_path", None) + if not source: + return None + try: + clock = read_movie_clock(source) + except (FileNotFoundError, ValueError, OSError) as exc: + log.debug("insitu: no frame time base for %s (%s)", source, exc) + return None + tree.insitu_clock = clock + return clock + + +def _current_lane(values: np.ndarray) -> tuple[str, np.ndarray]: + """Name and scale the current lane so its axis is readable.""" + finite = values[np.isfinite(values)] + peak = float(np.max(np.abs(finite))) if finite.size else 0.0 + if 0 < peak < _MICROAMP_CUTOFF_MA: + return f"{CURRENT_LANE} (µA)", values * 1e3 + return f"{CURRENT_LANE} (mA)", values + + +def _register_lanes(tree, columns: dict[str, np.ndarray]) -> list[str]: + """Ensure the potential and current exist as named navigator signals. + + A navigator signal must be exactly as long as the movie's navigation axis + (``_preprocess_navigator`` enforces it), and NaN would poison the display + levels — so frames outside the instrument record are filled with the + nearest in-record value here. The authoritative NaN-bearing arrays stay on + ``tree.insitu_channels``; this is a display copy. + + Attaching twice is an ordinary thing to do — auto-discovery runs on open + and the user may then pick a record by hand — so a lane that already + exists has its VALUES replaced in place rather than being skipped or + registered again. Re-registering would add a second plot state for the + same name; skipping would leave the old run's trace on screen while the + tree claims the new one. Mutating the existing signal's array keeps one + plot state and one truth. + + Returns every lane now present, whether it was created or updated — so the + caller can tell "this run has no E/I channel" (empty) from "nothing left + to do" (non-empty), which is not the same outcome. + """ + # The movie's OWN navigation calibration has to land on each lane's signal + # axis. A 1-D selector turns a widget position into a frame index with + # ``(x - offset) / scale`` read from the navigator plot's own signal axis, + # so an uncalibrated lane is indexed in seconds against a scale of 1 — the + # stacked view's x-axis reads 0…7913 instead of 0…259 s, and dragging its + # cursor resolves to the wrong frame. ``calibrated_nav_signal`` is the + # existing helper for exactly this; imported lazily so this package stays + # usable without the backend. + from spyde.backend._session_files import calibrated_nav_signal + + lanes: list[tuple[str, np.ndarray]] = [] + potential = next( + (columns[k] for k in ("Ewe/V", "/V", "|E|/V") if k in columns), None + ) + if potential is not None: + lanes.append((POTENTIAL_LANE, potential)) + current = next( + (columns[k] for k in ("/mA", "I/mA", "|I|/mA") if k in columns), None + ) + if current is not None: + lanes.append(_current_lane(current)) + + present: list[str] = [] + existing = getattr(tree, "navigator_signals", {}) + for name, values in lanes: + display = _fill_edges(values).astype(np.float32) + try: + if name in existing: + _replace_lane_data(existing[name], display) + else: + lane = calibrated_nav_signal(display, tree.root) + lane.metadata.General.title = name + tree.add_navigator_signal(name, lane) + present.append(name) + except Exception as exc: + # Warning, not debug: a lane that silently fails to register looks + # from the outside exactly like "this run has no E/I channel", and + # chasing that took a round trip through the real app. + log.warning("insitu: registering navigator %r failed: %s", name, exc) + return present + + +def _replace_lane_data(entry, values: np.ndarray) -> None: + """Overwrite an already-registered lane's samples in place. + + ``add_navigator_signal`` stores whatever ``_preprocess_navigator`` returned, + and that is ALWAYS a list — ``[signal]`` for a plain trace, or + ``[navigator, signal]`` for a navigated one — never the bare signal handed + in. So unwrap before touching ``.data``, and write through the entries + whose shape matches rather than rebinding the list. + """ + targets = entry if isinstance(entry, (list, tuple)) else [entry] + updated = 0 + for signal in targets: + data = getattr(signal, "data", None) + if data is None: + continue + shape = np.asarray(data).shape + if shape != values.shape: + continue + signal.data = values.reshape(shape) + updated += 1 + if not updated: + shapes = [getattr(getattr(s, "data", None), "shape", None) for s in targets] + raise ValueError( + f"no registered lane array matches {values.shape} (found {shapes})" + ) + + +def _fill_edges(values: np.ndarray) -> np.ndarray: + """Replace NaN with the nearest finite value (display copy only).""" + out = np.asarray(values, dtype=float).copy() + finite = np.isfinite(out) + if not finite.any(): + return np.zeros_like(out) + idx = np.arange(out.size) + out[~finite] = np.interp(idx[~finite], idx[finite], out[finite]) + return out + + +def _store(tree, clock, run, alignment, columns) -> None: + tree.insitu_clock = clock + tree.insitu_run = run + tree.insitu_alignment = alignment + tree.insitu_channels = columns + + +def attach_run(tree, clock: MovieClock, run: EcRun, alignment) -> AttachResult: + """Resample *run* onto *clock*'s frames and register the display lanes.""" + columns = resample_to_frames(clock, run, alignment) + nav = _nav_size(tree) + if nav is not None and nav != clock.n_frames: + return AttachResult( + reason=( + f"the movie has {nav} frames but " + f"{os.path.basename(clock.timestamps_path or '?')} lists " + f"{clock.n_frames} — cannot map instrument samples to frames" + ) + ) + _store(tree, clock, run, alignment, columns) + lanes = _register_lanes(tree, columns) + if not lanes: + return AttachResult( + run, alignment, (), + f"{run.name} has no potential or current channel — it records " + f"{', '.join(sorted(run.channels)) or 'nothing'}", + ) + return AttachResult(run, alignment, lanes) + + +def _nav_size(tree) -> int | None: + try: + shape = tree.root.axes_manager.navigation_shape + return int(shape[0]) if len(shape) == 1 else None + except Exception: + return None + + +def discover_and_attach(tree, path: str) -> AttachResult: + """Look beside *path* for an instrument record that explains this movie. + + Silent by design when nothing matches: an in-situ movie sitting in a folder + of unrelated records is the common case, and a false attach is worse than + none. :func:`~spyde.insitu.align.match_runs` only returns runs whose + duration agrees with the movie's, so "found nothing" here means "found + nothing that could plausibly have been recorded during this movie". + """ + clock = movie_clock_for(tree, path) + if clock is None: + return AttachResult(reason="no frame timestamps beside this movie") + directory = os.path.dirname(os.path.abspath(path)) or "." + try: + runs = find_ec_runs(directory) + except OSError as exc: + return AttachResult(reason=f"could not scan {directory}: {exc}") + if not runs: + return AttachResult(reason="no EC-Lab records in this folder") + ranked = match_runs(clock, runs) + if not ranked: + return AttachResult( + reason=( + f"{len(runs)} EC-Lab record(s) beside this movie, none matching " + f"its {clock.duration:.1f} s duration" + ) + ) + run, alignment = ranked[0] + return attach_run(tree, clock, run, alignment) + + +def attach_ec_file(tree, ec_path: str, *, movie_path: str | None = None, + method: str = "auto", lag_s: float | None = None, + ec_utc_offset_hours: float | None = None) -> AttachResult: + """Attach one explicitly chosen instrument record to *tree*.""" + clock = movie_clock_for(tree, movie_path) + if clock is None: + return AttachResult( + reason=("this dataset has no per-frame time base — a DE movie needs " + "its *_movie_timestamps.csv beside it") + ) + try: + run = read_ec_file(ec_path) + except (ValueError, OSError) as exc: + return AttachResult(reason=f"could not read {os.path.basename(ec_path)}: {exc}") + if run.n_points < 2: + return AttachResult(reason=f"{run.name} has no samples") + try: + alignment = align_clocks( + clock, run, method=method, lag_s=lag_s, + ec_utc_offset_hours=ec_utc_offset_hours, + ) + except ValueError as exc: + return AttachResult(reason=f"could not align {run.name}: {exc}") + return attach_run(tree, clock, run, alignment) diff --git a/spyde/insitu/de_movie.py b/spyde/insitu/de_movie.py new file mode 100644 index 00000000..26714576 --- /dev/null +++ b/spyde/insitu/de_movie.py @@ -0,0 +1,369 @@ +"""Direct Electron movie sidecars — the REAL per-frame time base. + +A DE acquisition writes, beside ``_movie.mrc``: + +* ``_movie_timestamps.csv`` — ``Frame Index, Timestamp (s), Electrons``, + one row per SAVED frame. ``Timestamp (s)`` is a free-running monotonic camera + clock (values in the ~10^6 s range: uptime, not an epoch), so it says exactly + *when each frame happened relative to every other frame* and nothing about + where that sits in wall-clock time. +* ``_info.txt`` — ``key = value`` acquisition metadata, including + ``Timestamp (seconds since Epoch)``, the single absolute anchor available. + +Two reasons this matters more than it looks: + +**The uniform axis is a guess, and here it is a wrong one.** RosettaSciIO's MRC +reader derives the time scale from the metadata as +``1 / (Frames Per Second * Autosave Movie Sum Count)`` (``rsciio/mrc/_api.py``). +Summing ``N`` camera frames into one saved frame makes the saved period +``N / fps``, not ``1 / (fps*N)`` — so a 2-frame sum at 61.05 fps is calibrated +0.00819 s/frame when the timestamps say 0.03276 s/frame, a factor of ``N**2`` +too fast. The CSV is ground truth and does not need the formula to be right. + +**The epoch anchor is coarse and LATE.** ``Timestamp (seconds since Epoch)`` has +1 s resolution and is written when the info file is written — after the +acquisition stops (the same file reports ``Acquisition Status = Stopped`` and an +``Acquisition Counter`` already incremented past this dataset's). So it dates +the END of the movie, give or take the flush, and it is a UTC epoch while the +instrument log it must be matched against is normally naive local time. Treat it +as a coarse hint, never as a precise sync — which is why +:mod:`spyde.insitu.align` prefers to match on span and keeps the absolute route +as a cross-check. +""" +from __future__ import annotations + +import csv +import datetime as dt +import glob +import logging +import os +import re +from dataclasses import dataclass, field + +import numpy as np + +log = logging.getLogger(__name__) + +TIMESTAMPS_SUFFIX = "_movie_timestamps.csv" +INFO_SUFFIX = "_info.txt" + +# The CSV column header, normalised (lowercased, stripped). Kept loose because +# the units suffix has changed between DE server versions. +_FRAME_COL = "frame index" +_TIME_COL = "timestamp" +_ELECTRONS_COL = "electrons" + + +@dataclass +class MovieClock: + """The per-frame time base of one DE movie, plus its acquisition metadata. + + ``t_camera`` is the raw monotonic camera clock in seconds — only DIFFERENCES + of it are meaningful. ``t`` is the same thing rebased to zero at the first + saved frame, which is what you want as the movie's own time axis. + """ + + frame_index: np.ndarray + t_camera: np.ndarray + electrons: np.ndarray | None = None + info: dict[str, str] = field(default_factory=dict) + timestamps_path: str | None = None + info_path: str | None = None + + @property + def n_frames(self) -> int: + return int(self.t_camera.size) + + @property + def t(self) -> np.ndarray: + """Frame times in seconds since the first saved frame.""" + if self.t_camera.size == 0: + return self.t_camera + return self.t_camera - self.t_camera[0] + + @property + def duration(self) -> float: + """First-frame-to-last-frame span in seconds (one period short of the + total exposed time — see :attr:`frame_period`).""" + if self.t_camera.size < 2: + return 0.0 + return float(self.t_camera[-1] - self.t_camera[0]) + + @property + def frame_period(self) -> float: + """Median saved-frame period in seconds (robust to a dropped frame).""" + if self.t_camera.size < 2: + return 0.0 + return float(np.median(np.diff(self.t_camera))) + + @property + def epoch_utc(self) -> float | None: + """``Timestamp (seconds since Epoch)`` from the info file, or None. + + Coarse (1 s) and stamped at acquisition STOP — see the module docstring. + """ + raw = self.info.get("Timestamp (seconds since Epoch)") + if raw is None: + return None + try: + return float(raw) + except ValueError: + log.warning("de_movie: uninterpretable epoch timestamp %r", raw) + return None + + def epoch_datetime(self, tz: dt.tzinfo | None = dt.timezone.utc) -> dt.datetime | None: + """:attr:`epoch_utc` as an aware datetime, or None.""" + ep = self.epoch_utc + if ep is None: + return None + return dt.datetime.fromtimestamp(ep, dt.timezone.utc).astimezone(tz) + + def dropped_frames(self, tol: float = 0.5) -> np.ndarray: + """Indices ``i`` where the gap to frame ``i+1`` exceeds ``1 + tol`` + periods — i.e. where the camera lost frames.""" + if self.t_camera.size < 3: + return np.empty(0, dtype=int) + gaps = np.diff(self.t_camera) + period = self.frame_period + if period <= 0: + return np.empty(0, dtype=int) + return np.flatnonzero(gaps > period * (1.0 + tol)) + + @property + def reader_frame_period(self) -> float | None: + """The period RosettaSciIO's MRC reader would derive from the metadata. + + Exposed so a caller can SEE the discrepancy rather than guess at it; + compare against :attr:`frame_period`. + """ + try: + fps = float(self.info["Frames Per Second"]) + n_sum = float(self.info.get("Autosave Movie Sum Count", 1) or 1) + except (KeyError, TypeError, ValueError): + return None + if fps <= 0 or n_sum <= 0: + return None + return 1.0 / (fps * n_sum) + + +def _strip_suffix(path: str) -> str: + """``…_movie.mrc`` / ``…_movie_timestamps.csv`` / ``…_info.txt`` → ``…``.""" + base = path + for suffix in (TIMESTAMPS_SUFFIX, INFO_SUFFIX): + if base.endswith(suffix): + return base[: -len(suffix)] + base = os.path.splitext(base)[0] + if base.endswith("_movie"): + base = base[: -len("_movie")] + return base + + +def find_movie_sidecars(path: str) -> tuple[str | None, str | None]: + """Locate ``(timestamps_csv, info_txt)`` for *path* (the .mrc, or either + sidecar). Returns None for whichever is absent. + + DE names the info file after the ACQUISITION while the movie file also + carries a per-autosave-session number (``…_88071_run1_2616_movie.mrc`` vs + ``…_88071_info.txt``), so an exact stem match is not enough — we widen to a + glob in the same directory and take the longest common prefix. + """ + stem = _strip_suffix(path) + ts = stem + TIMESTAMPS_SUFFIX + info = stem + INFO_SUFFIX + found_ts = ts if os.path.exists(ts) else None + found_info = info if os.path.exists(info) else None + + directory = os.path.dirname(os.path.abspath(path)) or "." + name = os.path.basename(stem) + for suffix, current in ((TIMESTAMPS_SUFFIX, found_ts), (INFO_SUFFIX, found_info)): + if current is not None: + continue + best, best_len = None, 0 + for cand in glob.glob(os.path.join(glob.escape(directory), "*" + suffix)): + cand_stem = os.path.basename(cand)[: -len(suffix)] + shared = len(os.path.commonprefix([name, cand_stem])) + # Require a real shared prefix, not just a leading date. + if shared > best_len and shared >= min(8, len(cand_stem)): + best, best_len = cand, shared + if suffix == TIMESTAMPS_SUFFIX: + found_ts = best + else: + found_info = best + return found_ts, found_info + + +def read_info(path: str) -> dict[str, str]: + """Parse a DE ``*_info.txt`` into ``{key: value}`` (both stripped).""" + out: dict[str, str] = {} + with open(path, encoding="utf-8", errors="replace") as fh: + for line in fh: + key, sep, value = line.partition("=") + if not sep: + continue + out[key.strip()] = value.strip() + return out + + +def read_timestamps(path: str) -> tuple[np.ndarray, np.ndarray, np.ndarray | None]: + """Parse a DE ``*_movie_timestamps.csv`` → ``(frame_index, t_camera, electrons)``. + + Column order is taken from the header rather than assumed, and a trailing + torn row (acquisition killed mid-write) is dropped rather than raising. + """ + frames: list[int] = [] + times: list[float] = [] + electrons: list[float] = [] + with open(path, newline="", encoding="utf-8", errors="replace") as fh: + reader = csv.reader(fh) + try: + header = next(reader) + except StopIteration: + raise ValueError(f"{path}: empty timestamps file") from None + norm = [c.strip().lower() for c in header] + + def _col(want: str) -> int | None: + for i, c in enumerate(norm): + if c.startswith(want): + return i + return None + + i_frame, i_time, i_el = _col(_FRAME_COL), _col(_TIME_COL), _col(_ELECTRONS_COL) + if i_time is None: + raise ValueError(f"{path}: no 'Timestamp' column in header {header!r}") + for row_no, row in enumerate(reader, start=2): + if not row or not row[0].strip(): + continue + try: + t = float(row[i_time]) + f = int(float(row[i_frame])) if i_frame is not None else len(times) + e = float(row[i_el]) if i_el is not None and i_el < len(row) else np.nan + except (IndexError, ValueError): + log.warning("de_movie: dropping unparseable row %d of %s", row_no, path) + continue + frames.append(f) + times.append(t) + electrons.append(e) + + t = np.asarray(times, dtype=float) + idx = np.asarray(frames, dtype=np.int64) + el = np.asarray(electrons, dtype=float) + return idx, t, (None if el.size == 0 or np.all(np.isnan(el)) else el) + + +def read_movie_clock(path: str) -> MovieClock: + """Read the frame time base for the movie at *path*. + + *path* may be the ``.mrc``, the timestamps ``.csv`` or the ``_info.txt`` — + the siblings are located automatically. Raises ``FileNotFoundError`` if no + timestamps file can be found, since without it there is no real time base. + """ + ts_path, info_path = find_movie_sidecars(path) + if ts_path is None: + raise FileNotFoundError( + f"no '*{TIMESTAMPS_SUFFIX}' sidecar found beside {path!r}; " + "the movie has no per-frame time base without it" + ) + idx, t, electrons = read_timestamps(ts_path) + info = read_info(info_path) if info_path else {} + + if t.size and np.any(np.diff(t) <= 0): + log.warning("de_movie: %s is not strictly increasing", ts_path) + + clock = MovieClock( + frame_index=idx, + t_camera=t, + electrons=electrons, + info=info, + timestamps_path=ts_path, + info_path=info_path, + ) + reader_period = clock.reader_frame_period + if reader_period and clock.frame_period > 0: + ratio = clock.frame_period / reader_period + if abs(ratio - 1.0) > 0.01: + log.info( + "de_movie: %s measured frame period %.6f s vs reader-derived " + "%.6f s (x%.3g) — using the timestamps", + os.path.basename(ts_path), clock.frame_period, reader_period, ratio, + ) + return clock + + +_FRAMES_WRITTEN_KEYS = ("Autosave Movie Frames Written", "Number of Frames Processed") + + +def info_matches_movie(clock: MovieClock) -> bool: + """True when the info file's frame count agrees with the timestamps file. + + DE writes ONE info file per acquisition but a new movie file per autosave + session, so a mismatched count means the info file (and therefore its epoch + stamp) belongs to a DIFFERENT session and must not be used as this movie's + absolute anchor. + """ + for key in _FRAMES_WRITTEN_KEYS: + raw = clock.info.get(key) + if raw is None: + continue + try: + return int(float(raw)) == clock.n_frames + except ValueError: + continue + return False + + +def _info_float(info: dict, key: str) -> float | None: + raw = info.get(key) + if raw is None: + return None + try: + return float(raw) + except (TypeError, ValueError): + return None + + +def spatial_calibration(info: dict) -> tuple[float, float, str] | None: + """``(scale_y, scale_x, units)`` for a DE frame, or None if undeterminable. + + DE records the real-space pixel size as ``Specimen Pixel Size X/Y + (nanometers)`` and the reciprocal one as ``Diffraction Pixel Size X/Y``, + with ``-1`` meaning "not applicable". Which pair applies is decided by + ``Instrument Project Camera Length (centimeters)``: a genuine diffraction + exposure has a POSITIVE camera length. + + RosettaSciIO gets that test wrong — it asks ``camera_length != -1``, so the + ``0`` an imaging exposure records reads as "has a camera length" and the + frame is calibrated as diffraction. The reciprocal pixel size is then the + unset ``-1``, which survives its own ``== -1`` guard (the value is a string + at that point), and a TEM image ends up at ``-1.00 nm^-1`` per pixel with + the correct ``1.14786 nm`` sitting unused in the same file. + + So the sentinel test here is ``> 0``, applied to both candidates: a scale + is only used if it is a real positive number. + """ + camera_length = _info_float(info, "Instrument Project Camera Length (centimeters)") + diff_x = _info_float(info, "Diffraction Pixel Size X") + diff_y = _info_float(info, "Diffraction Pixel Size Y") + spec_x = _info_float(info, "Specimen Pixel Size X (nanometers)") + spec_y = _info_float(info, "Specimen Pixel Size Y (nanometers)") + + diffracting = bool(camera_length and camera_length > 0) + if diffracting and diff_x and diff_x > 0 and diff_y and diff_y > 0: + return diff_y, diff_x, "nm^-1" + if spec_x and spec_x > 0 and spec_y and spec_y > 0: + return spec_y, spec_x, "nm" + # Imaging exposure with no specimen pixel size, or a diffraction one with + # no diffraction pixel size — nothing trustworthy to say. + if diff_x and diff_x > 0 and diff_y and diff_y > 0: + return diff_y, diff_x, "nm^-1" + return None + + +def parse_de_datetime(text: str) -> dt.datetime | None: + """Parse the ``YYYYMMDD`` prefix DE puts on dataset names, for a sanity date.""" + m = re.match(r"(\d{4})(\d{2})(\d{2})", text.strip()) + if not m: + return None + try: + return dt.datetime(int(m[1]), int(m[2]), int(m[3])) + except ValueError: + return None diff --git a/spyde/insitu/eclab.py b/spyde/insitu/eclab.py new file mode 100644 index 00000000..e9c2c5e8 --- /dev/null +++ b/spyde/insitu/eclab.py @@ -0,0 +1,621 @@ +"""BioLogic EC-Lab potentiostat records — ``.mpr``, ``.mpt``/``.txt``, ``.mps``. + +An EC-Lab experiment is one *settings* file (``.mps``, the recipe) plus one +*record* per linked technique, written twice over: a binary ``.mpr`` and, +whenever the operator remembers to export it, an ASCII ``.mpt`` (often saved as +``.txt``). Both carry the same samples; only the ``.mpr`` is guaranteed to +exist, so it is the primary path here and the ASCII reader is the cross-check. + +Time in an EC-Lab record is ``time/s``, seconds since the ACQUISITION started — +which is *before* the technique started, because a linked sequence keeps one +clock across all its techniques. A second technique therefore begins at a +``time/s`` of tens or hundreds of seconds, not at zero. The absolute origin for +that clock lives in the ``VMP LOG`` module as an OLE date (and in the ASCII +header as ``Acquisition started on``); it is the EC-Lab PC's **naive local wall +clock**, with no timezone recorded anywhere in the file. + +Format notes (reverse-engineered; cross-checked against an ASCII export of the +same run, see ``spyde/tests/migrated/test_eclab.py``): + +* A module header is ``b"MODULE"`` + 10-byte short name + 25-byte long name, + then either a 32-bit length, or — when that field reads ``0xFFFFFFFF`` — a + 64-bit length following it. Both layouts appear in the wild. +* The ``VMP data`` body is ``npts:u4, ncols:u2, ids:u2[ncols]`` and then a + fixed block of padding before the records. Rather than hard-code that padding + (it is version-dependent), the record start is DERIVED as + ``len(body) - npts * record_size``, which self-checks the column layout: if + the assumed widths were wrong the subtraction would not land on a plausible + offset over a run of zero bytes. +* Several column IDs are *flags* packed into a single leading ``u1`` rather than + columns of their own. + +Coverage: the column table below holds the IDs seen in real CV/OCV records. An +unrecognised ID is not guessed at — a wrong width silently shifts every later +column and corrupts the whole record — except in the one case where exactly one +ID is unknown and the byte arithmetic pins its width unambiguously. Anything +else raises :class:`UnsupportedColumns`, which names the IDs and points at the +ASCII export as the way through. +""" +from __future__ import annotations + +import datetime as dt +import glob +import logging +import os +import re +import struct +from dataclasses import dataclass, field + +import numpy as np + +log = logging.getLogger(__name__) + +MPR_MAGIC = b"BIO-LOGIC MODULAR FILE\x1a" +MPT_MAGIC = "EC-Lab ASCII FILE" + +_OLE_EPOCH = dt.datetime(1899, 12, 30) +# Candidate byte offsets of the acquisition-start OLE date inside VMP LOG. +# Which one is populated varies between EC-Lab versions; the first that decodes +# to a sane calendar date wins. +_LOG_DATE_OFFSETS = (585, 465, 469, 473, 993) +_OLE_MIN, _OLE_MAX = 36525.0, 73050.0 # 2000-01-01 .. 2100-01-01 + +# Column IDs that are BIT FLAGS sharing one leading u1 byte, not columns. +_FLAG_IDS: dict[int, tuple[str, int]] = { + 1: ("mode", 0x03), + 2: ("ox/red", 0x04), + 3: ("error", 0x08), + 21: ("control changes", 0x10), + 31: ("Ns changes", 0x20), + 65: ("counter inc.", 0x80), +} + +# Column ID -> (name, numpy dtype). Verified against an ASCII export. +_COLUMN_IDS: dict[int, tuple[str, str]] = { + 4: ("time/s", "/mA", "/V", " None: + self.time_s = np.asarray(self.time_s, dtype=float) + + @property + def n_points(self) -> int: + return int(self.time_s.size) + + @property + def duration(self) -> float: + """First-to-last sample span in seconds.""" + if self.time_s.size < 2: + return 0.0 + return float(self.time_s[-1] - self.time_s[0]) + + @property + def sample_period(self) -> float: + """Median sampling period in seconds.""" + if self.time_s.size < 2: + return 0.0 + return float(np.median(np.diff(self.time_s))) + + @property + def name(self) -> str: + return os.path.splitext(os.path.basename(self.path))[0] + + def start_utc(self, utc_offset_hours: float) -> dt.datetime | None: + """:attr:`start` reinterpreted as being *utc_offset_hours* ahead of UTC. + + The file records no timezone, so the offset has to come from the caller + (or from :func:`spyde.insitu.align.align`'s span match, which sidesteps + the question entirely). + """ + if self.start is None: + return None + return (self.start - dt.timedelta(hours=utc_offset_hours)).replace( + tzinfo=dt.timezone.utc + ) + + def potential(self) -> np.ndarray | None: + """The working-electrode potential, whichever column carries it.""" + for key in ("Ewe/V", "/V", "Ewe-Ece/V", "|E|/V"): + if key in self.channels: + return self.channels[key] + return None + + def current(self) -> np.ndarray | None: + """The current in mA, whichever column carries it.""" + for key in ("/mA", "I/mA", "|I|/mA"): + if key in self.channels: + return self.channels[key] + return None + + def describe(self) -> str: + when = self.start.strftime("%Y-%m-%d %H:%M:%S.%f")[:-3] if self.start else "?" + return ( + f"{self.name}: {self.technique}, {self.n_points} pts, " + f"{self.time_s[0]:.2f}–{self.time_s[-1]:.2f} s " + f"({self.duration:.2f} s @ {self.sample_period * 1e3:.1f} ms), " + f"acq start {when}" + ) if self.n_points else f"{self.name}: {self.technique}, empty" + + +# -------------------------------------------------------------------------- +# .mpr (binary) +# -------------------------------------------------------------------------- + +@dataclass +class _Module: + offset: int + short: str + long: str + version: int + date: str + body: bytes + + +def _read_modules(buf: bytes) -> list[_Module]: + mods: list[_Module] = [] + i = buf.find(b"MODULE") + while i >= 0: + short = buf[i + 6 : i + 16].decode("latin1").strip() + long = buf[i + 16 : i + 41].decode("latin1").strip() + (first,) = struct.unpack_from(" dt.datetime | None: + if not (_OLE_MIN < value < _OLE_MAX): + return None + try: + return _OLE_EPOCH + dt.timedelta(days=float(value)) + except (OverflowError, ValueError): + return None + + +def _log_start_date(mod: _Module | None) -> dt.datetime | None: + if mod is None: + return None + for off in _LOG_DATE_OFFSETS: + if off + 8 > len(mod.body): + continue + (raw,) = struct.unpack_from(" str: + """Technique name from the settings module's embedded strings, else the + filename's EC-Lab suffix (``…_02_CV_C01.mpr`` → ``CV``).""" + if settings is not None: + text = settings.body.decode("latin1", errors="replace") + for name in _TECHNIQUES: + if name in text: + return name + m = re.search(r"_\d+_([A-Za-z]+)_C\d+", os.path.basename(path)) + if not m: + return "unknown" + code = m.group(1).upper() + return _TECHNIQUE_CODES.get(code, code) + + +def _layout(ids: tuple[int, ...]) -> tuple[list[tuple[str, str]], dict[str, int], list[int]]: + """``ids`` → (numpy dtype fields, flag masks, unknown ids). + + Flag IDs collapse into a single leading ``flags`` field; everything else + becomes one field, in the order the IDs appear. + """ + fields: list[tuple[str, str]] = [] + flags: dict[str, int] = {} + unknown: list[int] = [] + for cid in ids: + if cid in _FLAG_IDS: + name, mask = _FLAG_IDS[cid] + if not flags: + fields.append(("flags", " bool: + """True when ``body[first:last]`` is the all-zero run that precedes the + records (bar the lone marker byte EC-Lab sets in some versions).""" + padding = bytearray(body[first:last]) + marker = _PADDING_MARKER - first + if 0 <= marker < len(padding): + padding[marker] = 0 + return not any(padding) + + +def _resolve_unknown(fields: list[tuple[str, str]], body: bytes, npts: int, + header_end: int, unknown: list[int], + path: str) -> list[tuple[str, str]]: + """Pin the width of a single unknown column from the record size. + + The records sit flush against the END of the data module behind a run of + zero padding, so a candidate width is right only if it puts the first + record where the padding stops. That is the whole test: try each plausible + width and keep the one whose implied start lands on clean padding. + + With two or more unknowns there is nothing to solve — several width + combinations would satisfy the same total — and a wrong width silently + shifts every column after it, so we refuse rather than guess. + """ + if not unknown: + return fields + if len(unknown) > 1: + raise UnsupportedColumns( + f"{os.path.basename(path)}: unknown EC-Lab column IDs {unknown} — " + "cannot determine their byte widths. Export the run as ASCII from " + "EC-Lab (File > Export as text) and load the .mpt/.txt instead." + ) + known = sum(np.dtype(d).itemsize for _, d in fields if d) + for width, dtype in _WIDTH_TO_DTYPE.items(): + start = len(body) - npts * (known + width) + if start >= header_end and _padding_is_clean(body, header_end, start): + log.warning( + "eclab: %s has unrecognised column ID %d; reading it as a " + "%d-byte 'unknown_%d' pinned by the record size", + os.path.basename(path), unknown[0], width, unknown[0], + ) + return [(n, d or dtype) for n, d in fields] + raise UnsupportedColumns( + f"{os.path.basename(path)}: unknown EC-Lab column ID {unknown[0]} — no " + f"byte width places {npts} records inside a {len(body)} B data module. " + "Export the run as ASCII from EC-Lab and load the .mpt/.txt instead." + ) + + +def read_mpr(path: str) -> EcRun: + """Read a BioLogic ``.mpr`` binary record.""" + with open(path, "rb") as fh: + buf = fh.read() + if not buf.startswith(MPR_MAGIC): + raise ValueError(f"{path}: not a BioLogic .mpr file") + + mods = _read_modules(buf) + by_short = {m.short: m for m in mods} + data_mod = next((m for m in mods if m.short.startswith("VMP data")), None) + if data_mod is None: + raise ValueError(f"{path}: no 'VMP data' module") + + body = data_mod.body + (npts,) = struct.unpack_from(" dict[str, str]: + """Best-effort ``key : value`` pairs from the settings module's ASCII.""" + if mod is None: + return {} + text = mod.body.decode("latin1", errors="replace") + out: dict[str, str] = {} + for line in re.split(r"[\r\n\x00]+", text): + key, sep, value = line.partition(" : ") + if sep and 0 < len(key.strip()) < 60 and key.strip().isprintable(): + out[key.strip()] = value.strip() + return out + + +# -------------------------------------------------------------------------- +# .mpt / .txt (ASCII export) +# -------------------------------------------------------------------------- + +_ACQ_START_RE = re.compile( + r"Acquisition started on\s*:\s*(\d{1,2}/\d{1,2}/\d{4}\s+[\d:.]+)" +) +_ABS_TIME_RE = re.compile(r"^\d{1,2}/\d{1,2}/\d{4}\s") + + +def _parse_ec_datetime(text: str) -> dt.datetime | None: + for fmt in ("%m/%d/%Y %H:%M:%S.%f", "%m/%d/%Y %H:%M:%S", + "%d/%m/%Y %H:%M:%S.%f", "%d/%m/%Y %H:%M:%S"): + try: + return dt.datetime.strptime(text.strip(), fmt) + except ValueError: + continue + return None + + +def _to_float(token: str) -> float: + """EC-Lab writes in the exporting PC's locale — decimal comma is common.""" + token = token.strip() + if not token: + return np.nan + try: + return float(token) + except ValueError: + try: + return float(token.replace(",", ".")) + except ValueError: + return np.nan + + +def read_mpt(path: str) -> EcRun: + """Read an EC-Lab ASCII export (``.mpt``, often saved as ``.txt``). + + Handles both time conventions EC-Lab can export: ``time/s`` as elapsed + seconds, or as an absolute ``MM/DD/YYYY HH:MM:SS.ffff`` stamp (which is + converted back to elapsed seconds so both readers agree). + """ + with open(path, encoding="latin1") as fh: + lines = fh.read().splitlines() + if not lines or MPT_MAGIC not in lines[0]: + raise ValueError(f"{path}: not an EC-Lab ASCII file") + + n_header = 3 + for line in lines[:10]: + m = re.search(r"Nb header lines\s*:\s*(\d+)", line) + if m: + n_header = int(m.group(1)) + break + header = lines[: n_header - 1] + technique = next( + (t for line in header for t in _TECHNIQUES if line.strip() == t), "unknown" + ) + start = None + for line in header: + m = _ACQ_START_RE.search(line) + if m: + start = _parse_ec_datetime(m.group(1)) + break + + columns = [c.strip() for c in lines[n_header - 1].split("\t") if c.strip()] + rows = [line.split("\t") for line in lines[n_header:] if line.strip()] + if not rows: + raise ValueError(f"{path}: no data rows after {n_header} header lines") + + time_idx = next((i for i, c in enumerate(columns) if c.startswith("time/s")), None) + absolute_time = ( + time_idx is not None + and len(rows[0]) > time_idx + and bool(_ABS_TIME_RE.match(rows[0][time_idx].strip())) + ) + + values: dict[str, list[float]] = {c: [] for c in columns} + stamps: list[dt.datetime] = [] + for row in rows: + for i, name in enumerate(columns): + token = row[i] if i < len(row) else "" + if i == time_idx and absolute_time: + when = _parse_ec_datetime(token) + stamps.append(when or dt.datetime.min) + values[name].append(np.nan) + else: + values[name].append(_to_float(token)) + + channels = {c: np.asarray(v, dtype=float) for c, v in values.items()} + if absolute_time: + origin = start or (stamps[0] if stamps else None) + time_s = np.asarray([(s - origin).total_seconds() for s in stamps], dtype=float) + channels.pop(columns[time_idx], None) + if start is None: + start = origin + else: + time_s = channels.pop(columns[time_idx]) if time_idx is not None else np.arange( + len(rows), dtype=float + ) + + settings = {} + for line in header: + key, sep, value = line.partition(" : ") + if sep: + settings[key.strip()] = value.strip() + + return EcRun(path=path, technique=technique, start=start, time_s=time_s, + channels=channels, settings=settings) + + +# -------------------------------------------------------------------------- +# .mps (settings) + discovery +# -------------------------------------------------------------------------- + +def read_mps(path: str) -> dict: + """Parse an EC-Lab ``.mps`` settings file into ``{header, techniques}``. + + The ``.mps`` holds no samples — it is the recipe. Useful for knowing which + techniques a sequence was *meant* to run, and in what order. + """ + with open(path, encoding="latin1") as fh: + lines = fh.read().splitlines() + header: dict[str, str] = {} + techniques: list[dict[str, str]] = [] + current: dict[str, str] | None = None + for line in lines: + stripped = line.strip() + m = re.match(r"Technique\s*:\s*(\d+)", stripped) + if m: + current = {"index": m.group(1), "name": ""} + techniques.append(current) + continue + if current is not None and not current["name"] and stripped: + current["name"] = stripped + continue + key, sep, value = stripped.partition(" : ") + if sep: + (current if current is not None else header)[key.strip()] = value.strip() + return {"path": path, "header": header, "techniques": techniques} + + +def read_ec_file(path: str) -> EcRun: + """Read any EC-Lab record, dispatching on content rather than extension + (the ASCII export is routinely saved as ``.txt``).""" + with open(path, "rb") as fh: + head = fh.read(len(MPR_MAGIC)) + if head.startswith(MPR_MAGIC): + return read_mpr(path) + return read_mpt(path) + + +def find_ec_runs(directory: str) -> list[EcRun]: + """Read every EC-Lab record in *directory*, oldest acquisition first. + + Prefers the ``.mpr`` when a run has both a binary and an ASCII export, so + the same technique is not returned twice. Identity is the run's *sample + window* — acquisition start plus point count plus first/last timestamp — + rather than the filename, because EC-Lab exports pick up suffixes + (``-et``, ``(2)``) and the same run can be exported more than once. + Unreadable files are logged and skipped rather than failing the whole scan. + """ + seen: dict[tuple, EcRun] = {} + candidates = sorted(glob.glob(os.path.join(glob.escape(directory), "*"))) + binaries = [p for p in candidates if p.lower().endswith(".mpr")] + ascii_files = [p for p in candidates if p.lower().endswith((".mpt", ".txt"))] + for path in binaries + ascii_files: + try: + run = read_ec_file(path) + except (ValueError, OSError, struct.error) as exc: + log.debug("eclab: skipping %s (%s)", os.path.basename(path), exc) + continue + key = ( + run.start, + run.n_points, + round(float(run.time_s[0]), 3) if run.n_points else 0.0, + round(float(run.time_s[-1]), 3) if run.n_points else 0.0, + ) + seen.setdefault(key, run) + runs = list(seen.values()) + runs.sort(key=lambda r: (r.start or dt.datetime.min, r.time_s[0] if r.n_points else 0)) + return runs diff --git a/spyde/metadata_extract.py b/spyde/metadata_extract.py index 18ecd0c9..ab2fba78 100644 --- a/spyde/metadata_extract.py +++ b/spyde/metadata_extract.py @@ -245,6 +245,12 @@ def build_metadata_dict(signal_tree: "BaseSignalTree") -> dict[str, dict[str, st # numbers instead of "--". We require a recognised time UNIT (not just a # "time"-ish name) and convert it to seconds — deriving from an unconvertible # unit (or a bare uncalibrated name) would show a wrong fps, worse than "--". + # + # The key stays authoritative on purpose. When a DE movie's per-frame + # timestamps disagree with it (a summed movie's key holds the CAMERA rate, + # not the saved-frame rate), the loader corrects the KEY at load time + # (`_session_files._apply_frame_timestamps`) rather than this precedence — + # fixing the recorded value beats teaching every reader of it to distrust it. try: movie = subsections.get("Movie / In-Situ") if movie is not None: diff --git a/spyde/tests/migrated/test_insitu_align.py b/spyde/tests/migrated/test_insitu_align.py new file mode 100644 index 00000000..47cc8a11 --- /dev/null +++ b/spyde/tests/migrated/test_insitu_align.py @@ -0,0 +1,514 @@ +"""In-situ auxiliary channels: DE frame timestamps, EC-Lab records, clock align. + +Everything here is synthesised into ``tmp_path`` — no instrument files on disk — +but the synthesis follows the real byte layout, so the ``.mpr`` builder below +doubles as the format's specification. The numbers in +:class:`TestRealWorldShape` are taken from a real paired acquisition (a DE +Apollo movie and a BioLogic SP-200 cyclic voltammogram recorded together) and +pin the two behaviours that make the alignment work at all: matching on span +rather than on the two disagreeing wall clocks, and refusing to invent +instrument data outside the record. +""" +from __future__ import annotations + +import datetime as dt +import struct + +import numpy as np +import pytest + +from spyde.insitu import align as align_mod +from spyde.insitu import de_movie, eclab + +# -------------------------------------------------------------------------- +# builders +# -------------------------------------------------------------------------- + +MPR_HEADER = eclab.MPR_MAGIC + b" " * 25 + b"\x00" * 4 # 52 bytes, as EC-Lab writes +_DATA_START = 1007 # padding before the records, EC-Lab v11 + + +def _module(short: str, long: str, version: int, body: bytes) -> bytes: + """One ``MODULE`` block in the 64-bit-length layout.""" + return ( + b"MODULE" + + short.encode().ljust(10) + + long.encode().ljust(25) + + struct.pack(" bytes: + """A ``VMP LOG`` carrying the acquisition start as an OLE date.""" + body = bytearray(8010) + days = (start - dt.datetime(1899, 12, 30)).total_seconds() / 86400.0 + struct.pack_into(" bytes: + """Assemble a minimal but structurally faithful ``.mpr``.""" + npts = len(records) + header = struct.pack("/mA", " np.ndarray: + rec = np.zeros(n, dtype=CV_DTYPE) + t = t0 + dt_s * np.arange(n) + rec["time/s"] = t + rec["Ewe/V"] = 0.02 * (t - t0) # a 20 mV/s ramp + rec["control/V"] = rec["Ewe/V"] + rec["/mA"] = 1e-4 * np.sin(t) + rec["cycle number"] = np.where(np.arange(n) < n // 2, 1.0, 2.0) + rec["I Range"] = np.where(np.arange(n) < n // 2, 52, 53) + rec["flags"] = np.where(np.arange(n) % 2, 0x04, 0x00) | 0x02 # ox/red + mode + return rec + + +def write_movie(tmp_path, *, stem="20251117_88071", n=200, period=0.03276, + t0=1133858.28439, epoch=None, info_extra=None, frames_written=None): + """Write a DE timestamps CSV (+ info.txt) and return the .mrc path.""" + csv_path = tmp_path / f"{stem}{de_movie.TIMESTAMPS_SUFFIX}" + lines = ["Frame Index, Timestamp (s), Electrons"] + for i in range(n): + lines.append(f"{i}, {t0 + i * period:.6f}, {31000000 + i}") + csv_path.write_text("\n".join(lines) + "\n") + + info = { + "Frames Per Second": "61.05006", + "Autosave Movie Sum Count": "2", + "Autosave Movie Frames Written": str(n if frames_written is None else frames_written), + "Acquisition Status": "Stopped", + } + if epoch is not None: + info["Timestamp (seconds since Epoch)"] = str(epoch) + info.update(info_extra or {}) + info_path = tmp_path / f"{stem}{de_movie.INFO_SUFFIX}" + info_path.write_text( + "\n".join(f"{k:<60} = {v}" for k, v in info.items()) + "\n" + ) + mrc = tmp_path / f"{stem}_movie.mrc" + mrc.write_bytes(b"") + return str(mrc) + + +EC_START = dt.datetime(2025, 11, 17, 14, 50, 59, 687000) + + +def _eu(value: float, fmt: str) -> str: + """A number as EC-Lab writes it on a decimal-comma locale.""" + return format(value, fmt).replace(".", ",") + + +def write_mpt(tmp_path, name="run.txt", *, absolute=False, n=50): + """An EC-Lab ASCII export. + + ``Nb header lines`` counts every line up to AND INCLUDING the column + header, so it is derived from the header list rather than hard-coded — + getting that off by one is the classic way to misread these files. + """ + header = [ + "EC-Lab ASCII FILE", + "Nb header lines : {n}", + "", + "Cyclic Voltammetry", + "", + "Acquisition started on : 11/17/2025 14:50:59.687", + "Device : SP-200 (SN 1593)", + "", + "ox/red\ttime/s\tcontrol/V\tEwe/V\t/mA\t", + ] + header[1] = header[1].format(n=len(header)) + + rows = [] + for i in range(n): + t = 31.56 + 0.08 * i + ramp = 0.02 * (t - 31.56) + # A datetime keeps its dot; only the numbers take a decimal comma. + when = (EC_START + dt.timedelta(seconds=t)).strftime("%m/%d/%Y %H:%M:%S.%f")[:-2] + t_col = when if absolute else _eu(t, ".15E") + rows.append("\t".join([ + str(i % 2), t_col, _eu(ramp, ".7E"), _eu(ramp, ".7E"), + _eu(1e-4 * np.sin(t), ".15E"), + ])) + + path = tmp_path / name + path.write_text("\n".join(header + rows) + "\n", encoding="latin1") + return str(path) + + +# -------------------------------------------------------------------------- +# DE movie sidecars +# -------------------------------------------------------------------------- + +class TestMovieClock: + def test_reads_frame_times(self, tmp_path): + mrc = write_movie(tmp_path, n=200) + clock = de_movie.read_movie_clock(mrc) + assert clock.n_frames == 200 + assert clock.frame_period == pytest.approx(0.03276, abs=1e-9) + assert clock.duration == pytest.approx(199 * 0.03276, abs=1e-6) + assert clock.t[0] == 0.0 + + def test_finds_sidecars_from_any_member(self, tmp_path): + mrc = write_movie(tmp_path) + for probe in (mrc, mrc.replace("_movie.mrc", de_movie.TIMESTAMPS_SUFFIX), + mrc.replace("_movie.mrc", de_movie.INFO_SUFFIX)): + ts, info = de_movie.find_movie_sidecars(probe) + assert ts is not None and info is not None + + def test_finds_sidecar_when_movie_carries_a_session_number(self, tmp_path): + """DE names the info file per ACQUISITION but the movie per autosave + session, so the stems do not match exactly.""" + write_movie(tmp_path, stem="20251117_88071") + movie = tmp_path / "20251117_88071_run1_2616_movie.mrc" + movie.write_bytes(b"") + ts, info = de_movie.find_movie_sidecars(str(movie)) + assert ts is not None and info is not None + + def test_reader_derived_period_disagrees_with_the_timestamps(self, tmp_path): + """RosettaSciIO derives 1/(fps*sum) where the true saved period is + sum/fps — a factor of sum**2. The CSV is ground truth; this pins that + the discrepancy is visible rather than silent.""" + clock = de_movie.read_movie_clock(write_movie(tmp_path)) + assert clock.reader_frame_period == pytest.approx(1 / (61.05006 * 2)) + assert clock.frame_period / clock.reader_frame_period == pytest.approx(4.0, rel=1e-3) + + def test_epoch_is_read_as_utc(self, tmp_path): + clock = de_movie.read_movie_clock(write_movie(tmp_path, epoch=1763387769)) + assert clock.epoch_utc == 1763387769.0 + assert clock.epoch_datetime().strftime("%Y-%m-%d %H:%M:%S") == "2025-11-17 13:56:09" + + def test_info_frame_count_gates_the_epoch_anchor(self, tmp_path): + """A DE acquisition writes ONE info file but a movie per autosave + session; a count mismatch means the epoch stamp is another session's.""" + good = de_movie.read_movie_clock(write_movie(tmp_path, n=50)) + assert de_movie.info_matches_movie(good) + other = tmp_path / "b" + other.mkdir() + bad = de_movie.read_movie_clock( + write_movie(other, n=50, frames_written=3005) + ) + assert not de_movie.info_matches_movie(bad) + + def test_detects_dropped_frames(self, tmp_path): + mrc = write_movie(tmp_path, n=10) + csv = mrc.replace("_movie.mrc", de_movie.TIMESTAMPS_SUFFIX) + lines = open(csv).read().splitlines() + # Blow a hole in the middle: frame 5 arrives three periods late. + rows = [lines[0]] + for i, line in enumerate(lines[1:]): + idx, t, e = line.split(",") + rows.append(f"{idx},{float(t) + (0.0655 if i >= 5 else 0):.6f},{e}") + open(csv, "w").write("\n".join(rows) + "\n") + clock = de_movie.read_movie_clock(mrc) + assert clock.dropped_frames().tolist() == [4] + + def test_missing_timestamps_raises(self, tmp_path): + (tmp_path / "x_movie.mrc").write_bytes(b"") + with pytest.raises(FileNotFoundError): + de_movie.read_movie_clock(str(tmp_path / "x_movie.mrc")) + + def test_torn_row_is_dropped_not_fatal(self, tmp_path): + mrc = write_movie(tmp_path, n=10) + csv = mrc.replace("_movie.mrc", de_movie.TIMESTAMPS_SUFFIX) + with open(csv, "a") as fh: + fh.write("10, \n") # acquisition killed mid-write + assert de_movie.read_movie_clock(mrc).n_frames == 10 + + +# -------------------------------------------------------------------------- +# EC-Lab records +# -------------------------------------------------------------------------- + +class TestMpr: + def test_round_trips_columns_and_flags(self, tmp_path): + rec = cv_records(n=100) + path = tmp_path / "r_02_CV_C01.mpr" + path.write_bytes(build_mpr(ids=CV_IDS, records=rec)) + run = eclab.read_mpr(str(path)) + + assert run.n_points == 100 + assert run.time_s == pytest.approx(rec["time/s"]) + assert run.channels["Ewe/V"] == pytest.approx(rec["Ewe/V"]) + assert run.channels["/mA"] == pytest.approx(rec["/mA"]) + assert run.channels["I Range"].tolist() == rec["I Range"].tolist() + # flags share one u1: ox/red alternates, mode is the low 2 bits + assert run.channels["ox/red"].tolist() == [bool(i % 2) for i in range(100)] + assert set(np.unique(run.channels["mode"])) == {2} + assert run.unknown_columns == () + + def test_reads_the_acquisition_start(self, tmp_path): + path = tmp_path / "r_02_CV_C01.mpr" + path.write_bytes(build_mpr(ids=CV_IDS, records=cv_records())) + run = eclab.read_mpr(str(path)) + assert run.start == dt.datetime(2025, 11, 17, 14, 50, 59, 687000) + + def test_technique_falls_back_to_the_filename_code(self, tmp_path): + path = tmp_path / "r_02_CV_C01.mpr" + path.write_bytes(build_mpr(ids=CV_IDS, records=cv_records(), + settings_text=b"\x00")) + assert eclab.read_mpr(str(path)).technique == "Cyclic Voltammetry" + + def test_one_unknown_column_is_pinned_by_the_record_size(self, tmp_path): + """An unrecognised ID is readable when the byte arithmetic leaves only + one possible width.""" + dtype = np.dtype(CV_DTYPE.descr + [("mystery", "= 0 + # every mapped frame is within half a frame period of its sample + t_frames = align_mod.ec_time_for_frames(clock, al) + err = np.abs(t_frames[frames[inside]] - run.time_s[inside]) + assert err.max() <= clock.frame_period / 2 + 1e-9 + + +class TestMatchRuns: + def test_picks_the_run_whose_duration_matches(self, tmp_path): + """Of several techniques in a session, the one that ran DURING the + movie is the one with the movie's duration.""" + mrc = write_movie(tmp_path, n=200, epoch=1763387769) + clock = de_movie.read_movie_clock(mrc) + n_match = int(np.ceil(clock.duration / 0.08)) + 1 + for name, n_ec in (("a_02_CV_C01.mpr", 30), + ("b_02_CV_C01.mpr", n_match), + ("c_02_CV_C01.mpr", 12)): + (tmp_path / name).write_bytes( + build_mpr(ids=CV_IDS, records=cv_records(n=n_ec)) + ) + runs = eclab.find_ec_runs(str(tmp_path)) + ranked = align_mod.match_runs(clock, runs) + assert ranked, "no run matched the movie" + best, al = ranked[0] + assert best.path.endswith("b_02_CV_C01.mpr") + assert al.method == "span" + assert al.trustworthy + + def test_no_candidates_returns_empty(self, tmp_path): + clock, run = _paired(tmp_path, n_ec=30) + assert align_mod.match_runs(clock, [run]) == [] + + +class TestRealWorldShape: + """Numbers from a real paired DE Apollo + BioLogic SP-200 acquisition. + + The point is the *relationship*: 7914 frames at 30.525 fps and 3256 samples + at 12.5 Hz cover the same 259.2 s, while the two PCs' wall clocks disagree + by very nearly an hour. Span matching gets the right answer without ever + consulting those clocks. + """ + + def test_span_beats_a_one_hour_clock_skew(self, tmp_path): + mrc = write_movie(tmp_path, n=7914, period=0.03276, epoch=1763387769) + clock = de_movie.read_movie_clock(mrc) + assert clock.duration == pytest.approx(259.23, abs=0.01) + + path = tmp_path / "floating_02_CV_C01.mpr" + path.write_bytes( + build_mpr(ids=CV_IDS, records=cv_records(n=3243, t0=31.56, dt_s=0.08)) + ) + run = eclab.read_ec_file(str(path)) + + al = align_mod.align_clocks(clock, run) + assert al.method == "span" + assert abs(al.duration_mismatch_s) < 1.0 + assert al.covered_fraction == pytest.approx(1.0, abs=1e-3) + # The camera's own epoch stamp puts the movie an hour off the EC clock; + # span matching is immune to that, and reports the skew instead. + assert round(al.implied_utc_offset_hours) == 1 + + cols = align_mod.resample_to_frames(clock, run, al) + assert np.isfinite(cols["Ewe/V"]).all() + assert cols["Ewe/V"].shape == (7914,) diff --git a/spyde/tests/migrated/test_insitu_attach.py b/spyde/tests/migrated/test_insitu_attach.py new file mode 100644 index 00000000..2a3e13fc --- /dev/null +++ b/spyde/tests/migrated/test_insitu_attach.py @@ -0,0 +1,444 @@ +"""Wiring: a movie's timestamps calibrate its time axis, and instrument data +recorded beside it attaches as navigator lanes. + +The two behaviours pinned here are the ones a green reader suite cannot see: +that the loader PREFERS the timestamps sidecar over the reader's derived +period (which is wrong by ``sum_count**2`` for a summed DE movie), and that +auto-discovery stays quiet unless a record really does explain the movie. +""" +from __future__ import annotations + +import numpy as np +import pytest +import hyperspy.api as hs + +from spyde.backend._session_files import FileLoaderMixin +from spyde.insitu import attach as attach_mod +from spyde.insitu import de_movie +from spyde.tests.migrated.test_insitu_align import ( + CV_IDS, build_mpr, cv_records, write_movie, write_mpt, +) + +N_FRAMES = 120 +PERIOD = 0.03276 + + +def movie_signal(n=N_FRAMES, frame=8): # noqa: D401 + """A minimal in-situ movie signal: nav-dim 1, 2-D frames, time axis.""" + sig = hs.signals.Signal2D(np.zeros((n, frame, frame), np.uint16)) + ax = sig.axes_manager.navigation_axes[0] + ax.name, ax.units = "time", "sec" + ax.scale = 1.0 / (61.05006 * 2) # what the MRC reader would have set + return sig + + +def paired(tmp_path, *, n_ec=None, ec_dt=0.08): + mrc = write_movie(tmp_path, n=N_FRAMES, period=PERIOD, epoch=1763387769) + duration = (N_FRAMES - 1) * PERIOD + if n_ec is None: + n_ec = int(np.ceil(duration / ec_dt)) + 1 + (tmp_path / "r_02_CV_C01.mpr").write_bytes( + build_mpr(ids=CV_IDS, records=cv_records(n=n_ec, dt_s=ec_dt)) + ) + return mrc + + +class _FakeTree: + """Enough tree surface for attach: nav shape + navigator registration. + + ``navigator_signals`` stores a LIST, because that is what the real + ``BaseSignalTree.add_navigator_signal`` stores — ``_preprocess_navigator`` + returns ``[signal]`` (or ``[navigator, signal]``), never the bare signal it + was handed. A fake that stored the signal directly let a re-attach bug + through this suite and all the way into the running app. + """ + + def __init__(self, n=N_FRAMES, source_path=None): + self.root = movie_signal(n) + self.navigator_signals: dict = {} + self.source_path = source_path + self.registered: list[tuple[str, np.ndarray]] = [] + + def add_navigator_signal(self, name, signal): + self.navigator_signals[name] = [signal] + self.registered.append((name, np.asarray(signal.data))) + + def lane_data(self, name) -> np.ndarray: + return np.asarray(self.navigator_signals[name][0].data) + + +class TestTimeAxisFromTimestamps: + def test_timestamps_override_the_readers_period(self, tmp_path): + """The reader's 1/(fps*sum) is 4× too fast for a 2-frame sum; the CSV + is ground truth and must win.""" + mrc = write_movie(tmp_path, n=N_FRAMES, period=PERIOD) + sig = movie_signal() + before = float(sig.axes_manager.navigation_axes[0].scale) + + clock = FileLoaderMixin._apply_frame_timestamps(sig, mrc) + + assert clock is not None + ax = sig.axes_manager.navigation_axes[0] + assert ax.scale == pytest.approx(PERIOD, abs=1e-9) + assert ax.units == "s" + assert before / ax.scale == pytest.approx(0.25, rel=1e-3) # was 4× fast + + def test_the_recorded_fps_is_corrected_too(self, tmp_path): + """The metadata panel prefers the explicit fps KEY over the axis, so + leaving the reader's camera rate there would report 61 fps beside a + 32.76 ms/frame axis. Fix the value, not the precedence.""" + from spyde.metadata_extract import build_metadata_dict + + mrc = write_movie(tmp_path, n=N_FRAMES, period=PERIOD) + sig = movie_signal() + sig.metadata.set_item("Acquisition_instrument.TEM.frames_per_second", 61.05006) + + FileLoaderMixin._apply_frame_timestamps(sig, mrc) + + recorded = sig.metadata.get_item( + "Acquisition_instrument.TEM.frames_per_second") + # Rounded for display — the metadata chip prints this number verbatim, + # so "30.525030441931396 fps" is noise. The AXIS keeps full precision. + assert recorded == pytest.approx(1 / PERIOD, rel=1e-4) + assert len(str(recorded)) <= 8, f"unrounded fps on the chip: {recorded}" + + from spyde.tests.migrated.test_movie_metadata import _Tree + panel = build_metadata_dict(_Tree(sig))["Movie / In-Situ"] + assert "30.5" in panel["FPS"], panel["FPS"] + assert "61" not in panel["FPS"], panel["FPS"] + + def test_frame_count_mismatch_keeps_the_period_but_not_the_mapping(self, tmp_path): + """A stale sidecar from another autosave session still knows the + camera's frame period — that is a camera property. What it cannot do is + map ITS frames onto THIS movie's, so no clock is returned.""" + mrc = write_movie(tmp_path, n=N_FRAMES + 40, period=PERIOD) + sig = movie_signal(n=N_FRAMES) + + clock = FileLoaderMixin._apply_frame_timestamps(sig, mrc) + + assert clock is None + assert sig.axes_manager.navigation_axes[0].scale == pytest.approx(PERIOD) + + def test_no_sidecar_leaves_the_signal_untouched(self, tmp_path): + (tmp_path / "bare_movie.mrc").write_bytes(b"") + sig = movie_signal() + before = float(sig.axes_manager.navigation_axes[0].scale) + assert FileLoaderMixin._apply_frame_timestamps( + sig, str(tmp_path / "bare_movie.mrc")) is None + assert sig.axes_manager.navigation_axes[0].scale == before + + def test_non_movie_signals_are_skipped(self, tmp_path): + """A 4D-STEM scan has no frame time base to fix.""" + mrc = write_movie(tmp_path, n=N_FRAMES, period=PERIOD) + scan = hs.signals.Signal2D(np.zeros((4, 5, 8, 8), np.uint16)) + assert FileLoaderMixin._apply_frame_timestamps(scan, mrc) is None + + +class TestPixelSize: + """RosettaSciIO decides imaging-vs-diffraction with ``camera_length != -1``, + but an IMAGING exposure records 0 — so a TEM image is calibrated as + diffraction (nm^-1) at the unset -1 pixel size.""" + + def _sig(self): + s = movie_signal(n=4, frame=8) + # What the reader leaves on a mis-detected imaging exposure. + for ax, name in zip(s.axes_manager.signal_axes, ("kx", "ky")): + ax.scale, ax.units, ax.name = -1.0, "nm^-1", name + return s + + def test_imaging_axes_are_renamed_out_of_reciprocal_space(self, tmp_path): + """The reader names them kx/ky for the diffraction branch it wrongly + took; nm axes called "kx" are just as wrong as nm^-1 ones.""" + mrc = write_movie(tmp_path, n=4, info_extra={ + "Instrument Project Camera Length (centimeters)": "0", + "Specimen Pixel Size X (nanometers)": "1.14786", + "Specimen Pixel Size Y (nanometers)": "1.14786", + }) + sig = self._sig() + FileLoaderMixin._apply_de_pixel_size(sig, mrc) + assert [ax.name for ax in sig.axes_manager.signal_axes] == ["x", "y"] + + def test_a_user_named_axis_is_not_clobbered(self, tmp_path): + mrc = write_movie(tmp_path, n=4, info_extra={ + "Instrument Project Camera Length (centimeters)": "0", + "Specimen Pixel Size X (nanometers)": "1.14786", + "Specimen Pixel Size Y (nanometers)": "1.14786", + }) + sig = self._sig() + for ax in sig.axes_manager.signal_axes: + ax.name = "my axis" + FileLoaderMixin._apply_de_pixel_size(sig, mrc) + assert [ax.name for ax in sig.axes_manager.signal_axes] == ["my axis"] * 2 + + def test_imaging_exposure_gets_nm_from_the_specimen_pixel_size(self, tmp_path): + mrc = write_movie(tmp_path, n=4, info_extra={ + "Instrument Project Camera Length (centimeters)": "0", + "Diffraction Pixel Size X": "-1", + "Diffraction Pixel Size Y": "-1", + "Specimen Pixel Size X (nanometers)": "1.14786", + "Specimen Pixel Size Y (nanometers)": "1.14786", + }) + sig = self._sig() + assert FileLoaderMixin._apply_de_pixel_size(sig, mrc) + for ax in sig.axes_manager.signal_axes: + assert ax.scale == pytest.approx(1.14786) + assert ax.units == "nm" + + def test_real_diffraction_keeps_reciprocal_units(self, tmp_path): + mrc = write_movie(tmp_path, n=4, info_extra={ + "Instrument Project Camera Length (centimeters)": "80", + "Diffraction Pixel Size X": "0.0031", + "Diffraction Pixel Size Y": "0.0031", + "Specimen Pixel Size X (nanometers)": "-1", + "Specimen Pixel Size Y (nanometers)": "-1", + }) + sig = self._sig() + assert FileLoaderMixin._apply_de_pixel_size(sig, mrc) + for ax in sig.axes_manager.signal_axes: + assert ax.scale == pytest.approx(0.0031) + assert ax.units == "nm^-1" + + def test_no_usable_pixel_size_changes_nothing(self, tmp_path): + mrc = write_movie(tmp_path, n=4, info_extra={ + "Instrument Project Camera Length (centimeters)": "0", + "Diffraction Pixel Size X": "-1", + "Diffraction Pixel Size Y": "-1", + "Specimen Pixel Size X (nanometers)": "-1", + "Specimen Pixel Size Y (nanometers)": "-1", + }) + sig = self._sig() + assert not FileLoaderMixin._apply_de_pixel_size(sig, mrc) + assert sig.axes_manager.signal_axes[0].units == "nm^-1" + + def test_a_1d_signal_is_left_alone(self, tmp_path): + import hyperspy.api as hs + mrc = write_movie(tmp_path, n=4, info_extra={ + "Specimen Pixel Size X (nanometers)": "1.1", + "Specimen Pixel Size Y (nanometers)": "1.1", + }) + line = hs.signals.Signal1D(np.zeros((4, 8), np.uint16)) + assert not FileLoaderMixin._apply_de_pixel_size(line, mrc) + + +class TestAutoDiscovery: + def test_attaches_a_matching_record_as_navigator_lanes(self, tmp_path): + mrc = paired(tmp_path) + tree = _FakeTree(source_path=mrc) + + result = attach_mod.discover_and_attach(tree, mrc) + + assert result, result.reason + names = [n for n, _ in tree.registered] + assert attach_mod.POTENTIAL_LANE in names + assert any(n.startswith(attach_mod.CURRENT_LANE) for n in names) + for _, values in tree.registered: + assert values.shape == (N_FRAMES,) + assert np.isfinite(values).all(), "a lane must not carry NaN" + assert result.alignment.method == "span" + + def test_lanes_carry_the_movies_time_calibration(self, tmp_path): + """A lane's signal axis MUST be the movie's time axis. A 1-D selector + maps a widget position to a frame with ``(x - offset) / scale`` read + off the shown navigator's own axis, so an uncalibrated lane plots in + frame index and its cursor resolves to the wrong frame.""" + mrc = paired(tmp_path) + tree = _FakeTree(source_path=mrc) + # The loader calibrates the root's time axis first; the lanes copy it. + FileLoaderMixin._apply_frame_timestamps(tree.root, mrc) + attach_mod.discover_and_attach(tree, mrc) + + root_ax = tree.root.axes_manager.navigation_axes[0] + assert root_ax.scale == pytest.approx(PERIOD, abs=1e-9) + assert tree.navigator_signals, "no lanes registered" + for name in tree.navigator_signals: + lane_ax = tree.navigator_signals[name][0].axes_manager.signal_axes[0] + assert lane_ax.scale == pytest.approx(root_ax.scale), name + assert lane_ax.offset == pytest.approx(root_ax.offset), name + assert lane_ax.units == root_ax.units, name + # …so the lane's x runs over the movie's DURATION in seconds, not + # over its frame COUNT. + span = lane_ax.scale * (N_FRAMES - 1) + assert span == pytest.approx((N_FRAMES - 1) * PERIOD, rel=1e-6), name + + def test_current_lane_is_named_for_the_unit_it_uses(self, tmp_path): + """A µA-scale current plotted as mA is an unreadable 0.0001 axis; the + rescale is fine but the NAME has to say so.""" + mrc = paired(tmp_path) + tree = _FakeTree(source_path=mrc) + attach_mod.discover_and_attach(tree, mrc) + current = [n for n, _ in tree.registered if n.startswith(attach_mod.CURRENT_LANE)] + assert current == ["I (µA)"] + + def test_silent_when_nothing_matches_the_duration(self, tmp_path): + """A folder of unrelated records must not produce a false attach.""" + mrc = paired(tmp_path, n_ec=12) # far too short for the movie + tree = _FakeTree(source_path=mrc) + + result = attach_mod.discover_and_attach(tree, mrc) + + assert not result + assert tree.registered == [] + assert "none matching" in result.reason + + def test_silent_with_no_records_at_all(self, tmp_path): + mrc = write_movie(tmp_path, n=N_FRAMES, period=PERIOD) + tree = _FakeTree(source_path=mrc) + result = attach_mod.discover_and_attach(tree, mrc) + assert not result + assert "no EC-Lab records" in result.reason + + def test_refuses_when_the_movie_has_no_time_base(self, tmp_path): + (tmp_path / "bare_movie.mrc").write_bytes(b"") + tree = _FakeTree() + result = attach_mod.discover_and_attach(tree, str(tmp_path / "bare_movie.mrc")) + assert not result + assert "timestamps" in result.reason + + def test_stores_the_full_table_on_the_tree(self, tmp_path): + """Only two channels become chips; everything else stays reachable.""" + mrc = paired(tmp_path) + tree = _FakeTree(source_path=mrc) + attach_mod.discover_and_attach(tree, mrc) + assert set(tree.insitu_channels) >= {"time/s", "Ewe/V", "/mA", + "cycle number", "I Range"} + assert tree.insitu_run is not None + assert tree.insitu_alignment.covered_fraction == pytest.approx(1.0, abs=0.02) + + def test_a_frame_count_mismatch_refuses_to_map(self, tmp_path): + mrc = paired(tmp_path) + tree = _FakeTree(n=N_FRAMES - 5, source_path=mrc) # movie shorter than CSV + result = attach_mod.discover_and_attach(tree, mrc) + assert not result + assert "cannot map" in result.reason + + +class TestManualAttach: + def test_attaches_an_explicitly_chosen_file(self, tmp_path): + mrc = write_movie(tmp_path, n=N_FRAMES, period=PERIOD) + elsewhere = tmp_path / "elsewhere" + elsewhere.mkdir() + duration = (N_FRAMES - 1) * PERIOD + n_ec = int(np.ceil(duration / 0.08)) + 1 + ec = elsewhere / "r_02_CV_C01.mpr" + ec.write_bytes(build_mpr(ids=CV_IDS, records=cv_records(n=n_ec))) + tree = _FakeTree(source_path=mrc) + + result = attach_mod.attach_ec_file(tree, str(ec), movie_path=mrc) + + assert result, result.reason + assert attach_mod.POTENTIAL_LANE in tree.navigator_signals + + def test_reads_the_ascii_export_too(self, tmp_path): + mrc = write_movie(tmp_path, n=N_FRAMES, period=PERIOD) + duration = (N_FRAMES - 1) * PERIOD + n_ec = int(np.ceil(duration / 0.08)) + 1 + ec = write_mpt(tmp_path, "export.txt", n=n_ec) + tree = _FakeTree(source_path=mrc) + assert attach_mod.attach_ec_file(tree, ec, movie_path=mrc) + + def test_manual_override_of_the_lag(self, tmp_path): + mrc = paired(tmp_path, n_ec=12) # too short to auto-align + tree = _FakeTree(source_path=mrc) + ec = str(tmp_path / "r_02_CV_C01.mpr") + + assert not attach_mod.attach_ec_file(tree, ec, movie_path=mrc) + result = attach_mod.attach_ec_file( + tree, ec, movie_path=mrc, method="manual", lag_s=31.56 + ) + assert result, result.reason + assert result.alignment.lag_s == pytest.approx(31.56) + + def test_reattaching_the_same_run_succeeds(self, tmp_path): + """Auto-discovery runs on open, so picking the same record by hand + afterwards is the NORMAL second call — it must read as success, not as + 'this run has no potential or current channel'.""" + mrc = paired(tmp_path) + tree = _FakeTree(source_path=mrc) + ec = str(tmp_path / "r_02_CV_C01.mpr") + + first = attach_mod.discover_and_attach(tree, mrc) + second = attach_mod.attach_ec_file(tree, ec, movie_path=mrc) + + assert first and second, second.reason + assert set(second.lanes) == set(first.lanes) + # ONE plot state per lane, not one per attach. + names = [n for n, _ in tree.registered] + assert len(names) == len(set(names)) + + def test_reattach_replaces_the_lane_values(self, tmp_path): + """A different record under the same lane name must REPLACE the trace, + not leave the previous run's on screen.""" + mrc = paired(tmp_path) + tree = _FakeTree(source_path=mrc) + attach_mod.discover_and_attach(tree, mrc) + before = tree.lane_data(attach_mod.POTENTIAL_LANE).copy() + + duration = (N_FRAMES - 1) * PERIOD + n_ec = int(np.ceil(duration / 0.08)) + 1 + other = tmp_path / "other_02_CV_C01.mpr" + records = cv_records(n=n_ec) + records["Ewe/V"] = records["Ewe/V"] * -1.0 - 0.25 # a distinguishable sweep + other.write_bytes(build_mpr(ids=CV_IDS, records=records)) + + assert attach_mod.attach_ec_file(tree, str(other), movie_path=mrc) + after = tree.lane_data(attach_mod.POTENTIAL_LANE) + assert not np.allclose(before, after), "the lane still shows the old run" + assert after.shape == before.shape + + def test_a_run_with_no_e_or_i_says_what_it_does_have(self, tmp_path): + mrc = write_movie(tmp_path, n=N_FRAMES, period=PERIOD) + clock = de_movie.read_movie_clock(mrc) + tree = _FakeTree(source_path=mrc) + run = type("R", (), {})() + # An OCV record carries /V, so strip to something that carries + # neither a potential nor a current. + from spyde.insitu.eclab import EcRun + run = EcRun(path="x_01_OCV_C01.mpr", technique="Open Circuit Voltage", + start=None, time_s=np.linspace(0, clock.duration, 40), + channels={"cycle number": np.zeros(40)}) + from spyde.insitu.align import align_clocks + result = attach_mod.attach_run(tree, clock, run, align_clocks(clock, run)) + assert not result + assert "cycle number" in result.reason + + def test_unreadable_file_reports_rather_than_raises(self, tmp_path): + mrc = write_movie(tmp_path, n=N_FRAMES, period=PERIOD) + junk = tmp_path / "junk.mpr" + junk.write_bytes(b"not an instrument record") + tree = _FakeTree(source_path=mrc) + result = attach_mod.attach_ec_file(tree, str(junk), movie_path=mrc) + assert not result + assert "could not read" in result.reason + + +class TestSessionEntryPoint: + def test_mps_is_reported_as_a_recipe_not_data(self, window, tmp_path): + """Picking the .mps is an easy mistake — it holds no samples. Say what + it planned instead of failing blankly.""" + session = window["window"] + mps = tmp_path / "seq.mps" + mps.write_text( + "EC-LAB SETTING FILE\n\nNumber of linked techniques : 2\n\n" + "Technique : 1\nOpen Circuit Voltage\ntR (h:m:s) 0:00:30\n\n" + "Technique : 2\nCyclic Voltammetry\nEi (V) 0,000\n", + encoding="latin1", + ) + session.load_insitu_data(str(mps)) + text = " ".join(str(m) for m in window["messages"]) + assert "settings file" in text + assert "Cyclic Voltammetry" in text + + def test_missing_file_errors(self, window, tmp_path): + session = window["window"] + session.load_insitu_data(str(tmp_path / "nope.mpr")) + text = " ".join(str(m) for m in window["messages"]) + assert "not found" in text.lower() + + def test_no_movie_open_is_reported(self, window, tmp_path): + session = window["window"] + (tmp_path / "r_02_CV_C01.mpr").write_bytes( + build_mpr(ids=CV_IDS, records=cv_records(n=50)) + ) + session.load_insitu_data(str(tmp_path / "r_02_CV_C01.mpr")) + text = " ".join(str(m) for m in window["messages"]) + assert "in-situ movie" in text.lower() From da78cf1de4a266bbcaceef53b7c73f1ba21eed00 Mon Sep 17 00:00:00 2001 From: Carter Francis Date: Thu, 6 Aug 2026 10:02:55 -0500 Subject: [PATCH 2/3] fix(nav): one master cursor on the stacked lanes, the rest follow Dragging a lane's time cursor made every lane jump to and fro for the whole drag. `_busy` is cleared synchronously when the drag handler returns, but the selector's index hook runs on the `_NavDispatcher` thread and its widget writes are marshalled onto the main thread -- so they land AFTER the guard is false and write the last committed (index-quantised) position onto the line still under the pointer. The pointer has moved on, the next `pointer_move` drags it forward again, and it oscillates. Whichever line is being dragged is now the MASTER for the length of the drag and `_set_all_lines` skips it; `pointer_up` hands the role back so the committed position can then settle every lane. Two guards because there are two races: `_busy` for the synchronous `set -> pointer_move` echo, `_master` for the asynchronous write-back. Verified with a real drag in the app: samples 86.5, 108.5, 130.5, 152.5, 174.5, 196.5 -- monotonic at exactly the 22 px/step the mouse moved. --- spyde/actions/navigator_views.py | 58 +++++- .../migrated/test_stacked_cursor_master.py | 195 ++++++++++++++++++ 2 files changed, 245 insertions(+), 8 deletions(-) create mode 100644 spyde/tests/migrated/test_stacked_cursor_master.py diff --git a/spyde/actions/navigator_views.py b/spyde/actions/navigator_views.py index 4391181f..ed6bedea 100644 --- a/spyde/actions/navigator_views.py +++ b/spyde/actions/navigator_views.py @@ -249,11 +249,26 @@ class _StackedNavCursor: selector commits a new index and fires this cursor's ``index_hook``, which sets every row's line ``x`` to match. - The guard (``_busy``) stops the ``set → pointer_move → handler → set`` echo - (widget ``.set`` fires ``pointer_move``) AND the drive→hook→drive loop. - Position writes to widgets are UI updates, so when the sync originates on the - ``_NavDispatcher`` thread (the index_hook) they are marshalled onto the - asyncio main thread via ``session._dispatch_to_main``.""" + Exactly ONE line is the master at a time. Whichever the user is dragging + holds that role for the length of the drag; every other line is a pure + follower and is never written back to by its own handler. + + That asymmetry is load-bearing, not tidiness. There are two guards here + because there are two different races: + + * ``_busy`` stops the SYNCHRONOUS echo — a widget's ``.set`` itself fires + ``pointer_move``, so mirroring would re-enter the handler immediately. + * ``_master`` stops the ASYNCHRONOUS write-back. The index hook runs on the + ``_NavDispatcher`` thread and is marshalled onto the main thread, so it + lands SOME TIME AFTER the drag handler has returned and cleared + ``_busy``. Without ``_master`` it then writes the last committed + (index-quantised) position onto the line the user is still holding; the + pointer has moved on, the next ``pointer_move`` yanks it back, and the + cursor visibly jumps to and fro for the whole drag. ``_busy`` cannot + cover this — it is long since false by the time the write-back arrives. + + ``pointer_up`` releases the master so the authoritative committed position + can then settle every line, including the one just let go.""" def __init__(self, session, window_id: int, widgets, sel): self.session = session @@ -261,6 +276,9 @@ def __init__(self, session, window_id: int, widgets, sel): self.widgets = list(widgets) self.sel = sel self._busy = False + # The widget currently under the pointer — the master. None when no + # drag is in progress, i.e. every line follows. + self._master = None self._handlers: list = [] # keep wrapper refs alive (weak registration) self._closed = False self._wire_drag() @@ -269,18 +287,34 @@ def __init__(self, session, window_id: int, widgets, sel): # ── row line dragged → drive the real selector + mirror the other rows ── def _wire_drag(self) -> None: for w in self.widgets: - h = self._make_drag_handler(w) - self._handlers.append(h) - for et in ("pointer_move", "pointer_up"): + move = self._make_drag_handler(w) + release = self._make_release_handler(w) + self._handlers.extend((move, release)) + for et, h in (("pointer_move", move), ("pointer_up", release)): try: w.add_event_handler(h, et) except Exception as e: log.debug("wiring stacked cursor %s handler failed: %s", et, e) + def _make_release_handler(self, src): + """``pointer_up``: drive the final position, then hand back the master + role so the committed index can settle every line.""" + drive = self._make_drag_handler(src) + + def handler(_ev=None): + try: + drive() + finally: + if self._master is src: + self._master = None + return handler + def _make_drag_handler(self, src): def handler(_ev=None): if self._busy: return + # Claim the master role for as long as this line is being dragged. + self._master = src self._busy = True try: x = float(src.get("x")) @@ -335,11 +369,18 @@ def _on_selector_index(self, indices) -> None: self.session._dispatch_to_main(lambda: self._set_all_lines(x)) def _set_all_lines(self, x: float) -> None: + """Write the committed position onto every FOLLOWER line. + + The master (the line under the pointer, if any) is deliberately skipped + — see the class docstring. Writing to it is what made the cursor jump + to and fro mid-drag.""" if self._closed or self._busy: return self._busy = True try: for w in self.widgets: + if w is self._master: + continue try: w.set(x=float(x)) except Exception as e: @@ -352,6 +393,7 @@ def close(self) -> None: keep syncing (and can be GC'd). Called on chip switch-back / window close.""" self._closed = True + self._master = None try: if self._index_hook in self.sel.index_hooks: self.sel.index_hooks.remove(self._index_hook) diff --git a/spyde/tests/migrated/test_stacked_cursor_master.py b/spyde/tests/migrated/test_stacked_cursor_master.py new file mode 100644 index 00000000..1c79a759 --- /dev/null +++ b/spyde/tests/migrated/test_stacked_cursor_master.py @@ -0,0 +1,195 @@ +"""Stacked 1-D navigator lanes: one master, the rest followers. + +The failure this pins is an interaction bug you cannot see in a screenshot and +cannot see in a synchronous unit test either — it needs the write-back to +arrive AFTER the drag handler has returned, which is exactly what +``_dispatch_to_main`` does in the app. + +While a lane is being dragged its line is the MASTER. The selector's index hook +fires on the ``_NavDispatcher`` thread with the committed (index-quantised) +position and is marshalled onto the main thread, landing some milliseconds +later — by which time the synchronous ``_busy`` guard is long since false. If +that write-back touches the held line, it snaps back to the last committed +frame while the pointer is somewhere else, the next ``pointer_move`` drags it +forward again, and the cursor oscillates for the whole drag. +""" +from __future__ import annotations + +import numpy as np +import pytest + +from spyde.actions.navigator_views import _StackedNavCursor + + +class _FakeWidget: + """A VLine stand-in. ``set`` fires ``pointer_move`` like the real one.""" + + def __init__(self, x=0.0): + self._x = float(x) + self._handlers: dict[str, list] = {} + + def add_event_handler(self, fn, event_type): + self._handlers.setdefault(event_type, []).append(fn) + + def get(self, key): + assert key == "x" + return self._x + + @property + def x(self): + return self._x + + @x.setter + def x(self, value): + self._x = float(value) + + def set(self, x=None, **_kw): + if x is None: + return + self._x = float(x) + # The real widget echoes a pointer_move on a programmatic set — the + # whole reason _busy exists. + self._fire("pointer_move") + + def _fire(self, event_type): + for fn in list(self._handlers.get(event_type, ())): + fn() + + def drag_to(self, x: float): + self._x = float(x) + self._fire("pointer_move") + + def release(self): + self._fire("pointer_up") + + +class _FakeSelector: + def __init__(self, scale=0.03276): + self.index_hooks: list = [] + self._widget = _FakeWidget() + self.current_indices = np.array([0]) + self.scale = scale + self.updates = 0 + self.current_plot = self + + # `_selector_axis` reads current_plot.plot_state.current_signal… + @property + def plot_state(self): + return self + + @property + def current_signal(self): + scale = self.scale + + class _Axis: + pass + + ax = _Axis() + ax.scale, ax.offset = scale, 0.0 + + class _AM: + signal_axes = [ax] + + class _Sig: + axes_manager = _AM() + + return _Sig() + + def delayed_update_data(self, force=False): + self.updates += 1 + + +class _FakeSession: + """Defers marshalled work, so a test can land it at a chosen moment — + the app's ``_dispatch_to_main`` is likewise asynchronous.""" + + def __init__(self): + self.pending: list = [] + + def _dispatch_to_main(self, fn): + self.pending.append(fn) + + def flush(self): + pending, self.pending = self.pending, [] + for fn in pending: + fn() + + +@pytest.fixture +def stacked(): + session = _FakeSession() + widgets = [_FakeWidget(), _FakeWidget(), _FakeWidget()] + sel = _FakeSelector() + cursor = _StackedNavCursor(session, 1, widgets, sel) + return session, widgets, sel, cursor + + +class TestMasterFollower: + def test_drag_mirrors_to_the_other_lanes(self, stacked): + _session, widgets, sel, _cursor = stacked + widgets[0].drag_to(4.2) + assert [w.x for w in widgets] == [4.2, 4.2, 4.2] + assert sel._widget.x == pytest.approx(4.2) + assert sel.updates == 1 + + def test_late_writeback_does_not_move_the_held_line(self, stacked): + """THE regression: the index hook lands after the handler returned.""" + session, widgets, sel, cursor = stacked + + widgets[0].drag_to(4.2) # user drags lane 0 to 4.2 + # The dispatcher commits frame 128 (128 * 0.03276 = 4.19328) and fires + # the hook on its own thread; the write-back is marshalled. + for hook in sel.index_hooks: + hook(np.array([128])) + widgets[0].drag_to(4.9) # user keeps dragging BEFORE it lands + session.flush() # …and now it lands + + assert widgets[0].x == pytest.approx(4.9), ( + "the held line was yanked back to the committed position" + ) + # The followers do take the committed position — that is their job. + assert widgets[1].x == pytest.approx(4.19328) + assert widgets[2].x == pytest.approx(4.19328) + + def test_release_hands_the_master_role_back(self, stacked): + """After pointer_up every line settles on the committed position.""" + session, widgets, sel, cursor = stacked + + widgets[0].drag_to(4.2) + widgets[0].release() + assert cursor._master is None + + for hook in sel.index_hooks: + hook(np.array([128])) + session.flush() + + assert [pytest.approx(w.x) for w in widgets] == [pytest.approx(4.19328)] * 3 + + def test_only_one_master_at_a_time(self, stacked): + """Mirroring fires the followers' own pointer_move handlers; they must + not steal the master role from the line actually under the pointer.""" + _session, widgets, _sel, cursor = stacked + widgets[1].drag_to(2.0) + assert cursor._master is widgets[1] + assert [w.x for w in widgets] == [2.0, 2.0, 2.0] + + def test_programmatic_move_with_no_drag_syncs_every_line(self, stacked): + """Playback moves the selector with nobody dragging — all lanes follow.""" + session, widgets, sel, _cursor = stacked + for hook in sel.index_hooks: + hook(np.array([64])) + session.flush() + assert [pytest.approx(w.x) for w in widgets] == [pytest.approx(64 * 0.03276)] * 3 + + def test_drag_does_not_re_enter_via_the_mirror_echo(self, stacked): + """`set` echoes a pointer_move; one drag must drive the selector once.""" + _session, widgets, sel, _cursor = stacked + widgets[2].drag_to(1.5) + assert sel.updates == 1 + + def test_close_releases_the_master_and_stops_syncing(self, stacked): + session, widgets, sel, cursor = stacked + widgets[0].drag_to(4.2) + cursor.close() + assert cursor._master is None + assert sel.index_hooks == [] From df6df70be8093dd9a70750fbd0de84032d2c1966 Mon Sep 17 00:00:00 2001 From: Carter Francis Date: Thu, 6 Aug 2026 10:03:20 -0500 Subject: [PATCH 3/3] feat(movie): one burn-in object, temporal integration, x32 playback BURN-IN. There were three ways to put text on a frame: free text in `annotations` (own button, lane, inspector), the timestamp as a bool param drawn at a hardcoded (6,4), and instrument channels in `text_overlays`. They are one object now -- a `text_overlays` entry differing only in where its string comes from (`builtin:"label"`, `builtin:"time"`, `insitu_channel`). One action `movie_add_burnin(source)`, one backend-owned source list, one Burn-in timeline lane, one inspector, one rail group. Legacy specs migrate in place ONCE (`burnin_migrated`), after which the overlay list is the only truth -- re-deriving it on every read left `params["timestamp"]` authoritative, so deleting the timestamp clip was undone by the next read. Four bugs this exposed or fixed: * `sync_overlay_widgets` cleared the text-overlay DICT but never removed the label widgets from the plot, so every resync stacked another copy -- one add read as two. * the labels had NO drag handler (only annotations did), so moving one in the editor never reached the spec and the export drew it at the original xy. This was the editor and the movie disagreeing about where things sit. * `_bake_poster` imported `_overlay_value_at` from `pipeline`, where it has never existed. Latent: it only ran when a movie had text overlays, which was uncommon until the timestamp became one -- then every poster silently failed to bake. * burnt-in text used an absolute px size while the timestamp scaled with the frame, so an 18 pt overlay was a speck on a 4k movie; and `.2f` rendered a uA current as "0.00". RESAMPLING. The three reducers now differ on purpose: spatial `downsample` box-means (already did); fps INTEGRATES the source frames each output frame stands for (30.5 -> 12 fps averages ~3, real SNR on noisy data); speed segments SUB-SELECT -- a 32x jump advances ~81 frames but keeps the fps-sized window, since integrating 81 would smear a second of change into one picture. Slow motion narrows it, or consecutive output frames double-count. `read_frame_integrated` accumulates one frame at a time, so the memory-safety contract holds for any window. ENCODER. Quality 7 -> 9 and a 24-frame GOP cap. A fast-forward's large inter-frame deltas starved the high-frequency detail, so the previous frame's timestamp ghosted through -- text tearing while the image looked fine. PLAYBACK. The speed cycle reaches x32 (a 7914-frame acquisition is still 32 s at x8) and the badge shows whenever a speed above 1x is selected, not only while the clock runs. --- .../src/components/FloatingToolbar.tsx | 21 +- .../renderer/src/components/MovieEditor.tsx | 94 +++- electron/src/renderer/src/kernel/protocol.ts | 32 +- spyde/actions/base.py | 2 +- spyde/actions/movie_export/encoder.py | 34 +- spyde/actions/movie_export/pipeline.py | 233 +++++++- spyde/actions/movie_export/traces.py | 89 +++- spyde/actions/playback.py | 17 +- spyde/actions/registry.py | 2 + spyde/actions/report/movie.py | 319 ++++++++++- spyde/tests/migrated/test_movie_block.py | 16 +- .../migrated/test_movie_insitu_overlay.py | 504 ++++++++++++++++++ .../tests/migrated/test_movie_integration.py | 149 ++++++ spyde/tests/migrated/test_playback.py | 17 +- spyde/toolbars.yaml | 2 +- 15 files changed, 1446 insertions(+), 85 deletions(-) create mode 100644 spyde/tests/migrated/test_movie_insitu_overlay.py create mode 100644 spyde/tests/migrated/test_movie_integration.py diff --git a/electron/src/renderer/src/components/FloatingToolbar.tsx b/electron/src/renderer/src/components/FloatingToolbar.tsx index f1c5d2a2..2bab4d77 100644 --- a/electron/src/renderer/src/components/FloatingToolbar.tsx +++ b/electron/src/renderer/src/components/FloatingToolbar.tsx @@ -314,8 +314,12 @@ export function FloatingToolbar({ // clock runs and un-lights when playback auto-stops at the movie end. const pb = state.playback const playbackActive = a.name === 'Play' && pb.playing - const ffSpeed = a.name === 'Fast Forward' && pb.playing && pb.speed > 1 - ? pb.speed : 0 + // The "×N" badge shows whenever a speed above 1× is SELECTED, not only + // while the clock happens to be running: Fast Forward is a cycle, so + // pausing at ×8 and pressing Play must still read as ×8. Gating it on + // `pb.playing` made the speed vanish the moment you paused, which + // looked like the speed change had not been applied at all. + const ffSpeed = a.name === 'Fast Forward' && pb.speed > 1 ? pb.speed : 0 const active = openName === a.name || live.has(a.name) || (state.subItems.get(windowId)?.get(a.name)?.length ?? 0) > 0 || playbackActive @@ -714,12 +718,17 @@ const styles: Record = { border: 'none', cursor: 'pointer', width: 30, height: 30, borderRadius: 6, display: 'flex', alignItems: 'center', justifyContent: 'center', }, - // Small "×N" corner badge on the Fast Forward button while speed > 1. + // "×N" badge on the Fast Forward button while a speed above 1× is selected. + // Sized to stay legible at "32x" — the cycle reaches ×32, and the previous + // 8px corner chip was too small to read the speed off at a glance, which is + // the badge's entire job. speedBadge: { - position: 'absolute', bottom: -3, right: -3, + position: 'absolute', bottom: -5, right: -7, background: '#f38ba8', color: '#11111b', - fontSize: 8, fontWeight: 700, lineHeight: 1, - padding: '1px 3px', borderRadius: 6, pointerEvents: 'none', + fontSize: 10, fontWeight: 800, lineHeight: 1, + padding: '2px 4px', borderRadius: 7, pointerEvents: 'none', + letterSpacing: '-0.2px', + boxShadow: '0 0 0 1.5px #11111b', }, subBar: { ...subBase, top: '100%', marginTop: 10, animation: 'spyde-pop 130ms ease-out' }, subBarUp: { ...subBase, bottom: '100%', marginBottom: 10, animation: 'spyde-pop-up 130ms ease-out' }, diff --git a/electron/src/renderer/src/components/MovieEditor.tsx b/electron/src/renderer/src/components/MovieEditor.tsx index 6b36f6d9..00acc05e 100644 --- a/electron/src/renderer/src/components/MovieEditor.tsx +++ b/electron/src/renderer/src/components/MovieEditor.tsx @@ -24,7 +24,7 @@ import React from 'react' import { useSpyDE } from '../kernel/SpyDEContext' import { SeamlessFigureFrame } from './ReportFigureCell' import { WINDOW_DRAG_MIME } from '../kernel/dnd' -import type { MovieStateMessage, MovieParams, MovieAnnotation, MovieSpeedSegment } from '../kernel/protocol' +import type { MovieStateMessage, MovieParams, MovieAnnotation, MovieSpeedSegment, MovieTextOverlay } from '../kernel/protocol' interface Props { cellId: string @@ -35,7 +35,17 @@ interface Props { const CMAPS = ['gray', 'viridis', 'magma', 'inferno', 'plasma', 'cividis', 'hot', 'jet'] const DOWNSAMPLES = [1, 2, 4, 8] // Speed presets (a segment's multiplier). 0 = hold (freeze); <1 slow-mo; >1 ff. -const SPEEDS = [0, 0.25, 0.5, 1, 2, 4, 8] +// Segment speed multipliers. Reaches 32x for the same reason the playback +// cycle does: an in-situ acquisition is thousands of frames of slow change, +// and a long uneventful stretch wants skipping hard, not merely fast. +const SPEEDS = [0, 0.25, 0.5, 1, 2, 4, 8, 16, 32] + +/** A burn-in's clip label: its literal text, or the quantity it tracks. */ +function burninLabel(o: MovieTextOverlay): string { + if (o.builtin === 'label') return o.text || 'Text' + if (o.builtin === 'time') return 'Timestamp' + return o.label || o.insitu_channel || 'value' +} const SPEED_LABEL = (s: number) => (s === 0 ? 'hold' : `${s}×`) // A selected timeline clip (which lane + index), so the inspector edits it. @@ -185,6 +195,11 @@ export function MovieEditor({ cellId, sendAction, onClose }: Props) { // ── overlay + speed lists ───────────────────────────────────────────────── const anns = st?.annotations ?? [] const textOverlays = st?.text_overlays ?? [] + // Everything addable as burnt-in text (static label, clock, instrument + // channels) — the backend owns the list so the rail has one source of truth. + const burninSources = st?.burnin_sources + ?? [{ source: 'label', label: 'Text', units: '' }, + { source: 'time', label: 'Timestamp', units: 's' }] const speedSegs = st?.speed_segments ?? [] const setAnnotations = (list: MovieAnnotation[]) => { setSt(s => (s ? { ...s, annotations: list } : s)); tune({ annotations: list }) @@ -254,11 +269,28 @@ export function MovieEditor({ cellId, sendAction, onClose }: Props) {
{/* Left rail — just the ADD buttons (compact). */}
-
Add overlay
- + {/* Burnt-in TEXT — one section, one button per source. Static text, + the clock and each attached instrument channel are the same kind + of object (they differ only in where the string comes from), so + they share this list, the Burn-in timeline lane and one + inspector. */} +
Burn-in text
+ {burninSources.map(s => ( + + ))} +
+
Shapes
+
+
Timing
-
+
Crop
) } @@ -509,6 +549,10 @@ function RenderControls({ params, patchParams }: { onChange={(b) => patchParams({ axes: b })} /> patchParams({ scalebar: b })} /> + {/* Timestamp is an ordinary burn-in overlay (`builtin: "time"`), so the + checkbox only adds/removes it — position, size and colour are edited + on the Signal lane like a voltage's, not with a second set of + controls here. */} patchParams({ timestamp: b })} />
@@ -644,6 +688,8 @@ const styles: Record = { body: { flex: 1, display: 'flex', minHeight: 0 }, rail: { display: 'flex', flexDirection: 'column', gap: 5, padding: 12, borderRight: '1px solid #313244', flexShrink: 0, width: 132, overflowY: 'auto' }, railHead: { fontSize: 10, fontWeight: 700, color: '#89b4fa', textTransform: 'uppercase', letterSpacing: 0.4 }, + // Hairline between the rail's groups (burn-in text / shapes / timing). + railRule: { height: 1, background: 'rgba(255,255,255,0.10)', margin: '10px 0 8px' }, toolBtn: { background: '#1e1e2e', color: '#cdd6f4', border: '1px solid #313244', borderRadius: 5, padding: '5px 8px', fontSize: 11.5, cursor: 'pointer', textAlign: 'left' }, toolBtnActive: { background: '#89b4fa', color: '#11111b', borderColor: '#89b4fa', fontWeight: 700 }, center: { flex: 1, display: 'flex', flexDirection: 'column', minWidth: 0, padding: 14, gap: 10 }, diff --git a/electron/src/renderer/src/kernel/protocol.ts b/electron/src/renderer/src/kernel/protocol.ts index e2795a75..ed914db6 100644 --- a/electron/src/renderer/src/kernel/protocol.ts +++ b/electron/src/renderer/src/kernel/protocol.ts @@ -300,7 +300,7 @@ export interface PlaybackStateMessage extends MsgBase { type: 'playback_state' /** True while the movie clock is running. */ playing: boolean - /** Current speed multiplier (1/2/4/8) — drives the Fast Forward "×N" badge. */ + /** Current speed multiplier (1/2/4/8/16/32) — drives the Fast Forward "×N" badge. */ speed?: number loop?: boolean } @@ -561,6 +561,14 @@ export interface ReportCell { * at render time; `fmt` is a Python-style format string over {label,value,units}. */ export interface MovieTextOverlay { source?: Record // a SignalRef dict (opaque to the renderer) + /** An attached instrument channel (spyde/insitu), e.g. "Ewe/V". */ + insitu_channel?: string + /** A built-in value needing no source: "time" is the elapsed-time burn-in, + * which is an ordinary overlay so it gets the same widget, timeline clip + * and inspector controls as any other. */ + builtin?: 'time' | 'label' + /** Literal string for a builtin:'label' overlay. */ + text?: string label?: string units?: string fmt?: string @@ -623,6 +631,8 @@ export interface MovieParams { cmap?: string clim?: [number, number] | null timestamp?: boolean + /** Burnt-in timestamp colour ("#rrggbb"); white when unset. */ + timestamp_color?: string scalebar?: boolean axes?: boolean // draw calibrated axis ticks (default true) t_start?: number @@ -659,6 +669,26 @@ export interface MovieStateMessage extends MsgBase { signal_window_id: number | null // the MDI window holding the signal figure nav_fig_id: string | null // the 1-D navigator figure (shown beside, opt) current_index: number // the navigator's current time index + /** Instrument channels aligned to this movie (electrochemistry potential / + * current, holder temperature…). Already per-frame on the tree, so the + * editor offers each as a one-click burn-in overlay; empty when none is + * attached. See spyde/insitu and movie_add_insitu_overlay. */ + insitu_channels?: InsituChannelOption[] + /** Everything addable as burnt-in text: static label, clock, channels. */ + burnin_sources?: BurninSource[] +} + +/** One attached instrument channel offered as a burn-in overlay. */ +export interface BurninSource { + source: string // "label" | "time" | an instrument channel key + label: string + units: string +} + +export interface InsituChannelOption { + channel: string // the backend key, e.g. "Ewe/V" + label: string // display name, e.g. "Ewe" + units: string // e.g. "V" } /** Export finished (spyde:movie_done). */ diff --git a/spyde/actions/base.py b/spyde/actions/base.py index c36103d3..60c68240 100644 --- a/spyde/actions/base.py +++ b/spyde/actions/base.py @@ -99,7 +99,7 @@ def play_pause(toolbar: "ActionContext", toggled=None, *args, **kwargs): def fast_forward(toolbar: "ActionContext", toggled=None, *args, **kwargs): - """Fast-forward = speed multiplier. Cycles 2x → 4x → 8x → back to 1x. Pressed + """Fast-forward = speed multiplier. Cycles 2x → 4x → 8x → 16x → 32x → back to 1x. Pressed while stopped, starts playback at 2x; while playing, bumps the speed one notch (staying at 1x after 8x).""" session = _session_of(toolbar) diff --git a/spyde/actions/movie_export/encoder.py b/spyde/actions/movie_export/encoder.py index 718799c0..dec57453 100644 --- a/spyde/actions/movie_export/encoder.py +++ b/spyde/actions/movie_export/encoder.py @@ -32,9 +32,23 @@ log = logging.getLogger(__name__) -# H.264 quality (imageio scale 0..10, higher = better). 7 is a good -# size/quality balance for scientific movies. -_H264_QUALITY = 7 +# H.264 quality (imageio scale 0..10, higher = better). +# +# 9, not 7. A fast-forward segment SUBSAMPLES the source (one output frame per +# N source frames — see pipeline.frame_indices_with_speed), so at 16×/32× two +# consecutive output frames can be ~80 source frames apart. Every P-frame then +# carries a near-total rewrite of the picture, and at quality 7 x264 spends its +# budget on the low-frequency bulk and smears the high-frequency detail — +# visibly, the burnt-in timestamp and scale bar TEAR and ghost while the image +# itself looks fine. The content is noisy electron-microscope data, which is +# expensive to encode at the best of times. +_H264_QUALITY = 9 + +# Cap the keyframe interval. Without it x264 picks a GOP of ~250 frames and, +# on a heavily subsampled sequence, a scene-cut-every-frame stream never gets a +# clean reference to recover against. One keyframe per ~1 s of output bounds +# how far any artifact can propagate. +_H264_GOP = 24 class Writer(Protocol): @@ -57,11 +71,21 @@ def __init__(self, path: str, fps: float, size: tuple[int, int]): # size is (W, H); imageio infers it from the first appended frame, but we # keep it for the GIF path and for validation. self._w, self._h = int(size[0]), int(size[1]) - self._writer = imageio.get_writer( - path, fps=float(fps), codec="libx264", + kwargs = dict( + fps=float(fps), codec="libx264", quality=_H264_QUALITY, macro_block_size=1, pixelformat="yuv420p", ffmpeg_log_level="error", ) + # `output_params` is imageio-ffmpeg's passthrough to the CLI. It is not + # in every imageio version's signature, so a TypeError falls back to + # the plain writer rather than failing the export — the GOP cap is a + # quality improvement, never a requirement. + try: + self._writer = imageio.get_writer( + path, output_params=["-g", str(_H264_GOP)], **kwargs) + except TypeError: + log.debug("imageio has no output_params; encoding without a GOP cap") + self._writer = imageio.get_writer(path, **kwargs) def append(self, rgb: np.ndarray) -> None: self._writer.append_data(np.ascontiguousarray(rgb, dtype=np.uint8)) diff --git a/spyde/actions/movie_export/pipeline.py b/spyde/actions/movie_export/pipeline.py index ba8ca305..dadf9135 100644 --- a/spyde/actions/movie_export/pipeline.py +++ b/spyde/actions/movie_export/pipeline.py @@ -171,11 +171,69 @@ def _text_with_shadow(draw, xy, text, font, fill, shadow): draw.text((x, y), text, font=font, fill=fill) -def _draw_timestamp(img, t_sec: float, font): +def label_overlay(frame_h: int = 512, *, text: str = "Label", + color: str = "#ffffff") -> dict: + """A STATIC text burn-in, as an ordinary text-overlay dict. + + Free text used to live in ``annotations`` (``kind: "text"``) while a live + value lived in ``text_overlays`` — two representations, two add paths, two + timeline lanes and two inspectors for what is, on the frame, the same + thing: some text at a position. Both are text overlays now; only the SOURCE + of the string differs (``builtin: "label"`` carries it literally, + ``builtin: "time"`` derives it from the frame, an instrument channel reads + it per frame). + """ + return { + "builtin": "label", + "text": str(text), + "xy": [8, max(4, int(frame_h * 0.08))], + "size": 18, + "color": color, + } + + +def time_overlay(frame_h: int = 512, *, color: str = "#ffffff") -> dict: + """The built-in elapsed-time overlay, as an ordinary text-overlay dict. + + The timestamp used to be a boolean param drawn at a fixed ``(6, 4)`` with + its own font rule and no presence in the editor — so toggling it changed + nothing on screen, and it could not be moved, recoloured or time-gated like + every other burnt-in value. Expressing it as a normal overlay makes one set + of controls cover all of them. + """ + return { + "builtin": "time", + "label": "t", + "units": "s", + "xy": [8, max(4, int(frame_h * 0.02))], + "size": 18, + "color": color, + } + + +def has_time_overlay(overlays) -> bool: + return any(isinstance(o, dict) and o.get("builtin") == "time" + for o in (overlays or ())) + + +def _hex_to_rgb(value, default=_TS_COLOR) -> tuple: + """``"#ff9100"`` → ``(255, 145, 0)``; anything unparseable → *default*.""" + text = str(value or "").strip().lstrip("#") + if len(text) == 3: + text = "".join(c * 2 for c in text) + if len(text) != 6: + return default + try: + return tuple(int(text[i:i + 2], 16) for i in (0, 2, 4)) + except ValueError: + return default + + +def _draw_timestamp(img, t_sec: float, font, color=None): from PIL import ImageDraw draw = ImageDraw.Draw(img) _text_with_shadow(draw, (6, 4), f"t = {t_sec:.2f} s", font, - _TS_COLOR, _TS_SHADOW) + _hex_to_rgb(color), _TS_SHADOW) def _draw_scalebar(img, scale_x: float, units: str, font): @@ -286,6 +344,48 @@ def sc(v, o=0.0): log.debug("annotation %r draw failed: %s", kind, e) +# A burnt-in text size is quoted at this output height and scaled from it. +# 512 px matches the timestamp's own rule (out_h // 28) closely enough that a +# default-sized overlay and the timestamp read as the same size. +_TEXT_REF_H = 512 + + +def _overlay_font_px(size, out_h: int) -> int: + """A burnt-in text size in OUTPUT pixels. + + The size on an overlay is quoted against a reference frame, not in absolute + pixels, because the same movie exports at wildly different resolutions — + the timestamp already scales itself (``out_h // 28``) while text overlays + did not, so on a 4096 px source an 18 pt overlay came out as an unreadable + speck beside a 36 px timestamp. That is what "the voltage doesn't show on + export" was: drawn, but hair-thin. + """ + try: + pt = float(size) + except (TypeError, ValueError): + pt = 18.0 + return max(10, int(round(pt * max(1, int(out_h)) / _TEXT_REF_H))) + + +def _overlay_number(val) -> str: + """Format a live overlay value legibly across the ranges these carry. + + A fixed ``.2f`` is right for a potential (0.56 V) and useless for the + current beside it (2.1e-04 mA renders as "0.00"). Small magnitudes get + significant figures instead. + """ + try: + v = float(val) + except (TypeError, ValueError): + return "—" + if v != v: # NaN — outside the instrument record + return "—" + mag = abs(v) + if mag and (mag < 0.01 or mag >= 1e5): + return f"{v:.3g}" + return f"{v:.2f}" + + def _draw_text_overlays(img, overlays, values_at_t, t_sec: float, k: int, crop_origin=(0, 0)): """Draw each 1-D-signal-as-text overlay: a live value formatted as text (e.g. @@ -304,12 +404,27 @@ def _draw_text_overlays(img, overlays, values_at_t, t_sec: float, k: int, def sc(v, o=0.0): return (float(v) - o) / max(1, k) + out_h = img.size[1] for ov, val in zip(overlays, values_at_t): if not isinstance(ov, dict) or not _in_time_range(t_sec, ov): continue + # The elapsed-time overlay is a BUILT-IN: its value is the frame's own + # time, so it needs no captured trace. Everything else about it — the + # position, size, colour, time gate, the editor's live widget — is the + # ordinary overlay machinery, which is the point of routing it here + # rather than drawing it from its own special case. + if ov.get("builtin") == "time": + val = t_sec try: xy = ov.get("xy", [8, 8]) - font = _load_font(int(ov.get("size", 18))) + font = _load_font(_overlay_font_px(ov.get("size", 18), out_h)) + if ov.get("builtin") == "label": + # A static label: the string IS the content, no value to format. + _text_with_shadow( + draw, (sc(xy[0], ox), sc(xy[1], oy)), + str(ov.get("text", "") or ""), font, + _color(ov.get("color", "#ffffff")), (0, 0, 0, 220)) + continue color = _color(ov.get("color", "#ffffff")) label = str(ov.get("label", "") or "") units = str(ov.get("units", "") or "") @@ -321,9 +436,9 @@ def sc(v, o=0.0): try: text = fmt.format(label=label, value=float(val), units=units) except (KeyError, IndexError, ValueError): - text = f"{label} = {float(val):.2f} {units}".strip() + text = f"{label} = {_overlay_number(val)} {units}".strip() else: - text = f"{label} = {float(val):.2f} {units}".strip() + text = f"{label} = {_overlay_number(val)} {units}".strip() _text_with_shadow(draw, (sc(xy[0], ox), sc(xy[1], oy)), text, font, color, (0, 0, 0, 220)) except Exception as e: @@ -581,10 +696,77 @@ def _segment_end_frame(segments, t_sec, sc, t1) -> float: return t_sec / sc -def _base_frame(raw, t: int, crop, k: int) -> np.ndarray: - """Read source frame *t*, apply the CROP (source px, before downsample), then - the spatial downsample. Memory-safe: one :func:`read_frame` slice.""" - return downsample(apply_crop(read_frame(raw, t), crop), k) +def speed_at_frame(speed_segments, src_frame: float, scale_s: float) -> float: + """The speed multiplier in force at source frame *src_frame* (1× outside + every segment).""" + t_sec = float(src_frame) * (float(scale_s) or 1.0) + for seg in (speed_segments or ()): + try: + tr = seg.get("time_range") + if tr and float(tr[0]) <= t_sec <= float(tr[1]): + return max(0.0, float(seg.get("speed", 1.0))) + except (TypeError, ValueError, AttributeError): + continue + return 1.0 + + +def integration_window(fps: float, scale_s: float, speed: float = 1.0) -> int: + """How many SOURCE frames to average into one output frame. + + Dropping from a 30.5 frame/s acquisition to a 12 frame/s movie means each + output frame stands for ~2.5 source frames. Picking one and discarding the + rest throws away real signal — on noisy electron-microscope data the + integrated frame is visibly cleaner — so the fps reduction INTEGRATES, the + same way the spatial ``downsample`` box-means rather than decimating. + + Speed never increases the window. A 32× segment advances the cursor ~81 + source frames per output frame, but integrating 81 would smear a second of + real change into one picture; a fast-forward is a SUB-SELECTION, so it + keeps the same short window and simply jumps further between them. + + Speed does reduce it: in slow motion the output advances a fraction of a + frame, and integrating more than it advances would double-count the same + source frames into consecutive output frames and blur the result. Hence + ``min(1, speed)`` — the window is what fps asks for, capped by what the + output frame actually spans. + """ + if fps <= 0 or scale_s <= 0: + return 1 + span = (1.0 / float(fps)) / float(scale_s) + return max(1, int(round(span * min(1.0, max(0.0, float(speed)))))) + + +def read_frame_integrated(raw, t: int, n: int, n_frames: int) -> np.ndarray: + """Mean of source frames ``[t, t+n)``, clamped to the dataset. + + Reads ONE FRAME AT A TIME into a running accumulator — never a stacked + slice — so the memory-safety contract holds exactly as it does for the + single-frame path, whatever ``n`` is. + """ + n = max(1, int(n)) + if n == 1: + return read_frame(raw, t) + stop = min(int(n_frames), int(t) + n) + start = max(0, min(int(t), stop - 1)) + acc = None + count = 0 + for i in range(start, stop): + frame = np.asarray(read_frame(raw, i), dtype=np.float32) + acc = frame if acc is None else acc + frame + count += 1 + if acc is None or count == 0: + return read_frame(raw, t) + return acc / float(count) + + +def _base_frame(raw, t: int, crop, k: int, n_int: int = 1, + n_frames: int = 0) -> np.ndarray: + """Read source frame *t* (optionally integrating *n_int* frames), apply the + CROP (source px, before downsample), then the spatial downsample. + Memory-safe: one :func:`read_frame` slice at a time.""" + frame = (read_frame(raw, t) if n_int <= 1 + else read_frame_integrated(raw, t, n_int, n_frames)) + return downsample(apply_crop(frame, crop), k) def _resolve_clim(first: np.ndarray, clim): @@ -605,6 +787,7 @@ def _resolve_clim(first: np.ndarray, clim): def _compose_frame(frame, lut, lo, hi, out_h, out_w, *, t_sec, k, anns, text_overlays, text_values, timestamp, scalebar, + ts_color=None, sb_scale, sig_units, ts_font, sb_font, inset=None, inset_i=0, overlay=None, raw_over=None, src_t=0, crop_origin=(0, 0)): """LUT + fit + the whole overlay stack for ONE already-read (cropped + @@ -627,8 +810,11 @@ def _compose_frame(frame, lut, lo, hi, out_h, out_w, *, t_sec, k, _draw_annotations(img, anns, t_sec, k, crop_origin) if text_overlays: _draw_text_overlays(img, text_overlays, text_values, t_sec, k, crop_origin) - if timestamp: - _draw_timestamp(img, t_sec, ts_font) + # Only the LEGACY path draws it here: a spec that has been migrated carries + # a `builtin: "time"` entry in text_overlays and gets it drawn there, with + # the position/size/colour/time-gate every other overlay has. + if timestamp and not has_time_overlay(text_overlays): + _draw_timestamp(img, t_sec, ts_font, ts_color) if scalebar: _draw_scalebar(img, sb_scale, sig_units, sb_font) if inset is not None: @@ -649,10 +835,16 @@ def render_single_frame(raw, t: int, *, params: dict, n_frames: int, crop = p.get("crop") cmap = str(p.get("cmap", "gray") or "gray") timestamp = bool(p.get("timestamp", True)) + ts_color = p.get("timestamp_color") scalebar = bool(p.get("scalebar", True)) and sig_scale_x > 0 anns = p.get("annotations") or [] lut = build_lut(cmap) - frame = _base_frame(raw, int(t), crop, k) + frame = _base_frame( + raw, int(t), crop, k, + integration_window(float(p.get("fps", 10.0) or 10.0), scale_s, + speed_at_frame(p.get("speed_segments") or [], + int(t), scale_s)), + n_frames) lo, hi = _resolve_clim(frame, p.get("clim")) rgb0 = even_crop(apply_lut(frame, lut, lo, hi)) out_h, out_w = rgb0.shape[:2] @@ -665,7 +857,8 @@ def render_single_frame(raw, t: int, *, params: dict, n_frames: int, return _compose_frame( frame, lut, lo, hi, out_h, out_w, t_sec=t_sec, k=k, anns=anns, text_overlays=(text_overlays or []), - text_values=(text_values or []), timestamp=timestamp, scalebar=scalebar, + text_values=(text_values or []), timestamp=timestamp, ts_color=ts_color, + scalebar=scalebar, sb_scale=sb_scale, sig_units=sig_units, ts_font=ts_font, sb_font=sb_font, crop_origin=crop_origin, overlay=overlay, raw_over=raw_over, src_t=int(t)) @@ -741,6 +934,7 @@ def export_movie(raw, *, path: str, params: dict, n_frames: int, clim = p.get("clim") crop = p.get("crop") timestamp = bool(p.get("timestamp", True)) + ts_color = p.get("timestamp_color") scalebar = bool(p.get("scalebar", True)) and sig_scale_x > 0 anns = p.get("annotations") or [] text_overlays = list(text_overlays or []) @@ -767,7 +961,13 @@ def export_movie(raw, *, path: str, params: dict, n_frames: int, raise _Cancelled() # Auto contrast from the FIRST rendered frame when clim is unset (robust 2-98%). - first = _base_frame(raw, idxs[0], crop, k) + # Frames to integrate per output frame. Sized by the FPS reduction and + # capped by the local speed (see integration_window) — a fast-forward + # sub-selects with the same short window, it does not average more. + segs = p.get("speed_segments") or [] + n_ints = [integration_window(fps, scale_s, + speed_at_frame(segs, i, scale_s)) for i in idxs] + first = _base_frame(raw, idxs[0], crop, k, n_ints[0], n_frames) # Cancel immediately AFTER the probe read too (a cancel that arrived while the # read was in flight stops us before we open the writer / encode any frame). @@ -820,12 +1020,13 @@ def export_movie(raw, *, path: str, params: dict, n_frames: int, for fi, t in enumerate(idxs): if should_cancel is not None and should_cancel(): raise _Cancelled() - frame = first if fi == 0 else _base_frame(raw, t, crop, k) + frame = (first if fi == 0 else + _base_frame(raw, t, crop, k, n_ints[fi], n_frames)) text_values = [(r[fi] if r is not None else None) for r in text_resampled] img = _compose_frame( frame, lut, lo, hi, out_h, out_w, t_sec=times[fi], k=k, anns=anns, text_overlays=text_overlays, text_values=text_values, - timestamp=timestamp, scalebar=scalebar, sb_scale=sb_scale, + timestamp=timestamp, ts_color=ts_color, scalebar=scalebar, sb_scale=sb_scale, sig_units=sig_units, ts_font=ts_font, sb_font=sb_font, inset=inset, inset_i=fi, overlay=overlay, raw_over=raw_over, src_t=t, diff --git a/spyde/actions/movie_export/traces.py b/spyde/actions/movie_export/traces.py index 6658ef8f..cf9d3ef2 100644 --- a/spyde/actions/movie_export/traces.py +++ b/spyde/actions/movie_export/traces.py @@ -122,12 +122,83 @@ def capture_from_plot(plot, *, color: str | None = None) -> "TraceSpec | None": x=x, y=y) -def from_metadata(signal, key: str): # pragma: no cover - documented seam only - """SEAM (NOT IMPLEMENTED): build a TraceSpec from per-frame values baked into - a movie's ``original_metadata`` (e.g. a temperature/pressure column from the - DE-MRC reader). The movie time base would be the trace's own x; this is the - obvious growth point for CSV import too. Kept as a stub so the source can be - added without reshaping :class:`TraceSpec`.""" - raise NotImplementedError( - "trace-from-original_metadata is a post-v1 seam; only 1-D plot capture " - "is implemented (see capture_from_plot).") +def from_insitu_channel(tree, channel: str, *, color: str | None = None): + """Capture a :class:`TraceSpec` from a movie's ATTACHED instrument channel. + + This is the second trace source (``capture_from_plot`` is the first): a + per-frame column that :mod:`spyde.insitu` already aligned and resampled + onto this movie's frame times, sitting on the tree as ``insitu_channels``. + It needs no 1-D plot window to exist — an electrochemistry potential lives + on the tree from the moment the movie opens, so it can be burnt into the + frame directly. + + The x base is the movie's OWN time axis in seconds (index × scale), not the + instrument clock: an overlay is resampled against ``movie_times`` at render + time, and the instrument clock has a lag offset that would shift every + value by that lag. The values are already per-frame, so this is an identity + resample — but going through the same path keeps one code path for both + sources. + + Frames outside the instrument record are NaN in ``insitu_channels`` and + stay NaN here; :func:`~spyde.actions.movie_export.pipeline._draw_text_overlays` + paints a dash for those rather than a made-up number. + + Returns None when the tree has no such channel. + """ + channels = getattr(tree, "insitu_channels", None) or {} + values = channels.get(channel) + if values is None: + log.debug("traces: no in-situ channel %r on this tree", channel) + return None + y = np.asarray(values, dtype=float) + if y.size == 0: + return None + try: + ax = tree.root.axes_manager.navigation_axes[0] + scale, offset = float(ax.scale), float(ax.offset) + except Exception: + scale, offset = 1.0, 0.0 + x = np.arange(y.size, dtype=float) * scale + offset + + label, units = _split_channel_name(channel) + return TraceSpec(id=new_trace_id(), label=label, + color=str(color or _TRACE_COLORS[0]), units=units, x=x, y=y) + + +def _split_channel_name(channel: str) -> tuple[str, str]: + """``"Ewe/V"`` → ``("Ewe", "V")``; ``"/mA"`` → ``("I", "mA")``. + + EC-Lab names a column ``quantity/unit`` and brackets an averaged one, which + reads badly burnt into a frame ("/mA = 0.0002"). Split it so the overlay + can render "I = 0.0002 mA" like every other trace. + """ + name, _, unit = str(channel).rpartition("/") + if not name: + name, unit = str(channel), "" + return name.strip().strip("<>") or str(channel), unit.strip() + + +def insitu_channel_options(tree) -> list[dict]: + """The instrument channels on *tree* worth offering as an overlay. + + Flags and bookkeeping columns (``mode``, ``error``, ``counter inc.``) are + excluded — they are booleans that read as "0"/"1" burnt into a frame. The + result is renderer-facing: ``[{"channel", "label", "units"}]``. + """ + channels = getattr(tree, "insitu_channels", None) or {} + out: list[dict] = [] + for name in channels: + if name == "time/s" or name in _SKIP_CHANNELS: + continue + values = np.asarray(channels[name], dtype=float) + if values.size == 0 or not np.isfinite(values).any(): + continue + label, units = _split_channel_name(name) + out.append({"channel": name, "label": label, "units": units}) + return out + + +# Per-frame columns that carry no meaning as a burnt-in number. +_SKIP_CHANNELS = frozenset({ + "mode", "error", "ox/red", "control changes", "Ns changes", "counter inc.", +}) diff --git a/spyde/actions/playback.py b/spyde/actions/playback.py index a6db21f7..0a094221 100644 --- a/spyde/actions/playback.py +++ b/spyde/actions/playback.py @@ -27,7 +27,7 @@ Fast-forward = speed multiplier ------------------------------- -``fast_forward()`` cycles the speed 1×→2×→4×→8×→1× (starting playback at 2× if +``fast_forward()`` cycles the speed 1×→2×→4×→8×→16×→32×→1× (starting playback at 2× if stopped). ``play()`` is a plain toggle at 1× (or the current speed). Both step the same selector via ``translate_pixels`` + ``delayed_update_data(force=True)`` — the same path a manual drag uses — so no new frame-read machinery is introduced. At the @@ -46,8 +46,12 @@ logger = logging.getLogger(__name__) -# Speed multipliers cycled by Fast Forward. -SPEED_CYCLE = (1, 2, 4, 8) +# Speed multipliers cycled by Fast Forward. Reaches ×32 because an in-situ +# movie is often thousands of frames of slow change — a 7914-frame acquisition +# is 4.3 minutes at ×1 and still 32 s at ×8. The clock is wall-clock-paced with +# a frame-skip (see `_tick`), so a high multiplier drops frames rather than +# demanding an impossible paint rate; ×32 costs no more per second than ×1. +SPEED_CYCLE = (1, 2, 4, 8, 16, 32) # Legacy fallback frame rate when the time axis carries no usable scale. DEFAULT_FPS = 10.0 @@ -87,10 +91,11 @@ class MoviePlaybackController: State machine (all guarded by ``self._lock``): • ``_playing`` — a clock thread is running. - • ``speed`` — current multiplier (1/2/4/8); the "×N" the UI shows. + • ``speed`` — current multiplier (1/2/4/8/16/32); the "×N" the UI shows. • ``loop`` — wrap to the start at the end instead of stopping. ``play()`` is a plain toggle at the current (or requested) speed; - ``fast_forward()`` starts at 2× if stopped, else bumps 1→2→4→8→1 while playing. + ``fast_forward()`` starts at 2× if stopped, else bumps 1→2→4→8→16→32→1 while + playing. """ def __init__(self, session) -> None: @@ -268,7 +273,7 @@ def fast_forward(self, loop: "bool | None" = None) -> bool: """Fast-forward = speed multiplier cycle. Stopped → start playing at 2×. - Playing → bump the speed 1→2→4→8→1 (stays playing at 1× after 8×). + Playing → bump the speed 1→2→4→8→16→32→1 (back to 1× after 32×). Returns True when playback is running afterwards.""" if loop is not None: self.loop = bool(loop) diff --git a/spyde/actions/registry.py b/spyde/actions/registry.py index 0aa04db2..f1b62fa5 100644 --- a/spyde/actions/registry.py +++ b/spyde/actions/registry.py @@ -225,6 +225,8 @@ "movie_tune": "spyde.actions.report.movie.movie_tune", "movie_crop_mode": "spyde.actions.report.movie.movie_crop_mode", "movie_add_text_overlay": "spyde.actions.report.movie.movie_add_text_overlay", + "movie_add_burnin": "spyde.actions.report.movie.movie_add_burnin", + "movie_add_insitu_overlay": "spyde.actions.report.movie.movie_add_insitu_overlay", "movie_add_overlay_image": "spyde.actions.report.movie.movie_add_overlay_image", "movie_drop_window": "spyde.actions.report.movie.movie_drop_window", "movie_export": "spyde.actions.report.movie.movie_export", diff --git a/spyde/actions/report/movie.py b/spyde/actions/report/movie.py index ff877dd7..b6854bf0 100644 --- a/spyde/actions/report/movie.py +++ b/spyde/actions/report/movie.py @@ -57,6 +57,7 @@ from spyde.actions.lifecycle import bump_generation, is_current, run_on_worker from spyde.actions.playback import _units_to_seconds from spyde.actions.movie_export import traces as _traces +from spyde.actions.movie_export import pipeline as _pipeline from spyde.actions.movie_export.pipeline import ( export_movie, render_single_frame, _Cancelled, ) @@ -75,7 +76,7 @@ # (seeded properly from the signal's time axis + the plot's cmap on movie_open). _DEFAULT_PARAMS = dict( fps=12, downsample=1, stride=1, cmap="gray", clim=None, - timestamp=True, scalebar=True, t_start=0, t_end=0, + timestamp=True, timestamp_color="#ffffff", scalebar=True, t_start=0, t_end=0, ) # fps auto-seed clamp band (real-time can be absurd) — mirrors the old wizard. @@ -235,12 +236,18 @@ def sync_overlay_widgets(self) -> None: if p2 is None: return # Drop the previous widgets (a rebuild supersedes them wholesale). + # BOTH sets: the text-overlay labels used to have their dict cleared + # without ever being removed from the plot, so every resync stacked + # another copy of every label on the figure — one add read as two. try: for w in self._ann_widgets.values(): p2._widgets.pop(getattr(w, "id", None), None) + for lw, _ov in getattr(self, "_text_overlay_widgets", {}).values(): + p2._widgets.pop(getattr(lw, "id", None), None) except Exception as e: log.debug("movie clear overlay widgets failed: %s", e) self._ann_widgets = {} + self._text_overlay_widgets = {} self._widget_handlers = [] cur_sec = self.current_index() * self.scale_seconds() anns = (self.cell.movie.annotations if self.cell.movie else None) or [] @@ -299,6 +306,13 @@ def sync_overlay_widgets(self) -> None: color=str(ov.get("color", "#ffffff") or "#ffffff"), show_handles=False) self._text_overlay_widgets[i] = (lw, ov) + # PERSIST a drag. Without this the label moves on screen and the + # spec keeps its original xy, so the export draws it somewhere + # else entirely — the editor and the movie disagreeing about + # where the timestamp and the voltage sit. + handler = _make_burnin_widget_handler(self, i) + lw.add_event_handler(handler, "pointer_up") + self._widget_handlers.append(handler) except Exception as e: log.debug("movie add text-overlay widget failed: %s", e) try: @@ -677,6 +691,125 @@ def _plot_clim(self): # ── text-overlay trace captures ────────────────────────────────────────────── + def _overlays_with_time(self, spec) -> list: + """The spec's burn-in overlays, MIGRATING the two legacy shapes into it. + + Everything that draws text on a frame is one kind of object now, so a + spec written before that has to be upgraded in place the first time it + is read: + + * ``params["timestamp"]`` — a bool with a hardcoded position — becomes + a ``builtin: "time"`` overlay; + * ``annotations`` entries with ``kind: "text"`` — free labels that had + their own add button, lane and inspector — become ``builtin: + "label"`` overlays. + + After this there is ONE list, ONE add path and ONE set of controls for + burnt-in text, which is the whole point; the shapes (rect/circle/arrow) + stay in ``annotations`` because they are not text. + """ + overlays = list(spec.text_overlays or []) + params = spec.params or {} + # Migrate ONCE. Re-deriving on every read means the legacy flags stay + # authoritative forever: deleting the timestamp clip from the timeline + # left `params["timestamp"]` true, so the very next read put it back and + # the removal never stuck. After this flag is set the overlay LIST is + # the only truth. + if params.get("burnin_migrated"): + return overlays + changed = False + + # Legacy free-text annotations → static label overlays. + anns = list(spec.annotations or []) + legacy_text = [a for a in anns + if isinstance(a, dict) and a.get("kind") == "text"] + if legacy_text: + for ann in legacy_text: + ov = _pipeline.label_overlay( + self.frame_size()[1], + text=str(ann.get("text", "") or "Label"), + color=str(ann.get("color", "#ffffff") or "#ffffff")) + ov["xy"] = [int(v) for v in (ann.get("xy") or ov["xy"])] + ov["size"] = int(ann.get("size", ov["size"]) or ov["size"]) + if ann.get("time_range"): + ov["time_range"] = list(ann["time_range"]) + overlays.append(ov) + spec.annotations = [a for a in anns + if not (isinstance(a, dict) + and a.get("kind") == "text")] + changed = True + + if bool(params.get("timestamp", True)) and not _pipeline.has_time_overlay(overlays): + overlays.insert(0, _pipeline.time_overlay( + self.frame_size()[1], + color=str(params.get("timestamp_color") or "#ffffff"))) + changed = True + + if changed: + spec.text_overlays = overlays + spec.params = {**params, "burnin_migrated": True} + return overlays + + # The burn-in sources the editor can add. `label` and `time` are always + # available; the rest are whatever instrument channels are attached. + def burnin_sources(self) -> list: + """``[{source, label, units}]`` — everything addable as burnt-in text.""" + out = [{"source": "label", "label": "Text", "units": ""}, + {"source": "time", "label": "Timestamp", "units": "s"}] + for opt in self.insitu_channel_options(): + out.append({"source": opt["channel"], "label": opt["label"], + "units": opt["units"]}) + return out + + def add_burnin(self, source: str, xy=None) -> bool: + """Add ONE burnt-in text overlay of any kind. The single add path. + + *source* is ``"label"``, ``"time"``, or an instrument channel key. All + three produce an entry in ``text_overlays`` differing only in where the + string comes from. + """ + spec = self.cell.movie + if spec is None: + return False + existing = self._overlays_with_time(spec) + fw, fh = self.frame_size() + if source == "label": + ov = _pipeline.label_overlay(fh) + elif source == "time": + if _pipeline.has_time_overlay(existing): + return False # only one clock makes sense + ov = _pipeline.time_overlay(fh) + else: + option = {o["channel"]: o + for o in self.insitu_channel_options()}.get(str(source)) + if option is None: + return False + ov = {"insitu_channel": option["channel"], "label": option["label"], + "units": option["units"], "size": 18} + ov["color"] = _traces.color_for_index(len(existing)) + ov["xy"] = ([int(xy[0]), int(xy[1])] if xy is not None + else [int(fw * 0.06), + int(fh * 0.9) - _overlay_row_step(fh) * len(existing)]) + spec.text_overlays = list(existing) + [ov] + return True + + def set_timestamp_enabled(self, on: bool) -> None: + """Add or remove the built-in time overlay (the Timestamp checkbox).""" + spec = self.cell.movie + if spec is None: + return + self._overlays_with_time(spec) # migrate first, then edit + overlays = [o for o in (spec.text_overlays or []) + if not (isinstance(o, dict) and o.get("builtin") == "time")] + if on: + params = spec.params or {} + overlays.insert(0, _pipeline.time_overlay( + self.frame_size()[1], + color=str(params.get("timestamp_color") or "#ffffff"))) + spec.text_overlays = overlays + # Keep the legacy flag honest for anything still reading it. + spec.params = {**(spec.params or {}), "timestamp": bool(on)} + def rebuild_text_traces(self) -> list: """For every 1-D-signal-as-text overlay on the spec, attach a live ``_trace`` (a captured :class:`TraceSpec`) so export/preview can resample @@ -689,10 +822,27 @@ def rebuild_text_traces(self) -> list: spec = self.cell.movie cache = getattr(self, "_overlay_traces", None) or {} out = [] - for ov in (spec.text_overlays or []): + for ov in self._overlays_with_time(spec): ov = dict(ov) + # The elapsed-time overlay's value is the frame's own time, so it + # carries no trace — the drawing/format paths read `builtin`. + if ov.get("builtin") == "time": + out.append(ov) + continue ref = ov.get("source") tr = None + # An INSTRUMENT channel (electrochemistry potential/current, holder + # temperature) is already per-frame on the tree — no source window + # has to exist, and it is re-read every rebuild rather than cached, + # so re-attaching a different record updates the overlay too. + channel = ov.get("insitu_channel") + if channel: + tr = _traces.from_insitu_channel( + self.tree, str(channel), color=ov.get("color")) + if tr is not None: + ov["_trace"] = tr + out.append(ov) + continue key = _overlay_key(ref) if key is not None and key in cache: tr = cache[key] @@ -881,12 +1031,48 @@ def state(self) -> dict: "signal_window_id": self.signal_window_id(), "nav_fig_id": self.nav_fig_id(), "current_index": self.current_index(), + # Instrument channels attached to this movie (electrochemistry + # potential/current, holder temperature…). Already per-frame, so + # the editor can offer them as burn-in overlays directly — no 1-D + # source window has to be open and dragged in. + "insitu_channels": self.insitu_channel_options(), + # Everything addable as burnt-in text, in one list — the editor + # renders one button per entry instead of a text button here, a + # checkbox there and a channel list somewhere else. + "burnin_sources": self.burnin_sources(), } + def insitu_channel_options(self) -> list: + """Instrument channels this movie can burn in (see + :func:`spyde.actions.movie_export.traces.insitu_channel_options`).""" + try: + return _traces.insitu_channel_options(self.tree) + except Exception as e: + log.debug("listing in-situ overlay channels failed: %s", e) + return [] + + def add_insitu_overlay(self, channel: str, xy=None, **kw) -> bool: + """Add a burn-in for one attached instrument channel. + + Kept as a name; :meth:`add_burnin` is the one implementation. + """ + return self.add_burnin(str(channel), xy=xy) + def emit(self) -> None: ipc.emit(self.state()) +def _overlay_row_step(frame_h: int) -> int: + """Vertical gap between stacked burn-in overlays, in SOURCE pixels. + + It has to scale with the frame. A fixed 30 px is a readable row on a 512 px + frame and 0.7 % of a 4096 px one — on a real DE movie two overlays landed + 30 px apart on a 4096 px frame, which on screen is the same line twice and + reads as a duplicated text box rather than as two overlays. + """ + return max(30, int(round(frame_h * 0.045))) + + def _public_overlay(ov: dict) -> dict: """A text-overlay dict WITHOUT the ephemeral ``_trace`` (for the wire / serialization).""" @@ -937,7 +1123,12 @@ def _format_overlay_value(ov: dict, frame: int, scale_s: float, label = str(ov.get("label", "") or "") units = str(ov.get("units", "") or "") fmt = str(ov.get("fmt", "") or "") - val = _overlay_value_at(ov.get("_trace"), frame, scale_s, n_frames) + if ov.get("builtin") == "time": + # Its value IS the frame's time — no trace to resolve, which is why it + # would otherwise render as a dash in the editor. + val = float(frame) * float(scale_s or 0.0) + else: + val = _overlay_value_at(ov.get("_trace"), frame, scale_s, n_frames) if val is None: return f"{label} = —" if label else "—" if fmt: @@ -945,7 +1136,9 @@ def _format_overlay_value(ov: dict, frame: int, scale_s: float, return fmt.format(label=label, value=val, units=units) except (KeyError, IndexError, ValueError): pass - return f"{label} = {val:.2f} {units}".strip() + # Same formatter the burnt-in frames use, so the editor preview and the + # export never disagree about how a value reads. + return f"{label} = {_pipeline._overlay_number(val)} {units}".strip() def _make_movie_crop_handler(st): @@ -977,6 +1170,41 @@ def _on_drag_end(event): return _on_drag_end +def _make_burnin_widget_handler(st, index: int): + """``pointer_up`` handler persisting a dragged BURN-IN label's position. + + The sibling of :func:`_make_movie_widget_handler` for ``text_overlays``. + Geometry is in IMAGE pixels, which for a signal frame ARE source pixels — + the same units the export's ``xy`` is in — so it maps straight across with + no conversion. + + ``rebuild_text_traces`` returns copies IN SPEC ORDER, so the widget index is + the spec index; the bounds check below is what keeps that assumption honest + if an overlay is removed between the build and the drag. + """ + def _on_drag_end(event): + try: + widget = getattr(event, "source", None) + g = getattr(widget, "_data", None) if widget is not None else None + if not isinstance(g, dict): + return + spec = st.cell.movie + overlays = (spec.text_overlays if spec else None) or [] + if not (0 <= index < len(overlays)): + return + nx = int(round(float(g.get("x", 0)))) + ny = int(round(float(g.get("y", 0)))) + if overlays[index].get("xy") != [nx, ny]: + overlays[index]["xy"] = [nx, ny] + spec.text_overlays = overlays + st.mgr.dirty = True + st.emit() + except Exception as e: + log.debug("burn-in drag persist failed (idx %s): %s", index, e) + + return _on_drag_end + + def _make_movie_widget_handler(st, ann_index: int, kind: str): """A module-level closure (NOT a bound method — anyplotlib sets ``fn._event_types`` on the handler) returning a ``pointer_up`` handler that @@ -1290,6 +1518,7 @@ def movie_tune(session, plot, payload) -> None: if cell is None or cell.cell_type != "movie" or cell.movie is None: return spec = cell.movie + want_timestamp = None if "params" in payload and isinstance(payload["params"], dict): p = dict(spec.params or {}) incoming = payload["params"] @@ -1306,9 +1535,20 @@ def movie_tune(session, plot, payload) -> None: p["clim"] = ([float(cl[0]), float(cl[1])] if cl and len(cl) == 2 and cl[0] is not None and cl[1] is not None else None) - for key in ("timestamp", "scalebar", "axes"): + for key in ("scalebar", "axes"): if key in incoming: p[key] = bool(incoming[key]) + if incoming.get("timestamp_color"): + p["timestamp_color"] = str(incoming["timestamp_color"]) + # The Timestamp checkbox adds/removes the built-in time OVERLAY, so the + # editor shows the change immediately (it used to flip a render-time + # param that nothing on screen reflected) and the timestamp is then + # movable, recolourable and time-gateable like every other overlay. + if "timestamp" in incoming: + p["timestamp"] = bool(incoming["timestamp"]) + # Applied AFTER the text_overlays replacement below — a payload + # carrying both would otherwise clobber the overlay we just added. + want_timestamp = p["timestamp"] # Clamp the time range to the dataset (n_frames known only with a source). n = st.n_frames() if st is not None and st.has_source else None if n: @@ -1331,10 +1571,16 @@ def movie_tune(session, plot, payload) -> None: if "out_size" in payload: os_ = payload["out_size"] spec.out_size = ([int(v) for v in os_] if os_ and len(os_) == 2 else None) + if want_timestamp is not None and st is not None: + st.set_timestamp_enabled(want_timestamp) mgr.dirty = True - # An annotation change → rebuild the draggable widgets on the live figure so a - # newly-added / removed / retimed overlay is immediately editable on the figure. - if st is not None and "annotations" in payload: + # An annotation / overlay change → rebuild the draggable widgets on the live + # figure so a newly-added, removed or retimed overlay is immediately visible + # and editable there. `text_overlays` is in this list because toggling the + # timestamp now adds/removes one, and without a resync the editor showed no + # change at all — the original "toggling Timestamp does nothing" report. + if st is not None and ("annotations" in payload or "text_overlays" in payload + or want_timestamp is not None): st.sync_overlay_widgets() # A render-param change (cmap / clim / axes) → push it to the live figure. if st is not None and "params" in payload: @@ -1449,7 +1695,10 @@ def movie_add_text_overlay(session, plot, payload) -> None: if tr is None: ipc.emit_error("Add text overlay: source window is not a 1-D plot.") return - xy = payload.get("xy") or [8, 8 + 26 * len(cell.movie.text_overlays)] + st_for_size = _sessions(mgr).get(cell.id) + fh = st_for_size.frame_size()[1] if st_for_size is not None else 512 + xy = payload.get("xy") or [ + 8, 8 + _overlay_row_step(fh) * len(cell.movie.text_overlays)] ov = { "source": SignalRef.from_plot(src).to_dict(), "label": str(payload.get("label") or tr.label or "value"), @@ -1469,6 +1718,52 @@ def movie_add_text_overlay(session, plot, payload) -> None: mgr.emit_state() +def movie_add_burnin(session, plot, payload) -> None: + """Add ONE burnt-in text overlay (``{cell_id, source, xy?}``). + + The single add path for everything that draws text on a frame: ``source`` + is ``"label"`` (static text), ``"time"`` (the elapsed-time clock), or an + attached instrument channel key such as ``"Ewe/V"``. They differ only in + where the string comes from — position, size, colour, time gate, the + draggable widget and the timeline clip are identical, which is why they are + one object with one action rather than three. Emits ``movie_state``. + """ + mgr = _manager(session) + if not mgr.open: + return + cell = mgr.doc.cell_by_id(payload.get("cell_id")) + if cell is None or cell.cell_type != "movie" or cell.movie is None: + return + st = _sessions(mgr).get(cell.id) + if st is None: + ipc.emit_error("Add burn-in: the movie editor is not open.") + return + source = str(payload.get("source") or payload.get("channel") or "label") + if not st.add_burnin(source, xy=payload.get("xy")): + if source == "time": + ipc.emit_error("The timestamp is already on this movie.") + else: + ipc.emit_error( + f"Add burn-in: no in-situ channel {source!r} on this movie. " + "Load its instrument data first (File ▸ Load In-Situ Data…)." + ) + return + if source == "time": + # Keep the legacy flag in step so a reopened spec doesn't re-add it. + st.cell.movie.params = {**(st.cell.movie.params or {}), "timestamp": True} + mgr.dirty = True + st.sync_overlay_widgets() # show it on the figure immediately + st.emit() + mgr.emit_state() + + +# Back-compat alias: the previous, channel-only entry point. +def movie_add_insitu_overlay(session, plot, payload) -> None: + payload = dict(payload or {}) + payload.setdefault("source", payload.get("channel")) + movie_add_burnin(session, plot, payload) + + def movie_export(session, plot, payload) -> None: """Render the movie to ``{cell_id, path}`` on a worker thread — memory-safe, generation-guarded, per-frame cancellable, with partial-file cleanup and a @@ -1631,7 +1926,11 @@ def _bake_poster(raw, params, n_frames, scale_s, sig_scale_x, sig_units, tvals = None overlays = list(text_overlays or []) if overlays: - from spyde.actions.movie_export.pipeline import _overlay_value_at + # `_overlay_value_at` is defined in THIS module — the old import + # pulled it from `pipeline`, where it has never existed, so this + # raised and the poster bake was silently skipped. It only ran when + # a movie had text overlays, which used to be the uncommon case; + # the timestamp being an overlay now makes it every movie. tvals = [_overlay_value_at(o.get("_trace"), t0, scale_s, n_frames) for o in overlays] img = render_single_frame(raw, t0, params=pp, n_frames=n_frames, diff --git a/spyde/tests/migrated/test_movie_block.py b/spyde/tests/migrated/test_movie_block.py index 0a85888c..85b630fa 100644 --- a/spyde/tests/migrated/test_movie_block.py +++ b/spyde/tests/migrated/test_movie_block.py @@ -568,13 +568,17 @@ def test_add_text_overlay_from_1d_window(self, movie_dataset): "cell_id": cell_id, "source_window_id": line_plot.window_id, "label": "T"}) cell = session._report.doc.cell_by_id(cell_id) - assert len(cell.movie.text_overlays) == 1 - ov = cell.movie.text_overlays[0] + # The timestamp is a burn-in overlay now, so a movie always has one + # before anything is added; count only the dragged-in signal. + added = [o for o in cell.movie.text_overlays if not o.get('builtin')] + assert len(added) == 1 + ov = added[0] assert ov["label"] == "T" assert isinstance(ov.get("source"), dict) # a SignalRef dict # The editor state ships the overlay WITHOUT the ephemeral _trace. st = _latest(messages, "movie_state") - assert len(st["text_overlays"]) == 1 + emitted = [o for o in st["text_overlays"] if not o.get("builtin")] + assert len(emitted) == 1 assert "_trace" not in st["text_overlays"][0] def test_text_overlay_live_value_on_figure(self, movie_dataset): @@ -606,8 +610,10 @@ def _is_temp(p): M.movie_add_text_overlay(session, None, { "cell_id": cell_id, "source_window_id": tline.window_id, "label": "T"}) sess = M._sessions(session._report)[cell_id] - assert len(sess._text_overlay_widgets) == 1, "no live text-overlay widget" - lw = sess._text_overlay_widgets[0][0] + widgets = [w for w in sess._text_overlay_widgets.values() + if not w[1].get("builtin")] + assert len(widgets) == 1, "no live text-overlay widget" + lw = widgets[0][0] assert "100" in lw.text, lw.text # frame 0 → 100 M.movie_scrub(session, None, {"cell_id": cell_id, "t": 5}) _wait_until(lambda: sess.current_index() == 5) diff --git a/spyde/tests/migrated/test_movie_insitu_overlay.py b/spyde/tests/migrated/test_movie_insitu_overlay.py new file mode 100644 index 00000000..d6afbfad --- /dev/null +++ b/spyde/tests/migrated/test_movie_insitu_overlay.py @@ -0,0 +1,504 @@ +"""Burn an attached instrument channel into the movie's frames. + +The movie editor's original text overlay needed a live 1-D plot window to drag +in. An instrument channel — an electrochemistry potential, a holder temperature +— is already per-frame on the tree once :mod:`spyde.insitu` has aligned it, so +it needs no window at all. This covers that second trace source (the +``from_metadata`` seam ``traces.py`` documented) and the editor plumbing on top. +""" +from __future__ import annotations + +import numpy as np +import pytest +import hyperspy.api as hs + +from spyde.actions.movie_export import traces as _traces + + +class _Tree: + def __init__(self, n=50, scale=0.03276, channels=None): + self.root = hs.signals.Signal2D(np.zeros((n, 4, 4), np.uint16)) + ax = self.root.axes_manager.navigation_axes[0] + ax.name, ax.units, ax.scale = "time", "s", scale + self.insitu_channels = channels if channels is not None else { + "time/s": np.arange(n) * scale + 31.56, + "Ewe/V": np.linspace(0.0, 0.8, n), + "/mA": np.linspace(-1e-3, 1e-3, n), + "cycle number": np.ones(n), + "I Range": np.full(n, 52.0), + "mode": np.full(n, 2.0), + "ox/red": np.zeros(n), + "error": np.zeros(n), + } + + +class TestFromInsituChannel: + def test_captures_a_channel_on_the_movie_time_base(self): + tree = _Tree(n=50, scale=0.03276) + tr = _traces.from_insitu_channel(tree, "Ewe/V") + assert tr is not None + assert tr.label == "Ewe" + assert tr.units == "V" + assert tr.y.shape == (50,) + # x is the MOVIE's own time axis, NOT the instrument clock — an overlay + # is resampled against movie_times, and the instrument clock carries the + # alignment lag (here 31.56 s) that would shift every value. + assert tr.x[0] == pytest.approx(0.0) + assert tr.x[-1] == pytest.approx(49 * 0.03276) + + def test_bracketed_average_names_are_cleaned(self): + """EC-Lab writes "/mA"; burnt into a frame that should read "I".""" + tr = _traces.from_insitu_channel(_Tree(), "/mA") + assert (tr.label, tr.units) == ("I", "mA") + + def test_missing_channel_returns_none(self): + assert _traces.from_insitu_channel(_Tree(), "Nope/V") is None + + def test_no_channels_at_all_returns_none(self): + tree = _Tree(channels={}) + assert _traces.from_insitu_channel(tree, "Ewe/V") is None + + def test_nan_outside_the_record_is_preserved(self): + """Frames the instrument never covered must stay NaN so the overlay + paints a dash, not a fabricated value.""" + n = 20 + values = np.linspace(0, 1, n) + values[-5:] = np.nan + tree = _Tree(n=n, channels={"Ewe/V": values}) + tr = _traces.from_insitu_channel(tree, "Ewe/V") + assert np.isnan(tr.y[-5:]).all() + assert np.isfinite(tr.y[:-5]).all() + + def test_resample_onto_movie_times_is_the_identity(self): + """The values are already per-frame, so resampling at the frame times + must return them unchanged — the overlay path is shared with dragged + 1-D traces and must not shift an in-situ channel.""" + tree = _Tree(n=40, scale=0.05) + tr = _traces.from_insitu_channel(tree, "Ewe/V") + got = tr.resample(tr.x) + assert got == pytest.approx(tr.y) + + +class TestChannelOptions: + def test_lists_the_meaningful_channels(self): + options = _traces.insitu_channel_options(_Tree()) + names = [o["channel"] for o in options] + assert "Ewe/V" in names and "/mA" in names + # time/s is the axis, not a reading; flags are booleans that read as + # "0"/"1" burnt into a frame. + for skipped in ("time/s", "mode", "error", "ox/red"): + assert skipped not in names + + def test_labels_and_units_are_split(self): + options = {o["channel"]: o for o in _traces.insitu_channel_options(_Tree())} + assert options["Ewe/V"]["label"] == "Ewe" + assert options["Ewe/V"]["units"] == "V" + + def test_all_nan_channel_is_not_offered(self): + tree = _Tree(n=10, channels={"Ewe/V": np.full(10, np.nan)}) + assert _traces.insitu_channel_options(tree) == [] + + def test_no_insitu_data_offers_nothing(self): + tree = _Tree() + del tree.insitu_channels + assert _traces.insitu_channel_options(tree) == [] + + +class _FakeSpec: + def __init__(self): + self.text_overlays: list = [] + self.annotations: list = [] + self.params: dict = {"timestamp": False} + + +class _FakeEditSession: + """The slice of MovieEditSession the burn-in add path touches.""" + from spyde.actions.report.movie import MovieEditSession + add_burnin = MovieEditSession.add_burnin + add_insitu_overlay = MovieEditSession.add_insitu_overlay + burnin_sources = MovieEditSession.burnin_sources + insitu_channel_options = MovieEditSession.insitu_channel_options + _overlays_with_time = MovieEditSession._overlays_with_time + set_timestamp_enabled = MovieEditSession.set_timestamp_enabled + + def __init__(self, tree): + self.tree = tree + + class _Cell: + pass + self.cell = _Cell() + self.cell.movie = _FakeSpec() + + def frame_size(self): + return (512, 512) + + +class TestAddOverlay: + def test_adds_an_overlay_bound_to_the_channel(self): + st = _FakeEditSession(_Tree()) + assert st.add_insitu_overlay("Ewe/V") + (ov,) = st.cell.movie.text_overlays + assert ov["insitu_channel"] == "Ewe/V" + assert ov["label"] == "Ewe" and ov["units"] == "V" + # No SignalRef — the whole point is that no source window is needed. + assert "source" not in ov + + def test_unknown_channel_is_refused(self): + st = _FakeEditSession(_Tree()) + assert not st.add_insitu_overlay("Nope/V") + assert st.cell.movie.text_overlays == [] + + def test_a_flag_channel_is_refused(self): + """Only what `insitu_channel_options` offers can be added.""" + st = _FakeEditSession(_Tree()) + assert not st.add_insitu_overlay("ox/red") + + def test_successive_overlays_stack_down_the_frame(self): + st = _FakeEditSession(_Tree()) + st.add_insitu_overlay("Ewe/V") + st.add_insitu_overlay("/mA") + ys = [o["xy"][1] for o in st.cell.movie.text_overlays] + assert ys[1] < ys[0], "the second overlay should not land on the first" + colors = {o["color"] for o in st.cell.movie.text_overlays} + assert len(colors) == 2, "each overlay should take its own colour" + + def test_the_row_gap_scales_with_the_frame(self): + """A fixed 30 px gap is a readable row on a 512 px frame and 0.7% of a + 4096 px one — two overlays then land on the same line and read as one + duplicated text box.""" + from spyde.actions.report.movie import _overlay_row_step + + class _Big(_FakeEditSession): + def frame_size(self): + return (4096, 4096) + + st = _Big(_Tree()) + st.add_insitu_overlay("Ewe/V") + st.add_insitu_overlay("/mA") + ys = [o["xy"][1] for o in st.cell.movie.text_overlays] + assert ys[0] - ys[1] == _overlay_row_step(4096) + assert ys[0] - ys[1] >= 180, "rows too close to tell apart on a 4k frame" + # …and a small frame keeps a sane minimum rather than collapsing. + assert _overlay_row_step(256) == 30 + + +class TestBurnInLegibility: + """What "the voltage doesn't show on export" actually was: drawn, but at an + absolute 18 px beside a frame-relative timestamp, so a speck on a 4k movie — + and formatted `.2f`, which renders a µA current as "0.00".""" + + def test_font_size_scales_with_the_output_frame(self): + from spyde.actions.movie_export.pipeline import _overlay_font_px + # The timestamp's own rule is out_h // 28; a default overlay should + # land in the same ballpark rather than staying at a fixed 18 px. + assert _overlay_font_px(18, 512) == 18 + assert _overlay_font_px(18, 1024) == 36 + assert _overlay_font_px(18, 1024) == pytest.approx(1024 // 28, abs=4) + # …and never collapses to nothing on a tiny export. + assert _overlay_font_px(18, 64) >= 10 + + def test_font_size_survives_junk(self): + from spyde.actions.movie_export.pipeline import _overlay_font_px + assert _overlay_font_px(None, 512) == 18 + assert _overlay_font_px("big", 512) == 18 + + def test_small_magnitudes_keep_their_significant_figures(self): + from spyde.actions.movie_export.pipeline import _overlay_number + assert _overlay_number(0.5603) == "0.56" + # A µA-scale current in mA — ".2f" would render this as "0.00". + assert _overlay_number(2.67e-4) == "0.000267" + assert _overlay_number(-6.1e-3) == "-0.0061" + assert _overlay_number(1.5e6) == "1.5e+06" + + def test_nan_and_junk_paint_a_dash(self): + from spyde.actions.movie_export.pipeline import _overlay_number + assert _overlay_number(float("nan")) == "—" + assert _overlay_number(None) == "—" + assert _overlay_number([1, 2]) == "—" + + def test_the_overlay_actually_lights_pixels_on_a_rendered_frame(self): + """End-to-end through the real compose path — the check a green unit + suite could not make, and the one that would have caught this.""" + from spyde.actions.movie_export import pipeline + + n, edge = 12, 256 + raw = np.full((n, edge, edge), 12000, np.uint16) + tree = _Tree(n=n) + tr = _traces.from_insitu_channel(tree, "Ewe/V") + overlay = { + "insitu_channel": "Ewe/V", "label": "Ewe", "units": "V", + "xy": [12, int(edge * 0.85)], "size": 18, "color": "#ffcc00", + "_trace": tr, + } + values = pipeline._resample_text_overlays( + [overlay], np.arange(n) * 0.03276, src_indices=np.arange(n)) + img = pipeline.render_single_frame( + raw, 5, params=dict(fps=12, downsample=1, stride=1, cmap="gray", + clim=None, timestamp=False, scalebar=False, + t_start=0, t_end=n - 1), + n_frames=n, scale_s=0.03276, sig_scale_x=1.0, sig_units="nm", + text_overlays=[overlay], + text_values=[None if v is None else v[5] for v in values], + ) + a = np.asarray(img) + lit = np.count_nonzero((a[..., 0] > 180) & (a[..., 1] > 140) & (a[..., 2] < 90)) + assert lit > 60, f"the burnt-in overlay is invisible ({lit} px lit)" + + +class TestOneAddPath: + """Static text, the clock and an instrument channel are the same object and + go through ONE action. They used to have three add paths landing in two + different lists across two timeline lanes.""" + + def test_every_source_produces_a_text_overlay(self): + st = _FakeEditSession(_Tree()) + for source in ("label", "time", "Ewe/V"): + assert st.add_burnin(source), source + kinds = [o.get("builtin") or o.get("insitu_channel") + for o in st.cell.movie.text_overlays] + assert kinds == ["time", "label", "Ewe/V"] or set(kinds) == { + "time", "label", "Ewe/V"} + # …and nothing landed in `annotations`, which is for shapes now. + assert st.cell.movie.annotations == [] + + def test_they_all_carry_the_same_editable_fields(self): + st = _FakeEditSession(_Tree()) + for source in ("label", "time", "Ewe/V"): + st.add_burnin(source) + for ov in st.cell.movie.text_overlays: + for key in ("xy", "size", "color"): + assert key in ov, f"{ov.get('builtin') or ov.get('insitu_channel')} lacks {key}" + + def test_the_source_list_covers_label_time_and_channels(self): + sources = [s["source"] for s in _FakeEditSession(_Tree()).burnin_sources()] + assert sources[:2] == ["label", "time"] + assert "Ewe/V" in sources and "/mA" in sources + + def test_only_one_clock(self): + st = _FakeEditSession(_Tree()) + assert st.add_burnin("time") + assert not st.add_burnin("time"), "a second timestamp makes no sense" + + def test_adding_twice_adds_exactly_two(self): + """The reported bug was one add showing as two.""" + st = _FakeEditSession(_Tree()) + st.add_burnin("Ewe/V") + st.add_burnin("Ewe/V") + assert len(st.cell.movie.text_overlays) == 2 + + def test_legacy_text_annotations_migrate_to_overlays(self): + st = _FakeEditSession(_Tree()) + st.cell.movie.annotations = [ + {"kind": "text", "text": "Before", "xy": [10, 20], "size": 24, + "color": "#ff0000", "time_range": [0.0, 1.0]}, + {"kind": "rect", "xy": [0, 0], "wh": [10, 10]}, + ] + overlays = st._overlays_with_time(st.cell.movie) + labels = [o for o in overlays if o.get("builtin") == "label"] + assert len(labels) == 1 + assert labels[0]["text"] == "Before" + assert labels[0]["xy"] == [10, 20] + assert labels[0]["size"] == 24 + assert labels[0]["time_range"] == [0.0, 1.0] + # The SHAPE stays an annotation — only text moved. + assert [a["kind"] for a in st.cell.movie.annotations] == ["rect"] + + def test_a_static_label_draws_its_literal_text(self): + from spyde.actions.movie_export import pipeline + n, edge = 4, 96 + raw = np.full((n, edge, edge), 9000, np.uint16) + ov = pipeline.label_overlay(edge, text="Hello", color="#ffcc00") + img = pipeline.render_single_frame( + raw, 1, params=dict(fps=12, downsample=1, stride=1, cmap="gray", + clim=None, timestamp=False, scalebar=False, + t_start=0, t_end=n - 1), + n_frames=n, scale_s=0.5, sig_scale_x=1.0, sig_units="nm", + text_overlays=[ov], text_values=[None]) + a = np.asarray(img) + lit = np.count_nonzero((a[..., 0] > 180) & (a[..., 1] > 140) & (a[..., 2] < 90)) + assert lit > 20, "a static label overlay drew nothing" + + +class TestDragPersists: + """Dragging a burn-in in the editor moved the WIDGET and left the spec's + `xy` untouched, so the export drew it somewhere else — the editor and the + movie disagreeing about where the timestamp and the voltage sit.""" + + def _dragged(self, index, x, y): + from spyde.actions.report.movie import _make_burnin_widget_handler + st = _FakeEditSession(_Tree()) + st.add_burnin("time") + st.add_burnin("Ewe/V") + st.mgr = type("M", (), {"dirty": False})() + st.emit = lambda: None + + class _Ev: + source = type("W", (), {"_data": {"x": x, "y": y}})() + _make_burnin_widget_handler(st, index)(_Ev()) + return st + + def test_a_drag_writes_the_new_position_to_the_spec(self): + st = self._dragged(0, 3100.4, 120.6) + assert st.cell.movie.text_overlays[0]["xy"] == [3100, 121] + + def test_it_moves_only_the_dragged_one(self): + st = self._dragged(1, 900, 40) + assert st.cell.movie.text_overlays[1]["xy"] == [900, 40] + assert st.cell.movie.text_overlays[0]["xy"] != [900, 40] + + def test_it_marks_the_report_dirty(self): + assert self._dragged(0, 10, 10).mgr.dirty is True + + def test_an_out_of_range_index_is_ignored(self): + from spyde.actions.report.movie import _make_burnin_widget_handler + st = _FakeEditSession(_Tree()) + st.add_burnin("time") + st.mgr = type("M", (), {"dirty": False})() + st.emit = lambda: None + + class _Ev: + source = type("W", (), {"_data": {"x": 5, "y": 5}})() + _make_burnin_widget_handler(st, 7)(_Ev()) # removed since the build + assert st.cell.movie.text_overlays[0]["xy"] != [5, 5] + + def test_every_burn_in_label_gets_a_handler_wired(self): + import inspect + from spyde.actions.report.movie import MovieEditSession + src = inspect.getsource(MovieEditSession.sync_overlay_widgets) + tail = src.split("_text_overlay_widgets[i]", 1)[1] + assert "_make_burnin_widget_handler" in tail + assert 'add_event_handler(handler, "pointer_up")' in tail + + +class TestWidgetLeak: + """`sync_overlay_widgets` cleared the text-overlay DICT but never removed + the label widgets from the plot, so every resync stacked another copy of + every label — one add read as two.""" + + def test_resync_removes_the_previous_labels(self): + import inspect + from spyde.actions.report.movie import MovieEditSession + src = inspect.getsource(MovieEditSession.sync_overlay_widgets) + head = src.split("cur_sec", 1)[0] + assert "_text_overlay_widgets" in head, ( + "text-overlay widgets are not popped off the plot before a rebuild" + ) + assert head.count("p2._widgets.pop") >= 2 + + +class TestTimestampIsAnOrdinaryOverlay: + """The timestamp used to be a bool param drawn at a fixed spot with its own + font rule and NO presence in the editor — so toggling it changed nothing on + screen and it could not be moved, recoloured or time-gated like the values + burnt in beside it. It is now a `builtin: "time"` text overlay. + """ + + def _session(self): + st = _FakeEditSession(_Tree()) + st.cell.movie.params = {"timestamp": True} + return st + + def test_a_legacy_param_migrates_to_a_real_overlay(self): + from spyde.actions.movie_export.pipeline import has_time_overlay + st = self._session() + overlays = st._overlays_with_time(st.cell.movie) + assert has_time_overlay(overlays) + # …and it is persisted, so it is only synthesised once. + assert has_time_overlay(st.cell.movie.text_overlays) + + def test_it_carries_the_same_fields_as_any_other_overlay(self): + st = self._session() + (ov,) = st._overlays_with_time(st.cell.movie) + for key in ("label", "units", "xy", "size", "color"): + assert key in ov, f"timestamp overlay is missing {key!r}" + + def test_toggling_off_removes_it_and_leaves_the_others(self): + st = self._session() + st._overlays_with_time(st.cell.movie) + st.add_insitu_overlay("Ewe/V") + st.set_timestamp_enabled(False) + kinds = [o.get("builtin") or o.get("insitu_channel") + for o in st.cell.movie.text_overlays] + assert kinds == ["Ewe/V"] + st.set_timestamp_enabled(True) + assert "time" in [o.get("builtin") for o in st.cell.movie.text_overlays] + + def test_removing_the_clip_STICKS(self): + """The migration must run ONCE. Re-deriving it on every read left the + legacy `params["timestamp"]` authoritative forever, so deleting the + timestamp clip from the timeline was undone by the next read.""" + st = self._session() + st._overlays_with_time(st.cell.movie) + assert st.cell.movie.text_overlays + st.cell.movie.text_overlays = [] # the timeline's remove + again = st._overlays_with_time(st.cell.movie) + assert [o.get("builtin") for o in again] == [] + + def test_migration_does_not_re_add_after_a_toggle_off(self): + st = self._session() + st.set_timestamp_enabled(False) + assert st._overlays_with_time(st.cell.movie) == [] + assert st.cell.movie.params["timestamp"] is False + + def test_legacy_text_annotations_migrate_only_once(self): + st = _FakeEditSession(_Tree()) + st.cell.movie.annotations = [{"kind": "text", "text": "A", "xy": [1, 2]}] + st._overlays_with_time(st.cell.movie) + assert len(st.cell.movie.text_overlays) == 1 + st.cell.movie.text_overlays = [] # user deletes it + assert st._overlays_with_time(st.cell.movie) == [] + + def test_its_value_is_the_frames_own_time(self): + """No trace to resolve — the editor label rendered a dash before.""" + from spyde.actions.report.movie import _format_overlay_value + ov = {"builtin": "time", "label": "t", "units": "s"} + assert _format_overlay_value(ov, 100, 0.03276, 200) == "t = 3.28 s" + + def test_the_burnt_in_frame_draws_it_from_the_overlay_path(self): + from spyde.actions.movie_export import pipeline + + n, edge = 6, 128 + raw = np.full((n, edge, edge), 9000, np.uint16) + ov = pipeline.time_overlay(edge, color="#ffcc00") + params = dict(fps=12, downsample=1, stride=1, cmap="gray", clim=None, + timestamp=True, scalebar=False, t_start=0, t_end=n - 1) + img = pipeline.render_single_frame( + raw, 3, params=params, n_frames=n, scale_s=0.5, + sig_scale_x=1.0, sig_units="nm", + text_overlays=[ov], text_values=[None]) + a = np.asarray(img) + lit = np.count_nonzero((a[..., 0] > 180) & (a[..., 1] > 140) & (a[..., 2] < 90)) + assert lit > 20, "the timestamp overlay drew nothing" + + def test_the_legacy_path_does_not_double_draw(self): + """A migrated spec still has `timestamp: True`; the old fixed-position + draw must stand down once the overlay exists, or the frame carries two + timestamps.""" + import inspect + from spyde.actions.movie_export import pipeline + src = inspect.getsource(pipeline._compose_frame) + assert "not has_time_overlay(text_overlays)" in src + + +class TestTimestampColour: + def test_hex_is_parsed(self): + from spyde.actions.movie_export.pipeline import _hex_to_rgb + assert _hex_to_rgb("#ff9100") == (255, 145, 0) + assert _hex_to_rgb("f90") == (255, 153, 0) + + def test_unset_or_junk_falls_back_to_white(self): + from spyde.actions.movie_export.pipeline import _TS_COLOR, _hex_to_rgb + for bad in (None, "", "not-a-colour", "#12345"): + assert _hex_to_rgb(bad) == _TS_COLOR + + def test_the_default_params_carry_a_timestamp_colour(self): + from spyde.actions.report.movie import _DEFAULT_PARAMS + assert _DEFAULT_PARAMS["timestamp_color"] == "#ffffff" + + def test_the_pipeline_threads_the_colour_through(self): + """The param must reach `_draw_timestamp`, not just sit in the dict.""" + import inspect + from spyde.actions.movie_export import pipeline + src = inspect.getsource(pipeline) + assert 'p.get("timestamp_color")' in src + assert "_draw_timestamp(img, t_sec, ts_font, ts_color)" in src diff --git a/spyde/tests/migrated/test_movie_integration.py b/spyde/tests/migrated/test_movie_integration.py new file mode 100644 index 00000000..3e7c8e8d --- /dev/null +++ b/spyde/tests/migrated/test_movie_integration.py @@ -0,0 +1,149 @@ +"""Export resampling: what bins, what sub-selects. + +Three controls reduce the data on the way out, and they must not all behave the +same way: + +* **downsample** (spatial) box-MEANS k×k blocks — it already did. +* **fps** (temporal) INTEGRATES the source frames each output frame stands for. + Dropping 30.5 frame/s to 12 frame/s means ~2.5 source frames per output one; + keeping one and binning the rest throws away real signal, which on noisy + in-situ data is visible as noise the integrated frame does not have. +* **speed segments** SUB-SELECT. A 32x segment jumps ~81 source frames per + output frame; integrating 81 would smear a second of real change into one + picture, so the window stays what fps asked for and the cursor simply jumps + further. +""" +from __future__ import annotations + +import numpy as np +import pytest + +from spyde.actions.movie_export import pipeline + +SCALE_S = 0.03276 # a real DE movie: 30.525 frame/s + + +class TestSpatialBinning: + def test_downsample_is_a_box_mean_not_a_decimation(self): + frame = np.arange(16, dtype=np.float32).reshape(4, 4) + got = pipeline.downsample(frame, 2) + # Top-left 2x2 block is 0,1,4,5 -> mean 2.5 (a decimation would give 0). + assert got[0, 0] == pytest.approx(2.5) + assert got.shape == (2, 2) + + def test_k_of_one_is_a_no_op(self): + frame = np.arange(9, dtype=np.float32).reshape(3, 3) + assert pipeline.downsample(frame, 1) is frame + + +class TestIntegrationWindow: + def test_fps_reduction_sets_the_window(self): + # 30.525 source frame/s -> 12 frame/s output = 2.54 source frames each. + assert pipeline.integration_window(12.0, SCALE_S) == 3 + assert pipeline.integration_window(30.525, SCALE_S) == 1 + + def test_speed_up_does_NOT_widen_the_window(self): + """The whole point: a fast-forward sub-selects, it does not average + more. 32x advances ~81 frames but still integrates the fps window.""" + base = pipeline.integration_window(12.0, SCALE_S, speed=1.0) + for speed in (2, 4, 8, 16, 32): + assert pipeline.integration_window(12.0, SCALE_S, speed=speed) == base + + def test_slow_motion_narrows_it(self): + """In slow-mo the output advances a fraction of a frame; integrating + more than it advances would double-count frames into consecutive output + frames and blur it.""" + assert pipeline.integration_window(12.0, SCALE_S, speed=0.25) == 1 + assert pipeline.integration_window(12.0, SCALE_S, speed=0.0) == 1 + + def test_degenerate_inputs_fall_back_to_one_frame(self): + assert pipeline.integration_window(0, SCALE_S) == 1 + assert pipeline.integration_window(12.0, 0) == 1 + assert pipeline.integration_window(12.0, SCALE_S, speed=-3) == 1 + + def test_never_below_one(self): + # A faster output than the source cannot integrate a fraction of a frame. + assert pipeline.integration_window(120.0, SCALE_S) == 1 + + +class TestSpeedAtFrame: + def test_inside_a_segment(self): + segs = [{"time_range": [1.0, 2.0], "speed": 8}] + assert pipeline.speed_at_frame(segs, 45, SCALE_S) == 8 # 1.47 s + assert pipeline.speed_at_frame(segs, 5, SCALE_S) == 1.0 # 0.16 s + + def test_no_segments_is_one(self): + assert pipeline.speed_at_frame([], 10, SCALE_S) == 1.0 + + def test_malformed_segments_are_skipped(self): + assert pipeline.speed_at_frame([{"speed": 4}, None], 10, SCALE_S) == 1.0 + + +class TestIntegratedRead: + def _raw(self, n=10, edge=4): + # Frame i is filled with i, so a mean over a window is exactly its + # arithmetic mean — an integration bug shows up as a wrong number. + return np.stack([np.full((edge, edge), i, np.uint16) for i in range(n)]) + + def test_it_averages_the_window(self): + got = pipeline.read_frame_integrated(self._raw(), 2, 4, 10) + assert got.mean() == pytest.approx(3.5) # (2+3+4+5)/4 + + def test_window_of_one_is_the_plain_read(self): + got = pipeline.read_frame_integrated(self._raw(), 7, 1, 10) + assert got.mean() == pytest.approx(7.0) + + def test_it_clamps_at_the_end(self): + got = pipeline.read_frame_integrated(self._raw(n=10), 8, 5, 10) + assert got.mean() == pytest.approx(8.5) # (8+9)/2, not out of range + + def test_it_reads_one_frame_at_a_time(self): + """The memory-safety contract: never a stacked slice, whatever n is.""" + seen = [] + + class _Probe: + def __init__(self, data): + self.data = data + + def __getitem__(self, key): + seen.append(key) + return self.data[key] + + raw = _Probe(self._raw(n=10)) + pipeline.read_frame_integrated(raw, 1, 5, 10) + assert len(seen) == 5 + for key in seen: + # Each read is a scalar frame index, never a slice object. + assert isinstance(key[0], (int, np.integer)), key + + def test_integrating_reduces_noise(self): + """The reason this exists, on data shaped like the real thing.""" + rng = np.random.default_rng(0) + raw = (rng.normal(1000, 50, (16, 32, 32))).astype(np.float32) + single = pipeline.read_frame_integrated(raw, 0, 1, 16) + integrated = pipeline.read_frame_integrated(raw, 0, 8, 16) + assert integrated.std() < single.std() * 0.6 + + +class TestExportUsesIt: + def test_a_fast_segment_sub_selects_rather_than_smears(self): + """End-to-end: the window used inside a 32x segment is the fps window, + not the ~81 frames the cursor advances.""" + n = 400 + idxs = pipeline.frame_indices_with_speed( + n, 0, n - 1, 1, [{"time_range": [0.0, n * SCALE_S], "speed": 32}], + fps=12, scale_s=SCALE_S) + assert len(idxs) >= 2 + step = idxs[1] - idxs[0] + assert step > 40, f"a 32x segment should jump far, stepped {step}" + window = pipeline.integration_window( + 12.0, SCALE_S, pipeline.speed_at_frame( + [{"time_range": [0.0, n * SCALE_S], "speed": 32}], idxs[0], SCALE_S)) + assert window == 3 + assert window < step, "integration must not span the whole jump" + + def test_the_preview_integrates_like_the_export(self): + """Editor preview and export must not disagree about the picture.""" + import inspect + src = inspect.getsource(pipeline.render_single_frame) + assert "integration_window" in src diff --git a/spyde/tests/migrated/test_playback.py b/spyde/tests/migrated/test_playback.py index 99c4d084..1c6ccbee 100644 --- a/spyde/tests/migrated/test_playback.py +++ b/spyde/tests/migrated/test_playback.py @@ -215,14 +215,29 @@ def test_fast_forward_cycles_speed(self): assert pb.fast_forward() is True assert pb.is_playing is True assert pb.speed == 2 - # Then 2→4→8→1 while playing. + # Then 2→4→8→16→32→1 while playing. The cycle reaches ×32 because an + # in-situ acquisition is often thousands of frames: 7914 frames of real + # data is 4.3 minutes at ×1 and still 32 s at ×8. pb.fast_forward(); assert pb.speed == 4 pb.fast_forward(); assert pb.speed == 8 + pb.fast_forward(); assert pb.speed == 16 + pb.fast_forward(); assert pb.speed == 32 pb.fast_forward(); assert pb.speed == 1 # wraps, still playing assert pb.is_playing is True pb.fast_forward(); assert pb.speed == 2 pb.pause() + def test_speed_cycle_is_the_declared_one(self): + from spyde.actions.playback import SPEED_CYCLE + pb, _ = _controller(n=100000, scale=1.0, units="s") + seen = [] + pb.fast_forward() + for _ in range(len(SPEED_CYCLE)): + seen.append(pb.speed) + pb.fast_forward() + pb.pause() + assert sorted(set(seen)) == sorted(SPEED_CYCLE) + def test_fast_forward_emits_speed_in_state(self): # The playback_state emit carries the current speed. emitted = [] diff --git a/spyde/toolbars.yaml b/spyde/toolbars.yaml index ca4db289..cf1ad628 100644 --- a/spyde/toolbars.yaml +++ b/spyde/toolbars.yaml @@ -34,7 +34,7 @@ functions: toggle: True toolbar_side: right Fast Forward: - description: Fast-forward — cycle the playback speed 2x → 4x → 8x → 1x. Starts playback if stopped. + description: Fast-forward — cycle the playback speed 2x → 4x → 8x → 16x → 32x → 1x. Starts playback if stopped. icon: drawing/toolbars/icons/fastforward.svg function: spyde.actions.base.fast_forward plot_dim: [ 1 ]