From 291a2236689017ea8fa484446a48d59c4df41e2f Mon Sep 17 00:00:00 2001 From: Josh de Leeuw Date: Fri, 19 Jun 2026 09:52:29 -0400 Subject: [PATCH] feat(metadata): accept { "trials": [...] } JSON wrapper via parseJsonData MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Some jsPsych exports (e.g. OSF) wrap the trials array as a single object { "trials": [...] } instead of a bare array, so every consumer's Array.isArray gate silently skipped them ("0 files read") and the library threw. Add a shared unwrapTrials helper (exported from @jspsych/metadata) that unwraps the array only for an exact single-key { trials: [...] } wrapper and returns every other shape unchanged. Rather than wiring it into each call site separately, fold it into parseJsonData's whole-document fast path — since #115 every data parse site (generate(), the CLI pipeline, the frontend uploader) already routes through parseJsonData, so they all get wrapper support through one shared parser. Wrapped originals are still preserved byte-for-byte under data/raw/. Integrates PR #116 onto current main (its branch predated #115's parseJsonData). Co-Authored-By: Claude Opus 4.8 --- .changeset/unwrap-trials-wrapper.md | 8 ++ packages/cli/tests/data.test.ts | 60 +++++++++++++++ packages/frontend/tests/unwrapTrials.test.ts | 24 ++++++ packages/metadata/src/index.ts | 2 +- packages/metadata/src/utils.ts | 40 +++++++++- packages/metadata/tests/unwrap-trials.test.ts | 74 +++++++++++++++++++ 6 files changed, 204 insertions(+), 4 deletions(-) create mode 100644 .changeset/unwrap-trials-wrapper.md create mode 100644 packages/frontend/tests/unwrapTrials.test.ts create mode 100644 packages/metadata/tests/unwrap-trials.test.ts diff --git a/.changeset/unwrap-trials-wrapper.md b/.changeset/unwrap-trials-wrapper.md new file mode 100644 index 0000000..cac7a25 --- /dev/null +++ b/.changeset/unwrap-trials-wrapper.md @@ -0,0 +1,8 @@ +--- +"@jspsych/metadata": minor +"@jspsych/metadata-cli": patch +--- + +Accept jsPsych data exported as a `{ "trials": [...] }` wrapper (e.g. from OSF), not just a bare array. A new `unwrapTrials` helper (exported from `@jspsych/metadata`) unwraps the array when the input is exactly that single-key wrapper; every other JSON shape is returned unchanged, so `generate()` still throws on non-array input and the CLI/frontend still skip it. An object with sibling keys (`{ trials: [...], meta: {...} }`) is deliberately left untouched rather than silently discarding its top-level metadata. + +`unwrapTrials` is folded into `parseJsonData`'s whole-document fast path, so every data parse site — `generate()`, the CLI directory pipeline, and the frontend uploader — accepts the wrapper through the one shared parser. A wrapped file is converted to a Psych-DS data CSV (with sidecars) and its literal wrapped original is still preserved under `data/raw/`. Previously such files were silently skipped ("0 files read"). diff --git a/packages/cli/tests/data.test.ts b/packages/cli/tests/data.test.ts index 580a760..f3d9492 100644 --- a/packages/cli/tests/data.test.ts +++ b/packages/cli/tests/data.test.ts @@ -644,3 +644,63 @@ describe("resolveJoinKeysNonInteractive", () => { expect(result.keys).toEqual(["trial_index", "subject_id"]); // "subject_id" < "uid" }); }); + +// Some jsPsych exports (e.g. OSF) wrap trials as { "trials": [...] }. The CLI should treat +// these exactly like a bare array: build the main CSV from the unwrapped trials and preserve +// the literal wrapped original under raw/. Wrapper support comes from parseJsonData (which the +// CLI uses at every JSON parse site), so no CLI-specific unwrap step is needed. +describe("{ trials: [...] } wrapper handling", () => { + test("processDirectory builds the main CSV from unwrapped trials and preserves the wrapped original", async () => { + const srcDir = path.join(tmpDir, "src"); + const dataDir = path.join(tmpDir, "data"); + fs.mkdirSync(srcDir); + fs.mkdirSync(dataDir); + const wrapped = JSON.stringify({ + trials: [ + { trial_type: "jsPsych-html-keyboard-response", trial_index: 0, rt: 450 }, + { trial_type: "jsPsych-html-keyboard-response", trial_index: 1, rt: 512 }, + ], + }); + fs.writeFileSync(path.join(srcDir, "subj-1_data.json"), wrapped); + + const { total, failed } = await processDirectory(new JsPsychMetadata(), srcDir, false, dataDir); + expect(total).toBe(1); + expect(failed).toBe(0); + + // Main CSV built from the unwrapped trials: header + one row per trial. + const csv = fs.readFileSync(path.join(dataDir, "subj-1_data.csv"), "utf8"); + const rows = csv.trim().split("\n"); + expect(rows.length).toBe(3); // header + 2 trials + expect(csv).toContain("450"); + expect(csv).toContain("512"); + + // The literal wrapped original is preserved byte-for-byte under raw/. + const raw = fs.readFileSync(path.join(dataDir, "raw", "subj-1_data.json"), "utf8"); + expect(raw).toBe(wrapped); + }); + + test("preAnalyzeDirectory no longer skips a wrapped file (analyzes its trials)", async () => { + fs.writeFileSync( + path.join(tmpDir, "subj-1_data.json"), + JSON.stringify({ trials: [{ trial_index: 0 }, { trial_index: 0 }] }) + ); + + const result = await preAnalyzeDirectory(tmpDir); + expect(result).not.toBeNull(); + expect(result!.fileName).toBe("subj-1_data.json"); + expect(result!.analysis.isUnique).toBe(false); + }); + + test("a { trials: {...} } object (trials not an array) is still skipped", async () => { + const srcDir = path.join(tmpDir, "src"); + const dataDir = path.join(tmpDir, "data"); + fs.mkdirSync(srcDir); + fs.mkdirSync(dataDir); + fs.writeFileSync(path.join(srcDir, "subj-1_data.json"), JSON.stringify({ trials: { a: 1 } })); + + const { total, failed } = await processDirectory(new JsPsychMetadata(), srcDir, false, dataDir); + expect(total).toBe(1); + expect(failed).toBe(1); + expect(fs.existsSync(path.join(dataDir, "subj-1_data.csv"))).toBe(false); + }); +}); diff --git a/packages/frontend/tests/unwrapTrials.test.ts b/packages/frontend/tests/unwrapTrials.test.ts new file mode 100644 index 0000000..e5a2972 --- /dev/null +++ b/packages/frontend/tests/unwrapTrials.test.ts @@ -0,0 +1,24 @@ +import { unwrapTrials } from "@jspsych/metadata"; + +// DataUpload's JSON path (preflight join-key analysis and the CSV writer) parses uploaded files +// with parseJsonData, which folds in unwrapTrials — so a wrapped { "trials": [...] } export +// (e.g. from OSF) is converted like a bare array instead of being skipped as "not a jsPsych +// trial array". The conversion itself (buildPsychDSDataFiles) is covered end-to-end by the CLI +// suite, which shares this exact helper; here we guard the frontend's contract with the helper — +// that it resolves through the workspace mapping and unwraps exactly the shapes DataUpload depends on. +describe("frontend unwrapTrials integration", () => { + test("unwraps a { trials: [...] } export so DataUpload treats it as a trial array", () => { + const parsed = unwrapTrials('{"trials":[{"trial_index":0},{"trial_index":1}]}'); + expect(Array.isArray(parsed)).toBe(true); + expect(parsed).toEqual([{ trial_index: 0 }, { trial_index: 1 }]); + }); + + test("leaves a bare array unchanged", () => { + expect(unwrapTrials('[{"trial_index":0}]')).toEqual([{ trial_index: 0 }]); + }); + + test("does NOT unwrap an object with sibling keys (DataUpload still skips it)", () => { + const parsed = unwrapTrials('{"trials":[{"a":1}],"meta":{"v":2}}'); + expect(Array.isArray(parsed)).toBe(false); + }); +}); diff --git a/packages/metadata/src/index.ts b/packages/metadata/src/index.ts index a1e17ad..1b3f32c 100644 --- a/packages/metadata/src/index.ts +++ b/packages/metadata/src/index.ts @@ -1090,5 +1090,5 @@ export { AuthorFields, VariableFields } -export { analyzeJoinKeys, parseCSV, parseJsonData, isValidPsychDSDataFilename, toPsychDSValue, deriveArrayFilename, objectsToCSV, disambiguateArrayFilename, deriveFallbackBase, buildPsychDSDataFiles, stripUnnamedColumns, PSYCHDS_IGNORE_FILENAME, PSYCHDS_IGNORE_CONTENT } from "./utils"; +export { analyzeJoinKeys, parseCSV, parseJsonData, unwrapTrials, isValidPsychDSDataFilename, toPsychDSValue, deriveArrayFilename, objectsToCSV, disambiguateArrayFilename, deriveFallbackBase, buildPsychDSDataFiles, stripUnnamedColumns, PSYCHDS_IGNORE_FILENAME, PSYCHDS_IGNORE_CONTENT } from "./utils"; export type { JoinKeyAnalysis, PsychDSDataFile, BuildPsychDSDataFilesArgs } from "./utils"; diff --git a/packages/metadata/src/utils.ts b/packages/metadata/src/utils.ts index 5581395..df06433 100644 --- a/packages/metadata/src/utils.ts +++ b/packages/metadata/src/utils.ts @@ -78,6 +78,37 @@ export function tryParseJSON(value: string): any | null { } } +/** + * Some jsPsych exports (e.g. from OSF) wrap the trials array as { "trials": [...] } + * instead of a bare array. Accepts the raw JSON string (or an already-parsed value) + * and returns the unwrapped trials array ONLY when the input is exactly that wrapper — + * an object whose single key is `trials` and whose value is an array. Otherwise returns + * the parsed value unchanged so the caller's existing Array.isArray gate keeps its + * current behavior (the library throws on a non-array; the CLI/frontend skip non-array JSON). + * + * The single-key check is deliberate: this supports the known wrapper shape, it does not + * treat `trials` as a magic key. A future export like { trials: [...], meta: {...} } is + * left untouched rather than silently discarding its top-level metadata. + * + * Folded into parseJsonData's whole-document fast path so every data parse site (generate(), + * the CLI pipeline, the frontend uploader) gets wrapper support through the one shared parser; + * also exported for direct use and testing. + */ +export function unwrapTrials(data: string | unknown): unknown { + const parsed = typeof data === "string" ? JSON.parse(data) : data; + if (parsed !== null && typeof parsed === "object" && !Array.isArray(parsed)) { + const keys = Object.keys(parsed); + if ( + keys.length === 1 && + keys[0] === "trials" && + Array.isArray((parsed as Record).trials) + ) { + return (parsed as Record).trials; + } + } + return parsed; +} + /** * Parses experiment data that is either a single JSON document (the standard jsPsych * export — one array of trials, possibly pretty-printed) or JSON-Lines: one JSON value @@ -85,8 +116,9 @@ export function tryParseJSON(value: string): any | null { * array per line). Returns a flat array of observations in both cases. * * A well-formed single document is returned as-is (arrays untouched, so existing - * single-array callers see no change). Only when whole-string parsing fails do we fall - * back to line-by-line parsing, flattening any per-line arrays into one observation + * single-array callers see no change), except an exact { "trials": [...] } wrapper is + * unwrapped to its array via {@link unwrapTrials}. Only when whole-string parsing fails do + * we fall back to line-by-line parsing, flattening any per-line arrays into one observation * stream. Throws a descriptive error when the input is neither valid JSON nor valid JSONL. * * When `tagSourceRecordId` is set, `stats.synthesizedSourceRecordId` is set to true iff a @@ -102,10 +134,12 @@ export function parseJsonData( ): any { // Fast path: a single, well-formed JSON document. Covers the standard single array // (including pretty-printed/multi-line) with no behaviour change for existing callers. + // unwrapTrials accepts an exact { "trials": [...] } wrapper (e.g. OSF exports) and returns + // every other shape untouched, so a bare array passes through unchanged. // Note: tagSourceRecordId never applies here — a single document has no line boundaries // to identify source records by, so its rows are returned untouched. const whole = tryParseJSON(content); - if (whole !== null) return whole; + if (whole !== null) return unwrapTrials(whole); // Fallback: JSON-Lines. Each non-empty line must be its own JSON value; per-line // arrays are concatenated so a multi-participant export becomes one observation array. diff --git a/packages/metadata/tests/unwrap-trials.test.ts b/packages/metadata/tests/unwrap-trials.test.ts new file mode 100644 index 0000000..257f85c --- /dev/null +++ b/packages/metadata/tests/unwrap-trials.test.ts @@ -0,0 +1,74 @@ +import JsPsychMetadata, { unwrapTrials } from "../src/index"; + +// Some jsPsych exports (e.g. from OSF) wrap the trials array as { "trials": [...] } instead +// of the bare array the pipeline expects. unwrapTrials accepts that exact single-key wrapper +// and returns the array, leaving every other shape untouched so existing Array.isArray gates +// behave as before. It is folded into parseJsonData's fast path, so generate() (and the CLI / +// frontend, which share that parser) accept the wrapper too. +describe("unwrapTrials", () => { + test("unwraps an exact { trials: [...] } wrapper string", () => { + const result = unwrapTrials('{"trials":[{"a":1},{"a":2}]}'); + expect(result).toEqual([{ a: 1 }, { a: 2 }]); + }); + + test("returns a bare array string unchanged", () => { + const result = unwrapTrials('[{"a":1},{"a":2}]'); + expect(result).toEqual([{ a: 1 }, { a: 2 }]); + }); + + test("unwraps an empty trials array to []", () => { + expect(unwrapTrials('{"trials":[]}')).toEqual([]); + }); + + test("does NOT unwrap when trials is present but not an array", () => { + expect(unwrapTrials('{"trials":{"a":1}}')).toEqual({ trials: { a: 1 } }); + expect(unwrapTrials('{"trials":5}')).toEqual({ trials: 5 }); + }); + + test("does NOT unwrap a wrapper with sibling keys (preserves top-level metadata)", () => { + const obj = { trials: [{ a: 1 }], meta: { v: 2 } }; + expect(unwrapTrials(JSON.stringify(obj))).toEqual(obj); + }); + + test("returns non-wrapper JSON values unchanged", () => { + expect(unwrapTrials("null")).toBeNull(); + expect(unwrapTrials("5")).toBe(5); + expect(unwrapTrials('{"foo":1}')).toEqual({ foo: 1 }); + }); + + test("accepts an already-parsed value without double-parsing", () => { + expect(unwrapTrials({ trials: [{ a: 1 }] })).toEqual([{ a: 1 }]); + expect(unwrapTrials([{ a: 1 }])).toEqual([{ a: 1 }]); + expect(unwrapTrials({ foo: 1 })).toEqual({ foo: 1 }); + }); + + test("throws on malformed JSON strings", () => { + expect(() => unwrapTrials("{not json")).toThrow(); + }); +}); + +describe("generate() accepts the { trials: [...] } wrapper", () => { + // Comparing wrapper vs. bare-array output makes the assertion independent of any plugin + // description fetch — both runs take the identical path, so any network result cancels out. + const trials = [ + { trial_type: "survey-text", trial_index: 0, rt: 100, response: "a" }, + { trial_type: "survey-text", trial_index: 1, rt: 200, response: "b" }, + ]; + + test("produces the same metadata as the equivalent bare array", async () => { + const wrapped = new JsPsychMetadata(); + await wrapped.generate(JSON.stringify({ trials }), {}, "json"); + + const bare = new JsPsychMetadata(); + await bare.generate(JSON.stringify(trials), {}, "json"); + + expect(wrapped.getMetadata()).toEqual(bare.getMetadata()); + }); + + test("still throws on a non-array, non-wrapper object", async () => { + const meta = new JsPsychMetadata(); + await expect(meta.generate('{"trials":{"a":1}}', {}, "json")).rejects.toThrow( + "Expected an array of observations" + ); + }); +});