diff --git a/.changeset/unwrap-trials-wrapper.md b/.changeset/unwrap-trials-wrapper.md new file mode 100644 index 0000000..cac7a25 --- /dev/null +++ b/.changeset/unwrap-trials-wrapper.md @@ -0,0 +1,8 @@ +--- +"@jspsych/metadata": minor +"@jspsych/metadata-cli": patch +--- + +Accept jsPsych data exported as a `{ "trials": [...] }` wrapper (e.g. from OSF), not just a bare array. A new `unwrapTrials` helper (exported from `@jspsych/metadata`) unwraps the array when the input is exactly that single-key wrapper; every other JSON shape is returned unchanged, so `generate()` still throws on non-array input and the CLI/frontend still skip it. An object with sibling keys (`{ trials: [...], meta: {...} }`) is deliberately left untouched rather than silently discarding its top-level metadata. + +`unwrapTrials` is folded into `parseJsonData`'s whole-document fast path, so every data parse site — `generate()`, the CLI directory pipeline, and the frontend uploader — accepts the wrapper through the one shared parser. A wrapped file is converted to a Psych-DS data CSV (with sidecars) and its literal wrapped original is still preserved under `data/raw/`. Previously such files were silently skipped ("0 files read"). diff --git a/packages/cli/tests/data.test.ts b/packages/cli/tests/data.test.ts index 580a760..f3d9492 100644 --- a/packages/cli/tests/data.test.ts +++ b/packages/cli/tests/data.test.ts @@ -644,3 +644,63 @@ describe("resolveJoinKeysNonInteractive", () => { expect(result.keys).toEqual(["trial_index", "subject_id"]); // "subject_id" < "uid" }); }); + +// Some jsPsych exports (e.g. OSF) wrap trials as { "trials": [...] }. The CLI should treat +// these exactly like a bare array: build the main CSV from the unwrapped trials and preserve +// the literal wrapped original under raw/. Wrapper support comes from parseJsonData (which the +// CLI uses at every JSON parse site), so no CLI-specific unwrap step is needed. +describe("{ trials: [...] } wrapper handling", () => { + test("processDirectory builds the main CSV from unwrapped trials and preserves the wrapped original", async () => { + const srcDir = path.join(tmpDir, "src"); + const dataDir = path.join(tmpDir, "data"); + fs.mkdirSync(srcDir); + fs.mkdirSync(dataDir); + const wrapped = JSON.stringify({ + trials: [ + { trial_type: "jsPsych-html-keyboard-response", trial_index: 0, rt: 450 }, + { trial_type: "jsPsych-html-keyboard-response", trial_index: 1, rt: 512 }, + ], + }); + fs.writeFileSync(path.join(srcDir, "subj-1_data.json"), wrapped); + + const { total, failed } = await processDirectory(new JsPsychMetadata(), srcDir, false, dataDir); + expect(total).toBe(1); + expect(failed).toBe(0); + + // Main CSV built from the unwrapped trials: header + one row per trial. + const csv = fs.readFileSync(path.join(dataDir, "subj-1_data.csv"), "utf8"); + const rows = csv.trim().split("\n"); + expect(rows.length).toBe(3); // header + 2 trials + expect(csv).toContain("450"); + expect(csv).toContain("512"); + + // The literal wrapped original is preserved byte-for-byte under raw/. + const raw = fs.readFileSync(path.join(dataDir, "raw", "subj-1_data.json"), "utf8"); + expect(raw).toBe(wrapped); + }); + + test("preAnalyzeDirectory no longer skips a wrapped file (analyzes its trials)", async () => { + fs.writeFileSync( + path.join(tmpDir, "subj-1_data.json"), + JSON.stringify({ trials: [{ trial_index: 0 }, { trial_index: 0 }] }) + ); + + const result = await preAnalyzeDirectory(tmpDir); + expect(result).not.toBeNull(); + expect(result!.fileName).toBe("subj-1_data.json"); + expect(result!.analysis.isUnique).toBe(false); + }); + + test("a { trials: {...} } object (trials not an array) is still skipped", async () => { + const srcDir = path.join(tmpDir, "src"); + const dataDir = path.join(tmpDir, "data"); + fs.mkdirSync(srcDir); + fs.mkdirSync(dataDir); + fs.writeFileSync(path.join(srcDir, "subj-1_data.json"), JSON.stringify({ trials: { a: 1 } })); + + const { total, failed } = await processDirectory(new JsPsychMetadata(), srcDir, false, dataDir); + expect(total).toBe(1); + expect(failed).toBe(1); + expect(fs.existsSync(path.join(dataDir, "subj-1_data.csv"))).toBe(false); + }); +}); diff --git a/packages/frontend/tests/unwrapTrials.test.ts b/packages/frontend/tests/unwrapTrials.test.ts new file mode 100644 index 0000000..e5a2972 --- /dev/null +++ b/packages/frontend/tests/unwrapTrials.test.ts @@ -0,0 +1,24 @@ +import { unwrapTrials } from "@jspsych/metadata"; + +// DataUpload's JSON path (preflight join-key analysis and the CSV writer) parses uploaded files +// with parseJsonData, which folds in unwrapTrials — so a wrapped { "trials": [...] } export +// (e.g. from OSF) is converted like a bare array instead of being skipped as "not a jsPsych +// trial array". The conversion itself (buildPsychDSDataFiles) is covered end-to-end by the CLI +// suite, which shares this exact helper; here we guard the frontend's contract with the helper — +// that it resolves through the workspace mapping and unwraps exactly the shapes DataUpload depends on. +describe("frontend unwrapTrials integration", () => { + test("unwraps a { trials: [...] } export so DataUpload treats it as a trial array", () => { + const parsed = unwrapTrials('{"trials":[{"trial_index":0},{"trial_index":1}]}'); + expect(Array.isArray(parsed)).toBe(true); + expect(parsed).toEqual([{ trial_index: 0 }, { trial_index: 1 }]); + }); + + test("leaves a bare array unchanged", () => { + expect(unwrapTrials('[{"trial_index":0}]')).toEqual([{ trial_index: 0 }]); + }); + + test("does NOT unwrap an object with sibling keys (DataUpload still skips it)", () => { + const parsed = unwrapTrials('{"trials":[{"a":1}],"meta":{"v":2}}'); + expect(Array.isArray(parsed)).toBe(false); + }); +}); diff --git a/packages/metadata/src/index.ts b/packages/metadata/src/index.ts index a1e17ad..1b3f32c 100644 --- a/packages/metadata/src/index.ts +++ b/packages/metadata/src/index.ts @@ -1090,5 +1090,5 @@ export { AuthorFields, VariableFields } -export { analyzeJoinKeys, parseCSV, parseJsonData, isValidPsychDSDataFilename, toPsychDSValue, deriveArrayFilename, objectsToCSV, disambiguateArrayFilename, deriveFallbackBase, buildPsychDSDataFiles, stripUnnamedColumns, PSYCHDS_IGNORE_FILENAME, PSYCHDS_IGNORE_CONTENT } from "./utils"; +export { analyzeJoinKeys, parseCSV, parseJsonData, unwrapTrials, isValidPsychDSDataFilename, toPsychDSValue, deriveArrayFilename, objectsToCSV, disambiguateArrayFilename, deriveFallbackBase, buildPsychDSDataFiles, stripUnnamedColumns, PSYCHDS_IGNORE_FILENAME, PSYCHDS_IGNORE_CONTENT } from "./utils"; export type { JoinKeyAnalysis, PsychDSDataFile, BuildPsychDSDataFilesArgs } from "./utils"; diff --git a/packages/metadata/src/utils.ts b/packages/metadata/src/utils.ts index 5581395..df06433 100644 --- a/packages/metadata/src/utils.ts +++ b/packages/metadata/src/utils.ts @@ -78,6 +78,37 @@ export function tryParseJSON(value: string): any | null { } } +/** + * Some jsPsych exports (e.g. from OSF) wrap the trials array as { "trials": [...] } + * instead of a bare array. Accepts the raw JSON string (or an already-parsed value) + * and returns the unwrapped trials array ONLY when the input is exactly that wrapper — + * an object whose single key is `trials` and whose value is an array. Otherwise returns + * the parsed value unchanged so the caller's existing Array.isArray gate keeps its + * current behavior (the library throws on a non-array; the CLI/frontend skip non-array JSON). + * + * The single-key check is deliberate: this supports the known wrapper shape, it does not + * treat `trials` as a magic key. A future export like { trials: [...], meta: {...} } is + * left untouched rather than silently discarding its top-level metadata. + * + * Folded into parseJsonData's whole-document fast path so every data parse site (generate(), + * the CLI pipeline, the frontend uploader) gets wrapper support through the one shared parser; + * also exported for direct use and testing. + */ +export function unwrapTrials(data: string | unknown): unknown { + const parsed = typeof data === "string" ? JSON.parse(data) : data; + if (parsed !== null && typeof parsed === "object" && !Array.isArray(parsed)) { + const keys = Object.keys(parsed); + if ( + keys.length === 1 && + keys[0] === "trials" && + Array.isArray((parsed as Record).trials) + ) { + return (parsed as Record).trials; + } + } + return parsed; +} + /** * Parses experiment data that is either a single JSON document (the standard jsPsych * export — one array of trials, possibly pretty-printed) or JSON-Lines: one JSON value @@ -85,8 +116,9 @@ export function tryParseJSON(value: string): any | null { * array per line). Returns a flat array of observations in both cases. * * A well-formed single document is returned as-is (arrays untouched, so existing - * single-array callers see no change). Only when whole-string parsing fails do we fall - * back to line-by-line parsing, flattening any per-line arrays into one observation + * single-array callers see no change), except an exact { "trials": [...] } wrapper is + * unwrapped to its array via {@link unwrapTrials}. Only when whole-string parsing fails do + * we fall back to line-by-line parsing, flattening any per-line arrays into one observation * stream. Throws a descriptive error when the input is neither valid JSON nor valid JSONL. * * When `tagSourceRecordId` is set, `stats.synthesizedSourceRecordId` is set to true iff a @@ -102,10 +134,12 @@ export function parseJsonData( ): any { // Fast path: a single, well-formed JSON document. Covers the standard single array // (including pretty-printed/multi-line) with no behaviour change for existing callers. + // unwrapTrials accepts an exact { "trials": [...] } wrapper (e.g. OSF exports) and returns + // every other shape untouched, so a bare array passes through unchanged. // Note: tagSourceRecordId never applies here — a single document has no line boundaries // to identify source records by, so its rows are returned untouched. const whole = tryParseJSON(content); - if (whole !== null) return whole; + if (whole !== null) return unwrapTrials(whole); // Fallback: JSON-Lines. Each non-empty line must be its own JSON value; per-line // arrays are concatenated so a multi-participant export becomes one observation array. diff --git a/packages/metadata/tests/unwrap-trials.test.ts b/packages/metadata/tests/unwrap-trials.test.ts new file mode 100644 index 0000000..257f85c --- /dev/null +++ b/packages/metadata/tests/unwrap-trials.test.ts @@ -0,0 +1,74 @@ +import JsPsychMetadata, { unwrapTrials } from "../src/index"; + +// Some jsPsych exports (e.g. from OSF) wrap the trials array as { "trials": [...] } instead +// of the bare array the pipeline expects. unwrapTrials accepts that exact single-key wrapper +// and returns the array, leaving every other shape untouched so existing Array.isArray gates +// behave as before. It is folded into parseJsonData's fast path, so generate() (and the CLI / +// frontend, which share that parser) accept the wrapper too. +describe("unwrapTrials", () => { + test("unwraps an exact { trials: [...] } wrapper string", () => { + const result = unwrapTrials('{"trials":[{"a":1},{"a":2}]}'); + expect(result).toEqual([{ a: 1 }, { a: 2 }]); + }); + + test("returns a bare array string unchanged", () => { + const result = unwrapTrials('[{"a":1},{"a":2}]'); + expect(result).toEqual([{ a: 1 }, { a: 2 }]); + }); + + test("unwraps an empty trials array to []", () => { + expect(unwrapTrials('{"trials":[]}')).toEqual([]); + }); + + test("does NOT unwrap when trials is present but not an array", () => { + expect(unwrapTrials('{"trials":{"a":1}}')).toEqual({ trials: { a: 1 } }); + expect(unwrapTrials('{"trials":5}')).toEqual({ trials: 5 }); + }); + + test("does NOT unwrap a wrapper with sibling keys (preserves top-level metadata)", () => { + const obj = { trials: [{ a: 1 }], meta: { v: 2 } }; + expect(unwrapTrials(JSON.stringify(obj))).toEqual(obj); + }); + + test("returns non-wrapper JSON values unchanged", () => { + expect(unwrapTrials("null")).toBeNull(); + expect(unwrapTrials("5")).toBe(5); + expect(unwrapTrials('{"foo":1}')).toEqual({ foo: 1 }); + }); + + test("accepts an already-parsed value without double-parsing", () => { + expect(unwrapTrials({ trials: [{ a: 1 }] })).toEqual([{ a: 1 }]); + expect(unwrapTrials([{ a: 1 }])).toEqual([{ a: 1 }]); + expect(unwrapTrials({ foo: 1 })).toEqual({ foo: 1 }); + }); + + test("throws on malformed JSON strings", () => { + expect(() => unwrapTrials("{not json")).toThrow(); + }); +}); + +describe("generate() accepts the { trials: [...] } wrapper", () => { + // Comparing wrapper vs. bare-array output makes the assertion independent of any plugin + // description fetch — both runs take the identical path, so any network result cancels out. + const trials = [ + { trial_type: "survey-text", trial_index: 0, rt: 100, response: "a" }, + { trial_type: "survey-text", trial_index: 1, rt: 200, response: "b" }, + ]; + + test("produces the same metadata as the equivalent bare array", async () => { + const wrapped = new JsPsychMetadata(); + await wrapped.generate(JSON.stringify({ trials }), {}, "json"); + + const bare = new JsPsychMetadata(); + await bare.generate(JSON.stringify(trials), {}, "json"); + + expect(wrapped.getMetadata()).toEqual(bare.getMetadata()); + }); + + test("still throws on a non-array, non-wrapper object", async () => { + const meta = new JsPsychMetadata(); + await expect(meta.generate('{"trials":{"a":1}}', {}, "json")).rejects.toThrow( + "Expected an array of observations" + ); + }); +});