feat(metadata,cli,frontend): accept { "trials": [...] } JSON wrapper

.changeset/unwrap-trials-wrapper.md

@@ -0,0 +1,8 @@
+---
+"@jspsych/metadata": minor
+"@jspsych/metadata-cli": patch
+---
+
+Accept jsPsych data exported as a `{ "trials": [...] }` wrapper (e.g. from OSF), not just a bare array. A new `unwrapTrials` helper (exported from `@jspsych/metadata`) unwraps the array when the input is exactly that single-key wrapper; every other JSON shape is returned unchanged, so `generate()` still throws on non-array input and the CLI/frontend still skip it. An object with sibling keys (`{ trials: [...], meta: {...} }`) is deliberately left untouched rather than silently discarding its top-level metadata.
+
+`generate()`, the CLI directory pipeline, and the frontend uploader all route JSON through this helper, so a wrapped file is converted to a Psych-DS data CSV (with sidecars) and its literal wrapped original is still preserved under `data/raw/`. Previously such files were silently skipped ("0 files read"). When the JSON-Lines ingestion work lands, its parsing path should reuse `unwrapTrials` so wrapper support is consistent across JSON and JSONL inputs.

packages/cli/src/data.ts

@@ -1,6 +1,6 @@
 import fs from "fs";
 import path from "path";
-import JsPsychMetadata, { analyzeJoinKeys, JoinKeyAnalysis, parseCSV, objectsToCSV, isValidPsychDSDataFilename, buildPsychDSDataFiles, PSYCHDS_IGNORE_FILENAME, PSYCHDS_IGNORE_CONTENT } from "@jspsych/metadata";
+import JsPsychMetadata, { analyzeJoinKeys, JoinKeyAnalysis, parseCSV, unwrapTrials, objectsToCSV, isValidPsychDSDataFilename, buildPsychDSDataFiles, PSYCHDS_IGNORE_FILENAME, PSYCHDS_IGNORE_CONTENT } from "@jspsych/metadata";
 import { expandHomeDir, disambiguateFilename, fileStem } from "./utils";
 import { PlannedFile } from "./rename";
 
@@ -114,7 +114,7 @@ export async function preAnalyzeDirectory(
       let parsedData: Array<Record<string, any>>;
 
       if (ext === '.json') {
-        const raw = JSON.parse(content);
+        const raw = unwrapTrials(content);
         if (!Array.isArray(raw)) continue;
         parsedData = raw as Array<Record<string, any>>;
       } else {
@@ -191,7 +191,7 @@ export async function analyzeOutputColumns(
         continue;
       }
       if (ext === '.json') {
-        if (!Array.isArray(JSON.parse(content))) continue; // non-array JSON is skipped by the writer too
+        if (!Array.isArray(unwrapTrials(content))) continue; // non-array JSON is skipped by the writer too
         await metadata.generate(content, {}, 'json', options);
       } else {
         await metadata.generate(content, {}, 'csv', options);
@@ -277,7 +277,7 @@ const processFile = async (metadata: JsPsychMetadata, directoryPath: string, fil
       // a later valid file that maps to the same base.
       let parsed: Array<Record<string, any>> | null = null;
       if (fileExtension === '.json') {
-        const json = JSON.parse(content);
+        const json = unwrapTrials(content);
         if (!Array.isArray(json)) {
           console.error(`"${file}" is not a JSON array of jsPsych trials; skipping CSV conversion.`);
           return false;

packages/cli/tests/data.test.ts

@@ -492,3 +492,62 @@ describe("processDirectory JSON → CSV conversion", () => {
     ).rejects.toThrow(RenamePlanError);
   });
 });
+
+// Some jsPsych exports (e.g. OSF) wrap trials as { "trials": [...] }. The CLI should treat
+// these exactly like a bare array: build the main CSV from the unwrapped trials and preserve
+// the literal wrapped original under raw/.
+describe("{ trials: [...] } wrapper handling", () => {
+  test("processDirectory builds the main CSV from unwrapped trials and preserves the wrapped original", async () => {
+    const srcDir = path.join(tmpDir, "src");
+    const dataDir = path.join(tmpDir, "data");
+    fs.mkdirSync(srcDir);
+    fs.mkdirSync(dataDir);
+    const wrapped = JSON.stringify({
+      trials: [
+        { trial_type: "jsPsych-html-keyboard-response", trial_index: 0, rt: 450 },
+        { trial_type: "jsPsych-html-keyboard-response", trial_index: 1, rt: 512 },
+      ],
+    });
+    fs.writeFileSync(path.join(srcDir, "subj-1_data.json"), wrapped);
+
+    const { total, failed } = await processDirectory(new JsPsychMetadata(), srcDir, false, dataDir);
+    expect(total).toBe(1);
+    expect(failed).toBe(0);
+
+    // Main CSV built from the unwrapped trials: header + one row per trial.
+    const csv = fs.readFileSync(path.join(dataDir, "subj-1_data.csv"), "utf8");
+    const rows = csv.trim().split("\n");
+    expect(rows.length).toBe(3); // header + 2 trials
+    expect(csv).toContain("450");
+    expect(csv).toContain("512");
+
+    // The literal wrapped original is preserved byte-for-byte under raw/.
+    const raw = fs.readFileSync(path.join(dataDir, "raw", "subj-1_data.json"), "utf8");
+    expect(raw).toBe(wrapped);
+  });
+
+  test("preAnalyzeDirectory no longer skips a wrapped file (analyzes its trials)", async () => {
+    fs.writeFileSync(
+      path.join(tmpDir, "subj-1_data.json"),
+      JSON.stringify({ trials: [{ trial_index: 0 }, { trial_index: 0 }] })
+    );
+
+    const result = await preAnalyzeDirectory(tmpDir);
+    expect(result).not.toBeNull();
+    expect(result!.fileName).toBe("subj-1_data.json");
+    expect(result!.analysis.isUnique).toBe(false);
+  });
+
+  test("a { trials: {...} } object (trials not an array) is still skipped", async () => {
+    const srcDir = path.join(tmpDir, "src");
+    const dataDir = path.join(tmpDir, "data");
+    fs.mkdirSync(srcDir);
+    fs.mkdirSync(dataDir);
+    fs.writeFileSync(path.join(srcDir, "subj-1_data.json"), JSON.stringify({ trials: { a: 1 } }));
+
+    const { total, failed } = await processDirectory(new JsPsychMetadata(), srcDir, false, dataDir);
+    expect(total).toBe(1);
+    expect(failed).toBe(1);
+    expect(fs.existsSync(path.join(dataDir, "subj-1_data.csv"))).toBe(false);
+  });
+});

packages/frontend/src/pages/DataUpload.tsx

@@ -1,6 +1,6 @@
 import { useState, useRef, useEffect } from 'react';
 import JSZip from 'jszip';
-import JsPsychMetadata, { analyzeJoinKeys, deriveFallbackBase, buildPsychDSDataFiles, isValidPsychDSDataFilename, PSYCHDS_IGNORE_FILENAME, PSYCHDS_IGNORE_CONTENT } from '@jspsych/metadata';
+import JsPsychMetadata, { analyzeJoinKeys, unwrapTrials, deriveFallbackBase, buildPsychDSDataFiles, isValidPsychDSDataFilename, PSYCHDS_IGNORE_FILENAME, PSYCHDS_IGNORE_CONTENT } from '@jspsych/metadata';
 import PageHeader from '../components/PageHeader';
 import styles from './DataUpload.module.css';
 
@@ -187,7 +187,7 @@ const DataUpload: React.FC<DataUploadProps> = ({
       if (type !== 'json') continue;
       if (name === 'dataset_description.json' || name.endsWith('/dataset_description.json')) continue;
       try {
-        const parsed = JSON.parse(content);
+        const parsed = unwrapTrials(content);
         if (!Array.isArray(parsed) || parsed.length === 0) continue;
         const analysis = analyzeJoinKeys(parsed, ['trial_index']);
         if (!analysis.isUnique) {
@@ -268,7 +268,7 @@ const DataUpload: React.FC<DataUploadProps> = ({
         let mainRows: Array<Record<string, any>> = [];
         let mainContent: string | undefined;
         if (type === 'json') {
-          const json = JSON.parse(content);
+          const json = unwrapTrials(content);
           if (!Array.isArray(json)) {
             update(i, { status: 'skipped', detail: 'not a jsPsych trial array' });
             continue;

packages/frontend/tests/unwrapTrials.test.ts

@@ -0,0 +1,25 @@
+import { unwrapTrials } from "@jspsych/metadata";
+
+// DataUpload's JSON path (preflight join-key analysis at ~line 190 and the CSV writer at
+// ~line 271) parses uploaded files with unwrapTrials instead of JSON.parse, so a wrapped
+// { "trials": [...] } export (e.g. from OSF) is converted like a bare array instead of being
+// skipped as "not a jsPsych trial array". The conversion itself (buildPsychDSDataFiles) is
+// covered end-to-end by the CLI suite, which shares this exact helper; here we guard the
+// frontend's contract with the helper — that it resolves through the workspace mapping and
+// unwraps exactly the shapes DataUpload depends on.
+describe("frontend unwrapTrials integration", () => {
+  test("unwraps a { trials: [...] } export so DataUpload treats it as a trial array", () => {
+    const parsed = unwrapTrials('{"trials":[{"trial_index":0},{"trial_index":1}]}');
+    expect(Array.isArray(parsed)).toBe(true);
+    expect(parsed).toEqual([{ trial_index: 0 }, { trial_index: 1 }]);
+  });
+
+  test("leaves a bare array unchanged", () => {
+    expect(unwrapTrials('[{"trial_index":0}]')).toEqual([{ trial_index: 0 }]);
+  });
+
+  test("does NOT unwrap an object with sibling keys (DataUpload still skips it)", () => {
+    const parsed = unwrapTrials('{"trials":[{"a":1}],"meta":{"v":2}}');
+    expect(Array.isArray(parsed)).toBe(false);
+  });
+});

packages/metadata/src/index.ts

@@ -1,6 +1,6 @@
 import { AuthorFields, AuthorsMap } from "./AuthorsMap";
 import { PluginCache } from "./PluginCache";
-import { saveTextToFile, parseCSV, tryParseJSON, analyzeJoinKeys, JoinKeyAnalysis, SYSTEM_COLUMNS } from "./utils";
+import { saveTextToFile, parseCSV, tryParseJSON, unwrapTrials, analyzeJoinKeys, JoinKeyAnalysis, SYSTEM_COLUMNS } from "./utils";
 import { VariableFields, VariablesMap } from "./VariablesMap";
 
 /**
@@ -439,7 +439,9 @@ export default class JsPsychMetadata {
     }
 
     if (ext === 'json') {
-      parsed_data = JSON.parse(data);
+      // unwrapTrials accepts both a bare jsPsych array and a { "trials": [...] } wrapper
+      // (e.g. OSF exports); non-array, non-wrapper JSON falls through to the throw below.
+      parsed_data = unwrapTrials(data);
     }
 
     if (!Array.isArray(parsed_data)) {
@@ -1034,5 +1036,5 @@ export {
   AuthorFields,
   VariableFields
 }
-export { analyzeJoinKeys, parseCSV, isValidPsychDSDataFilename, toPsychDSValue, deriveArrayFilename, objectsToCSV, disambiguateArrayFilename, deriveFallbackBase, buildPsychDSDataFiles, PSYCHDS_IGNORE_FILENAME, PSYCHDS_IGNORE_CONTENT } from "./utils";
+export { analyzeJoinKeys, parseCSV, unwrapTrials, isValidPsychDSDataFilename, toPsychDSValue, deriveArrayFilename, objectsToCSV, disambiguateArrayFilename, deriveFallbackBase, buildPsychDSDataFiles, PSYCHDS_IGNORE_FILENAME, PSYCHDS_IGNORE_CONTENT } from "./utils";
 export type { JoinKeyAnalysis, PsychDSDataFile, BuildPsychDSDataFilesArgs } from "./utils";

packages/metadata/src/utils.ts

@@ -78,6 +78,37 @@ export function tryParseJSON(value: string): any | null {
   }
 }
 
+/**
+ * Some jsPsych exports (e.g. from OSF) wrap the trials array as { "trials": [...] }
+ * instead of a bare array. Accepts the raw JSON string (or an already-parsed value)
+ * and returns the unwrapped trials array ONLY when the input is exactly that wrapper —
+ * an object whose single key is `trials` and whose value is an array. Otherwise returns
+ * the parsed value unchanged so the caller's existing Array.isArray gate keeps its
+ * current behavior (the library throws on a non-array; the CLI/frontend skip non-array JSON).
+ *
+ * The single-key check is deliberate: this supports the known wrapper shape, it does not
+ * treat `trials` as a magic key. A future export like { trials: [...], meta: {...} } is
+ * left untouched rather than silently discarding its top-level metadata.
+ *
+ * TODO(feat/jsonl-ingestion): when the JSON-Lines ingestion branch lands, its parsing path
+ * (parseJsonData) should call this helper / reuse this logic so wrapper support behaves
+ * consistently across both JSON and JSON-Lines inputs.
+ */
+export function unwrapTrials(data: string | unknown): unknown {
+  const parsed = typeof data === "string" ? JSON.parse(data) : data;
+  if (parsed !== null && typeof parsed === "object" && !Array.isArray(parsed)) {
+    const keys = Object.keys(parsed);
+    if (
+      keys.length === 1 &&
+      keys[0] === "trials" &&
+      Array.isArray((parsed as Record<string, unknown>).trials)
+    ) {
+      return (parsed as Record<string, unknown>).trials;
+    }
+  }
+  return parsed;
+}
+
 /** System columns excluded from join-key candidate detection; also used to initialise ignored_variables in JsPsychMetadata. */
 export const SYSTEM_COLUMNS = new Set([
   'trial_type', 'trial_index', 'time_elapsed', 'extension_type', 'extension_version',

packages/metadata/tests/unwrap-trials.test.ts

@@ -0,0 +1,73 @@
+import JsPsychMetadata, { unwrapTrials } from "../src/index";
+
+// Some jsPsych exports (e.g. from OSF) wrap the trials array as { "trials": [...] } instead
+// of the bare array the pipeline expects. unwrapTrials accepts that exact single-key wrapper
+// and returns the array, leaving every other shape untouched so existing Array.isArray gates
+// behave as before.
+describe("unwrapTrials", () => {
+  test("unwraps an exact { trials: [...] } wrapper string", () => {
+    const result = unwrapTrials('{"trials":[{"a":1},{"a":2}]}');
+    expect(result).toEqual([{ a: 1 }, { a: 2 }]);
+  });
+
+  test("returns a bare array string unchanged", () => {
+    const result = unwrapTrials('[{"a":1},{"a":2}]');
+    expect(result).toEqual([{ a: 1 }, { a: 2 }]);
+  });
+
+  test("unwraps an empty trials array to []", () => {
+    expect(unwrapTrials('{"trials":[]}')).toEqual([]);
+  });
+
+  test("does NOT unwrap when trials is present but not an array", () => {
+    expect(unwrapTrials('{"trials":{"a":1}}')).toEqual({ trials: { a: 1 } });
+    expect(unwrapTrials('{"trials":5}')).toEqual({ trials: 5 });
+  });
+
+  test("does NOT unwrap a wrapper with sibling keys (preserves top-level metadata)", () => {
+    const obj = { trials: [{ a: 1 }], meta: { v: 2 } };
+    expect(unwrapTrials(JSON.stringify(obj))).toEqual(obj);
+  });
+
+  test("returns non-wrapper JSON values unchanged", () => {
+    expect(unwrapTrials("null")).toBeNull();
+    expect(unwrapTrials("5")).toBe(5);
+    expect(unwrapTrials('{"foo":1}')).toEqual({ foo: 1 });
+  });
+
+  test("accepts an already-parsed value without double-parsing", () => {
+    expect(unwrapTrials({ trials: [{ a: 1 }] })).toEqual([{ a: 1 }]);
+    expect(unwrapTrials([{ a: 1 }])).toEqual([{ a: 1 }]);
+    expect(unwrapTrials({ foo: 1 })).toEqual({ foo: 1 });
+  });
+
+  test("throws on malformed JSON strings", () => {
+    expect(() => unwrapTrials("{not json")).toThrow();
+  });
+});
+
+describe("generate() accepts the { trials: [...] } wrapper", () => {
+  // Comparing wrapper vs. bare-array output makes the assertion independent of any plugin
+  // description fetch — both runs take the identical path, so any network result cancels out.
+  const trials = [
+    { trial_type: "survey-text", trial_index: 0, rt: 100, response: "a" },
+    { trial_type: "survey-text", trial_index: 1, rt: 200, response: "b" },
+  ];
+
+  test("produces the same metadata as the equivalent bare array", async () => {
+    const wrapped = new JsPsychMetadata();
+    await wrapped.generate(JSON.stringify({ trials }), {}, "json");
+
+    const bare = new JsPsychMetadata();
+    await bare.generate(JSON.stringify(trials), {}, "json");
+
+    expect(wrapped.getMetadata()).toEqual(bare.getMetadata());
+  });
+
+  test("still throws on a non-array, non-wrapper object", async () => {
+    const meta = new JsPsychMetadata();
+    await expect(meta.generate('{"trials":{"a":1}}', {}, "json")).rejects.toThrow(
+      "Expected an array of observations"
+    );
+  });
+});