test: add nested-data & filename-normalization stress guards to CI

.changeset/stress-tests-in-ci.md

@@ -0,0 +1,13 @@
+---
+"@jspsych/metadata": patch
+"@jspsych/metadata-cli": patch
+---
+
+Add stress-test regression guards to the automated suite so previously-fixed nested-data and filename-normalization behavior can't silently regress.
+
+Four Jest suites, ported from the standalone `stress-tests/` harnesses so they run under plain `npm test` (and CI) without a build step:
+
+- `@jspsych/metadata`: `generate()` coherence over a comprehensive nested-data fixture (deep objects, arrays of objects/arrays, mixed-type columns, a `trial_type`-less row, unicode, empties), plus the Psych-DS filename-normalization helper invariants.
+- `@jspsych/metadata-cli`: the `processDirectory` conversion end-to-end (compliant main CSV, `data/raw/` preservation, two-way `variableMeasured` ↔ CSV-column cross-check, and a best-effort Psych-DS validation pass), plus the refusal to write a non-compliant filename non-interactively.
+
+Test-only change; no library or CLI behavior is modified. The shared fixture lives at `dev/stress/`.

dev/stress/nested-all-cases/subject-nested.json

@@ -0,0 +1,101 @@
+[
+  {
+    "trial_type": "html-keyboard-response",
+    "trial_index": 0,
+    "time_elapsed": 1500,
+    "rt": 432.5,
+    "response": "f",
+    "correct": true,
+    "always_null": null,
+    "empty_string": "",
+    "numeric_string": "42",
+    "bool_string": "TRUE",
+    "mixed_col": 10,
+    "json_string_object": "{\"nested\": {\"deep\": 1}}",
+    "json_string_array": "[5, 6, 7]",
+    "flat_object": { "a": 1, "b": "x" },
+    "deep_object": {
+      "l1": {
+        "l2": {
+          "l3": { "l4_leaf": 4, "l4_arr": [1, 2] },
+          "l3_leaf": "first"
+        },
+        "l2_leaf": true
+      }
+    },
+    "object_with_array_of_objects": {
+      "trials": [ { "x": 1, "y": 2 }, { "x": 3, "y": 4 } ]
+    },
+    "array_primitives": [1, 2, 3],
+    "array_objects": [
+      { "x": 0.1, "y": 0.2, "t": 5 },
+      { "x": 0.3, "y": 0.4, "t": 10 }
+    ],
+    "array_of_arrays": [ [1, 2], [3, 4] ],
+    "array_mixed": [1, "two", { "three": 3 }],
+    "array_deep_objects": [
+      { "meta": { "tag": "a", "score": { "raw": 9, "norm": 0.9 } } },
+      { "meta": { "tag": "b", "score": { "raw": 7, "norm": 0.7 } } }
+    ],
+    "empty_object": {},
+    "empty_array": [],
+    "varying_object": { "only_row0": 1 },
+    "unicode_col": "héllo wörld 👋"
+  },
+  {
+    "trial_type": "survey-text",
+    "trial_index": 1,
+    "time_elapsed": 3200,
+    "rt": 1001,
+    "response": { "Q0": "free text answer" },
+    "correct": false,
+    "always_null": null,
+    "empty_string": "",
+    "numeric_string": "7",
+    "bool_string": "false",
+    "mixed_col": "oops",
+    "flat_object": { "a": 2, "b": "y" },
+    "deep_object": {
+      "l1": {
+        "l2": {
+          "l3": { "l4_leaf": 8, "l4_arr": [3] },
+          "l3_leaf": "second"
+        },
+        "l2_leaf": false
+      }
+    },
+    "object_with_array_of_objects": {
+      "trials": [ { "x": 5, "y": 6 } ]
+    },
+    "array_primitives": [4, 5],
+    "array_objects": [ { "x": 0.5, "y": 0.6, "t": 15 } ],
+    "array_of_arrays": [ [5, 6] ],
+    "array_mixed": [true, 2.5],
+    "array_deep_objects": [
+      { "meta": { "tag": "c", "score": { "raw": 5, "norm": 0.5 } } }
+    ],
+    "empty_object": {},
+    "empty_array": [],
+    "varying_object": { "only_row1": "different keys per row" },
+    "unicode_col": "中文データ"
+  },
+  {
+    "trial_type": "made-up-plugin",
+    "trial_index": 2,
+    "time_elapsed": 4100,
+    "rt": "98.6",
+    "response": "j",
+    "correct": "true",
+    "always_null": "null",
+    "mixed_col": 3,
+    "flat_object": { "a": 3, "b": "z" },
+    "varying_object": {},
+    "unicode_col": "plain ascii"
+  },
+  {
+    "trial_index": 3,
+    "time_elapsed": 5000,
+    "orphan_col": "row with no trial_type at all",
+    "rt": 9999
+  }
+]

packages/cli/tests/nested-cli.stress.test.ts

@@ -0,0 +1,114 @@
+import fs from "fs";
+import os from "os";
+import path from "path";
+import JsPsychMetadata from "@jspsych/metadata";
+import { processDirectory } from "../src/data";
+
+/**
+ * Stress regression guard: run the CLI's real conversion pipeline (processDirectory) on the
+ * comprehensive nested-data fixture and assert the full Psych-DS output is coherent —
+ * a compliant main CSV, the original JSON preserved under data/raw/, and a clean two-way match
+ * between variableMeasured and the actual CSV columns. Ported from stress-tests/run-nested.mjs
+ * (Passes 2-4). The Psych-DS validator pass needs network (it fetches the schema), so it is
+ * best-effort: it asserts 0 errors when it can run and is skipped offline, while the structural
+ * and column-cross-check assertions run unconditionally.
+ */
+
+const fixtureDir = path.resolve(__dirname, "../../../dev/stress/nested-all-cases");
+
+// Minimal RFC-4180 header parser (handles quoted fields containing commas).
+function parseHeader(line: string): string[] {
+  const cols: string[] = [];
+  let cur = "", inQ = false;
+  for (let i = 0; i < line.length; i++) {
+    const c = line[i];
+    if (inQ) {
+      if (c === '"' && line[i + 1] === '"') { cur += '"'; i++; }
+      else if (c === '"') inQ = false;
+      else cur += c;
+    } else if (c === '"') inQ = true;
+    else if (c === ",") { cols.push(cur); cur = ""; }
+    else cur += c;
+  }
+  cols.push(cur);
+  return cols;
+}
+
+describe("nested-data CLI end-to-end (stress)", () => {
+  let projectDir: string;
+  let dataDir: string;
+  let total: number;
+  let failed: number;
+  let writtenCsvs: string[];
+
+  beforeAll(async () => {
+    jest.spyOn(console, "warn").mockImplementation(() => {});
+    projectDir = fs.mkdtempSync(path.join(os.tmpdir(), "stress-nested-"));
+    dataDir = path.join(projectDir, "data");
+    fs.mkdirSync(dataDir, { recursive: true });
+
+    const metadata = new JsPsychMetadata();
+    metadata.setMetadataField("name", "nested-stress");
+    ({ total, failed } = await processDirectory(metadata, fixtureDir, false, dataDir));
+    fs.writeFileSync(
+      path.join(projectDir, "dataset_description.json"),
+      JSON.stringify(metadata.getMetadata(), null, 2),
+    );
+    writtenCsvs = fs.readdirSync(dataDir).filter((f) => f.endsWith(".csv"));
+  }, 120_000);
+
+  afterAll(() => {
+    jest.restoreAllMocks();
+    fs.rmSync(projectDir, { recursive: true, force: true });
+  });
+
+  test("processes the fixture with no failures", () => {
+    expect(total).toBe(1);
+    expect(failed).toBe(0);
+  });
+
+  test("writes a compliant main CSV from the source filename", () => {
+    expect(writtenCsvs).toContain("subject-nested_data.csv");
+  });
+
+  test("preserves the original JSON under data/raw/", () => {
+    expect(fs.existsSync(path.join(dataDir, "raw", "subject-nested.json"))).toBe(true);
+  });
+
+  test("writes sidecar CSVs for the nested array/object columns", () => {
+    // The fixture has many nested columns; expect more than just the main CSV.
+    expect(writtenCsvs.length).toBeGreaterThan(1);
+  });
+
+  test("every variableMeasured name is a CSV column and vice versa", () => {
+    const allColumns = new Set<string>();
+    for (const csv of writtenCsvs) {
+      const firstLine = fs.readFileSync(path.join(dataDir, csv), "utf8").split(/\r?\n/)[0];
+      parseHeader(firstLine).forEach((c) => allColumns.add(c));
+    }
+    const meta = JSON.parse(fs.readFileSync(path.join(projectDir, "dataset_description.json"), "utf8"));
+    const varNames = new Set(
+      (meta.variableMeasured ?? []).map((v: any) => (typeof v === "string" ? v : v.name)),
+    );
+
+    const varsWithoutColumn = [...varNames].filter((n) => !allColumns.has(n as string));
+    const columnsWithoutVar = [...allColumns].filter((c) => !varNames.has(c));
+    expect(varsWithoutColumn).toEqual([]);
+    expect(columnsWithoutVar).toEqual([]);
+  });
+
+  test("the written dataset passes Psych-DS validation (best-effort; needs network)", async () => {
+    let ran = false;
+    let errors: string[] = [];
+    try {
+      const { validate } = await import("psychds-validator");
+      const result: any = await validate(path.relative(process.cwd(), projectDir).replace(/\\/g, "/"));
+      ran = true;
+      for (const [, issue] of result.issues) if (issue.severity === "error") errors.push(issue.key);
+    } catch {
+      // No network / validator could not run — the structural checks above are the source of truth.
+    }
+    if (ran) expect(errors).toEqual([]);
+    else console.warn("Psych-DS validation skipped: validator could not run (needs network).");
+  }, 120_000);
+});

packages/cli/tests/rename-reject.stress.test.ts

@@ -0,0 +1,63 @@
+import fs from "fs";
+import os from "os";
+import path from "path";
+import JsPsychMetadata from "@jspsych/metadata";
+import { processDirectory } from "../src/data";
+
+/**
+ * Stress regression guard: without a rename plan (the non-interactive path), the conversion must
+ * REFUSE a data file whose name can't form a Psych-DS-compliant base — rather than silently
+ * inventing a keyword — and write nothing for it. Ported from stress-tests/run-rename.mjs (Pass 2).
+ * (Inventing a keyword is the interactive pre-pass's job; it can't be driven without a TTY.)
+ */
+
+describe("CLI rejects a non-compliant filename (stress)", () => {
+  let tmpDir: string;
+  let dataDir: string;
+  let badDataDir: string;
+  const badFile = "weird name!.json"; // stem can't form a compliant base -> "weird name!_data.csv" is invalid
+
+  let total: number;
+  let failed: number;
+  let errorOutput: string;
+
+  beforeAll(async () => {
+    jest.spyOn(console, "warn").mockImplementation(() => {});
+    const errSpy = jest.spyOn(console, "error").mockImplementation(() => {});
+
+    tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "stress-rename-"));
+    dataDir = path.join(tmpDir, "data");
+    badDataDir = path.join(tmpDir, "input");
+    fs.mkdirSync(dataDir, { recursive: true });
+    fs.mkdirSync(badDataDir, { recursive: true });
+    fs.writeFileSync(
+      path.join(badDataDir, badFile),
+      JSON.stringify([{ trial_type: "html-keyboard-response", trial_index: 0, rt: 100 }]),
+    );
+
+    const metadata = new JsPsychMetadata();
+    metadata.setMetadataField("name", "rename-stress");
+    ({ total, failed } = await processDirectory(metadata, badDataDir, false, dataDir));
+    errorOutput = errSpy.mock.calls.map((args) => args.join(" ")).join("\n");
+  }, 60_000);
+
+  afterAll(() => {
+    jest.restoreAllMocks();
+    fs.rmSync(tmpDir, { recursive: true, force: true });
+  });
+
+  test("counts the non-compliant file as failed", () => {
+    expect(total).toBe(1);
+    expect(failed).toBe(1);
+  });
+
+  test("writes no data CSVs (fails before writing, never invents a keyword)", () => {
+    const csvs = fs.existsSync(dataDir) ? fs.readdirSync(dataDir).filter((f) => f.endsWith(".csv")) : [];
+    expect(csvs).toEqual([]);
+  });
+
+  test("explains the Psych-DS naming requirement and names the offending file", () => {
+    expect(errorOutput).toMatch(/does not follow the Psych-DS naming pattern/);
+    expect(errorOutput).toContain(badFile);
+  });
+});