From a0be278a30ef4c866cb13cc10ccf66214b63509a Mon Sep 17 00:00:00 2001 From: Mandyx22 <1915537307@qq.com> Date: Fri, 12 Jun 2026 14:15:24 -0400 Subject: [PATCH 1/2] test: add nested-data & filename-normalization stress guards to CI Port the standalone stress-tests/ harnesses into the automated Jest suite so regressions in already-fixed behavior are caught by plain `npm test` (and CI). The tests import from source (the CI test job runs no build) and add no library or CLI behavior. - metadata: nested-generation coherence over a comprehensive fixture, and the Psych-DS filename-normalization helper invariants. - cli: processDirectory end-to-end (compliant main CSV, data/raw/ preservation, variableMeasured <-> CSV-column cross-check, best-effort Psych-DS validation), and refusal to write a non-compliant filename non-interactively. Shared fixture lives at dev/stress/. Co-Authored-By: Claude Opus 4.8 --- .changeset/stress-tests-in-ci.md | 13 ++ .../nested-all-cases/subject-nested.json | 101 +++++++++++++ packages/cli/tests/nested-cli.stress.test.ts | 114 +++++++++++++++ .../cli/tests/rename-reject.stress.test.ts | 63 ++++++++ .../tests/nested-generation.stress.test.ts | 134 ++++++++++++++++++ .../tests/rename-normalization.stress.test.ts | 112 +++++++++++++++ 6 files changed, 537 insertions(+) create mode 100644 .changeset/stress-tests-in-ci.md create mode 100644 dev/stress/nested-all-cases/subject-nested.json create mode 100644 packages/cli/tests/nested-cli.stress.test.ts create mode 100644 packages/cli/tests/rename-reject.stress.test.ts create mode 100644 packages/metadata/tests/nested-generation.stress.test.ts create mode 100644 packages/metadata/tests/rename-normalization.stress.test.ts diff --git a/.changeset/stress-tests-in-ci.md b/.changeset/stress-tests-in-ci.md new file mode 100644 index 0000000..49251e0 --- /dev/null +++ b/.changeset/stress-tests-in-ci.md @@ -0,0 +1,13 @@ +--- +"@jspsych/metadata": patch +"@jspsych/metadata-cli": patch +--- + +Add stress-test regression guards to the automated suite so previously-fixed nested-data and filename-normalization behavior can't silently regress. + +Four Jest suites, ported from the standalone `stress-tests/` harnesses so they run under plain `npm test` (and CI) without a build step: + +- `@jspsych/metadata`: `generate()` coherence over a comprehensive nested-data fixture (deep objects, arrays of objects/arrays, mixed-type columns, a `trial_type`-less row, unicode, empties), plus the Psych-DS filename-normalization helper invariants. +- `@jspsych/metadata-cli`: the `processDirectory` conversion end-to-end (compliant main CSV, `data/raw/` preservation, two-way `variableMeasured` ↔ CSV-column cross-check, and a best-effort Psych-DS validation pass), plus the refusal to write a non-compliant filename non-interactively. + +Test-only change; no library or CLI behavior is modified. The shared fixture lives at `dev/stress/`. diff --git a/dev/stress/nested-all-cases/subject-nested.json b/dev/stress/nested-all-cases/subject-nested.json new file mode 100644 index 0000000..48804ad --- /dev/null +++ b/dev/stress/nested-all-cases/subject-nested.json @@ -0,0 +1,101 @@ +[ + { + "trial_type": "html-keyboard-response", + "trial_index": 0, + "time_elapsed": 1500, + "rt": 432.5, + "response": "f", + "correct": true, + "always_null": null, + "empty_string": "", + "numeric_string": "42", + "bool_string": "TRUE", + "mixed_col": 10, + "json_string_object": "{\"nested\": {\"deep\": 1}}", + "json_string_array": "[5, 6, 7]", + "flat_object": { "a": 1, "b": "x" }, + "deep_object": { + "l1": { + "l2": { + "l3": { "l4_leaf": 4, "l4_arr": [1, 2] }, + "l3_leaf": "first" + }, + "l2_leaf": true + } + }, + "object_with_array_of_objects": { + "trials": [ { "x": 1, "y": 2 }, { "x": 3, "y": 4 } ] + }, + "array_primitives": [1, 2, 3], + "array_objects": [ + { "x": 0.1, "y": 0.2, "t": 5 }, + { "x": 0.3, "y": 0.4, "t": 10 } + ], + "array_of_arrays": [ [1, 2], [3, 4] ], + "array_mixed": [1, "two", { "three": 3 }], + "array_deep_objects": [ + { "meta": { "tag": "a", "score": { "raw": 9, "norm": 0.9 } } }, + { "meta": { "tag": "b", "score": { "raw": 7, "norm": 0.7 } } } + ], + "empty_object": {}, + "empty_array": [], + "varying_object": { "only_row0": 1 }, + "unicode_col": "héllo wörld 👋" + }, + { + "trial_type": "survey-text", + "trial_index": 1, + "time_elapsed": 3200, + "rt": 1001, + "response": { "Q0": "free text answer" }, + "correct": false, + "always_null": null, + "empty_string": "", + "numeric_string": "7", + "bool_string": "false", + "mixed_col": "oops", + "flat_object": { "a": 2, "b": "y" }, + "deep_object": { + "l1": { + "l2": { + "l3": { "l4_leaf": 8, "l4_arr": [3] }, + "l3_leaf": "second" + }, + "l2_leaf": false + } + }, + "object_with_array_of_objects": { + "trials": [ { "x": 5, "y": 6 } ] + }, + "array_primitives": [4, 5], + "array_objects": [ { "x": 0.5, "y": 0.6, "t": 15 } ], + "array_of_arrays": [ [5, 6] ], + "array_mixed": [true, 2.5], + "array_deep_objects": [ + { "meta": { "tag": "c", "score": { "raw": 5, "norm": 0.5 } } } + ], + "empty_object": {}, + "empty_array": [], + "varying_object": { "only_row1": "different keys per row" }, + "unicode_col": "中文データ" + }, + { + "trial_type": "made-up-plugin", + "trial_index": 2, + "time_elapsed": 4100, + "rt": "98.6", + "response": "j", + "correct": "true", + "always_null": "null", + "mixed_col": 3, + "flat_object": { "a": 3, "b": "z" }, + "varying_object": {}, + "unicode_col": "plain ascii" + }, + { + "trial_index": 3, + "time_elapsed": 5000, + "orphan_col": "row with no trial_type at all", + "rt": 9999 + } +] diff --git a/packages/cli/tests/nested-cli.stress.test.ts b/packages/cli/tests/nested-cli.stress.test.ts new file mode 100644 index 0000000..f6c2839 --- /dev/null +++ b/packages/cli/tests/nested-cli.stress.test.ts @@ -0,0 +1,114 @@ +import fs from "fs"; +import os from "os"; +import path from "path"; +import JsPsychMetadata from "@jspsych/metadata"; +import { processDirectory } from "../src/data"; + +/** + * Stress regression guard: run the CLI's real conversion pipeline (processDirectory) on the + * comprehensive nested-data fixture and assert the full Psych-DS output is coherent — + * a compliant main CSV, the original JSON preserved under data/raw/, and a clean two-way match + * between variableMeasured and the actual CSV columns. Ported from stress-tests/run-nested.mjs + * (Passes 2-4). The Psych-DS validator pass needs network (it fetches the schema), so it is + * best-effort: it asserts 0 errors when it can run and is skipped offline, while the structural + * and column-cross-check assertions run unconditionally. + */ + +const fixtureDir = path.resolve(__dirname, "../../../dev/stress/nested-all-cases"); + +// Minimal RFC-4180 header parser (handles quoted fields containing commas). +function parseHeader(line: string): string[] { + const cols: string[] = []; + let cur = "", inQ = false; + for (let i = 0; i < line.length; i++) { + const c = line[i]; + if (inQ) { + if (c === '"' && line[i + 1] === '"') { cur += '"'; i++; } + else if (c === '"') inQ = false; + else cur += c; + } else if (c === '"') inQ = true; + else if (c === ",") { cols.push(cur); cur = ""; } + else cur += c; + } + cols.push(cur); + return cols; +} + +describe("nested-data CLI end-to-end (stress)", () => { + let projectDir: string; + let dataDir: string; + let total: number; + let failed: number; + let writtenCsvs: string[]; + + beforeAll(async () => { + jest.spyOn(console, "warn").mockImplementation(() => {}); + projectDir = fs.mkdtempSync(path.join(os.tmpdir(), "stress-nested-")); + dataDir = path.join(projectDir, "data"); + fs.mkdirSync(dataDir, { recursive: true }); + + const metadata = new JsPsychMetadata(); + metadata.setMetadataField("name", "nested-stress"); + ({ total, failed } = await processDirectory(metadata, fixtureDir, false, dataDir)); + fs.writeFileSync( + path.join(projectDir, "dataset_description.json"), + JSON.stringify(metadata.getMetadata(), null, 2), + ); + writtenCsvs = fs.readdirSync(dataDir).filter((f) => f.endsWith(".csv")); + }, 120_000); + + afterAll(() => { + jest.restoreAllMocks(); + fs.rmSync(projectDir, { recursive: true, force: true }); + }); + + test("processes the fixture with no failures", () => { + expect(total).toBe(1); + expect(failed).toBe(0); + }); + + test("writes a compliant main CSV from the source filename", () => { + expect(writtenCsvs).toContain("subject-nested_data.csv"); + }); + + test("preserves the original JSON under data/raw/", () => { + expect(fs.existsSync(path.join(dataDir, "raw", "subject-nested.json"))).toBe(true); + }); + + test("writes sidecar CSVs for the nested array/object columns", () => { + // The fixture has many nested columns; expect more than just the main CSV. + expect(writtenCsvs.length).toBeGreaterThan(1); + }); + + test("every variableMeasured name is a CSV column and vice versa", () => { + const allColumns = new Set(); + for (const csv of writtenCsvs) { + const firstLine = fs.readFileSync(path.join(dataDir, csv), "utf8").split(/\r?\n/)[0]; + parseHeader(firstLine).forEach((c) => allColumns.add(c)); + } + const meta = JSON.parse(fs.readFileSync(path.join(projectDir, "dataset_description.json"), "utf8")); + const varNames = new Set( + (meta.variableMeasured ?? []).map((v: any) => (typeof v === "string" ? v : v.name)), + ); + + const varsWithoutColumn = [...varNames].filter((n) => !allColumns.has(n as string)); + const columnsWithoutVar = [...allColumns].filter((c) => !varNames.has(c)); + expect(varsWithoutColumn).toEqual([]); + expect(columnsWithoutVar).toEqual([]); + }); + + test("the written dataset passes Psych-DS validation (best-effort; needs network)", async () => { + let ran = false; + let errors: string[] = []; + try { + const { validate } = await import("psychds-validator"); + const result: any = await validate(path.relative(process.cwd(), projectDir).replace(/\\/g, "/")); + ran = true; + for (const [, issue] of result.issues) if (issue.severity === "error") errors.push(issue.key); + } catch { + // No network / validator could not run — the structural checks above are the source of truth. + } + if (ran) expect(errors).toEqual([]); + else console.warn("Psych-DS validation skipped: validator could not run (needs network)."); + }, 120_000); +}); diff --git a/packages/cli/tests/rename-reject.stress.test.ts b/packages/cli/tests/rename-reject.stress.test.ts new file mode 100644 index 0000000..a1d8cbd --- /dev/null +++ b/packages/cli/tests/rename-reject.stress.test.ts @@ -0,0 +1,63 @@ +import fs from "fs"; +import os from "os"; +import path from "path"; +import JsPsychMetadata from "@jspsych/metadata"; +import { processDirectory } from "../src/data"; + +/** + * Stress regression guard: without a rename plan (the non-interactive path), the conversion must + * REFUSE a data file whose name can't form a Psych-DS-compliant base — rather than silently + * inventing a keyword — and write nothing for it. Ported from stress-tests/run-rename.mjs (Pass 2). + * (Inventing a keyword is the interactive pre-pass's job; it can't be driven without a TTY.) + */ + +describe("CLI rejects a non-compliant filename (stress)", () => { + let tmpDir: string; + let dataDir: string; + let badDataDir: string; + const badFile = "weird name!.json"; // stem can't form a compliant base -> "weird name!_data.csv" is invalid + + let total: number; + let failed: number; + let errorOutput: string; + + beforeAll(async () => { + jest.spyOn(console, "warn").mockImplementation(() => {}); + const errSpy = jest.spyOn(console, "error").mockImplementation(() => {}); + + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "stress-rename-")); + dataDir = path.join(tmpDir, "data"); + badDataDir = path.join(tmpDir, "input"); + fs.mkdirSync(dataDir, { recursive: true }); + fs.mkdirSync(badDataDir, { recursive: true }); + fs.writeFileSync( + path.join(badDataDir, badFile), + JSON.stringify([{ trial_type: "html-keyboard-response", trial_index: 0, rt: 100 }]), + ); + + const metadata = new JsPsychMetadata(); + metadata.setMetadataField("name", "rename-stress"); + ({ total, failed } = await processDirectory(metadata, badDataDir, false, dataDir)); + errorOutput = errSpy.mock.calls.map((args) => args.join(" ")).join("\n"); + }, 60_000); + + afterAll(() => { + jest.restoreAllMocks(); + fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + test("counts the non-compliant file as failed", () => { + expect(total).toBe(1); + expect(failed).toBe(1); + }); + + test("writes no data CSVs (fails before writing, never invents a keyword)", () => { + const csvs = fs.existsSync(dataDir) ? fs.readdirSync(dataDir).filter((f) => f.endsWith(".csv")) : []; + expect(csvs).toEqual([]); + }); + + test("explains the Psych-DS naming requirement and names the offending file", () => { + expect(errorOutput).toMatch(/does not follow the Psych-DS naming pattern/); + expect(errorOutput).toContain(badFile); + }); +}); diff --git a/packages/metadata/tests/nested-generation.stress.test.ts b/packages/metadata/tests/nested-generation.stress.test.ts new file mode 100644 index 0000000..208db3a --- /dev/null +++ b/packages/metadata/tests/nested-generation.stress.test.ts @@ -0,0 +1,134 @@ +import fs from "fs"; +import path from "path"; +import JsPsychMetadata from "../src/index"; + +/** + * Stress regression guard: generate() over a fixture that exercises every nested-data shape + * (deep objects, arrays of objects, arrays of arrays, mixed-type columns, a trial_type-less + * row, unicode, empties) and assert each variable's stored type / levels / range stays coherent. + * + * Ported from stress-tests/run-nested.mjs (Pass 1). Three documented findings (F1a, F1b, F2 — + * see the comments below) are asserted as *current* behavior so this stays green; each is a + * deviation pending its own intent decision and must not be "fixed" here by loosening. + */ + +const fixturePath = path.resolve(__dirname, "../../../dev/stress/nested-all-cases/subject-nested.json"); + +// Plugin descriptions are fetched from unpkg; stub fetch so the suite is offline-deterministic. +// Nothing this suite asserts (types, levels, ranges) depends on the human-readable descriptions. +const mockFetch = jest.fn().mockResolvedValue({ ok: false, status: 404 }); + +// Expected variable -> expected stored type. 'numeric' = "registered, any numeric type"; '*' = +// "registered, any type". Derived from the fixture design. +const EXPECTED: Record = { + trial_type: "string", trial_index: "numeric", time_elapsed: "numeric", rt: "number", + response: "string", "response.Q0": "string", correct: "boolean", always_null: "unknown", + empty_string: "unknown", numeric_string: "number", + // Post-#90: "true"/"false" STRINGS stay strings (levels); only genuine JSON booleans are boolean. + bool_string: "string", + mixed_col: "string", // mixed numeric/string -> downgraded to categorical + json_string_object: "object", "json_string_object.nested": "object", "json_string_object.nested.deep": "number", + json_string_array: "array", "json_string_array.value": "number", + flat_object: "object", "flat_object.a": "number", "flat_object.b": "string", + deep_object: "object", "deep_object.l1": "object", "deep_object.l1.l2": "object", + "deep_object.l1.l2.l3": "object", "deep_object.l1.l2.l3.l4_leaf": "number", + "deep_object.l1.l2.l3.l4_arr": "array", "deep_object.l1.l2.l3.l4_arr.value": "number", + "deep_object.l1.l2.l3_leaf": "string", "deep_object.l1.l2_leaf": "boolean", + object_with_array_of_objects: "object", "object_with_array_of_objects.trials": "array", + "object_with_array_of_objects.trials.x": "number", "object_with_array_of_objects.trials.y": "number", + array_primitives: "array", "array_primitives.value": "number", + array_objects: "array", "array_objects.x": "number", "array_objects.y": "number", "array_objects.t": "number", + array_of_arrays: "array", "array_of_arrays.value": "array", "array_of_arrays.value.value": "number", + "array_of_arrays.element_index": "number", + array_mixed: "array", "array_mixed.value": "string", "array_mixed.three": "number", + array_deep_objects: "array", "array_deep_objects.meta": "object", "array_deep_objects.meta.tag": "string", + "array_deep_objects.meta.score": "object", "array_deep_objects.meta.score.raw": "number", + "array_deep_objects.meta.score.norm": "number", + empty_object: "object", empty_array: "array", + varying_object: "object", "varying_object.only_row0": "number", "varying_object.only_row1": "string", + unicode_col: "string", + orphan_col: "*", // column from a trial_type-less row: any type, but it must exist + element_index: "number", +}; + +// F2 (run-nested RESULTS.md): this boolean column also carries a `levels` array. Asserted as +// known current behavior for this column only, so a NEW boolean-with-levels regression is caught. +const F2_BOOLEAN_WITH_LEVELS = new Set(["correct"]); + +describe("nested-data generation coherence (stress)", () => { + let metadata: JsPsychMetadata; + let variableMeasured: any[]; + let vars: Map; + + beforeAll(async () => { + (global as any).fetch = mockFetch; + jest.spyOn(console, "warn").mockImplementation(() => {}); + metadata = new JsPsychMetadata(); + await metadata.generate(fs.readFileSync(fixturePath, "utf8"), {}, "json"); + variableMeasured = metadata.getMetadata().variableMeasured; + vars = new Map(variableMeasured.map((v: any) => [v.name, v])); + }); + + afterAll(() => jest.restoreAllMocks()); + + test("registers every expected variable with the right stored type", () => { + const mismatches: string[] = []; + for (const [name, type] of Object.entries(EXPECTED)) { + const v = vars.get(name); + if (!v) { mismatches.push(`${name}: MISSING from variableMeasured`); continue; } + if (type === "*" || type === "numeric") continue; + if (v.value !== type) mismatches.push(`${name}: expected "${type}", got "${v.value}"`); + } + expect(mismatches).toEqual([]); + }); + + test("produces no unexpected variables", () => { + const unexpected = [...vars.keys()].filter((n) => !(n in EXPECTED)); + expect(unexpected).toEqual([]); + }); + + test("every variable's type/level/range fields are mutually coherent", () => { + const incoherent: string[] = []; + for (const v of variableMeasured) { + const issues: string[] = []; + const hasRange = v.minValue !== undefined || v.maxValue !== undefined; + if (v.value === "number") { + if (v.minValue !== undefined && v.maxValue !== undefined && v.minValue > v.maxValue) issues.push(`min ${v.minValue} > max ${v.maxValue}`); + if (typeof v.minValue === "number" && !Number.isFinite(v.minValue)) issues.push("non-finite minValue"); + if (v.levels) issues.push("numeric but has levels"); + } else if (v.value === "boolean") { + if (v.levels && !F2_BOOLEAN_WITH_LEVELS.has(v.name)) issues.push("boolean but has levels"); + if (hasRange) issues.push("boolean but has min/max"); + } else if (v.value === "string") { + if (hasRange) issues.push("string but has min/max"); + } else if (v.value === "object" || v.value === "array") { + if (hasRange) issues.push(`${v.value} but has min/max`); + if (v.levels) issues.push(`${v.value} but has levels`); + } + if (issues.length) incoherent.push(`${v.name}: ${issues.join("; ")}`); + } + expect(incoherent).toEqual([]); + }); + + test("coerces a numeric-string range and keeps mixed values as levels", () => { + expect(vars.get("rt").minValue).toBe(98.6); // "98.6" string coerced + const mixedLevels = ([] as string[]).concat(vars.get("mixed_col").levels ?? []); + expect(mixedLevels).toEqual(expect.arrayContaining(["10", "oops", "3"])); + }); + + test("F1a: values in a trial_type-less row are dropped from min/max", () => { + // rt's 9999 lives in the trial_type-less row and is NOT counted, so max stays 1001. + expect(vars.get("rt").maxValue).toBe(1001); + }); + + test("F1b: a column appearing only in a trial_type-less row stays \"unknown\"", () => { + expect(vars.get("orphan_col").value).toBe("unknown"); + }); + + test("extracts deeply nested array/object columns into sidecars", () => { + const arrays = metadata.getExtractedArrays(); + expect(arrays.has("deep_object.l1.l2.l3.l4_arr")).toBe(true); // 4 levels down + expect(arrays.has("array_of_arrays.value")).toBe(true); + expect(arrays.has("empty_array")).toBe(false); // empty array -> no sidecar rows + }); +}); diff --git a/packages/metadata/tests/rename-normalization.stress.test.ts b/packages/metadata/tests/rename-normalization.stress.test.ts new file mode 100644 index 0000000..592fbe2 --- /dev/null +++ b/packages/metadata/tests/rename-normalization.stress.test.ts @@ -0,0 +1,112 @@ +import { + toPsychDSValue, + isValidPsychDSDataFilename, + deriveArrayFilename, + disambiguateArrayFilename, +} from "../src/utils"; + +/** + * Stress regression guard for the Psych-DS filename-normalization helpers: throw a battery of + * nasty inputs (spaces, symbols, unicode, empty, collisions) at the four exported functions and + * assert (a) their documented output and (b) the core invariant — every name they produce is a + * fully Psych-DS-compliant data filename. Ported from stress-tests/run-rename.mjs (Pass 1). + */ + +describe("toPsychDSValue (stress)", () => { + // [input, expected]. Runs of non-alphanumerics are word boundaries -> camelCase; inputs with + // no alphanumerics fall back to "value". + const cases: [string, string][] = [ + ["mouse_tracking", "mouseTracking"], + ["RT (ms)", "RTMs"], + ["snake_case_thing", "snakeCaseThing"], + ["a.b.c", "aBC"], + [" spaced ", "spaced"], + ["trailing-", "trailing"], + ["-leading", "leading"], + ["CamelCase", "CamelCase"], + ["simple", "simple"], + ["123", "123"], + ["héllo wörld", "hLloWRld"], // non-ASCII letters are boundaries, not kept + ["👋", "value"], + ["", "value"], + ["!!!", "value"], + ]; + + test.each(cases)("toPsychDSValue(%j) -> %j and is a legal value segment", (input, expected) => { + const got = toPsychDSValue(input); + expect(got).toBe(expected); + expect(got).toMatch(/^[a-zA-Z0-9]+$/); + }); + + test("honors a custom fallback when the input has no alphanumerics", () => { + expect(toPsychDSValue("!!!", "col")).toBe("col"); + }); +}); + +describe("isValidPsychDSDataFilename (stress)", () => { + const valid = [ + "subject-001_data.csv", + "subject-nested_measure-rt_data.csv", + "task-stroop_session-1_data.tsv", + "a-b_data.csv", + ]; + const invalid: [string, string][] = [ + ["subject_data.csv", "no keyword-value pair"], + ["Subject-001_data.csv", "uppercase keyword"], + ["subject-001_data.json", "wrong extension"], + ["subject-001.csv", "missing _data"], + ["_data.csv", "empty base"], + ["subject-001_measure-_data.csv", "empty value segment"], + ["sub-1_2-x_data.csv", "second keyword has a digit"], + ]; + + test.each(valid)("accepts %s", (name) => expect(isValidPsychDSDataFilename(name)).toBe(true)); + test.each(invalid)("rejects %s (%s)", (name) => expect(isValidPsychDSDataFilename(name)).toBe(false)); +}); + +describe("deriveArrayFilename (stress)", () => { + const cases: [string, string, string][] = [ + ["subject-001", "mouse_tracking", "subject-001_measure-mouseTracking_data.csv"], + ["subject-001", "!!!", "subject-001_measure-col_data.csv"], // unusable column -> "col" fallback + ["task-stroop_session-1", "RT (ms)", "task-stroop_session-1_measure-RTMs_data.csv"], + ["subject-001", "héllo", "subject-001_measure-hLlo_data.csv"], + ]; + test.each(cases)("deriveArrayFilename(%j, %j) -> compliant %j", (base, col, expected) => { + const got = deriveArrayFilename(base, col); + expect(got).toBe(expected); + expect(isValidPsychDSDataFilename(got)).toBe(true); + }); +}); + +describe("disambiguateArrayFilename (stress)", () => { + test("appends a separator-less counter on collision, staying Psych-DS valid", () => { + const base = "subject-001_measure-x_data.csv"; + const used = new Set(); + expect(disambiguateArrayFilename(base, used)).toBe(base); + + used.add(base); + const second = disambiguateArrayFilename(base, used); + expect(second).toBe("subject-001_measure-x2_data.csv"); + + used.add(second); + const third = disambiguateArrayFilename(base, used); + expect(third).toBe("subject-001_measure-x3_data.csv"); + + // Counter has no separator, so it stays inside the value segment rather than creating a bad pair. + expect(isValidPsychDSDataFilename(second)).toBe(true); + expect(isValidPsychDSDataFilename(third)).toBe(true); + }); + + test("invariant sweep: every derived+disambiguated name from the value battery is valid", () => { + const columns = ["mouse_tracking", "RT (ms)", "snake_case_thing", "a.b.c", " spaced ", "👋", "", "!!!"]; + const used = new Set(); + const offenders: string[] = []; + for (const col of columns) { + const finalName = disambiguateArrayFilename(deriveArrayFilename("subject-001", col), used); + used.add(finalName); + if (!isValidPsychDSDataFilename(finalName)) offenders.push(`${JSON.stringify(col)} -> ${finalName}`); + } + expect(offenders).toEqual([]); + expect(used.size).toBe(columns.length); // all collisions disambiguated to unique names + }); +}); From 8a98394b4ba9e9cd5af54ac7929090056c9f98c3 Mon Sep 17 00:00:00 2001 From: Mandyx22 <1915537307@qq.com> Date: Fri, 12 Jun 2026 16:24:56 -0400 Subject: [PATCH 2/2] test: add CSV-ingestion, scale, and cross-file-collision stress guards Stacked on the nested-data/filename stress suites (PR #104). Three more Jest suites, all test-only (no library/CLI behavior change): - metadata csv-input.stress: type re-inference from CSV string cells (numeric coercion, Infinity/NaN rejection, mixed-column downgrade, RFC-4180 quoting, unicode, empty/literal-null, 50-char level cap, JSON-in-cell extraction) + CSV/JSON parity. - metadata scale.stress: 5,000-row exact extremes, dedup, high-cardinality levels, boolean handling, throughput ceiling. - cli array-collision.stress: two same-stem files in different subdirs sharing a nested array column; asserts main CSV, sidecar, and raw originals each disambiguate without overwrite, staying Psych-DS valid. Co-Authored-By: Claude Opus 4.8 --- .changeset/more-stress-tests.md | 12 ++ .../cli/tests/array-collision.stress.test.ts | 131 +++++++++++++ .../metadata/tests/csv-input.stress.test.ts | 183 ++++++++++++++++++ packages/metadata/tests/scale.stress.test.ts | 85 ++++++++ 4 files changed, 411 insertions(+) create mode 100644 .changeset/more-stress-tests.md create mode 100644 packages/cli/tests/array-collision.stress.test.ts create mode 100644 packages/metadata/tests/csv-input.stress.test.ts create mode 100644 packages/metadata/tests/scale.stress.test.ts diff --git a/.changeset/more-stress-tests.md b/.changeset/more-stress-tests.md new file mode 100644 index 0000000..c47a272 --- /dev/null +++ b/.changeset/more-stress-tests.md @@ -0,0 +1,12 @@ +--- +"@jspsych/metadata": patch +"@jspsych/metadata-cli": patch +--- + +Extend the stress-test regression guards with three more Jest suites covering the CSV ingestion path, generation at scale, and cross-file output-name collisions. + +- `@jspsych/metadata` — `csv-input.stress`: pins how `generate(data, {}, "csv")` re-infers types from string cells (numeric coercion incl. whitespace/scientific-notation/`Infinity`/`NaN` rejection, mixed-column downgrade, `"true"`/`"false"` staying categorical, RFC-4180 quoting, unicode, empty/literal-`null` cells, the 50-char level cap, JSON-in-a-cell extraction), and asserts CSV/JSON parity for unambiguously-typed columns. +- `@jspsych/metadata` — `scale.stress`: feeds a 5,000-row dataset and checks exact numeric extremes, categorical dedup, high-cardinality level accumulation, boolean handling, and a throughput ceiling that guards against accidental O(n²) regressions. +- `@jspsych/metadata-cli` — `array-collision.stress`: two same-stem files in different subdirectories sharing a nested array column, asserting `processDirectory` disambiguates every main CSV, sidecar, and preserved raw original (no overwrites, all still Psych-DS compliant) — the cross-file collision gap left by the earlier rename suite. + +Test-only change; no library or CLI behavior is modified. diff --git a/packages/cli/tests/array-collision.stress.test.ts b/packages/cli/tests/array-collision.stress.test.ts new file mode 100644 index 0000000..30040ba --- /dev/null +++ b/packages/cli/tests/array-collision.stress.test.ts @@ -0,0 +1,131 @@ +import fs from "fs"; +import os from "os"; +import path from "path"; +import JsPsychMetadata from "@jspsych/metadata"; +import { + isValidPsychDSDataFilename, +} from "@jspsych/metadata"; +import { processDirectory } from "../src/data"; + +/** + * Stress regression guard for cross-file output-name collisions — the coverage gap left by the + * original rename suite. Two source files in different subdirectories share the same stem + * ("subject-001") AND the same nested array column ("mouse"), so without the run-wide + * disambiguation sets every one of {main CSV, preserved raw JSON, array sidecar} would collide and + * silently overwrite its twin. This asserts that processDirectory threads `usedArrayFilenames` / + * `usedRawFilenames` across files: every output lands under a distinct, still-Psych-DS-compliant + * name, nothing is overwritten, and the union of CSV columns still round-trips against + * variableMeasured. + */ + +// Minimal RFC-4180 header parser (handles quoted fields containing commas). +function parseHeader(line: string): string[] { + const cols: string[] = []; + let cur = "", inQ = false; + for (let i = 0; i < line.length; i++) { + const c = line[i]; + if (inQ) { + if (c === '"' && line[i + 1] === '"') { cur += '"'; i++; } + else if (c === '"') inQ = false; + else cur += c; + } else if (c === '"') inQ = true; + else if (c === ",") { cols.push(cur); cur = ""; } + else cur += c; + } + cols.push(cur); + return cols; +} + +// One source file's worth of trials, each with a nested array-of-objects "mouse" column that +// becomes its own sidecar CSV. `seed` keeps the two files' values distinct so an accidental +// overwrite would be detectable, not masked by identical content. +function makeTrials(seed: number) { + return [ + { trial_type: "html-keyboard-response", trial_index: 0, time_elapsed: 100, rt: 100 + seed, mouse: [{ x: seed, y: 1 }, { x: seed + 1, y: 2 }] }, + { trial_type: "html-keyboard-response", trial_index: 1, time_elapsed: 200, rt: 200 + seed, mouse: [{ x: seed + 2, y: 3 }] }, + ]; +} + +describe("cross-file output-name collision (stress)", () => { + let projectDir: string; + let dataDir: string; + let total: number; + let failed: number; + let csvs: string[]; + let rawFiles: string[]; + + beforeAll(async () => { + jest.spyOn(console, "warn").mockImplementation(() => {}); + jest.spyOn(console, "log").mockImplementation(() => {}); + + projectDir = fs.mkdtempSync(path.join(os.tmpdir(), "stress-collision-")); + const inputDir = path.join(projectDir, "input"); + dataDir = path.join(projectDir, "data"); + fs.mkdirSync(path.join(inputDir, "a"), { recursive: true }); + fs.mkdirSync(path.join(inputDir, "b"), { recursive: true }); + fs.mkdirSync(dataDir, { recursive: true }); + + // Same filename, same nested column, different subdirectory -> guaranteed three-way collision. + fs.writeFileSync(path.join(inputDir, "a", "subject-001.json"), JSON.stringify(makeTrials(0))); + fs.writeFileSync(path.join(inputDir, "b", "subject-001.json"), JSON.stringify(makeTrials(10))); + + const metadata = new JsPsychMetadata(); + metadata.setMetadataField("name", "collision-stress"); + ({ total, failed } = await processDirectory(metadata, inputDir, false, dataDir)); + fs.writeFileSync( + path.join(projectDir, "dataset_description.json"), + JSON.stringify(metadata.getMetadata(), null, 2), + ); + csvs = fs.readdirSync(dataDir).filter((f) => f.endsWith(".csv")); + rawFiles = fs.existsSync(path.join(dataDir, "raw")) ? fs.readdirSync(path.join(dataDir, "raw")) : []; + }, 120_000); + + afterAll(() => { + jest.restoreAllMocks(); + fs.rmSync(projectDir, { recursive: true, force: true }); + }); + + test("processes both files with no failures", () => { + expect(total).toBe(2); + expect(failed).toBe(0); + }); + + test("writes two distinct main CSVs instead of overwriting one", () => { + const mains = csvs.filter((f) => !f.includes("measure-")).sort(); + expect(mains).toEqual(["subject-0012_data.csv", "subject-001_data.csv"]); + }); + + test("writes two distinct mouse sidecars instead of overwriting one", () => { + const sidecars = csvs.filter((f) => f.includes("measure-mouse")).sort(); + expect(sidecars).toEqual(["subject-001_measure-mouse2_data.csv", "subject-001_measure-mouse_data.csv"]); + }); + + test("preserves both originals under data/raw/ under distinct names", () => { + expect(rawFiles.filter((f) => f.endsWith(".json")).sort()).toEqual(["subject-001.json", "subject-0012.json"]); + }); + + test("every written CSV name is unique and Psych-DS compliant", () => { + expect(new Set(csvs).size).toBe(csvs.length); // no two outputs share a name + expect(csvs.length).toBe(4); // 2 mains + 2 sidecars + for (const name of csvs) expect(isValidPsychDSDataFilename(name)).toBe(true); + }); + + test("no original's content was clobbered (each raw file matches one of the two inputs)", () => { + const contents = rawFiles + .filter((f) => f.endsWith(".json")) + .map((f) => fs.readFileSync(path.join(dataDir, "raw", f), "utf8")); + expect(contents).toEqual(expect.arrayContaining([JSON.stringify(makeTrials(0)), JSON.stringify(makeTrials(10))])); + }); + + test("every variableMeasured name is a column across the written CSVs", () => { + const allColumns = new Set(); + for (const csv of csvs) { + const firstLine = fs.readFileSync(path.join(dataDir, csv), "utf8").split(/\r?\n/)[0]; + parseHeader(firstLine).forEach((c) => allColumns.add(c)); + } + const meta = JSON.parse(fs.readFileSync(path.join(projectDir, "dataset_description.json"), "utf8")); + const varNames = (meta.variableMeasured ?? []).map((v: any) => (typeof v === "string" ? v : v.name)); + const missing = varNames.filter((n: string) => !allColumns.has(n)); + expect(missing).toEqual([]); + }); +}); diff --git a/packages/metadata/tests/csv-input.stress.test.ts b/packages/metadata/tests/csv-input.stress.test.ts new file mode 100644 index 0000000..1d73c52 --- /dev/null +++ b/packages/metadata/tests/csv-input.stress.test.ts @@ -0,0 +1,183 @@ +import JsPsychMetadata from "../src/index"; + +/** + * Stress regression guard for the CSV ingestion path (generate(data, {}, "csv")). + * + * Where nested-generation.stress.test.ts feeds richly-typed JSON, this suite feeds CSV — where + * every cell arrives as a *string* — and pins how generateObservation re-infers types from those + * strings: numeric coercion (incl. whitespace, scientific notation, Infinity/NaN rejection), + * mixed-column downgrade, "true"/"false" staying categorical (post-#90), RFC-4180 quoting + * (embedded commas / quotes / newlines), unicode, empty / literal-"null" cells, the 50-char level + * cap, and JSON-in-a-cell extraction. A final case asserts CSV and the equivalent JSON agree on + * type for the columns where they should. + */ + +// Plugin descriptions come from unpkg; stub fetch so the suite is offline-deterministic. Nothing +// asserted here (types / levels / ranges) depends on the human-readable descriptions. +const mockFetch = jest.fn().mockResolvedValue({ ok: false, status: 404 }); + +/** Minimal RFC-4180 serializer: quote a field iff it contains a comma, quote, CR or LF. */ +function toCSV(headers: string[], rows: Record[]): string { + const enc = (v: string) => (/[",\r\n]/.test(v) ? `"${v.replace(/"/g, '""')}"` : v); + const lines = [headers.join(",")]; + for (const row of rows) lines.push(headers.map((h) => enc(row[h] ?? "")).join(",")); + return lines.join("\n"); +} + +const LONG = "x".repeat(80); // > MAX_LENGTH (50) so it must be truncated to first-50 + "..." + +// Three observations. Every row carries trial_type so no column is dropped by the trial_type-less +// behavior pinned in nested-generation.stress.test.ts (findings F1a/F1b). +const HEADERS = [ + "trial_type", "trial_index", + "int_col", "float_col", "ws_num", "sci_num", "neg_num", + "inf_col", "nan_col", "bool_str", "mixed_col", + "quoted_comma", "quoted_newline", "quoted_quote", "unicode_col", + "empty_col", "null_word_col", "long_level_col", + "json_obj_col", "json_arr_col", +]; +const ROWS: Record[] = [ + { + trial_type: "html-keyboard-response", trial_index: "0", + int_col: "42", float_col: "1.5", ws_num: " 10 ", sci_num: "1e3", neg_num: "-5", + inf_col: "Infinity", nan_col: "NaN", bool_str: "TRUE", mixed_col: "10", + quoted_comma: "a,b", quoted_newline: "line1\nline2", quoted_quote: 'say "hi"', unicode_col: "café", + empty_col: "", null_word_col: "null", long_level_col: LONG, + json_obj_col: '{"a": 1, "b": "x"}', json_arr_col: "[1, 2, 3]", + }, + { + trial_type: "html-keyboard-response", trial_index: "1", + int_col: "7", float_col: "2.25", ws_num: " 20 ", sci_num: "2e3", neg_num: "-1", + inf_col: "Infinity", nan_col: "NaN", bool_str: "FALSE", mixed_col: "oops", + quoted_comma: "c,d", quoted_newline: "x\ny", quoted_quote: 'a""b', unicode_col: "日本語", + empty_col: "", null_word_col: "null", long_level_col: "short", + json_obj_col: '{"a": 9, "b": "y"}', json_arr_col: "[4, 5]", + }, + { + trial_type: "html-keyboard-response", trial_index: "2", + int_col: "100", float_col: "0.5", ws_num: " 30 ", sci_num: "1.5e3", neg_num: "-10", + inf_col: "Infinity", nan_col: "NaN", bool_str: "true", mixed_col: "3", + quoted_comma: "e,f", quoted_newline: "p\nq", quoted_quote: 'plain', unicode_col: "emoji👋", + empty_col: "", null_word_col: "null", long_level_col: "short", + json_obj_col: '{"a": 50, "b": "z"}', json_arr_col: "[6]", + }, +]; + +describe("CSV ingestion type-inference (stress)", () => { + let vars: Map; + let metadata: JsPsychMetadata; + + beforeAll(async () => { + (global as any).fetch = mockFetch; + jest.spyOn(console, "warn").mockImplementation(() => {}); + metadata = new JsPsychMetadata(); + await metadata.generate(toCSV(HEADERS, ROWS), {}, "csv"); + vars = new Map(metadata.getMetadata().variableMeasured.map((v: any) => [v.name, v])); + }); + + afterAll(() => jest.restoreAllMocks()); + + test("coerces integers, floats, scientific notation and negatives to numeric ranges", () => { + expect(vars.get("int_col")).toMatchObject({ value: "number", minValue: 7, maxValue: 100 }); + expect(vars.get("float_col")).toMatchObject({ value: "number", minValue: 0.5, maxValue: 2.25 }); + expect(vars.get("sci_num")).toMatchObject({ value: "number", minValue: 1000, maxValue: 2000 }); + expect(vars.get("neg_num")).toMatchObject({ value: "number", minValue: -10, maxValue: -1 }); + // No numeric column should carry levels. + for (const n of ["int_col", "float_col", "sci_num", "neg_num"]) expect(vars.get(n).levels).toBeUndefined(); + }); + + test("trims surrounding whitespace before the numeric test (Number(' 10 ') === 10)", () => { + expect(vars.get("ws_num")).toMatchObject({ value: "number", minValue: 10, maxValue: 30 }); + }); + + test("rejects Infinity / NaN as non-numeric and keeps them as string levels", () => { + // Number.isFinite (not !isNaN) is the gate, so these never leak into a numeric range. + expect(vars.get("inf_col").value).toBe("string"); + expect(vars.get("inf_col").minValue).toBeUndefined(); + expect(vars.get("inf_col").levels).toEqual(["Infinity"]); + expect(vars.get("nan_col").value).toBe("string"); + expect(vars.get("nan_col").levels).toEqual(["NaN"]); + }); + + test('keeps "true"/"false" strings categorical (only genuine JSON booleans are boolean)', () => { + const v = vars.get("bool_str"); + expect(v.value).toBe("string"); + expect(v.levels).toEqual(expect.arrayContaining(["TRUE", "FALSE", "true"])); + expect(v.minValue).toBeUndefined(); + }); + + test("downgrades a numeric-then-string column to categorical, preserving the numeric boundary as a level", () => { + const v = vars.get("mixed_col"); + expect(v.value).toBe("string"); + expect(v.minValue).toBeUndefined(); + // "10" seen first (numeric boundary), then "oops", then "3". + expect(v.levels).toEqual(["10", "oops", "3"]); + }); + + test("parses RFC-4180 quoted fields (embedded comma, quote, newline) without corruption", () => { + expect(vars.get("quoted_comma").levels).toEqual(["a,b", "c,d", "e,f"]); + expect(vars.get("quoted_newline").levels).toEqual(["line1\nline2", "x\ny", "p\nq"]); + expect(vars.get("quoted_quote").levels).toEqual(['say "hi"', 'a""b', "plain"]); + }); + + test("preserves unicode in level strings", () => { + expect(vars.get("unicode_col").levels).toEqual(["café", "日本語", "emoji👋"]); + }); + + test('treats empty cells and the literal string "null" as no-value (column stays "unknown")', () => { + for (const n of ["empty_col", "null_word_col"]) { + const v = vars.get(n); + expect(v.value).toBe("unknown"); + expect(v.levels).toBeUndefined(); + } + }); + + test("caps an over-long level at 50 chars + ellipsis", () => { + const v = vars.get("long_level_col"); + const truncated = "x".repeat(50) + "..."; + expect(v.levels).toEqual(expect.arrayContaining([truncated, "short"])); + expect(v.levels).not.toContain(LONG); // the full 80-char string is never stored + }); + + test("parses a JSON object / array embedded in a CSV cell and extracts its sub-columns", () => { + expect(vars.get("json_obj_col").value).toBe("object"); + expect(vars.get("json_obj_col.a")).toMatchObject({ value: "number", minValue: 1, maxValue: 50 }); + expect(vars.get("json_obj_col.b").value).toBe("string"); + expect(vars.get("json_arr_col").value).toBe("array"); + const arrays = metadata.getExtractedArrays(); + expect(arrays.has("json_arr_col")).toBe(true); + }); +}); + +describe("CSV / JSON parity for unambiguously-typed columns (stress)", () => { + // Booleans and nulls intentionally differ between the two formats (a CSV "true" is a string + // level; a JSON true is a boolean), so this parity check is restricted to numeric and plain + // string columns, where CSV coercion must reproduce exactly what native JSON typing produces. + const headers = ["trial_type", "trial_index", "num", "word"]; + const rows = [ + { trial_type: "t", trial_index: "0", num: "5", word: "alpha" }, + { trial_type: "t", trial_index: "1", num: "9", word: "beta" }, + { trial_type: "t", trial_index: "2", num: "1", word: "alpha" }, + ]; + + test("CSV and the equivalent JSON yield identical type/range/levels for num & word", async () => { + (global as any).fetch = mockFetch; + jest.spyOn(console, "warn").mockImplementation(() => {}); + + const fromCsv = new JsPsychMetadata(); + await fromCsv.generate(toCSV(headers, rows), {}, "csv"); + + const json = rows.map((r) => ({ ...r, trial_index: Number(r.trial_index), num: Number(r.num) })); + const fromJson = new JsPsychMetadata(); + await fromJson.generate(JSON.stringify(json), {}, "json"); + + const pick = (m: JsPsychMetadata, name: string) => { + const v = m.getMetadata().variableMeasured.find((x: any) => x.name === name); + return { value: v.value, minValue: v.minValue, maxValue: v.maxValue, levels: v.levels }; + }; + expect(pick(fromCsv, "num")).toEqual(pick(fromJson, "num")); + expect(pick(fromCsv, "word")).toEqual(pick(fromJson, "word")); + + jest.restoreAllMocks(); + }); +}); diff --git a/packages/metadata/tests/scale.stress.test.ts b/packages/metadata/tests/scale.stress.test.ts new file mode 100644 index 0000000..a1e6b9e --- /dev/null +++ b/packages/metadata/tests/scale.stress.test.ts @@ -0,0 +1,85 @@ +import JsPsychMetadata from "../src/index"; + +/** + * Stress regression guard for generate() at scale: feed a large synthetic dataset and assert the + * accumulator stays exact and bounded — numeric min/max reflect the true extremes over thousands + * of rows, a low-cardinality categorical column dedups to its real distinct set, a high-cardinality + * column accumulates one level per distinct value (there is no cap on the *number* of levels — only + * on each level's length), booleans never accrue levels, and the whole pass finishes well within a + * generous time budget. Complements the correctness-focused nested/CSV suites with a volume check. + */ + +const mockFetch = jest.fn().mockResolvedValue({ ok: false, status: 404 }); + +const N = 5000; +const CATEGORIES = ["alpha", "bravo", "charlie", "delta"]; + +// Build the dataset deterministically so expected extremes are known exactly. `signed` swings +// positive and negative so min/max can't be faked by a single-sign assumption. +function buildRows(): any[] { + const rows: any[] = []; + for (let i = 0; i < N; i++) { + rows.push({ + trial_type: "html-keyboard-response", + trial_index: i, + rt: i, // 0 .. N-1 + signed: i - Math.floor(N / 2), // spans negative and positive + category: CATEGORIES[i % CATEGORIES.length], // exactly 4 distinct levels + uid: `id_${i}`, // N distinct levels + correct: i % 2 === 0, // genuine boolean -> no levels, no range + }); + } + return rows; +} + +describe("generate() at scale (stress)", () => { + let vars: Map; + let elapsedMs: number; + + beforeAll(async () => { + (global as any).fetch = mockFetch; + jest.spyOn(console, "warn").mockImplementation(() => {}); + const metadata = new JsPsychMetadata(); + const start = Date.now(); + await metadata.generate(JSON.stringify(buildRows()), {}, "json"); + elapsedMs = Date.now() - start; + vars = new Map(metadata.getMetadata().variableMeasured.map((v: any) => [v.name, v])); + }, 60_000); + + afterAll(() => jest.restoreAllMocks()); + + test(`tracks exact numeric extremes across ${N} rows`, () => { + expect(vars.get("rt")).toMatchObject({ value: "number", minValue: 0, maxValue: N - 1 }); + expect(vars.get("signed")).toMatchObject({ + value: "number", + minValue: -Math.floor(N / 2), + maxValue: N - 1 - Math.floor(N / 2), + }); + }); + + test("dedups a low-cardinality categorical column to its real distinct set", () => { + const levels = vars.get("category").levels; + expect(new Set(levels)).toEqual(new Set(CATEGORIES)); + expect(levels.length).toBe(CATEGORIES.length); // no duplicates despite N/4 occurrences each + }); + + test("accumulates one level per distinct value for a high-cardinality column (no count cap)", () => { + // Documents current behavior: the 50-char cap is per-level, not a cap on how many levels exist. + expect(vars.get("uid").value).toBe("string"); + expect(vars.get("uid").levels.length).toBe(N); + }); + + test("a genuine boolean column carries neither levels nor a numeric range", () => { + const v = vars.get("correct"); + expect(v.value).toBe("boolean"); + expect(v.levels).toBeUndefined(); + expect(v.minValue).toBeUndefined(); + expect(v.maxValue).toBeUndefined(); + }); + + test(`completes the ${N}-row pass within the time budget`, () => { + // Pure in-memory accumulation (fetch stubbed); generous ceiling guards against accidental + // O(n^2) regressions in the hot loop without being flaky on slow CI. + expect(elapsedMs).toBeLessThan(15_000); + }); +});