From c97bf76745e5d29815ba859221fc6a21643fe0f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Micha=C3=ABl=20Celerier?= Date: Sat, 4 Jul 2026 21:50:40 -0400 Subject: [PATCH] testing: golden differential harness across backends Cross-backend golden differential testing: the golden backend records each object's inputs + correct outputs, and per-backend harnesses replay those inputs and diff the captured output against the oracle. - binding/golden/run.hpp: one-at-a-time case sweep (base case + per enum mode + per numeric range endpoint) per object. - tooling/golden_compare.py: shared audio/control/texture compare + case aggregation. - tooling/run_{python,gst,pd,max}_golden.py + tooling/max driver: the per-backend replay harnesses. - tooling/gen_tester_patches.py, tooling/td/run_td_sweep.py: TD runner generation + sweep (records the crash stage; texture output reported as not verifiable headless rather than a false mismatch). Depends at runtime on the backend fixes in fix/backend-audio-texture-correctness and fix/python-binding-improvements for the harnesses to report clean. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01JH7HS58wPSA22HztHZsakd --- include/avnd/binding/golden/run.hpp | 162 ++++++++-- tooling/gen_tester_patches.py | 258 ++++++++------- tooling/golden_compare.py | 136 ++++++++ tooling/max/avnd_max_driver.js | 482 ++++++++++++++++++++++++++++ tooling/max/avnd_max_driver.maxpat | 132 ++++++++ tooling/run_gst_golden.py | 353 ++++++++++++++++++++ tooling/run_max_golden.py | 400 +++++++++++++++++++++++ tooling/run_pd_golden.py | 341 ++++++++++++++++++++ tooling/run_python_golden.py | 331 +++++++++++++++++++ tooling/td/run_td_sweep.py | 144 ++++----- 10 files changed, 2511 insertions(+), 228 deletions(-) create mode 100644 tooling/golden_compare.py create mode 100644 tooling/max/avnd_max_driver.js create mode 100644 tooling/max/avnd_max_driver.maxpat create mode 100644 tooling/run_gst_golden.py create mode 100644 tooling/run_max_golden.py create mode 100644 tooling/run_pd_golden.py create mode 100644 tooling/run_python_golden.py diff --git a/include/avnd/binding/golden/run.hpp b/include/avnd/binding/golden/run.hpp index 9564aa0a..a0aedd35 100644 --- a/include/avnd/binding/golden/run.hpp +++ b/include/avnd/binding/golden/run.hpp @@ -68,23 +68,37 @@ std::string clean_name(int index) return std::string{nm}; } -// Set a deterministic value on an input control field, then append -// {index, name, value} to the ordered `arr` (order matches the dump). +// One test case = override control #index with `value` (index < 0 = base case, +// all controls at default). We sweep one control at a time: base, then each enum +// mode and each numeric range endpoint. `value` is the enum ordinal or endpoint. +struct case_override +{ + int index = -1; + double value = 0.0; +}; + +// Set a deterministic value on an input control field (honouring `ov` when it +// targets this control), then append {index, name, value} to the ordered `arr`. template -void set_and_record_control(F& field, dump_json::document& jdoc, dump_json::value arr, int index) +void set_and_record_control( + F& field, dump_json::document& jdoc, dump_json::value arr, int index, + const case_override& ov) { using vt = std::decay_t; + const bool over = (ov.index == index); auto node = jdoc.make_node(); node["index"] = index; node["name"] = clean_name(index); if constexpr(std::is_same_v) { - field.value = true; - node["value"] = true; + field.value = over ? (ov.value != 0.0) : true; + node["value"] = (bool)field.value; } else if constexpr(std::floating_point) { - if constexpr(avnd::parameter_with_minmax_range) + if(over) + field.value = static_cast(ov.value); + else if constexpr(avnd::parameter_with_minmax_range) { constexpr auto r = avnd::get_range(); field.value = static_cast(r.min + 0.5 * (r.max - r.min)); @@ -95,12 +109,14 @@ void set_and_record_control(F& field, dump_json::document& jdoc, dump_json::valu } else if constexpr(std::is_enum_v) { - field.value = static_cast(0); + field.value = over ? static_cast((int)ov.value) : static_cast(0); node["value"] = static_cast(field.value); } else if constexpr(std::integral) { - if constexpr(avnd::parameter_with_minmax_range) + if(over) + field.value = static_cast((std::int64_t)ov.value); + else if constexpr(avnd::parameter_with_minmax_range) { constexpr auto r = avnd::get_range(); field.value = static_cast(r.min + (r.max - r.min) / 2); @@ -121,6 +137,46 @@ void set_and_record_control(F& field, dump_json::document& jdoc, dump_json::valu arr.push_back(node); } +// Enumerate the OAT test cases for T: the base case, then for every enum input a +// case per non-zero mode, and for every ranged numeric input a case at min and +// at max. Bounded (sum, not product), so it stays small. +template +std::vector enumerate_cases() +{ + std::vector cases; + cases.push_back({-1, 0.0}); // base + if constexpr(avnd::parameter_input_introspection::size > 0) + { + avnd::effect_container probe; + avnd::init_controls(probe); + int idx = 0; + avnd::parameter_input_introspection::for_all( + avnd::get_inputs(probe), [&](auto& field) { + using F = std::decay_t; + using vt = std::decay_t; + if constexpr(std::is_enum_v) + { + constexpr int C = (int)avnd::get_enum_choices_count(); + for(int c = 1; c < C; ++c) + cases.push_back({idx, (double)c}); + } + else if constexpr( + (std::floating_point || std::integral)&&avnd:: + parameter_with_minmax_range) + { + constexpr auto r = avnd::get_range(); + if((double)r.min != (double)r.max) + { + cases.push_back({idx, (double)r.min}); + cases.push_back({idx, (double)r.max}); + } + } + idx++; + }); + } + return cases; +} + // Append an output control field's value as {index, name, value}. template void record_output_control(F& field, dump_json::document& jdoc, dump_json::value arr, int index) @@ -140,26 +196,11 @@ void record_output_control(F& field, dump_json::document& jdoc, dump_json::value arr.push_back(node); } +// Run ONE test case (fresh effect, override `ov` applied) and write its inputs +// and outputs into `case_node`. Throws on failure (caught by run()). template -int run(std::string_view path) +void run_case(dump_json::document& jdoc, dump_json::value case_node, const case_override& ov) { - dump_json::document jdoc; - auto root = jdoc.root(); - root["c_name"] = avnd::get_c_name(); - auto meta = root["meta"]; - meta["seed"] = k_seed; - meta["frames"] = k_frames; - meta["rate"] = k_rate; - - auto write = [&](std::string_view status) { - root["meta"]["status"] = status; - std::ofstream f{std::string{path}}; - f << jdoc.dump(); - return 0; - }; - - try - { avnd::effect_container effect; avnd::process_adapter processor; avnd::audio_channel_manager channels{effect}; @@ -227,7 +268,7 @@ int run(std::string_view path) control_buffers.reserve_space(effect, k_frames); // --- inputs --- - auto in = root["inputs"]; + auto in = case_node["inputs"]; auto in_controls = in["controls"]; in_controls.ensure_array(); if constexpr(avnd::parameter_input_introspection::size > 0) @@ -235,7 +276,7 @@ int run(std::string_view path) int idx = 0; avnd::parameter_input_introspection::for_all( avnd::get_inputs(effect), [&](auto& field) { - set_and_record_control(field, jdoc, in_controls, idx++); + set_and_record_control(field, jdoc, in_controls, idx++, ov); }); } @@ -282,7 +323,7 @@ int run(std::string_view path) avnd::span{out_ptrs.data(), (std::size_t)out_N}, k_frames); // --- outputs --- - auto out = root["outputs"]; + auto out = case_node["outputs"]; auto out_controls = out["controls"]; out_controls.ensure_array(); if constexpr(avnd::parameter_output_introspection::size > 0) @@ -333,17 +374,68 @@ int run(std::string_view path) out_tex.push_back(node); }); } +} - return write("ok"); - } - catch(const std::exception& e) +template +int run(std::string_view path) +{ + dump_json::document jdoc; + auto root = jdoc.root(); + root["c_name"] = avnd::get_c_name(); + auto meta = root["meta"]; + meta["seed"] = k_seed; + meta["frames"] = k_frames; + meta["rate"] = k_rate; + + auto finish = [&](std::string_view status) { + root["meta"]["status"] = status; + std::ofstream f{std::string{path}}; + f << jdoc.dump(); + return 0; + }; + + auto cases_arr = root["cases"]; + cases_arr.ensure_array(); + + std::vector overrides; + try { - root["meta"]["error"] = std::string_view{e.what()}; - return write("error"); + overrides = enumerate_cases(); } catch(...) { - return write("error"); + // If we cannot even enumerate, fall back to the single base case. + overrides = {{-1, 0.0}}; + } + meta["num_cases"] = (int)overrides.size(); + + bool all_ok = true; + int ci = 0; + for(const auto& ov : overrides) + { + auto cnode = jdoc.make_node(); + cnode["index"] = ci++; + cnode["override_control"] = ov.index; + cnode["override_value"] = ov.value; + try + { + run_case(jdoc, cnode, ov); + cnode["status"] = "ok"; + } + catch(const std::exception& e) + { + cnode["status"] = "error"; + cnode["error"] = std::string_view{e.what()}; + all_ok = false; + } + catch(...) + { + cnode["status"] = "error"; + all_ok = false; + } + cases_arr.push_back(cnode); } + + return finish(all_ok ? "ok" : "partial"); } } diff --git a/tooling/gen_tester_patches.py b/tooling/gen_tester_patches.py index c91af3f0..c9a11e94 100644 --- a/tooling/gen_tester_patches.py +++ b/tooling/gen_tester_patches.py @@ -472,6 +472,7 @@ def gen_td_runner(dumps): md = dump.get("metadatas", {}) ins = dump.get("inputs", []) is_top = any(port_kind(p) == "texture" for p in ins + dump.get("outputs", [])) + has_tex_in = any(port_kind(p) == "texture" for p in ins) pars = {sanitize_td_name(p["name"]): init_value(p) for p in ins if is_control(p) and "name" in p} # The registered opType is sanitize_td_name(C_NAME) -- the TD binding's @@ -480,6 +481,10 @@ def gen_td_runner(dumps): specs.append({"name": md.get("c_name", name), "type": sanitize_td_name(md.get("c_name", name)), "family": "TOP" if is_top else "CHOP", + # int, not bool: SPECS is embedded via json.dumps and then + # exec()'d in TD, where JSON `false`/`true` are not valid + # Python identifiers. + "has_tex_in": 1 if has_tex_in else 0, "file": name, "pars": pars}) return ( "# Run inside TouchDesigner (an Execute DAT). For each avendish Custom OP:\n" @@ -526,125 +531,152 @@ def gen_td_runner(dumps): " golden = json.load(open(os.path.join(GDIR, s['file'] + '.json')))\n" " except Exception:\n" " golden = None\n" - " entry = {'name': s['name'], 'ok': False}\n" - " try:\n" - " n = container.create(cls, 'test_' + s['name'])\n" - " # Apply golden input controls as params; ones that aren't params\n" - " # (value-input ports become input CHANNELS in TD) are collected to\n" - " # feed via a source CHOP below.\n" - " in_names = []\n" - " in_vals = []\n" - " applied = {}\n" - " if golden:\n" - " for c in golden.get('inputs', {}).get('controls', []):\n" - " if c.get('value') == 'unrecorded': continue # non-value (pulse/aggregate)\n" - " as_par = False\n" - " try:\n" - " setattr(n.par, c['name'], c['value']); as_par = True\n" - " except Exception:\n" + " entry = {'name': s['name'], 'ok': False, 'cases': []}\n" + " # Golden may be multi-case ({cases:[{inputs,outputs},...]}) or, for a\n" + " # missing/old-schema golden, a single implicit case. Replay each: a\n" + " # FRESH op per case (so a stateful object isn't tainted across cases,\n" + " # matching the golden which re-runs a fresh effect per case).\n" + " gcases = (golden or {}).get('cases')\n" + " if not gcases:\n" + " if golden and (golden.get('inputs') or golden.get('outputs')):\n" + " gcases = [{'inputs': golden.get('inputs', {}), 'outputs': golden.get('outputs', {})}]\n" + " else:\n" + " gcases = [{}]\n" + " gi = 0\n" + " for gcase in gcases:\n" + " tag = s['name'] + ':case' + str(gi)\n" + " open(CUR, 'w').write(tag) # crash breadcrumb\n" + " cres = {'index': gi, 'ok': False}\n" + " try:\n" + " uid = s['name'] + '_c' + str(gi)\n" + " n = container.create(cls, 'test_' + uid)\n" + " in_names = []\n" + " in_vals = []\n" + " applied = {}\n" + " gc = gcase.get('inputs', {}).get('controls', [])\n" + " if gc:\n" + " for c in gc:\n" + " if c.get('value') == 'unrecorded': continue # non-value (pulse/aggregate)\n" + " as_par = False\n" " try:\n" - " setattr(n.par, c['name'].capitalize(), c['value'])\n" - " as_par = True\n" + " setattr(n.par, c['name'], c['value']); as_par = True\n" " except Exception:\n" - " as_par = False\n" - " if as_par:\n" - " applied[c['name']] = c['value']\n" - " else:\n" - " in_names.append(c['name'])\n" - " in_vals.append(c['value'])\n" - " else:\n" - " for k, v in s['pars'].items():\n" - " try: setattr(n.par, k, v)\n" + " try:\n" + " setattr(n.par, c['name'].capitalize(), c['value']); as_par = True\n" + " except Exception:\n" + " as_par = False\n" + " if as_par:\n" + " applied[c['name']] = c['value']\n" + " else:\n" + " in_names.append(c['name'])\n" + " in_vals.append(c['value'])\n" + " else:\n" + " for k, v in s['pars'].items():\n" + " try: setattr(n.par, k, v)\n" + " except Exception: pass\n" + " cres['applied'] = applied\n" + " if gi == 0:\n" + " par_names = []\n" + " try:\n" + " for p in n.pars(): par_names.append(p.name)\n" " except Exception: pass\n" - " entry['applied'] = applied\n" - " par_names = []\n" - " try:\n" - " for p in n.pars(): par_names.append(p.name)\n" - " except Exception: pass\n" - " entry['par_names'] = par_names[:30]\n" - " gin = None\n" - " if golden: gin = golden.get('inputs', {}).get('audio')\n" - " entry['has_audio_in'] = bool(gin)\n" - " # Feed a source CHOP so value-IO objects can cook. Value inputs\n" - " # (ports that aren't params) become input channels -> Constant CHOP.\n" - " # Audio (time-varying) inputs need a Script CHOP -- via samples in\n" - " # a Table DAT the script reads.\n" - " src = None\n" - " try:\n" - " if gin:\n" - " tab = container.create(tableDAT, 'srcd_' + s['name'])\n" - " tab.clear()\n" - " for chn in gin:\n" - " row = []\n" - " for x in chn: row.append(repr(float(x)))\n" - " tab.appendRow(row)\n" - " src = container.create(scriptCHOP, 'src_' + s['name'])\n" - " cb = container.create(textDAT, 'srccb_' + s['name'])\n" - " cb.text = ('def onCook(scriptOp):\\n'\n" - " ' scriptOp.clear()\\n'\n" - " ' t = op(scriptOp.fetch(\"tab\",\"\"))\\n'\n" - " ' if not t: return\\n'\n" - " ' scriptOp.numSamples = t.numCols\\n'\n" - " ' for r in range(t.numRows):\\n'\n" - " ' c = scriptOp.appendChan(\"c\"+str(r))\\n'\n" - " ' for j in range(t.numCols):\\n'\n" - " ' c[j] = float(t[r,j].val)\\n')\n" - " src.store('tab', tab.path)\n" - " src.par.callbacks = cb\n" - " elif in_vals:\n" - " src = container.create(constantCHOP, 'src_' + s['name'])\n" - " for j in range(len(in_vals)):\n" + " entry['par_names'] = par_names[:30]\n" + " gin = gcase.get('inputs', {}).get('audio')\n" + " src = None\n" + " try:\n" + " if gin:\n" + " tab = container.create(tableDAT, 'srcd_' + uid)\n" + " tab.clear()\n" + " for chn in gin:\n" + " row = []\n" + " for x in chn: row.append(repr(float(x)))\n" + " tab.appendRow(row)\n" + " src = container.create(scriptCHOP, 'src_' + uid)\n" + " cb = container.create(textDAT, 'srccb_' + uid)\n" + " cb.text = ('def onCook(scriptOp):\\n'\n" + " ' scriptOp.clear()\\n'\n" + " ' t = op(scriptOp.fetch(\"tab\",\"\"))\\n'\n" + " ' if not t: return\\n'\n" + " ' scriptOp.numSamples = t.numCols\\n'\n" + " ' for r in range(t.numRows):\\n'\n" + " ' c = scriptOp.appendChan(\"c\"+str(r))\\n'\n" + " ' for j in range(t.numCols):\\n'\n" + " ' c[j] = float(t[r,j].val)\\n')\n" + " src.store('tab', tab.path)\n" + " src.par.callbacks = cb\n" + " elif in_vals:\n" + " src = container.create(constantCHOP, 'src_' + uid)\n" + " for j in range(len(in_vals)):\n" + " try:\n" + " setattr(src.par, 'value' + str(j), float(in_vals[j]))\n" + " setattr(src.par, 'name' + str(j), str(in_names[j]))\n" + " except Exception: pass\n" + " if src is not None:\n" + " src.cook(force=True)\n" + " if len(n.inputConnectors) > 0:\n" + " n.inputConnectors[0].connect(src)\n" + " except Exception as e:\n" + " cres['input_err'] = str(e)\n" + " if s.get('family') == 'TOP' and s.get('has_tex_in'):\n" + " try:\n" + " tsrc = container.create(noiseTOP, 'tsrc_' + uid)\n" " try:\n" - " setattr(src.par, 'value' + str(j), float(in_vals[j]))\n" - " setattr(src.par, 'name' + str(j), str(in_names[j]))\n" + " tsrc.par.resolutionw = 16; tsrc.par.resolutionh = 16\n" " except Exception: pass\n" - " if src is not None:\n" - " src.cook(force=True)\n" - " if len(n.inputConnectors) > 0:\n" - " n.inputConnectors[0].connect(src)\n" + " tsrc.cook(force=True)\n" + " if len(n.inputConnectors) > 0:\n" + " n.inputConnectors[0].connect(tsrc)\n" + " except Exception as e:\n" + " cres['tex_in_err'] = str(e)\n" + " open(CUR, 'w').write(tag + ':precook')\n" + " n.cook(force=True)\n" + " cres['ok'] = not n.errors()\n" + " cres['errors'] = n.errors()\n" + " try:\n" + " cres['dbg_nc'] = n.numChans\n" + " cres['dbg_ns'] = n.numSamples\n" + " except Exception: pass\n" + " td_out = []\n" + " try:\n" + " ns = n.numSamples\n" + " for ch in n.chans():\n" + " samples = []\n" + " for i in range(ns): samples.append(float(ch[i]))\n" + " td_out.append({'name': ch.name, 'samples': samples})\n" + " cres['td_out'] = td_out\n" + " except Exception:\n" + " cres['td_out'] = None\n" + " ti = {}\n" + " try:\n" + " info = container.create(infoCHOP, 'info_' + uid)\n" + " info.par.op = n\n" + " info.cook()\n" + " for ci in info.chans():\n" + " ti[ci.name] = float(ci[0])\n" + " info.destroy()\n" + " except Exception:\n" + " pass\n" + " cres['td_info'] = ti\n" + " td_tex = []\n" + " try:\n" + " if s.get('family') == 'TOP':\n" + " import zlib\n" + " a = n.numpyArray() # (h, w, ch), TD's normalized readback\n" + " hh = int(a.shape[0]); ww = int(a.shape[1])\n" + " td_tex.append({'width': ww, 'height': hh,\n" + " 'hash': int(zlib.crc32(a.tobytes()) & 0xffffffff)})\n" + " except Exception as e:\n" + " cres['tex_err'] = str(e)\n" + " cres['td_tex'] = td_tex\n" + " n.destroy()\n" " except Exception as e:\n" - " entry['input_err'] = str(e)\n" - " n.cook(force=True)\n" - " entry['ok'] = not n.errors()\n" - " entry['errors'] = n.errors()\n" - " entry['warnings'] = n.warnings()\n" - " # Capture output channels WITH names (control outputs surface as\n" - " # named channels, audio as ordered channels). Read n directly and\n" - " # cook the downstream reads WITHOUT force, so the op is cooked\n" - " # exactly ONCE -- extra forced pulls re-cook it, which corrupts\n" - " # stateful objects (counters, oscillators): they'd advance N times.\n" - " try:\n" - " entry['dbg_nc'] = n.numChans\n" - " entry['dbg_ns'] = n.numSamples\n" - " except Exception: pass\n" - " td_out = []\n" - " try:\n" - " ns = n.numSamples\n" - " for ch in n.chans():\n" - " samples = []\n" - " for i in range(ns): samples.append(float(ch[i]))\n" - " td_out.append({'name': ch.name, 'samples': samples})\n" - " entry['td_out'] = td_out\n" - " except Exception:\n" - " entry['td_out'] = None\n" - " # Control outputs that aren't signal channels surface via\n" - " # getInfoCHOPChan -- read them through an Info CHOP, cooked WITHOUT\n" - " # force so it reuses n's single cook (generic info channels are\n" - " # captured too but ignored by name-matching downstream).\n" - " ti = {}\n" - " try:\n" - " info = container.create(infoCHOP, 'info_' + s['name'])\n" - " info.par.op = n\n" - " info.cook()\n" - " for ci in info.chans():\n" - " ti[ci.name] = float(ci[0])\n" - " info.destroy()\n" - " except Exception:\n" - " pass\n" - " entry['td_info'] = ti\n" - " n.destroy()\n" - " except Exception as e:\n" - " entry['exception'] = str(e)\n" + " cres['exception'] = str(e)\n" + " entry['cases'].append(cres)\n" + " gi += 1\n" + " okall = len(entry['cases']) > 0\n" + " for c in entry['cases']:\n" + " if not c.get('ok'): okall = False\n" + " entry['ok'] = okall\n" " report.append(entry)\n" " open(RP, 'w').write(json.dumps(report, indent=2))\n" "open(CUR, 'w').write('DONE')\n" diff --git a/tooling/golden_compare.py b/tooling/golden_compare.py new file mode 100644 index 00000000..94696dcf --- /dev/null +++ b/tooling/golden_compare.py @@ -0,0 +1,136 @@ +"""Shared golden-differential comparison helpers. + +Every backend harness (TouchDesigner sweep, Python driver, ...) reads a golden +JSON produced by the `golden` backend, replays its recorded inputs, captures the +backend's outputs and diffs them with these functions, so tolerance and +name-matching semantics stay identical across backends. +""" + + +def compare_audio(out, golden, atol, rtol): + """Diff captured output channels (positional) against the golden audio + channels over their overlapping prefix (a backend's cook sample-count can + differ from the golden's frame count). `out` is [{name, samples}].""" + if not golden: + return ("no-golden-audio", "") + if not out: + return ("no-backend-audio", "") + ch = [e["samples"] for e in out] + nch = min(len(ch), len(golden)) + if nch == 0: + return ("empty", "") + maxdiff = 0.0 + for c in range(nch): + ns = min(len(ch[c]), len(golden[c])) + for i in range(ns): + maxdiff = max(maxdiff, abs(ch[c][i] - golden[c][i])) + peak = max((abs(x) for gch in golden for x in gch), default=0.0) + tol = atol + rtol * peak + verdict = "match" if maxdiff <= tol else "MISMATCH" + detail = (f"maxdiff={maxdiff:.2e} tol={tol:.2e} " + f"got={len(ch)}ch gold={len(golden)}ch") + return (verdict, detail) + + +def compare_controls(named, golden, atol, rtol): + """Diff golden output controls against backend values matched by name. + `named` is {name -> value}; exact, lowercase and capitalized forms of the + golden name are tried.""" + if not golden: + return ("no-golden-controls", "") + if not named: + return ("no-backend-controls", "") + checked, maxdiff, worst = 0, 0.0, "" + for gc in golden: + if not isinstance(gc, dict): + continue + nm, gv = gc.get("name", ""), gc.get("value") + if gv == "unrecorded" or not isinstance(gv, (int, float, str)): + continue + tv = None + for k in (nm, nm.lower(), nm.capitalize()): + if k in named: + tv = named[k] + break + if tv is None: + continue + if isinstance(gv, str): + # String outputs: exact equality. + checked += 1 + if str(tv) != gv and maxdiff == 0.0: + maxdiff, worst = 1.0, f"{nm}({tv!r}!={gv!r})" + continue + if isinstance(tv, bool): + tv = float(tv) + if not isinstance(tv, (int, float)): + continue + checked += 1 + if abs(tv - gv) > maxdiff: + maxdiff, worst = abs(tv - gv), nm + if checked == 0: + return ("no-name-match", f"got={list(named)[:4]}") + peak = max((abs(gc["value"]) for gc in golden + if isinstance(gc.get("value"), (int, float))), default=0.0) + tol = atol + rtol * peak + return ("match" if maxdiff <= tol else "MISMATCH", + f"{checked} ctrls maxdiff={maxdiff:.2e}@{worst} tol={tol:.2e}") + + +def compare_textures(textures, golden, atol, rtol, content="hash"): + """Diff captured texture outputs against the golden's recorded + {width, height, hash}. `textures` is [{width, height, hash}] in port order. + + content="hash" (default): dimensions AND the FNV-1a byte hash must match -- + valid only when the backend reads pixels back in the object's NATIVE byte + layout (the golden hashes native bytes; the Python binding does too). + content="dims": dimensions are authoritative, the hash is informational -- + for hosts (TouchDesigner, GStreamer) that hand back a normalized/converted + representation whose bytes never equal the native golden hash. Wrong output + SIZE is still a real failure and is caught.""" + if not golden: + return ("no-golden-texture", "") + if not textures: + return ("no-backend-texture", "") + n = min(len(textures), len(golden)) + hashes_match = True + for i in range(n): + t, g = textures[i], golden[i] + if (t.get("width"), t.get("height")) != (g.get("width"), g.get("height")): + return ("MISMATCH", + f"tex{i} size {t.get('width')}x{t.get('height')} " + f"vs gold {g.get('width')}x{g.get('height')}") + if t.get("hash") != g.get("hash"): + hashes_match = False + if content == "hash": + return ("MISMATCH", f"tex{i} content hash differs") + if content == "dims": + note = "content byte-identical" if hashes_match else "dims only" + return ("match", f"{n} textures ({note})") + return ("match", f"{n} textures") + + +def fnv1a64(data): + """FNV-1a 64-bit over raw bytes, returned as a SIGNED int64 -- the exact + hash the golden generator records (binding/golden/run.hpp).""" + h = 1469598103934665603 + for b in data: + h ^= b + h = (h * 1099511628211) & 0xFFFFFFFFFFFFFFFF + return h - 0x10000000000000000 if h >= 0x8000000000000000 else h + + +def aggregate_case_verdicts(verdicts): + """Fold per-case (index, verdict, detail) tuples into one per-object + verdict: any MISMATCH wins (reporting the first failing case), else match + if any case matched, else the first non-match category.""" + mm = [x for x in verdicts if x[1] == "MISMATCH"] + if mm: + detail = f"case{mm[0][0]} {mm[0][2]}" + if len(mm) > 1: + detail += f" (+{len(mm)-1} more cases)" + return ("MISMATCH", detail) + if any(x[1] == "match" for x in verdicts): + return ("match", "") + if verdicts: + return (verdicts[0][1], verdicts[0][2]) + return ("no-golden-output", "") diff --git a/tooling/max/avnd_max_driver.js b/tooling/max/avnd_max_driver.js new file mode 100644 index 00000000..27f38073 --- /dev/null +++ b/tooling/max/avnd_max_driver.js @@ -0,0 +1,482 @@ +// avnd_max_driver.js -- in-Max golden-differential driver. +// +// Loaded by avnd_max_driver.maxpat (loadbang -> "run"). For every golden object +// (data staged in avnd_max_data.js by run_max_golden.py) it: +// * control objects: instantiates the message_processor external, wires each +// output outlet through [prepend cap] back into this js inlet, replays the +// golden input controls and captures the committed output values; +// * audio objects: builds count~ -> index~(avin) -> mc.pack~ -> object -> +// mc.unpack~ -> record~(avout), fills the input buffers with the golden +// samples, runs one real DSP block and reads the recorded output samples; +// * none/texture: instantiate-only smoke (nothing feasible to capture). +// Results are written one JSON line per object to AVND_CFG.report; a per-object +// breadcrumb (AVND_CFG.breadcrumb) lets the Python harness relaunch past a +// crasher and resume (objects already in the report are skipped). +// +// Everything is deliberately async/Task-driven with a delay between "build" and +// "read" so committed/recorded values have settled regardless of Max's +// synchronous-vs-deferred message timing. + +autowatch = 0; +inlets = 1; +outlets = 1; + +include("avnd_max_config.js"); // -> AVND_CFG {report, breadcrumb, goldenDir} +include("avnd_max_data.js"); // -> AVND_DATA [{name, kind, cases:[...]}] + +var FRAMES = 64; +var RATE = 44100; +var BUFSAMPS = 64; // == FRAMES: record~ (loop off) auto-stops after one block +var CHANS = 2; + +var allLines = []; // raw report lines (prior + new), rewritten each object +var doneNames = {}; // objects already reported (resume after crash) +var WORK = []; // flattened [{oi, ci, first, last}] +var wi = -1; // current work index +var cur = []; // graph objects to clean up for the current case +var capture = {}; // cap -> value (control capture, current case) +var objCases = []; // accumulated case-result strings for the current object +var TASKS = []; // keep Task refs alive +var dspWorks = false; // set by the DSP self-test: does audio actually render? +var adDriver = null; // [adstatus driver] (persistent) for set + readback +var dsptimeObj = null; // [dsptime~] (persistent): samples rendered since dsp start +var setupObjs = []; // persistent NRT-setup objects (never cleaned up) +var pollTarget = 0, pollThen = "", pollAttempts = 0; // dsptime~ render gate + +function post_(s) { post(s + "\n"); } + +// ---- tiny JSON serialization (we only ever emit, never parse, JSON) -------- +function jnum(x) { + if (x === null || x === undefined) return "0"; + var n = Number(x); + if (!isFinite(n)) return "0"; + return "" + n; +} +function jstr(s) { + s = "" + s; + s = s.replace(/\\/g, "\\\\").replace(/"/g, "\\\""); + return "\"" + s + "\""; +} +function jarr(a) { + var p = []; + for (var i = 0; i < a.length; i++) p.push(jnum(a[i])); + return "[" + p.join(",") + "]"; +} + +// ---- files ---------------------------------------------------------------- +function crumb(s) { + // Max's File "write" mode overwrites in place WITHOUT truncating, so a shorter + // string leaves a stale tail; pad to a fixed width so the reader (which + // .strip()s) always sees exactly `s`. + while (s.length < 240) s += " "; + var f = new File(AVND_CFG.breadcrumb, "write"); + if (f.isopen) { f.writestring(s); f.close(); } +} +function loadExisting() { + var f = new File(AVND_CFG.report, "read"); + if (!f.isopen) return; + while (!f.eof) { + var ln = f.readline(2000000); + if (ln && ln.length > 0) { + allLines.push(ln); + var m = ln.indexOf("\"name\":\""); + if (m >= 0) { + var s = m + 8, e = ln.indexOf("\"", s); + if (e > s) doneNames[ln.substring(s, e)] = 1; + } + } + } + f.close(); +} +function writeReport() { + var f = new File(AVND_CFG.report, "write"); + if (!f.isopen) { post_("ERR cannot open report for write"); return; } + for (var i = 0; i < allLines.length; i++) f.writeline(allLines[i]); + f.close(); +} + +// ---- scheduling ------------------------------------------------------------ +// Max js Task callbacks don't reliably capture closures across engine versions, +// so dispatch by mode through a stable named callback + Task.arguments. +function defer(mode, ms) { + var t = new Task(taskCallback, this); + t.arguments = [mode]; + t.schedule(ms); + TASKS.push(t); + if (TASKS.length > 64) TASKS.splice(0, 32); +} +function taskCallback(mode) { + if (mode === "readStep") readStep(); + else if (mode === "buildStep") buildStep(); + else if (mode === "selfTest") selfTestDSP(); + else if (mode === "selfTestRead") selfTestRead(); + else if (mode === "poll") pollStep(); +} + +// ---- Non-Real-Time DSP: render MSP with no physical audio device ----------- +// Max has no headless mode, but its NonRealTime audio driver computes the DSP +// graph on demand (no hardware). Required companions (else the scheduler -- +// which drives our poll Tasks / metro / dsptime~ -- does not advance under NRT): +// Overdrive ON and Scheduler-in-Audio-Interrupt ON. We also pin the sample rate +// and signal vector size to the golden's (64 frames @ 44100) so block-based +// objects see exactly the oracle's one-block buffer. +function setupAudioNRT() { + var p = this.patcher; + var X = 660; + stopDSP(); + try { messnamed("max", "overdrive", 1); } catch (e) {} + var over = p.newdefault(X, 300, "adstatus", "overdrive"); over.message(1); + var sched = p.newdefault(X, 326, "adstatus", "scheduler"); sched.message(1); // SIAI + adDriver = p.newdefault(X, 352, "adstatus", "driver"); + var adp = p.newdefault(X + 170, 352, "prepend", "curdrv"); + p.connect(adDriver, 0, adp, 0); + p.connect(adp, 0, p.getnamed("driver"), 0); + adDriver.message("NonRealTime"); // set the driver + try { messnamed("dsp", "setdriver", "NonRealTime"); } catch (e) {} + var sr = p.newdefault(X, 378, "adstatus", "sr"); sr.message(RATE); + var vs = p.newdefault(X, 404, "adstatus", "sigvs"); vs.message(FRAMES); + var io = p.newdefault(X, 430, "adstatus", "iovs"); io.message(FRAMES); + dsptimeObj = p.newdefault(X, 456, "dsptime~"); + var dtp = p.newdefault(X + 170, 456, "prepend", "dsptime"); + p.connect(dsptimeObj, 0, dtp, 0); + p.connect(dtp, 0, p.getnamed("driver"), 0); + setupObjs = [over, sched, adDriver, adp, sr, vs, io, dsptimeObj, dtp]; +} +function startDSP() { + var ds = this.patcher.getnamed("dspstart"); + if (ds) ds.message("bang"); + try { messnamed("dsp", "start"); } catch (e) {} +} +function stopDSP() { + var ds = this.patcher.getnamed("dspstop"); + if (ds) ds.message("bang"); + try { messnamed("dsp", "stop"); } catch (e) {} +} +// Gate on dsptime~ (samples rendered), NOT wall-clock: NRT free-runs, so poll +// until at least `target` samples have elapsed, then dispatch `then`. +function startPoll(target, then) { + pollTarget = target; pollThen = then; pollAttempts = 0; + defer("poll", 8); +} +function pollStep() { + pollAttempts++; + if (dsptimeObj) dsptimeObj.message("bang"); + var t = capture["dsptime"]; + if ((typeof t === "number" && t >= pollTarget) || pollAttempts > 120) { + taskCallback(pollThen); + return; + } + defer("poll", 8); +} + +// One-shot: prove NRT DSP + record~ actually capture a signal here. Sets the +// global dspWorks flag; the per-object audio path only activates when true. +function selfTestDSP() { + var p = this.patcher; + capture = {}; + var b = new Buffer("avout0"); + for (var z = 0; z < BUFSAMPS; z++) b.poke(1, z, 0.0); + if (adDriver) adDriver.message("bang"); // read back the driver name + var osc = p.newdefault(40, 520, "cycle~", 441); + var snap = p.newdefault(40, 545, "snapshot~", 5); + var pre = p.newdefault(200, 545, "prepend", "stpeak"); + var rec = p.newdefault(40, 580, "record~", "avout0"); + cur = [osc, snap, pre, rec]; + p.connect(osc, 0, snap, 0); + p.connect(snap, 0, pre, 0); + p.connect(pre, 0, this.patcher.getnamed("driver"), 0); + p.connect(osc, 0, rec, 0); + rec.message(1); + startDSP(); + startPoll(FRAMES, "selfTestRead"); +} +function selfTestRead() { + var b = new Buffer("avout0"); + var peak = 0.0; + for (var i = 0; i < BUFSAMPS; i++) { + var v = b.peek(1, i); + if (v < 0) v = -v; + if (v > peak) peak = v; + } + stopDSP(); + var snap = capture["stpeak"]; + dspWorks = (peak > 0.0) || (typeof snap === "number" && snap !== 0.0); + var dbg = new File(AVND_CFG.breadcrumb + ".dbg", "readwrite"); + if (dbg.isopen) { + dbg.position = dbg.eof; + dbg.writeline("dsp_works=" + dspWorks); + dbg.writeline("audio_driver=" + (capture["curdrv"] === undefined ? "none" : capture["curdrv"])); + dbg.writeline("selftest_record_peak=" + peak); + dbg.writeline("selftest_dsptime=" + (capture["dsptime"] === undefined ? "none" : capture["dsptime"])); + dbg.close(); + } + cleanup(); + buildStep(); +} + +// ---- entry ----------------------------------------------------------------- +function run() { + loadExisting(); + WORK = []; + for (var oi = 0; oi < AVND_DATA.length; oi++) { + var o = AVND_DATA[oi]; + if (doneNames[o.name]) continue; + var nc = o.cases.length; + for (var ci = 0; ci < nc; ci++) + WORK.push({ oi: oi, ci: ci, first: ci === 0, last: ci === nc - 1 }); + } + post_("avnd max driver: " + WORK.length + " cases over " + + (AVND_DATA.length - countKeys(doneNames)) + " objects"); + wi = -1; + // size the static capture buffers once. + for (var ch = 0; ch < CHANS; ch++) { + sizeBuf("buf_avin" + ch); + sizeBuf("buf_avout" + ch); + } + // one-shot debug: which static boxes did we resolve by scripting name? + var dbg = new File(AVND_CFG.breadcrumb + ".dbg", "write"); + if (dbg.isopen) { + dbg.writeline("driver=" + (this.patcher.getnamed("driver") ? "ok" : "NULL")); + dbg.writeline("dspstart=" + (this.patcher.getnamed("dspstart") ? "ok" : "NULL")); + dbg.writeline("dspstop=" + (this.patcher.getnamed("dspstop") ? "ok" : "NULL")); + dbg.writeline("buf_avout0=" + (this.patcher.getnamed("buf_avout0") ? "ok" : "NULL")); + var tb = new Buffer("avout0"); + dbg.writeline("avout0.framecount=" + tb.framecount()); + dbg.close(); + } + setupAudioNRT(); + // give the driver switch a moment to apply before the self-test starts DSP. + defer("selfTest", 300); +} +function countKeys(o) { var n = 0; for (var k in o) n++; return n; } +function sizeBuf(varname) { + var b = this.patcher.getnamed(varname); + if (b) b.message("sizeinsamps", BUFSAMPS); +} + +// ---- per-case build -------------------------------------------------------- +function buildStep() { + wi++; + if (wi >= WORK.length) { finalize(); return; } + var w = WORK[wi]; + var obj = AVND_DATA[w.oi]; + crumb(obj.name + ":case" + w.ci); + if (w.first) objCases = []; + capture = {}; + cur = []; + + var kind = obj.kind; + var ok = false; + try { + if (kind === "audio") ok = buildAudio(obj, obj.cases[w.ci]); + else if (kind === "control") ok = buildControl(obj, obj.cases[w.ci]); + else ok = buildNone(obj, obj.cases[w.ci]); + } catch (e) { + ok = false; + } + if (!ok) { + objCases.push("{\"index\":" + w.ci + ",\"error\":\"could not instantiate\"}"); + cleanup(); + if (w.last) commitObject(obj); + defer("buildStep", 20); + return; + } + if (kind === "audio" && dspWorks) + startPoll(FRAMES, "readStep"); // gate the read on dsptime~ (samples rendered) + else + defer("readStep", 40); +} + +function buildControl(obj, c) { + var p = this.patcher; + var o = p.newdefault(220, 300, obj.name); + if (!o) return false; + cur.push(o); + var driver = p.getnamed("driver"); + var nout = c.outCtrlNames.length; + for (var i = 0; i < nout; i++) { + var pre = p.newdefault(220 + i * 60, 400, "prepend", "cap" + i); + cur.push(pre); + p.connect(o, i, pre, 0); + p.connect(pre, 0, driver, 0); + } + // Set each explicit control by INLET INDEX, not by name: the message_processor + // maps proxy inlet i -> parameter i, and its name-matching path fails for + // unnamed ports (whose golden name is a positional "p" placeholder). A + // message box feeding inlet i sets exactly parameter i. Inlet 0 also triggers + // process()+commit, so set the higher inlets first, then inlet 0 / a bang. + var msgs = []; + for (var k = 0; k < c.controls.length; k++) { + var idx = c.controls[k][0], val = c.controls[k][2]; + var mb = p.newdefault(20 + k * 70, 230, "message"); + mb.message("set", val); // the numeric content is NOT taken from a ctor arg + cur.push(mb); + p.connect(mb, 0, o, idx); + msgs.push(mb); + } + for (var m = msgs.length - 1; m >= 0; m--) { + var idx2 = c.controls[m][0]; + if (idx2 !== 0) msgs[m].message("bang"); + } + for (var m2 = 0; m2 < msgs.length; m2++) + if (c.controls[m2][0] === 0) msgs[m2].message("bang"); + o.message("bang"); // force a final process()+commit on inlet 0 + return true; +} + +function buildAudio(obj, c) { + var p = this.patcher; + var o = p.newdefault(420, 300, obj.name); + if (!o) return false; + cur.push(o); + // If DSP can't render in this session (no audio driver -- Max has no headless + // mode), instantiating the audio external is all we can verify. Skip the futile + // signal graph; readAudio will report dsp:false so it isn't a false MISMATCH. + if (!dspWorks) return true; + var nin = c.audioIn.length; + var nout = c.nAudioOut > 0 ? c.nAudioOut : nin; + if (nout < 1) nout = 1; + if (nout > CHANS) nout = CHANS; + + if (nin > 0) { + // count~ only advances while its left inlet carries a NONZERO signal; with + // nothing connected its gate is 0 and it stays stuck at index 0 (the object + // would see a constant input). Feed it a constant 1 so it counts 0,1,2,... + var one = p.newdefault(20, 110, "sig~", 1); + cur.push(one); + var cnt = p.newdefault(50, 140, "count~"); + cur.push(cnt); + p.connect(one, 0, cnt, 0); + var pack = (nin > 1) ? p.newdefault(300, 240, "mc.pack~", nin) : null; + if (pack) cur.push(pack); + for (var ch = 0; ch < nin && ch < CHANS; ch++) { + var b = new Buffer("avin" + ch); + var src = c.audioIn[ch]; + for (var i = 0; i < FRAMES && i < src.length; i++) b.poke(1, i, src[i]); + for (var j = src.length; j < BUFSAMPS; j++) b.poke(1, j, 0.0); + var idx = p.newdefault(50, 190 + ch * 40, "index~", "avin" + ch); + cur.push(idx); + p.connect(cnt, 0, idx, 0); + if (pack) p.connect(idx, 0, pack, ch); + else p.connect(idx, 0, o, 0); + } + if (pack) p.connect(pack, 0, o, 0); + } + // audio_processor has only the signal inlet 0 and matches controls by NAME + // (no proxy inlets / index fallback), so set them via " ". + for (var k = 0; k < c.controls.length; k++) + o.message(c.controls[k][1], c.controls[k][2]); + + var unpack = (nout > 1) ? p.newdefault(420, 400, "mc.unpack~", nout) : null; + if (unpack) { cur.push(unpack); p.connect(o, 0, unpack, 0); } + for (var oc = 0; oc < nout; oc++) { + var bo = new Buffer("avout" + oc); + for (var z = 0; z < BUFSAMPS; z++) bo.poke(1, z, 0.0); // clear stale samples + var rec = p.newdefault(420 + oc * 80, 470, "record~", "avout" + oc); + cur.push(rec); + if (unpack) p.connect(unpack, oc, rec, 0); + else p.connect(o, 0, rec, 0); + rec.message(1); // arm: begins recording the moment DSP starts + } + var ds = p.getnamed("dspstart"); + if (ds) ds.message("bang"); + return true; +} + +function buildNone(obj, c) { + var o = this.patcher.newdefault(220, 300, obj.name); + if (!o) return false; + cur.push(o); + return true; +} + +// ---- per-case read --------------------------------------------------------- +function readStep() { + var w = WORK[wi]; + var obj = AVND_DATA[w.oi]; + var c = obj.cases[w.ci]; + var line; + try { + if (obj.kind === "audio") line = readAudio(obj, c, w.ci); + else if (obj.kind === "control") line = readControl(obj, c, w.ci); + else line = "{\"index\":" + w.ci + ",\"instantiated\":true}"; + } catch (e) { + line = "{\"index\":" + w.ci + ",\"error\":\"read failed\"}"; + } + objCases.push(line); + cleanup(); + if (w.last) commitObject(obj); + defer("buildStep", 20); +} + +function readControl(obj, c, ci) { + var parts = []; + var nout = c.outCtrlNames.length; + for (var i = 0; i < nout; i++) { + var key = "cap" + i; + if (capture[key] !== undefined) { + var nm = c.outCtrlNames[i]; + parts.push(jstr(nm) + ":" + jnum(capture[key])); + } + } + return "{\"index\":" + ci + ",\"controls\":{" + parts.join(",") + "}}"; +} + +function readAudio(obj, c, ci) { + if (!dspWorks) + return "{\"index\":" + ci + ",\"instantiated\":true,\"dsp\":false}"; + var nout = c.nAudioOut > 0 ? c.nAudioOut : c.audioIn.length; + if (nout < 1) nout = 1; + if (nout > CHANS) nout = CHANS; + var ds = this.patcher.getnamed("dspstop"); + var chans = []; + for (var oc = 0; oc < nout; oc++) { + var b = new Buffer("avout" + oc); + var arr = []; + for (var i = 0; i < FRAMES; i++) { + var v = b.peek(1, i); + arr.push((v === undefined || v === null) ? 0.0 : v); + } + chans.push(jarr(arr)); + } + if (ds) ds.message("bang"); + return "{\"index\":" + ci + ",\"audio\":[" + chans.join(",") + "]}"; +} + +// ---- capture handler: [prepend cap] -> here ----------------------------- +function anything() { + // Captures everything wired back into inlet 0: control outputs (cap), the + // DSP self-test signal (stpeak), the audio driver name (curdrv) and the + // rendered-sample count (dsptime). + var sel = "" + messagename; + var a = arrayfromargs(arguments); + // Prefer the first numeric arg (scalar control / signal value); fall back to + // the first arg as-is (string/enum/symbol outputs, driver name). + var v = (a.length > 0) ? a[0] : 0; + for (var i = 0; i < a.length; i++) { + if (typeof a[i] === "number") { v = a[i]; break; } + } + capture[sel] = v; +} +function msg_float(v) { /* bare float outputs shouldn't reach us (prepended) */ } +function msg_int(v) { } + +// ---- object bookkeeping ---------------------------------------------------- +function commitObject(obj) { + var line = "{\"name\":" + jstr(obj.name) + ",\"kind\":" + jstr(obj.kind) + + ",\"cases\":[" + objCases.join(",") + "]}"; + allLines.push(line); + doneNames[obj.name] = 1; + writeReport(); +} +function cleanup() { + var p = this.patcher; + for (var i = 0; i < cur.length; i++) { + try { p.remove(cur[i]); } catch (e) {} + } + cur = []; +} +function finalize() { + crumb("DONE"); + post_("avnd max driver: DONE (" + allLines.length + " objects)"); +} diff --git a/tooling/max/avnd_max_driver.maxpat b/tooling/max/avnd_max_driver.maxpat new file mode 100644 index 00000000..236bbe1a --- /dev/null +++ b/tooling/max/avnd_max_driver.maxpat @@ -0,0 +1,132 @@ +{ + "patcher": { + "fileversion": 1, + "appversion": { "major": 8, "minor": 5, "revision": 0, "architecture": "x64", "modernui": 1 }, + "classnamespace": "box", + "rect": [80.0, 80.0, 900.0, 600.0], + "boxes": [ + { + "box": { + "id": "obj-driver", + "maxclass": "newobj", + "text": "js avnd_max_driver.js", + "varname": "driver", + "numinlets": 1, + "numoutlets": 1, + "outlettype": [""], + "patching_rect": [40.0, 160.0, 180.0, 22.0] + } + }, + { + "box": { + "id": "obj-lb", + "maxclass": "newobj", + "text": "loadbang", + "numinlets": 1, + "numoutlets": 1, + "outlettype": ["bang"], + "patching_rect": [40.0, 40.0, 70.0, 22.0] + } + }, + { + "box": { + "id": "obj-del", + "maxclass": "newobj", + "text": "del 800", + "numinlets": 2, + "numoutlets": 1, + "outlettype": ["bang"], + "patching_rect": [40.0, 80.0, 70.0, 22.0] + } + }, + { + "box": { + "id": "obj-run", + "maxclass": "message", + "text": "run", + "numinlets": 2, + "numoutlets": 1, + "outlettype": [""], + "patching_rect": [40.0, 120.0, 50.0, 22.0] + } + }, + { + "box": { + "id": "obj-dspstart", + "maxclass": "message", + "text": "; dsp start", + "varname": "dspstart", + "numinlets": 2, + "numoutlets": 1, + "outlettype": [""], + "patching_rect": [300.0, 40.0, 90.0, 22.0] + } + }, + { + "box": { + "id": "obj-dspstop", + "maxclass": "message", + "text": "; dsp stop", + "varname": "dspstop", + "numinlets": 2, + "numoutlets": 1, + "outlettype": [""], + "patching_rect": [300.0, 80.0, 90.0, 22.0] + } + }, + { + "box": { + "id": "obj-avin0", + "maxclass": "newobj", + "text": "buffer~ avin0", + "varname": "buf_avin0", + "numinlets": 1, + "numoutlets": 2, + "outlettype": ["float", "bang"], + "patching_rect": [500.0, 40.0, 110.0, 22.0] + } + }, + { + "box": { + "id": "obj-avin1", + "maxclass": "newobj", + "text": "buffer~ avin1", + "varname": "buf_avin1", + "numinlets": 1, + "numoutlets": 2, + "outlettype": ["float", "bang"], + "patching_rect": [500.0, 80.0, 110.0, 22.0] + } + }, + { + "box": { + "id": "obj-avout0", + "maxclass": "newobj", + "text": "buffer~ avout0", + "varname": "buf_avout0", + "numinlets": 1, + "numoutlets": 2, + "outlettype": ["float", "bang"], + "patching_rect": [500.0, 120.0, 110.0, 22.0] + } + }, + { + "box": { + "id": "obj-avout1", + "maxclass": "newobj", + "text": "buffer~ avout1", + "varname": "buf_avout1", + "numinlets": 1, + "numoutlets": 2, + "outlettype": ["float", "bang"], + "patching_rect": [500.0, 160.0, 110.0, 22.0] + } + } + ], + "lines": [ + { "patchline": { "source": ["obj-lb", 0], "destination": ["obj-del", 0] } }, + { "patchline": { "source": ["obj-del", 0], "destination": ["obj-run", 0] } }, + { "patchline": { "source": ["obj-run", 0], "destination": ["obj-driver", 0] } } + ] + } +} diff --git a/tooling/run_gst_golden.py b/tooling/run_gst_golden.py new file mode 100644 index 00000000..4d0e37c4 --- /dev/null +++ b/tooling/run_gst_golden.py @@ -0,0 +1,353 @@ +#!/usr/bin/env python3 +r"""Golden differential testing for the GStreamer binding (headless). + +For every golden JSON (produced by the `golden` backend), if the object built a +GStreamer element (element name == c_name), replay each recorded case through a +one-shot gst-launch pipeline: + + filesrc(golden input, raw F32LE interleaved) -> rawaudioparse -> audioconvert + -> -> filesink(out, raw F32LE interleaved) + +then de-interleave the output and diff it against the golden's recorded output +audio. Input controls are set as GObject properties (the binding names them by +the lowercased port symbol). Generators (no input audio) are fed silence of the +golden's output frame count so they cook. + +GStreamer is an audio/video streaming framework: control-only objects (no audio +ports) and non-audio outputs have no headless read-back path through gst-launch, +so they are reported skipped, not failed -- the same graceful gap Pd has for +shapes it can't represent. + +The GStreamer binding only compiles under clang64/MSYS2, so point --gst-bin at +the MSYS2 clang64 bin dir and --plugins at the built plugin dir. The diff runs +under any Python with numpy. + +Example: + + python run_gst_golden.py ^ + --gst-bin D:/msys64/clang64/bin ^ + --plugins D:/build/build-gst-clang64/gstreamer ^ + --goldens D:/build/build-avendish-msvc/golden/Debug +""" + +import argparse +import glob +import json +import os +import re +import subprocess +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from golden_compare import (aggregate_case_verdicts, compare_audio, + compare_textures) + +import numpy as np + + +def gst_env(gst_bin, plugins): + env = dict(os.environ) + env["PATH"] = gst_bin + os.pathsep + env.get("PATH", "") + env["GST_PLUGIN_PATH"] = plugins + # Keep the registry out of the user's home so a stale cache can't hide a + # freshly-built plugin. + env["GST_REGISTRY"] = os.path.join(plugins, ".avnd_gst_registry.bin") + return env + + +def norm(name): + """Match a golden control name to a gst property: the binding lowercases the + port symbol and turns every non-alnum char into '-' (canonical_name), so + compare on lowercase-alnum-only ('Combo_A' -> 'comboa' == 'combo-a').""" + return re.sub(r"[^a-z0-9]", "", str(name).lower()) + + +def inspect_element(gst_bin, env, element): + """Return (properties, has_sink, media) for `element`, or (None, False, None) + if it does not exist. `has_sink` is False for a pure source (generator); + `media` is 'audio' or 'video' (from the pad caps). Parses + `gst-inspect-1.0 `.""" + exe = os.path.join(gst_bin, "gst-inspect-1.0.exe") + try: + p = subprocess.run([exe, element], capture_output=True, text=True, + env=env, timeout=30) + except Exception: + return (None, False, None) + if p.returncode != 0: + return (None, False, None) + props, in_props = set(), False + has_sink = False + media = None + for line in p.stdout.splitlines(): + if re.match(r"^\s+SINK template:", line): + has_sink = True + if "video/x-raw" in line: + media = "video" + elif "audio/x-raw" in line and media is None: + media = "audio" + if line.startswith("Element Properties"): + in_props = True + continue + if not in_props: + continue + m = re.match(r"^ (\w[\w-]*)\s+:", line) + if m: + props.add(m.group(1)) + props -= {"name", "parent", "qos"} + return (props, has_sink, media) + + +def prop_arg(name, value): + """Format one `name=value` token for gst-launch, or None to skip.""" + if isinstance(value, bool): + return f"{name}={'true' if value else 'false'}" + if isinstance(value, (int, float)): + return f"{name}={value}" + if isinstance(value, str): + if value == "unrecorded": + return None + return f'{name}="{value}"' + return None + + +def interleave(channels): + """[[ch0..],[ch1..]] -> interleaved little-endian float32 bytes.""" + arr = np.asarray(channels, dtype=" [[ch0..],[ch1..]] (nch channels).""" + flat = np.frombuffer(raw, dtype=" gst_in.raw lost). + base = tmp.rstrip("/\\").replace("\\", "/") + out_path = base + "/gst_out.raw" + in_path = base + "/gst_in.raw" + if os.path.exists(out_path): + os.remove(out_path) + + prop_tokens = build_prop_tokens(props, gcase) + + if not has_sink: + # Pure source (generator): no input pad. Produce one block and capture. + out_frames = len(gout[0]) if (gout and gout[0]) else int(meta.get("frames", 64)) + pipeline = [ + exe, "-q", + element, *prop_tokens, "num-buffers=1", "!", + "filesink", f"location={out_path}", + ] + else: + if gin: + in_nch = len(gin) + in_frames = len(gin[0]) if gin[0] else 0 + raw = interleave(gin) + else: + # A filter with no recorded input audio: feed silence of the golden + # output length so it cooks that many frames. + in_nch = 1 + in_frames = len(gout[0]) if (gout and gout[0]) else int(meta.get("frames", 64)) + raw = np.zeros(in_frames * in_nch, dtype=" {args.report}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tooling/run_max_golden.py b/tooling/run_max_golden.py new file mode 100644 index 00000000..d0be70bc --- /dev/null +++ b/tooling/run_max_golden.py @@ -0,0 +1,400 @@ +#!/usr/bin/env python3 +"""Golden-differential testing for the avendish Max/MSP backend. + +Max has NO headless mode (see run_backend_tests.py::run_max), so -- exactly like +the TouchDesigner sweep (tooling/td/run_td_sweep.py) -- this drives the GUI host: + + 1. generate a self-contained data file (avnd_max_data.js) holding every + golden's recorded inputs/outputs, so the in-Max driver needs no JSON parser; + 2. stage the driver patch + js + data next to the built *.mxe64 externals + (Max searches the patcher's own folder, so co-location resolves both the + externals AND the driver's include()/File paths); + 3. launch `Max.exe avnd_max_driver.maxpat`; a loadbang runs the [js] driver, + which for every object instantiates it, replays the golden inputs and + CAPTURES what Max can expose -- control-output objects via their message + outlets, audio objects via a count~/index~ -> object -> record~ buffer chain + driven by a real DSP block -- writing one JSON line per object to a report; + 4. auto-dismiss Max's modal dialogs (reuse the TD Win32 EnumWindows+BM_CLICK); + 5. poll for the report; if an object crashes Max mid-sweep, record it from the + per-object breadcrumb and relaunch -- the driver resumes past everything + already reported, so one bad object doesn't block the rest; + 6. diff every captured output against the golden oracle via golden_compare. + +Capture reality (Max's limits, reported honestly per object): + * control objects -> message_processor commits output controls to outlets: + real value diff. + * audio objects -> audio_processor exposes only its MC signal outlet (it does + NOT commit control outlets -- known TODO), captured with + record~; real sample diff when DSP runs. + * none/texture -> nothing feasible to capture: instantiate-only smoke + (verdict 'instantiated' / 'no-backend-*'), never a false + pass. + +Windows only. Example: + + C:/ossia-sdk-x86_64/python/python.exe tooling/run_max_golden.py ^ + --max "C:/Program Files/Cycling '74/Max 8/Max.exe" ^ + --externals D:/build/build-avendish-msvc/max/Debug ^ + --goldens D:/build/build-avendish-msvc/golden/Debug +""" + +import argparse +import ctypes +import glob +import json +import os +import shutil +import subprocess +import sys +import time + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from golden_compare import (aggregate_case_verdicts, compare_audio, + compare_controls) + +user32 = ctypes.windll.user32 if sys.platform == "win32" else None + +# --- Win32: dismiss Max's modal dialogs (recovery, "already running", errors) -- +if user32: + EnumWindowsProc = ctypes.WINFUNCTYPE( + ctypes.c_bool, ctypes.c_void_p, ctypes.c_void_p) + + def _windows(): + found = [] + + def cb(hwnd, _): + if not user32.IsWindowVisible(hwnd): + return True + buf = ctypes.create_unicode_buffer(256) + user32.GetWindowTextW(hwnd, buf, 256) + found.append((hwnd, buf.value)) + return True + user32.EnumWindows(EnumWindowsProc(cb), 0) + return found + + def _click_button(dlg, labels): + """BM_CLICK the first child Button whose text matches one of labels.""" + target = [] + + def cb(hwnd, _): + cls = ctypes.create_unicode_buffer(64) + user32.GetClassNameW(hwnd, cls, 64) + txt = ctypes.create_unicode_buffer(64) + user32.GetWindowTextW(hwnd, txt, 64) + t = txt.value.replace("&", "").strip().lower() + if cls.value == "Button" and t in labels: + target.append(hwnd) + return False + return True + user32.EnumChildWindows(dlg, EnumWindowsProc(cb), 0) + if target: + user32.SendMessageW(target[0], 0x00F5, 0, 0) # BM_CLICK + return True + return False + + # Dialog titles Max raises during an automated launch, and the button we want. + _OK = {"ok", "continue", "no", "don't save", "dont save", "close"} + + def dismiss_dialogs(): + n = 0 + for hwnd, title in _windows(): + t = (title or "").lower() + if any(k in t for k in ("error", "max", "warning", "recover", + "save", "crash", "unresolved", "missing")): + if _click_button(hwnd, _OK): + n += 1 + return n +else: + def dismiss_dialogs(): + return 0 + + +def _round(x): + try: + return float(x) + except Exception: + return 0.0 + + +def gen_data_js(goldens, out_path): + """Emit avnd_max_data.js: `var AVND_DATA = [...]` -- the in-Max driver + include()s it, so it never has to parse JSON. One entry per object with its + kind and, per case, the input controls/audio and the golden output control + names (so captured outlet values can be matched back to a port name).""" + objs = [] + for g in goldens: + cases_out = [] + cases = g.get("cases") or [{"inputs": g.get("inputs", {}), + "outputs": g.get("outputs", {})}] + kind = "none" + for c in cases: + ins, outs = c.get("inputs", {}), c.get("outputs", {}) + controls = [[cc.get("index", k), cc.get("name", ""), cc.get("value")] + for k, cc in enumerate(ins.get("controls", [])) + if cc.get("value") != "unrecorded" + and isinstance(cc.get("value"), (int, float))] + audio_in = ins.get("audio") or [] + out_ctrl_names = [cc.get("name", "") + for cc in outs.get("controls", [])] + has_aout = bool(outs.get("audio")) + has_cout = bool(outs.get("controls")) + has_tex = bool(outs.get("texture")) + if has_aout: + kind = "audio" + elif has_cout and kind != "audio": + kind = "control" + elif has_tex and kind not in ("audio", "control"): + kind = "texture" + cases_out.append({ + "controls": controls, + "audioIn": audio_in, + "nAudioOut": len(outs.get("audio") or []), + "outCtrlNames": out_ctrl_names, + }) + objs.append({"name": g["c_name"], "kind": kind, "cases": cases_out}) + with open(out_path, "w", encoding="utf-8", newline="\n") as f: + f.write("// Auto-generated by run_max_golden.py -- golden inputs/outputs\n") + f.write("// for the in-Max driver (include()'d, so no JSON parser needed).\n") + f.write("var AVND_DATA = ") + json.dump(objs, f) + f.write(";\n") + return objs + + +def load_goldens(goldens_dir): + out = [] + for f in sorted(glob.glob(os.path.join(goldens_dir, "*.json"))): + try: + g = json.load(open(f, encoding="utf-8")) + if g.get("c_name"): + out.append(g) + except Exception: + pass + return out + + +def parse_report(path): + """Report is JSON-lines (one object per line), written by the js driver and + appended-to by us on crash. Returns {name: entry}.""" + entries = {} + if not os.path.exists(path): + return entries + for line in open(path, encoding="utf-8"): + line = line.strip() + if not line: + continue + try: + e = json.loads(line) + entries[e["name"]] = e + except Exception: + pass + return entries + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--max", required=True, help="Max.exe") + ap.add_argument("--externals", required=True, help="dir of built *.mxe64") + ap.add_argument("--goldens", required=True, help="golden JSON dir") + ap.add_argument("--atol", type=float, default=1e-3) + ap.add_argument("--rtol", type=float, default=1e-3) + ap.add_argument("--timeout", type=int, default=240, + help="seconds per launch before giving up") + ap.add_argument("--max-launches", type=int, default=80) + ap.add_argument("--only", help="comma-separated c_names to restrict to") + ap.add_argument("--report", default=None, + help="report path (default: /avnd_max_report.jsonl)") + args = ap.parse_args() + + here = os.path.dirname(os.path.abspath(__file__)) + asset_dir = os.path.join(here, "max") + ext_dir = os.path.abspath(args.externals).replace("\\", "/") + + goldens = load_goldens(args.goldens) + if args.only: + keep = set(args.only.split(",")) + goldens = [g for g in goldens if g["c_name"] in keep] + if not goldens: + sys.exit("no goldens matched") + + # 1. stage driver assets next to the externals (Max searches that folder). + staged = [] + for fn in ("avnd_max_driver.maxpat", "avnd_max_driver.js"): + dst = os.path.join(ext_dir, fn) + shutil.copy2(os.path.join(asset_dir, fn), dst) + staged.append(dst) + data_js = os.path.join(ext_dir, "avnd_max_data.js") + gen_data_js(goldens, data_js) + staged.append(data_js) + + report = args.report or os.path.join(ext_dir, "avnd_max_report.jsonl") + report = report.replace("\\", "/") + breadcrumb = os.path.join(ext_dir, "avnd_max_current.txt").replace("\\", "/") + goldir = os.path.abspath(args.goldens).replace("\\", "/") + + # config the driver reads (paths + object order). + cfg = os.path.join(ext_dir, "avnd_max_config.js") + with open(cfg, "w", encoding="utf-8", newline="\n") as f: + f.write("var AVND_CFG = ") + json.dump({"report": report, "breadcrumb": breadcrumb, + "goldenDir": goldir}, f) + f.write(";\n") + staged.append(cfg) + + for f in (report, breadcrumb): + if os.path.exists(f): + os.remove(f) + + patch = os.path.join(ext_dir, "avnd_max_driver.maxpat") + kill = ["taskkill", "/F", "/IM", "Max.exe"] + + def _read(p): + try: + return open(p, encoding="utf-8").read().strip() + except Exception: + return "" + + subprocess.run(kill, capture_output=True) + dismissed, crashers = 0, [] + for attempt in range(1, args.max_launches + 1): + done = parse_report(report) + remaining = [g for g in goldens if g["c_name"] not in done] + if not remaining: + break + print(f"launch #{attempt}: {len(done)}/{len(goldens)} done, " + f"{len(remaining)} remaining -> next '{remaining[0]['c_name']}'") + proc = subprocess.Popen([args.max, patch]) + deadline = time.time() + args.timeout + last_progress = len(done) + stall_since = time.time() + # Max is slow to start (~30-45s) and may hand off to another process + # instance (so proc.poll() exiting is NOT a reason to stop). Give a + # generous grace period until the driver first writes its breadcrumb, + # then apply a stall timeout on lack of new objects. + started = False + while time.time() < deadline: + dismissed += dismiss_dialogs() + crumb_val = _read(breadcrumb) + if crumb_val == "DONE": + break + if crumb_val and not started: + started = True + stall_since = time.time() + cur = parse_report(report) + if len(cur) != last_progress: + last_progress = len(cur) + stall_since = time.time() + elif started and time.time() - stall_since > 60: + break # driver was running but wedged for 60s -> crash/hang + time.sleep(1) + try: + proc.kill() + except Exception: + pass + subprocess.run(kill, capture_output=True) + time.sleep(2) + + if _read(breadcrumb) == "DONE": + break + # crashed / stalled: record the breadcrumb object so the relaunch skips it. + culprit = _read(breadcrumb) + base = culprit.split(":", 1)[0] if culprit else "" + done_now = parse_report(report) + if base and base not in done_now: + crashers.append(culprit) + with open(report, "a", encoding="utf-8", newline="\n") as f: + f.write(json.dumps({"name": base, "kind": "?", "cases": [], + "crash": culprit}) + "\n") + print(f" *** '{culprit}' crashed/stalled Max -> recorded; relaunching") + elif not base: + print(" no breadcrumb progress; stopping.") + break + + subprocess.run(kill, capture_output=True) + + # clean up staged assets (leave the externals + report). + for _ in range(8): + left = [f for f in staged if os.path.exists(f)] + for f in left: + try: + os.remove(f) + except Exception: + pass + if not [f for f in staged if os.path.exists(f)]: + break + time.sleep(1) + + if dismissed: + print(f"(auto-dismissed {dismissed} Max dialog(s))") + + # ---- diff vs golden ----------------------------------------------------- + goldmap = {g["c_name"]: g for g in goldens} + entries = parse_report(report) + counts, details, rows = {}, [], [] + for g in goldens: + name = g["c_name"] + e = entries.get(name) + gcases = g.get("cases") or [{"inputs": g.get("inputs", {}), + "outputs": g.get("outputs", {})}] + if e is None: + v, d = "no-report", "object never reported" + elif e.get("crash"): + v, d = "CRASH", f"crashed Max at {e.get('crash')}" + else: + rcases = e.get("cases", []) + verdicts = [] + for ci, gc in enumerate(gcases): + rc = next((r for r in rcases if r.get("index") == ci), None) + gout = gc.get("outputs", {}) + if rc is None: + verdicts.append((ci, "no-case", "")) + continue + if rc.get("error"): + verdicts.append((ci, "error", str(rc["error"])[:80])) + continue + if gout.get("audio"): + if rc.get("dsp") is False: + # Max rendered no audio (no live DSP driver in this + # session): the external instantiated but we can't do a + # sample diff -- do NOT report a false mismatch. + verdicts.append((ci, "audio-dsp-unavailable", + "instantiated; DSP did not run")) + continue + cap = rc.get("audio") + out = ([{"name": f"c{i}", "samples": ch} + for i, ch in enumerate(cap)] if cap else None) + verdicts.append((ci,) + compare_audio( + out, gout["audio"], args.atol, args.rtol)) + elif gout.get("controls"): + verdicts.append((ci,) + compare_controls( + rc.get("controls") or {}, gout["controls"], + args.atol, args.rtol)) + elif gout.get("texture"): + verdicts.append((ci, "texture-uncaptured", + "jitter matrix not captured")) + else: + # no output to verify: object at least instantiated + ran. + ok = rc.get("instantiated") or rc.get("ok") + verdicts.append((ci, "instantiated" if ok else "error", + "" if ok else "did not instantiate")) + v, d = aggregate_case_verdicts(verdicts) + counts[v] = counts.get(v, 0) + 1 + rows.append((name, v, d)) + if v in ("MISMATCH", "CRASH", "error", "no-report"): + details.append(f" {v:10} {name}: {d}") + + print(f"\n=== Max/MSP golden diff ({len(goldens)} objects) ===") + for k in sorted(counts): + print(f" {k:20} {counts[k]}") + if crashers: + print(f"\n*** {len(crashers)} object(s) crashed/stalled Max: " + + ", ".join(crashers)) + if details: + print("\n--- attention ---") + for line in details[:60]: + print(line) + print(f"\nreport -> {report}") + + +if __name__ == "__main__": + main() diff --git a/tooling/run_pd_golden.py b/tooling/run_pd_golden.py new file mode 100644 index 00000000..b657515b --- /dev/null +++ b/tooling/run_pd_golden.py @@ -0,0 +1,341 @@ +#!/usr/bin/env python3 +r"""Golden differential testing for the Pure Data binding (headless). + +For every golden JSON (produced by the `golden` backend at build time), if the +object built a Pd external (``.dll``), replay each recorded case through +a head-less Pd instance and diff the captured AUDIO output against the golden's +recorded output audio. + +Mechanism (one deterministic DSP block, fresh instance per case) +---------------------------------------------------------------- +Pd's default block size is 64 samples -- exactly the golden's frame count -- so +one DSP block == one golden frame. For each case we generate a driver ``.pd`` +that: + + 1. preloads each golden input channel into a ``[table inK 64]`` via a + ``; inK 0 v0 v1 ... v63`` message fired on ``loadbang``; + 2. feeds the object's signal inlet(s) from those tables with + ``[tabreceive~ inK]`` (reads one block per DSP tick); + 3. sets the input controls by sending `` `` to the left + inlet (the audio binding routes control messages there); + 4. captures each output channel with ``[tabwrite~ outK]`` into a + ``[table outK 64]`` -- armed by a bang BEFORE DSP is turned on so it records + the very FIRST block from the fresh instance (matching the golden, which + runs one block of a freshly-constructed effect); + 5. after the block has run, writes each output table to a text file + (``; outK write cr``) and quits (``; pd quit``). + +Ordering is enforced with a ``[trigger]``: controls are set and ``tabwrite~`` is +armed first, DSP is enabled last, so the recorded block is block 1 with fresh +state. Each (object, case) runs in its OWN pd process for crash isolation and so +that a stateful object never carries state across cases. + +Scope / honest limitations +--------------------------- +Only AUDIO outputs are diffed: + + * The Pd AUDIO processor does not commit output controls (a known binding + TODO), so audio objects that emit only control/value outputs (e.g. Peak, + analyzers) have nothing to read back -- reported ``no-control-commit``. + * Control-only / MIDI / texture / geometry objects have no head-less audio + read-back path here -- reported ``no-audio-output`` (the same graceful gap + the GStreamer harness has for shapes it cannot represent). Message objects + do send their outputs out outlets, but robustly capturing/parsing every + heterogeneous control-output type head-less is out of scope for this harness. + +Also note the Pd audio binding defaults a *dynamic* audio bus to a single +channel, so a stereo golden is compared against Pd's channel 0 only (the +overlapping-channel prefix -- exactly what ``compare_audio`` does). Objects that +sum/route across the golden's extra input channels therefore legitimately +differ; that is reported as MISMATCH, not hidden. + +Example: + + python run_pd_golden.py ^ + --pd "C:/Program Files/Pd/bin/pd.exe" ^ + --externals D:/build/build-avendish-msvc/pd/Debug ^ + --goldens D:/build/build-avendish-msvc/golden/Debug ^ + --tmp %TEMP%/pd_golden +""" + +import argparse +import glob +import json +import math +import os +import re +import subprocess +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from golden_compare import aggregate_case_verdicts, compare_audio + + +def pd_symbol(name): + """The symbol the Pd input binding matches a control against: + ``pd::get_name_symbol`` keeps only [A-Za-z0-9.~] from the port name (see + binding/pd/helpers.hpp valid_char_for_name).""" + return re.sub(r"[^A-Za-z0-9.~]", "", str(name)) + + +def pd_float(x): + """Format a number as a Pd float atom (finite; non-finite -> 0).""" + v = float(x) + if not math.isfinite(v): + return "0" + return repr(v) + + +def build_driver(c_name, gcase, frames, tmp): + """Return (patch_text, out_paths) for one golden case, or (None, []) if the + case has no audio output to capture.""" + gin = gcase.get("inputs", {}).get("audio") or [] + gout = gcase.get("outputs", {}).get("audio") or [] + in_nch, out_nch = len(gin), len(gout) + if out_nch == 0: + return (None, []) + + lines = ["#N canvas 0 0 900 700 12;"] + idx = 0 + + def add(s): + nonlocal idx + lines.append(s) + i = idx + idx += 1 + return i + + lb = add("#X obj 20 20 loadbang;") + # trigger: right outlet (1) = "prep" (fires first), left outlet (0) = "go". + trig = add("#X obj 20 60 trigger bang bang;") + + trecv = [add(f"#X obj 400 {160 + 18 * k} tabreceive~ in{k};") + for k in range(in_nch)] + twrite = [add(f"#X obj 400 {360 + 18 * k} tabwrite~ out{k};") + for k in range(out_nch)] + add(f"#X obj 400 280 {c_name};") # keep index explicit below + # (obj index computed:) + obj = idx - 1 + + # tables holding input / output blocks + tin = [add(f"#X obj 20 {120 + 18 * k} table in{k} {frames};") + for k in range(in_nch)] + tout = [add(f"#X obj 220 {120 + 18 * k} table out{k} {frames};") + for k in range(out_nch)] + + # preload input blocks (message fired by trigger's prep outlet) + setmsgs = [] + for k in range(in_nch): + vals = " ".join(pd_float(x) for x in gin[k]) + setmsgs.append(add(f"#X msg 440 {120 + 18 * k} \\; in{k} 0 {vals};")) + + # control inputs (numeric; sent to the left inlet) + cmsgs = [] + for c in gcase.get("inputs", {}).get("controls", []): + v = c.get("value") + if v == "unrecorded": + continue + if isinstance(v, bool): + v = int(v) + if not isinstance(v, (int, float)): + continue # strings/other: not sent through the audio left inlet + cmsgs.append(add(f"#X msg 660 {60 + 22 * len(cmsgs)} " + f"{pd_symbol(c.get('name', ''))} {v};")) + + dspon = add("#X msg 20 460 \\; pd dsp 1;") + d1 = add("#X obj 20 500 delay 20;") + out_paths = [f"{tmp}/pdout_{k}.txt" for k in range(out_nch)] + wl = "".join(f"\\; out{k} write {out_paths[k]} cr" for k in range(out_nch)) + savemsg = add(f"#X msg 20 540 {wl};") + d2 = add("#X obj 20 580 delay 60;") + quitmsg = add("#X msg 20 620 \\; pd quit;") + + conns = [] + # audio signal wiring + for k in range(in_nch): + conns.append((trecv[k], 0, obj, k)) + for k in range(out_nch): + conns.append((obj, k, twrite[k], 0)) + # loadbang -> trigger + conns.append((lb, 0, trig, 0)) + # prep (trigger outlet 1): preload inputs, set controls, arm tabwrite~ + for sm in setmsgs: + conns.append((trig, 1, sm, 0)) + for cm in cmsgs: + conns.append((trig, 1, cm, 0)) + conns.append((cm, 0, obj, 0)) + for tw in twrite: + conns.append((trig, 1, tw, 0)) + # go (trigger outlet 0): DSP on, then delayed save + quit + conns.append((trig, 0, dspon, 0)) + conns.append((trig, 0, d1, 0)) + conns.append((d1, 0, savemsg, 0)) + conns.append((d1, 0, d2, 0)) + conns.append((d2, 0, quitmsg, 0)) + + for (s, so, d, di) in conns: + lines.append(f"#X connect {s} {so} {d} {di};") + return ("\n".join(lines) + "\n", out_paths) + + +def run_case(pd_exe, externals, c_name, gcase, meta, tmp, timeout): + """Run one golden case head-less; return (out_channels, error). + out_channels is [samples or None] in port order (None = channel not + produced, e.g. Pd exposed fewer channels than the golden).""" + frames = int(meta.get("frames", 64)) + patch, out_paths = build_driver(c_name, gcase, frames, tmp) + if patch is None: + return (None, "no-audio-output") + + driver = f"{tmp}/pd_driver.pd" + with open(driver, "w", newline="\n") as f: + f.write(patch) + for p in out_paths: + if os.path.exists(p): + os.remove(p) + + cmd = [pd_exe, "-nogui", "-batch", "-noprefs", + "-path", externals, "-open", driver] + try: + p = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout) + except subprocess.TimeoutExpired: + # A wedged pd never reaches `; pd quit`: kill any stragglers. + subprocess.run(["taskkill", "/F", "/IM", "pd.exe"], + capture_output=True) + return (None, f"timeout {timeout}s") + + # Every other object in the driver is stock Pd, so a "couldn't create" means + # the external itself failed to instantiate. Catch it: otherwise the never- + # written output tables read back as silence and masquerade as a (possibly + # matching, if the golden is silent too) capture. + if "couldn't create" in (p.stderr or ""): + return (None, "object-not-created") + + out = [] + for p in out_paths: + if os.path.exists(p): + try: + out.append([float(x) for x in open(p).read().split()]) + except Exception: + out.append(None) + else: + out.append(None) + if all(x is None for x in out): + return (None, "no-output-captured") + return (out, None) + + +def run_object(pd_exe, externals, g, atol, rtol, tmp, timeout): + c_name = g["c_name"] + entry = {"name": c_name, "cases": []} + if not os.path.exists(os.path.join(externals, c_name + ".dll")): + entry["verdict"] = "no-external" + return entry + + gcases = g.get("cases") + if gcases is None: # old single-case schema + gcases = [{"inputs": g.get("inputs", {}), + "outputs": g.get("outputs", {})}] + verdicts = [] + for ci, gcase in enumerate(gcases): + rec = {"index": ci} + gout = gcase.get("outputs", {}).get("audio") or [] + if not gout: + # No audio output recorded: audio processors don't commit control + # outputs, other families have no head-less audio read-back here. + gc = gcase.get("outputs", {}).get("controls") or [] + gin_a = gcase.get("inputs", {}).get("audio") or [] + v = "no-control-commit" if (gin_a and gc) else "no-audio-output" + verdicts.append((ci, v, "")) + rec["verdict"] = v + entry["cases"].append(rec) + continue + out_ch, err = run_case(pd_exe, externals, c_name, gcase, + g.get("meta", {}), tmp, timeout) + if err: + verdicts.append((ci, "error", err)) + rec["verdict"], rec["error"] = "error", err + else: + produced = [{"name": f"c{i}", "samples": s} + for i, s in enumerate(out_ch) if s is not None] + v, d = compare_audio(produced, gout, atol, rtol) + verdicts.append((ci, v, d)) + rec["verdict"], rec["detail"] = v, d + entry["cases"].append(rec) + + v, detail = aggregate_case_verdicts(verdicts) + entry["verdict"] = v + entry["detail"] = detail + return entry + + +def load_goldens(goldens_dir): + goldens = [] + for f in sorted(glob.glob(os.path.join(goldens_dir, "*.json"))): + try: + g = json.load(open(f)) + if g.get("c_name"): + goldens.append(g) + except Exception: + pass + return goldens + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--pd", default="C:/Program Files/Pd/bin/pd.exe", + help="path to pd.exe") + ap.add_argument("--externals", required=True, + help="dir of built pd externals (.dll)") + ap.add_argument("--goldens", required=True, help="golden JSON dir") + ap.add_argument("--atol", type=float, default=1e-3, help="abs tolerance") + ap.add_argument("--rtol", type=float, default=1e-3, help="rel tolerance") + ap.add_argument("--report", default="pd_golden_report.json") + ap.add_argument("--only", help="run a single object (c_name)") + ap.add_argument("--timeout", type=int, default=30, + help="per-case pd invocation timeout (seconds)") + ap.add_argument("--tmp", default=os.path.join( + os.environ.get("TEMP", "."), "pd_golden"), + help="scratch dir for driver patches + captured output") + args = ap.parse_args() + + # Absolute + forward-slash: Pd resolves an array `write` path relative to the + # patch's own directory, and a backslash gets mangled in message-box parsing. + tmp = os.path.abspath(args.tmp).replace("\\", "/") + os.makedirs(tmp, exist_ok=True) + + goldens = load_goldens(args.goldens) + if args.only: + goldens = [g for g in goldens if g["c_name"] == args.only] + + counts, mism, report = {}, [], [] + for g in goldens: + entry = run_object(args.pd, args.externals, g, args.atol, args.rtol, + tmp, args.timeout) + report.append(entry) + v = entry.get("verdict", "?") + counts[v] = counts.get(v, 0) + 1 + if v in ("MISMATCH", "error"): + mism.append(f" {v} {entry['name']} " + f"({len(entry.get('cases', []))} cases): " + f"{entry.get('detail', '')[:140]}") + json.dump(report, open(args.report, "w"), indent=1) + + ncases = sum(len(e.get("cases", [])) for e in report) + print(f"\n=== pd diff vs golden ({len(report)} objects, " + f"{ncases} cases) ===") + for k in sorted(counts): + print(f" {k}: {counts[k]}") + if mism: + print("mismatches / errors:") + for m in mism: + print(m) + matched = [e["name"] for e in report if e.get("verdict") == "match"] + if matched: + print(f"matched ({len(matched)}): " + ", ".join(matched)) + print(f"report -> {args.report}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tooling/run_python_golden.py b/tooling/run_python_golden.py new file mode 100644 index 00000000..351a8872 --- /dev/null +++ b/tooling/run_python_golden.py @@ -0,0 +1,331 @@ +#!/usr/bin/env python3 +"""Golden differential testing for the Python binding (in-process). + +For every golden JSON (produced by the `golden` backend at build time), import +the object's Python module (py.pyd), replay each recorded test case -- +set the input controls as attributes, feed the recorded audio through +process_audio() or call process() -- capture the outputs (control attributes, +audio channels, CPU textures) and diff them against the golden's recorded +outputs. A fresh instance is created per case, matching the golden generator's +fresh-effect-per-case semantics. + +A crashing native module takes its whole process down, so the default (parent) +mode spawns one --child subprocess per object: a crash costs that object only, +and the child's stage breadcrumb (construct/apply/run/read) says where it died. + +Example: + + python run_python_golden.py ^ + --modules D:/build/build-avendish-msvc/python/Debug ^ + --goldens D:/build/build-avendish-msvc/golden/Debug +""" + +import argparse +import glob +import importlib +import json +import os +import re +import subprocess +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from golden_compare import (aggregate_case_verdicts, compare_audio, + compare_controls, compare_textures, fnv1a64) + + +def norm(name): + """Names differ per backend (golden records the port's display name, the + python binding uses its C identifier): compare lowercase alphanumerics.""" + return re.sub(r"[^a-z0-9]", "", str(name).lower()) + + +def find_class(mod, c_name): + cls = getattr(mod, c_name, None) + if isinstance(cls, type): + return cls + for v in vars(mod).values(): + if isinstance(v, type) and getattr(v, "__module__", "") == mod.__name__: + return v + return None + + +def attr_map(obj): + return {norm(a): a for a in dir(obj) if not a.startswith("_")} + + +def sub_accessor(obj, which): + """Return the obj.inputs / obj.outputs proxy and its attr map, or (None, {}). + + Newer modules expose nested `inputs`/`outputs` sub-objects that carry one + property per port -- the unambiguous path when an input and an output share + a symbol (they collide in the flat namespace). Older modules lack them, so + callers fall back to the flat attributes.""" + proxy = getattr(obj, which, None) + if proxy is None or isinstance(proxy, (bool, int, float, str)): + return (None, {}) + try: + return (proxy, attr_map(proxy)) + except Exception: + return (None, {}) + + +def set_control(obj, attrs, name, value): + """Set one golden input control on the instance. Returns (ok, error).""" + a = attrs.get(norm(name)) + if a is None: + return (False, "no-attr") + try: + setattr(obj, a, value) + return (True, None) + except Exception: + pass + # Enum properties reject raw ints; construct the bound enum type. + try: + cur = getattr(obj, a) + if hasattr(type(cur), "__members__"): + setattr(obj, a, type(cur)(int(value))) + return (True, None) + except Exception as e: + return (False, str(e)) + return (False, "type-mismatch") + + +def capture_textures(obj, attrs, count): + """Read CPU-texture outputs: ndarray-valued attributes, hashed exactly like + the golden generator (FNV-1a over the raw row-major pixel bytes).""" + import numpy as np + if count <= 0: + return [] + textures = [] + for a in sorted(attrs.values()): + try: + v = getattr(obj, a) + except Exception: + continue + if isinstance(v, np.ndarray) and v.ndim == 3: + textures.append({ + "width": int(v.shape[1]), + "height": int(v.shape[0]), + "hash": fnv1a64(np.ascontiguousarray(v).tobytes()), + }) + return textures + + +def run_case(cls, gcase, meta, stage): + """Replay one golden case on a fresh instance; returns the case record.""" + import numpy as np + rec = {"ok": False, "applied": {}, "skipped": []} + stage("construct") + obj = cls() + attrs = attr_map(obj) + # Prefer the nested obj.inputs / obj.outputs proxies (unambiguous when an + # input and an output share a symbol); fall back to flat attributes. + ins, in_attrs = sub_accessor(obj, "inputs") + outs, out_attrs = sub_accessor(obj, "outputs") + + stage("apply") + for c in gcase.get("inputs", {}).get("controls", []): + nm, val = c.get("name", ""), c.get("value") + if val == "unrecorded": + continue + if ins is not None and norm(nm) in in_attrs: + ok, err = set_control(ins, in_attrs, nm, val) + else: + ok, err = set_control(obj, attrs, nm, val) + if ok: + rec["applied"][nm] = val + else: + rec["skipped"].append(f"{nm}:{err}") + + gin = gcase.get("inputs", {}).get("audio") or [] + frames = int(meta.get("frames", 64)) + rate = float(meta.get("rate", 44100.0)) + stage("run") + if hasattr(obj, "process_audio"): + arr = (np.asarray(gin, dtype=np.float32) if gin + else np.zeros((0, frames), dtype=np.float32)) + out = obj.process_audio(arr, rate) + rec["audio"] = [{"name": f"c{i}", "samples": [float(x) for x in ch]} + for i, ch in enumerate(np.asarray(out))] + elif hasattr(obj, "process"): + obj.process() + else: + rec["runner"] = "none" + return rec + + stage("read") + named = {} + for gc in gcase.get("outputs", {}).get("controls", []): + nm = gc.get("name", "") + # Prefer obj.outputs. (unambiguous even when an input shares the + # symbol). Fall back to the flat attribute: unnamed output ports get an + # "out_" prefix from the binding, so try that before the bare name so an + # unnamed input p doesn't shadow the output we want. + if outs is not None and norm(nm) in out_attrs: + src, a = outs, out_attrs.get(norm(nm)) + else: + src = obj + a = attrs.get(norm("out_" + nm)) or attrs.get(norm(nm)) + if a is None: + continue + try: + v = getattr(src, a) + except Exception: + continue + if isinstance(v, (bool, int, float)): + named[gc["name"]] = float(v) + elif isinstance(v, str): + named[gc["name"]] = v + rec["controls"] = named + rec["textures"] = capture_textures( + obj, attrs, len(gcase.get("outputs", {}).get("texture", []))) + rec["ok"] = True + return rec + + +def compare_case(rec, gout, atol, rtol): + """Pick the comparison matching what the golden recorded for this case.""" + if rec.get("runner") == "none": + return ("no-runner", "") + if gout.get("audio"): + return compare_audio(rec.get("audio"), gout["audio"], atol, rtol) + if gout.get("controls"): + return compare_controls(rec.get("controls"), gout["controls"], atol, rtol) + if gout.get("texture"): + return compare_textures(rec.get("textures"), gout["texture"], atol, rtol) + return ("no-golden-output", "") + + +def run_object(g, atol, rtol, stagefile): + """Import one object's module and replay all its golden cases.""" + c_name = g["c_name"] + + def stage(s): + open(stagefile, "w").write(f"{c_name}:{s}") + + entry = {"name": c_name, "cases": []} + stage("import") + try: + mod = importlib.import_module("py" + c_name) + except ImportError: + entry["verdict"] = "no-module" + return entry + cls = find_class(mod, c_name) + if cls is None: + entry["verdict"] = "no-class" + return entry + + gcases = g.get("cases") + if gcases is None: # old single-case schema + gcases = [{"inputs": g.get("inputs", {}), + "outputs": g.get("outputs", {})}] + verdicts = [] + for ci, gcase in enumerate(gcases): + try: + rec = run_case(cls, gcase, g.get("meta", {}), + lambda s: stage(f"case{ci}:{s}")) + except Exception as e: + rec = {"ok": False, "exception": str(e)} + rec["index"] = ci + if "exception" in rec: + verdicts.append((ci, "exception", rec["exception"])) + else: + v, d = compare_case(rec, gcase.get("outputs", {}), atol, rtol) + verdicts.append((ci, v, d)) + rec["verdict"] = v + entry["cases"].append(rec) + v, detail = aggregate_case_verdicts(verdicts) + entry["verdict"] = v + entry["detail"] = detail + return entry + + +def load_goldens(goldens_dir): + goldens = [] + for f in sorted(glob.glob(os.path.join(goldens_dir, "*.json"))): + try: + g = json.load(open(f)) + if g.get("c_name"): + goldens.append(g) + except Exception: + pass + return goldens + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--modules", required=True, help="dir of built py*.pyd") + ap.add_argument("--goldens", required=True, help="golden JSON dir") + ap.add_argument("--atol", type=float, default=1e-3, help="abs tolerance") + ap.add_argument("--rtol", type=float, default=1e-3, help="rel tolerance") + ap.add_argument("--report", default="py_golden_report.json") + ap.add_argument("--only", help="run a single object (c_name)") + ap.add_argument("--child", action="store_true", + help="internal: run --only's object in this process") + ap.add_argument("--timeout", type=int, default=120, + help="per-object subprocess timeout (seconds)") + args = ap.parse_args() + + sys.path.insert(0, args.modules) + goldens = load_goldens(args.goldens) + if args.only: + goldens = [g for g in goldens if g["c_name"] == args.only] + + if args.child: + if not goldens: + return 2 + entry = run_object(goldens[0], args.atol, args.rtol, + args.report + ".stage") + json.dump(entry, open(args.report, "w"), indent=1) + return 0 + + counts, mism, report = {}, [], [] + for g in goldens: + c_name = g["c_name"] + objreport = args.report + ".obj" + stagefile = objreport + ".stage" + for f in (objreport, stagefile): + if os.path.exists(f): + os.remove(f) + cmd = [sys.executable, os.path.abspath(__file__), + "--modules", args.modules, "--goldens", args.goldens, + "--atol", str(args.atol), "--rtol", str(args.rtol), + "--report", objreport, "--only", c_name, "--child"] + try: + p = subprocess.run(cmd, capture_output=True, text=True, + timeout=args.timeout) + crashed = p.returncode != 0 or not os.path.exists(objreport) + how = f"exit {p.returncode}" + except subprocess.TimeoutExpired: + crashed = True + how = f"timeout {args.timeout}s" + if crashed: + st = "" + if os.path.exists(stagefile): + st = open(stagefile).read() + entry = {"name": c_name, "verdict": "CRASH", + "detail": f"{how} at {st or 'startup'}"} + else: + entry = json.load(open(objreport)) + report.append(entry) + v = entry.get("verdict", "?") + counts[v] = counts.get(v, 0) + 1 + if v in ("MISMATCH", "CRASH", "exception"): + mism.append(f" {v} {c_name} ({len(entry.get('cases', []))} cases):" + f" {entry.get('detail', '')[:140]}") + json.dump(report, open(args.report, "w"), indent=1) + + ncases = sum(len(e.get("cases", [])) for e in report) + print(f"\n=== python diff vs golden ({len(report)} objects, " + f"{ncases} cases) ===") + for k in sorted(counts): + print(f" {k}: {counts[k]}") + for m in mism: + print(m) + print(f"report -> {args.report}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tooling/td/run_td_sweep.py b/tooling/td/run_td_sweep.py index 78b2520c..82ed154b 100644 --- a/tooling/td/run_td_sweep.py +++ b/tooling/td/run_td_sweep.py @@ -32,6 +32,10 @@ import sys import time +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from golden_compare import (aggregate_case_verdicts, compare_audio, + compare_controls) + user32 = ctypes.windll.user32 if sys.platform == "win32" else None # --- Win32: find and OK the "Plugin Load Error" dialog ----------------------- @@ -98,64 +102,6 @@ def gen_runner(dumps, out_dir): sys.exit(f"runner not generated under {out_dir}") -def compare_audio(td_out, golden, atol, rtol): - """Diff TD output channels (positional) against the golden audio channels - over their overlapping prefix (TD's cook sample-count can differ from the - golden's frame count). td_out is [{name, samples}].""" - if not golden: - return ("no-golden-audio", "") - if not td_out: - return ("no-td-audio", "") - td = [e["samples"] for e in td_out] - nch = min(len(td), len(golden)) - if nch == 0: - return ("empty", "") - maxdiff = 0.0 - for c in range(nch): - ns = min(len(td[c]), len(golden[c])) - for i in range(ns): - maxdiff = max(maxdiff, abs(td[c][i] - golden[c][i])) - peak = max((abs(x) for ch in golden for x in ch), default=0.0) - tol = atol + rtol * peak - verdict = "match" if maxdiff <= tol else "MISMATCH" - detail = (f"maxdiff={maxdiff:.2e} tol={tol:.2e} " - f"td={len(td)}ch gold={len(golden)}ch") - return (verdict, detail) - - -def compare_controls(named, golden, atol, rtol): - """Diff golden output controls against TD values matched by name. `named` is - {channel/info-name -> value}, built from output channels + Info-CHOP.""" - if not golden: - return ("no-golden-controls", "") - if not named: - return ("no-td-controls", "") - checked, maxdiff, worst = 0, 0.0, "" - for gc in golden: - if not isinstance(gc, dict): - continue - nm, gv = gc.get("name", ""), gc.get("value") - if not isinstance(gv, (int, float)): - continue - tv = None - for k in (nm, nm.lower(), nm.capitalize()): - if k in named: - tv = named[k] - break - if tv is None: - continue - checked += 1 - if abs(tv - gv) > maxdiff: - maxdiff, worst = abs(tv - gv), nm - if checked == 0: - return ("no-name-match", f"td={list(named)[:4]}") - peak = max((abs(gc["value"]) for gc in golden - if isinstance(gc.get("value"), (int, float))), default=0.0) - tol = atol + rtol * peak - return ("match" if maxdiff <= tol else "MISMATCH", - f"{checked} ctrls maxdiff={maxdiff:.2e}@{worst} tol={tol:.2e}") - - def main(): ap = argparse.ArgumentParser() ap.add_argument("--td", required=True, help="TouchDesigner.exe") @@ -267,15 +213,20 @@ def _read(p): culprit = _read(cur) if not culprit or culprit == "DONE": break # timed out with no progress, or actually finished + # The breadcrumb may carry a ':' suffix (created/precook/cooked) + # for diagnostics -- strip it to get the object name used for resume + # matching, but keep the full breadcrumb so we can see where it died. + base = culprit.split(":", 1)[0] crashers.append(culprit) # Record the crasher so the relaunch skips it. try: rep = json.load(open(report, encoding="utf-8")) except Exception: rep = [] - if culprit not in {r["name"] for r in rep}: - rep.append({"name": culprit, "ok": False, - "exception": "crashed TouchDesigner (instantiation/cook)"}) + if base not in {r["name"] for r in rep}: + rep.append({"name": base, "ok": False, + "exception": "crashed TouchDesigner (instantiation/cook)", + "crash_stage": culprit}) open(report, "w", encoding="utf-8").write(json.dumps(rep, indent=2)) print(f" *** crashed on '{culprit}' -> recorded; relaunching to continue") @@ -323,33 +274,66 @@ def _read(p): goldens[g.get("c_name")] = g except Exception: pass - counts, mism = {}, [] - for r in results: - g = goldens.get(r.get("name")) - if not g: - continue - td_out = r.get("td_out") - gaud = g.get("outputs", {}).get("audio") - gctl = g.get("outputs", {}).get("controls") - # Prefer audio comparison when the golden has audio output, else the - # control outputs (which surface as named channels in TD). + # Compare one captured case (td_out + td_info) against a golden case's + # outputs. Prefer audio when the golden has audio output, else controls. + # + # TEXTURE OUTPUT IS NOT VERIFIABLE HEADLESS. Investigation (2026-07): + # a Custom TOP's GPU upload (createOutputBuffer/uploadBuffer) is NOT + # rendered/flushed for numpyArray() readback in a scripted, no-viewer + # cook -- every avendish TOP reads back as TD's empty 128x128 default + # (identical hash across all objects), and forcing the node's Output + # Resolution to Custom does not produce a surface. This is NOT an + # avendish bug: the object produces correct output (verified via the + # Python binding: pyavnd_test_tex_generator -> real 512x512), the same + # invoke_effect/get_outputs path drives the working CHOP binding, and + # built-in TOPs (noiseTOP/constantTOP) DO read back headless. So it is + # a TouchDesigner headless-Custom-OP limitation. Report it as an honest + # gap rather than a false MISMATCH. (GStreamer texture output IS verified + # -- see run_gst_golden.py -- so the object contract is still covered.) + def compare_one(rc, gout): + gaud = (gout or {}).get("audio") + gctl = (gout or {}).get("controls") + gtex = (gout or {}).get("texture") if gaud: - v, detail = compare_audio(td_out, gaud, args.atol, args.rtol) - elif gctl: - # Prefer named output channels; fall back to Info-CHOP values. + return compare_audio(rc.get("td_out"), gaud, args.atol, args.rtol) + if gctl: named = {} - for e in (td_out or []): + for e in (rc.get("td_out") or []): if e.get("samples"): named[e["name"]] = e["samples"][0] - for k, val in (r.get("td_info") or {}).items(): + for k, val in (rc.get("td_info") or {}).items(): named.setdefault(k, val) - v, detail = compare_controls(named, gctl, args.atol, args.rtol) + return compare_controls(named, gctl, args.atol, args.rtol) + if gtex: + return ("no-headless-texture-readback", + "TD renders Custom-OP GPU uploads only with a viewer") + return ("no-golden-output", "") + + counts, mism = {}, [] + for r in results: + g = goldens.get(r.get("name")) + if not g: + continue + gcases = g.get("cases") + rcases = r.get("cases") + verdicts = [] # (case_index, verdict, detail) + if gcases is not None and rcases is not None: + for rc in rcases: + ci = rc.get("index", 0) + if ci >= len(gcases): + continue + gout = gcases[ci].get("outputs", {}) + v, d = compare_one(rc, gout) + verdicts.append((ci, v, d)) else: - v, detail = "no-golden-output", "" + # old single-case schema (golden or result without `cases`) + v, d = compare_one(r, g.get("outputs", {})) + verdicts.append((0, v, d)) + v, detail = aggregate_case_verdicts(verdicts) counts[v] = counts.get(v, 0) + 1 if v == "MISMATCH": - mism.append(f" MISMATCH {r['name']}: {detail}") - print(f"\n=== output diff vs golden ({len(goldens)} goldens) ===") + mism.append(f" MISMATCH {r['name']} ({len(verdicts)} cases): {detail}") + print(f"\n=== output diff vs golden ({len(goldens)} goldens, per-case) ===") for k in sorted(counts): print(f" {k}: {counts[k]}") for m in mism[:40]: