From 58c3d9cac66494628fed22250258eac25995e781 Mon Sep 17 00:00:00 2001 From: Reuben Brooks Date: Mon, 27 Jul 2026 15:18:08 -0500 Subject: [PATCH 1/7] fasl: record shen.*lambdatable* deltas by name instead of serializing live lambdas MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On the S41.2 kernel every (define ...) in a loaded file runs shen.update-lambdatable, which (set)s shen.*lambdatable* to an assoc list whose entries hold live curried lambda functions. The fasl recorder captured that as an ordinary "g" (set) record, kdata_ser hit the closure ("unserializable KDATA value: function"), and the whole file became fasl-uncacheable — in practice ~every stdlib and user file, so the ~1s stdlib load re-compiled on every boot and every (load) paid full reader+macro+typecheck cost on every run. Record a new "lt" record instead: diff the new table against the current global (the recorder runs before the original set, so the old table is still live) and store only the NAMES whose entry was added/replaced. On replay, rebuild each entry from the live defun the same way shen.update-lambdatable does — shen.lambda-entry (which reads the arity property, already replayed by the preceding "p" record in stream order) + shen.assoc->. Mirrors the existing "lf" special case for the older kernel's shen.lambda-form put path. Bump FASL_FORMAT to SHENFASL5 so stale caches miss cleanly. Measured (arm64 macOS, LuaJIT 2.1 rolling): trivial-script startup 1.5s -> 0.34s warm; all 21 StLib files now fasl-hit (previously 19/21 uncacheable). make test 500/0, run-kernel-tests 134/0, warm/cold suite stdout identical modulo the by-design 'run time' banner drop. Co-Authored-By: Claude Fable 5 --- boot.lua | 54 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) diff --git a/boot.lua b/boot.lua index 7520f31..dadc2d6 100644 --- a/boot.lua +++ b/boot.lua @@ -520,7 +520,8 @@ end -- Known: (destroy ...) at the REPL between loads is not in the key. -- SHEN_FASL=off disables; SHEN_FASL_DIR overrides ~/.cache/shen-lua-fasl; -- SHEN_FASL_DEBUG=1 logs hits/misses to stderr. -local FASL_FORMAT = "SHENFASL4" -- 4: added "e" (per-form value/type echo) +local FASL_FORMAT = "SHENFASL5" -- 5: "lt" (shen.*lambdatable* delta by name); + -- 4: "e" (per-form value/type echo) -- records; 3: "pc" (shen.compile-prolog) local FASL_STACK = {} local FASL_ROLL = 2166136261 @@ -569,6 +570,8 @@ end -- | M\n shen.record-macro (fn rebuilt by name) -- | P\n (put ... *property-vector*) -- | E\n #bytes\n bytes per-form value/type echo (stoutput) +-- | A\n shen.*lambdatable* delta (entries +-- rebuilt by name via shen.lambda-entry) -- | G\n }* (set ...) outside any chunk -- narity\n {ar SP name\n}* kbase\n nkdata\n entries gensym\n local function fasl_write(path, rec, arity0) @@ -588,6 +591,9 @@ local function fasl_write(path, rec, arity0) elseif r.k == "lf" then parts[#parts+1] = "L\n" kdata_ser(r.name, parts) + elseif r.k == "lt" then + parts[#parts+1] = "A\n" + kdata_ser(r.names, parts) elseif r.k == "dt" then parts[#parts+1] = "T\n" kdata_ser(r.name, parts) @@ -670,6 +676,9 @@ local function fasl_read(path) elseif k == "L" then local v = de_n(1); if not v then return nil end recs[i] = { k = "lf", name = v[1] } + elseif k == "A" then + local v = de_n(1); if not v then return nil end + recs[i] = { k = "lt", names = v[1] } elseif k == "T" then local v = de_n(2); if not v then return nil end recs[i] = { k = "dt", name = v[1], rules = v[2] } @@ -729,6 +738,23 @@ local function fasl_replay(cached) local val = R.is_cons(entry) and entry[2] or entry P.F["put"](r.name, R.intern("shen.lambda-form"), val, P.GLOBALS["*property-vector*"]) + elseif r.k == "lt" then + -- 41.2 kernel path: (set shen.*lambdatable* ...) carries live curried + -- lambdas, so the recording stored only the NAMES whose entries the set + -- added/replaced. Rebuild each entry from the live defun exactly the way + -- shen.update-lambdatable does (shen.lambda-entry reads the arity + -- property, replayed by the preceding "p" record in stream order). + local names = r.names + while R.is_cons(names) do + local name = names[1] + local entry = P.F["shen.lambda-entry"](name) + if R.is_cons(entry) then + P.F["set"](R.intern("shen.*lambdatable*"), + P.F["shen.assoc->"](name, entry[2], + P.GLOBALS["shen.*lambdatable*"])) + end + names = names[2] + end elseif r.k == "dt" then P.F["shen.process-datatype"](r.name, r.rules) elseif r.k == "sy" then @@ -848,6 +874,32 @@ local function install_fasl() -- the gensym counter churns on every expansion-time gensym; the replay -- fast-forward (max) covers it without hundreds of noise records if nm == "shen.*gensym*" then return nil end + if nm == "shen.*lambdatable*" then + -- 41.2 kernel: shen.update-lambdatable / update-lambda-table set the + -- whole assoc list, whose entries are (name . live-curried-lambda) — + -- unserializable, and the reason every stdlib/user load used to be + -- fasl-uncacheable. The recorder runs BEFORE the original set, so the + -- global still holds the old table: diff it against `val` and record + -- only the names whose entry was added/replaced (an "lt" record; the + -- fns are rebuilt on replay via shen.lambda-entry). + local old = {} + local t = P.GLOBALS["shen.*lambdatable*"] + while R.is_cons(t) do + local e = t[1] + if R.is_cons(e) and R.is_symbol(e[1]) then old[e[1].name] = e[2] end + t = t[2] + end + local names = R.NIL + t = val + while R.is_cons(t) do + local e = t[1] + if R.is_cons(e) and R.is_symbol(e[1]) and old[e[1].name] ~= e[2] then + names = R.cons(e[1], names) + end + t = t[2] + end + return { k = "lt", names = names } + end return { k = "g", name = name, val = val } end) From 913bc9907e223352580dfa332c718a71ee428fa2 Mon Sep 17 00:00:00 2001 From: Reuben Brooks Date: Mon, 27 Jul 2026 16:26:14 -0500 Subject: [PATCH 2/7] =?UTF-8?q?feat:=20--hush-load=20/=20SHEN=5FHUSH=5FLOA?= =?UTF-8?q?D=3D1=20=E2=80=94=20silence=20only=20load's=20echo,=20keep=20us?= =?UTF-8?q?er=20output?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit -q sets *hush*, which on the 41.2 kernel gates pr itself, so a suite run under -q prints NOTHING including the tests' own (output ...) lines (issue #46 item 3) — unusable for golden-output batch runners. Meanwhile default mode echoes every loaded form's value/type (~100 lines on urdr suites) plus nondeterministic 'run time: N secs' banners. --hush-load (bin/shen flag, or env SHEN_HUSH_LOAD=1) drops ONLY what load itself prints to standard output: the per-form value/type echo (shen.eval-and-print / shen.work-through) and the run-time/typechecked banners. User (output ...)/pr from loaded forms is untouched, as are file streams and stderr. Mechanism reuses the distinction the fasl 'e' capture already relies on: load's chatter is pr'd OUTSIDE any eval-kl chunk, at the chunk nesting depth load was entered at, while user output runs INSIDE a chunk. boot.lua wraps load to publish that depth (P.LOADPR_DEPTH, saved/restored so nested loads compose); prims' native pr drops stdout writes at exactly that depth; prims.compile_and_load maintains the depth counter (only when recording or the flag is on — the default path stays a bare tail call). Fasl integration: the load wrapper sits inside install_fasl's wrappers, so a miss recorded under --hush-load still captures the full 'e' echo bytes (replayable to a later non-hushed run); a replay hit is gated at the 'e' re-emit site. Verified: default-mode stdout byte-identical (warm hit) / identical mod timing lines (cold) on urdr prng+world suites; hush warm==cold==env form; every hush-mode line present in shen-cl's output for the same suite; cache recorded under hush replays full echo non-hushed; -q still fully silent; make test 500/0; run-kernel-tests 134/0 ok. Co-Authored-By: Claude Fable 5 --- bin/shen | 12 +++++++++++- boot.lua | 45 ++++++++++++++++++++++++++++++++++++++++++++- prims.lua | 39 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 94 insertions(+), 2 deletions(-) diff --git a/bin/shen b/bin/shen index 7f77a82..3e3f44f 100755 --- a/bin/shen +++ b/bin/shen @@ -21,21 +21,29 @@ end local shen = require("shen") local USAGE = [[ -usage: shen [-q] [-e EXPR | FILE] ... +usage: shen [-q] [--hush-load] [-e EXPR | FILE] ... no arguments start the interactive Shen REPL FILE (load) the file (full Shen syntax), then exit -e, --eval EXPR evaluate EXPR and print its value -q, --quiet (set *hush* true) — silence load echo and hushable output + (on the 41.2 kernel *hush* gates pr itself, so -q also + silences the program's own (output ...) lines) + --hush-load silence only load's chatter — the per-form value/type + echo and "run time"/"typechecked" banners; (output ...) + from the loaded program still prints. Env: SHEN_HUSH_LOAD=1 ]] -- parse: a list of {kind="eval"|"load", arg=...} in command-line order local actions = {} local quiet = false +local hush_load = false local i = 1 while i <= #arg do local a = arg[i] if a == "-q" or a == "--quiet" then quiet = true + elseif a == "--hush-load" then + hush_load = true elseif a == "-e" or a == "--eval" then i = i + 1 if arg[i] == nil then @@ -60,6 +68,7 @@ if #actions == 0 then -- the kernel's own toplevel loop. shen.boot{quiet = quiet} if quiet then shen.prims.GLOBALS["*hush*"] = true end + if hush_load then shen.prims.HUSH_LOAD_ECHO = true end local ok, repl = pcall(require, "repl") if ok and type(repl) == "table" and type(repl.run) == "function" then repl.run{ P = shen.prims, quiet = quiet } @@ -87,6 +96,7 @@ end shen.boot{quiet=true} if quiet then shen.prims.GLOBALS["*hush*"] = true end +if hush_load then shen.prims.HUSH_LOAD_ECHO = true end for _, act in ipairs(actions) do local ok, err = pcall(function() if act.kind == "load" then diff --git a/boot.lua b/boot.lua index dadc2d6..9b26406 100644 --- a/boot.lua +++ b/boot.lua @@ -772,7 +772,12 @@ local function fasl_replay(cached) elseif r.k == "e" then -- re-emit the per-form echo through the live `pr` so replay-time *hush* -- still gates it (a -q hit stays silent, exactly as a -q cold load). - P.F["pr"](r.bytes, P.GLOBALS["*stoutput*"]) + -- hush-load likewise gates it here: on a replay `load`'s wrapper never + -- runs (orig_load is skipped entirely), so the depth-based pr gate + -- can't see these; check the flag directly for the same effect. + if not P.HUSH_LOAD_ECHO then + P.F["pr"](r.bytes, P.GLOBALS["*stoutput*"]) + end else -- "g" P.F["set"](r.name, r.val) end @@ -785,6 +790,43 @@ local function fasl_replay(cached) end end +-- ---- hush-load quiet mode (issue #46 item 3) ------------------------------- +-- --hush-load / SHEN_HUSH_LOAD=1 silences ONLY what `load` itself prints to +-- standard output — the per-form value/type echo (shen.eval-and-print on the +-- raw path, shen.work-through on the tc path) and the "run time: N secs" / +-- "typechecked in N inferences" banners — while user (output ...)/pr from the +-- loaded forms still writes. That makes `bin/shen --hush-load FILE` output +-- deterministic and cross-port comparable (only the program's own output), +-- which -q cannot do: on the 41.2 kernel *hush* gates pr itself, so -q +-- silences the tests' (output ...) lines too. +-- +-- Mechanism: load's own chatter is pr'd to (stoutput) OUTSIDE any eval-kl +-- chunk, i.e. at the same chunk-nesting depth `load` was entered at, whereas +-- everything a loaded form prints runs INSIDE its chunk (depth+1) — the same +-- distinction the fasl "e" capture uses. Wrap `load` to publish that entry +-- depth in P.LOADPR_DEPTH (saved/restored, so nested loads work: a nested +-- load sits inside its caller's chunk, one level deeper); the native pr +-- (prims.lua) drops stdout writes at exactly that depth while the flag is on. +-- P.CHUNK_DEPTH is maintained by prims.compile_and_load only when recording +-- or when the flag is on — default-mode behavior and codepaths are unchanged. +-- Installed BEFORE install_fasl so the fasl wrappers stay outermost: a fasl +-- miss still records the echo "e" bytes (its pr wrapper runs before the +-- native pr drops the write), so a cache written under --hush-load replays +-- the full echo to a later non-hushed run; a fasl hit never reaches this +-- wrapper and is gated at the "e" replay site instead. +local function install_hush_load() + local orig_load = P.F["load"] + P.F["load"] = function(fname) + if not P.HUSH_LOAD_ECHO then return orig_load(fname) end + local saved = P.LOADPR_DEPTH + P.LOADPR_DEPTH = P.CHUNK_DEPTH + local ok, res = pcall(orig_load, fname) + P.LOADPR_DEPTH = saved + if not ok then error(res, 0) end + return res + end +end + local function install_fasl() local dir = fasl_dir() if not dir then return end @@ -1093,6 +1135,7 @@ end P.load_kernel = function(verbose) load_kernel(verbose) + install_hush_load() -- before install_fasl: fasl wrappers stay outermost install_fasl() -- after native overrides so the declare wrapper composes -- Lua<->Shen interop surface (lua_interop.lua). Installed LAST so the -- typed bridge (lua.function) sees the fully composed F["declare"] diff --git a/prims.lua b/prims.lua index b4a4f3d..1b3b410 100644 --- a/prims.lua +++ b/prims.lua @@ -754,6 +754,17 @@ function P.install_native_stdlib() local orig_pr = F["pr"] local function pr(s, st) if GLOBALS["*hush*"] and st == GLOBALS["*stoutput*"] then return s end + -- hush-load (issue #46 item 3): while a (load ...) is in progress and + -- P.HUSH_LOAD_ECHO is on, boot.lua's load wrapper records the chunk + -- nesting depth at load entry in P.LOADPR_DEPTH. A pr to standard output + -- at exactly that depth is `load`'s own chatter — the per-form value/type + -- echo (shen.eval-and-print / shen.work-through) and the "run time"/ + -- "typechecked in N inferences" banners, all emitted AFTER (outside) the + -- form's eval-kl chunk — whereas user (output ...) runs INSIDE a chunk + -- (depth+1) and is left alone. Same policy as *hush*: only standard + -- output is gated; file streams and *sterror* always write. + if P.LOADPR_DEPTH ~= nil and P.CHUNK_DEPTH == P.LOADPR_DEPTH + and st == GLOBALS["*stoutput*"] then return s end if GLOBALS["*hush*"] then -- non-stdout stream under *hush*: write unconditionally. Temporarily -- clear *hush* so the original kernel pr takes its write branch, then @@ -930,6 +941,20 @@ local function load_chunk(code, chunkname) end P.load_chunk = load_chunk +-- hush-load quiet mode (issue #46 item 3). P.HUSH_LOAD_ECHO gates it +-- (bin/shen --hush-load, or SHEN_HUSH_LOAD=1); boot.lua wraps `load` to mark +-- the echo depth (P.LOADPR_DEPTH) and the native pr drops load's own +-- standard-output chatter at that depth. P.CHUNK_DEPTH counts the eval-kl +-- chunk nesting maintained by compile_and_load below. Unlike -q/*hush* +-- (which on the 41.2 kernel gates pr itself, i.e. ALL standard output), +-- hush-load leaves user (output ...)/pr from inside loaded forms alive. +P.CHUNK_DEPTH = 0 +P.LOADPR_DEPTH = nil +do + local hl = os.getenv("SHEN_HUSH_LOAD") + P.HUSH_LOAD_ECHO = (hl == "1" or hl == "on" or hl == "true") or nil +end + -- P.FASL_REC: active fasl recording context (boot.lua's user-program cache). -- While a (load ...) is being recorded, every TOP-LEVEL compile (a form the -- kernel hands to eval-kl) is dumped into the record; compiles triggered @@ -943,8 +968,22 @@ local function compile_and_load(luasrc, chunkname) rec.n = rec.n + 1 rec[rec.n] = { k = "c", name = chunkname, dump = string.dump(fn) } rec.in_chunk = true + P.CHUNK_DEPTH = P.CHUNK_DEPTH + 1 local ok, res = pcall(fn) rec.in_chunk = false + P.CHUNK_DEPTH = P.CHUNK_DEPTH - 1 + if not ok then error(res, 0) end + return res + end + if P.HUSH_LOAD_ECHO then + -- hush-load needs the chunk depth tracked even when fasl isn't recording + -- (SHEN_FASL=off, or a nested eval-kl) so the pr gate above can tell + -- `load`'s own echo (at the depth of load entry) from user output emitted + -- inside a chunk. Only paid when the mode is on: the default path below + -- stays a bare tail call. + P.CHUNK_DEPTH = P.CHUNK_DEPTH + 1 + local ok, res = pcall(fn) + P.CHUNK_DEPTH = P.CHUNK_DEPTH - 1 if not ok then error(res, 0) end return res end From d3ecc91278ada4a17fbce99095f66fc166ce7898 Mon Sep 17 00:00:00 2001 From: Reuben Brooks Date: Mon, 27 Jul 2026 16:32:39 -0500 Subject: [PATCH 3/7] perf: inline exact-arity (cons A B) as setmetatable({A,B},CMT) in codegen MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit jit.p profiling of urdr's software-SHA-256 suite (shen/tests/prng, LuaJIT 2.1 rolling, arm64) showed 27% of time in the interpreter with cons/append/list.take at the top, and jit.v showed why: 282 of the ~350 aborted trace attempts were 'blacklisted at prims.lua:239' — the tiny F["cons"] wrapper proto. cons is hot enough that traces root in it, those aborts blacklist the proto, and from then on EVERY caller trace that reaches a cons call aborts, demoting the whole workload to the interpreter. Emit the construction inline instead — ENV.CMT is the runtime Cons metatable — so there is no proto to blacklist and hot traces can sink the allocation. Same late-binding tradeoff as the existing ARITH2 fast paths (a runtime redefinition of cons is not seen by compiled exact-arity call sites). urdr prng suite wall (warm fasl, best of 5): 3.20s -> 1.43s (-55%). world suite: 4.4s -> 3.08s. After: 0 blacklisted traces, interpreter share 27% -> 8%, compiled share 37% -> 57%. Verified: suite stdout byte-identical (warm and cold); world golden exact; make test 500/0; run-kernel-tests 134/0 ok. Co-Authored-By: Claude Fable 5 --- compiler.lua | 15 +++++++++++++++ prims.lua | 5 +++++ 2 files changed, 20 insertions(+) diff --git a/compiler.lua b/compiler.lua index 64e2acc..c58f958 100644 --- a/compiler.lua +++ b/compiler.lua @@ -509,6 +509,21 @@ local function ccall(form, env) if helper and ar == 2 then return helper .. "(" .. argstr .. ")" end + -- Inline exact-arity (cons A B) as a bare table+metatable construction + -- (ENV.CMT is the runtime Cons metatable). Rationale: cons is the + -- single hottest call in list-heavy workloads (urdr's software + -- SHA-256: bits are cons lists), and routing it through the F["cons"] + -- wrapper proto makes that tiny function a JIT trace hotspot in its + -- own right — on LuaJIT it ends up blacklisted (repeated aborts of + -- traces rooted in it), after which EVERY caller's trace that reaches + -- it aborts too ("blacklisted at prims.lua"), demoting whole suites to + -- the interpreter. Emitted inline there is no proto to blacklist, and + -- hot traces can sink the allocation. Same late-binding tradeoff as + -- the ARITH2 fast paths above: a runtime redefinition of `cons` is + -- not seen by already-compiled exact-arity call sites. + if name == "cons" and ar == 2 then + return "setmetatable({" .. argstr .. "}, CMT)" + end return ftab_ref(name) .. "(" .. argstr .. ")" else -- Arity mismatch at compile time: route through APP so dispatch uses diff --git a/prims.lua b/prims.lua index 1b3b410..464003e 100644 --- a/prims.lua +++ b/prims.lua @@ -903,6 +903,11 @@ local ENV = { GE = function(a,b) if type(a)=="number" and type(b)=="number" then return a>=b end return F[">="](a,b) end, LE = function(a,b) if type(a)=="number" and type(b)=="number" then return a<=b end return F["<="](a,b) end, EQ = function(a,b) if type(a)=="number" then return a==b end return equal(a,b) end, + -- CMT: the runtime Cons metatable, for the compiler's inlined exact-arity + -- (cons A B) codegen — `setmetatable({A, B}, CMT)` (see ccall in + -- compiler.lua). Keeps the hottest allocation out of the tiny F["cons"] + -- wrapper proto, which LuaJIT otherwise blacklists under cons-heavy loads. + CMT = R.Cons, -- allow compiled code to reach a few Lua builtins safely pcall = pcall, select = select, error = error, setmetatable = setmetatable, getmetatable = getmetatable, From 082e87236b1786701d0b9fbc3ba93f511dcf71e8 Mon Sep 17 00:00:00 2001 From: Reuben Brooks Date: Mon, 27 Jul 2026 16:45:05 -0500 Subject: [PATCH 4/7] perf: default GC pause 400 (SHEN_GC env to override/disable) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After the cons-inline fix, jit.p on urdr's prng suite still shows ~27% of wall time in the GC — the SHA-256/bigint workload is cons churn the port cannot avoid (bits are Shen lists). Interleaved A/B on CPU time: pause=400 -15% prng, -20% world, peak RSS 30MB -> 60MB pause=400 mul=100 -16% (stepmul adds ~nothing) pause=800 mul=100 -25-28% peak RSS 108MB pause=200 mul=100 no change (stepmul alone is not the lever) Ship pause=400 as the boot default: solid suite-level win for a bounded 2x peak-RSS cost. SHEN_GC=off preserves the host's defaults (embedders managing their own GC); SHEN_GC="pause[,stepmul]" sets explicit values (800,100 documented for batch runs). Also rejected: jit.opt loopunroll=60/callunroll=10 — helps only warm in-process re-runs, cold-process suite wall got 30% WORSE (trace compile churn); left at current defaults. Verified: prng warm stdout byte-identical; world golden exact; make test 500/0; run-kernel-tests 134/0 ok. Co-Authored-By: Claude Fable 5 --- boot.lua | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/boot.lua b/boot.lua index 9b26406..4e4ac3c 100644 --- a/boot.lua +++ b/boot.lua @@ -56,6 +56,22 @@ do end end end + -- GC tuning. Compiled-KL workloads are cons-churn-heavy (jit.p on urdr's + -- software SHA-256 suite: ~27% of wall time in the GC at LuaJIT's default + -- pause=200), and most of that churn is short-lived list cells. Raising + -- the pause to 400 (heap may grow to 4x live before a full cycle) cuts + -- suite CPU ~15-20% for about 2x peak RSS (30MB -> 60MB on that suite). + -- SHEN_GC=off keeps the host's defaults (embedders that manage the GC + -- themselves should set it); SHEN_GC="pause[,stepmul]" sets explicit + -- values (e.g. SHEN_GC=800,100 buys another ~10% on batch runs at ~110MB; + -- SHEN_GC=200 is LuaJIT's default pause). + local gc = os.getenv("SHEN_GC") + if gc ~= "off" then + local pause, stepmul + if gc and gc ~= "" then pause, stepmul = gc:match("^(%d+),?(%d*)$") end + collectgarbage("setpause", tonumber(pause) or 400) + if stepmul and stepmul ~= "" then collectgarbage("setstepmul", tonumber(stepmul)) end + end end local function find_kldir() From b396fed097da7e212c257572960a1923bd568d6f Mon Sep 17 00:00:00 2001 From: Reuben Brooks Date: Mon, 27 Jul 2026 16:59:49 -0500 Subject: [PATCH 5/7] =?UTF-8?q?perf:=20native=20fn=20fast=20path=20?= =?UTF-8?q?=E2=80=94=20verify=20lambdatable=20once,=20return=20the=20raw?= =?UTF-8?q?=20function?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 41.2 shen->kl translator compiles every call to a function whose arity is unknown at translation time (chiefly forward references within a file) as ((fn name) args...). Kernel fn (reader.kl) then pays an arity property get PLUS an assoc over the whole shen.*lambdatable* per call, and returns the table's curried lambda, which APP applies one MKFUN closure at a time. jit.p F3 stacks on urdr's prng suite put ~35% of samples under fn (is_cons/equal < assoc < fn < urdr.prng.word.add-all/add2/...). Native fn_fast (prims.install_native_stdlib): for a symbol naming a live F entry with known positive arity, verify the lambdatable entry ONCE per (name, lambdatable identity) through the original fn — a miss still raises the kernel's "fn: X is undefined" — then return the RAW F[name] function. APP dispatches a raw function with recorded arity identically to the curried chain (exact, partial, and over-application) without the per-argument closure allocations. Every (set shen.*lambdatable* ...) builds a fresh spine, so table identity invalidates the verification cache; arity-0 (kernel fn CALLS those) and unknown-arity names always take the original path. Interleaved A/B (user+sys CPU, 6 runs each, sorted): prng: 3.14 3.34 3.43 3.61 3.68 3.88 -> 2.32 2.47 2.47 2.51 2.54 3.38 (-27%) world: 4.59 4.70 4.82 5.01 5.09 5.39 -> 3.92 4.16 4.35 4.54 4.61 5.35 (-12%) Verified: prng warm stdout byte-identical; world golden exact; make test 500/0; run-kernel-tests 134/0 ok. Co-Authored-By: Claude Fable 5 --- prims.lua | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/prims.lua b/prims.lua index 464003e..cf02485 100644 --- a/prims.lua +++ b/prims.lua @@ -816,7 +816,43 @@ function P.install_native_stdlib() return h end + -- fn (reader.kl): the 41.2 shen->kl translator compiles every call to a + -- function whose arity is unknown at translation time (forward references + -- within a file, mainly) as ((fn name) args...), and the kernel `fn` pays + -- an arity property get PLUS an assoc over the whole shen.*lambdatable* + -- on EVERY such call — measured at ~35% of urdr's software-SHA-256 suite + -- (jit.p F3: is_cons/equal < assoc < fn). Fast path: for a symbol naming + -- a live function with known positive arity, verify the lambdatable entry + -- ONCE per (name, lambdatable identity) through the original `fn` — so an + -- unregistered name still raises the kernel's "fn: X is undefined" — and + -- from then on return the RAW F[name] function instead of the table's + -- curried lambda. APP dispatches a raw function with recorded arity + -- identically to the curried chain (exact call, partial application, + -- over-application), but without one MKFUN closure allocation per applied + -- argument. Every (set shen.*lambdatable* ...) builds a fresh cons spine, + -- so table identity is the correct invalidation key; arity-0 and + -- unknown-arity names always take the original path (kernel `fn` CALLS an + -- arity-0 function rather than returning it). + local orig_fn = F["fn"] + local fn_seen, fn_seen_lt = {}, nil + local function fn_fast(v) + if is_symbol(v) then + local f = F[v.name] + local ar = f ~= nil and FA[f] or nil + if ar and ar > 0 then + local lt = GLOBALS["shen.*lambdatable*"] + if lt ~= fn_seen_lt then fn_seen = {}; fn_seen_lt = lt end + if fn_seen[v.name] then return f end + orig_fn(v) -- errors exactly like the kernel on a miss + fn_seen[v.name] = true + return f + end + end + return orig_fn(v) + end + local function install(name, fn, arity) F[name] = fn; FA[fn] = arity end + install("fn", fn_fast, 1) install("hash", hash, 2) install("pr", pr, 2) install("variable?", variable_q, 1) From 34d2c0576ef803ccb35084d8e9dbe62e39a96e8a Mon Sep 17 00:00:00 2001 From: Reuben Brooks Date: Mon, 27 Jul 2026 17:57:22 -0500 Subject: [PATCH 6/7] perf: native iterative append (urdr SHA list hot path) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit jit.p Fl on urdr's prng suite put ~10% of leaf samples in the kernel append (sys.kl): non-tail-recursive, one F["hd"]/F["tl"]/F["append"] lookup per element of the first list. word.rotr / word.bits concatenate 32-bit bit-lists through append on every SHA round. Native install_native_stdlib override: walk A once building a fresh spine iteratively (exactly |A| new cons cells — same allocation as the recursive form), splice B onto the last cell. Fresh cells are unshared until return, so in-place tail assignment is unobservable. Improper / non-list inputs delegate to the original for a byte-identical simple-error. Serial warm A/B (user+sys CPU, dual SHEN_FASL_DIR caches, N=8–10): prng: med 0.93 -> 0.91 (-2%), mean 1.00 -> 0.92 (-8%); final recheck med 0.90 -> 0.87 (-3%), mean 0.95 -> 0.91 (-4%) world: med 2.28 -> 1.85 (-19%) Also tried (not shipped): pure_tail_self bare f-error relaxation — correctly emits LOOP for multi-clause defines (list.drop/nth) but mean prng regressed ~+14% on short-list SHA walks; left documented in doc/PERF-URDR-RESULTS.md. Verified: prng/world warm stdout golden-identical; make test 500/0; run-kernel-tests 134/0 ok. Co-Authored-By: Claude Fable 5 --- doc/PERF-URDR-RESULTS.md | 99 ++++++++++++++++++++++++++++++++++++++++ prims.lua | 33 ++++++++++++++ 2 files changed, 132 insertions(+) create mode 100644 doc/PERF-URDR-RESULTS.md diff --git a/doc/PERF-URDR-RESULTS.md b/doc/PERF-URDR-RESULTS.md new file mode 100644 index 0000000..c3c8e13 --- /dev/null +++ b/doc/PERF-URDR-RESULTS.md @@ -0,0 +1,99 @@ +# Urdr pure-Shen SHA workload — port-side performance results + +Branch: `perf/urdr-workload` +Host: macOS arm64, LuaJIT 2.1 rolling (`/Users/reuben/.local/Homebrew/bin/luajit`) +Pins: urdr suites under `shen/tests/{prng,world,search}`; shen-cl baseline +`urdr-shen-cl-41.2/bin/sbcl/shen script …`. + +Invocation (warm fasl; no `script` subcommand on shen-lua): + +```bash +export PATH="/Users/reuben/.local/Homebrew/bin:$PATH" +export SHEN_LUA=/Users/reuben/projects/shen-lua/bin/shen +export SHEN_CL=/Users/reuben/projects/urdr-shen-cl-41.2/bin/sbcl/shen +cd /Users/reuben/projects/urdr +/usr/bin/time -lp $SHEN_LUA shen/tests/prng/run-tests.shen +/usr/bin/time -lp $SHEN_CL script shen/tests/prng/run-tests.shen +``` + +Wall times are thermally noisy on this host; prefer **user+sys CPU**, +**min-of-N / median of serial warm runs**, and dual-cache interleaved A/B +when comparing two SHAs. + +## Series on this branch (vs `main` @ c85ad7e) + +| Commit | Change | Primary measured effect | +|--------|--------|-------------------------| +| `58c3d9c` | fasl lambdatable-by-name (SHENFASL5) | trivial startup **1.5s → 0.34s** warm; 21/21 StLib fasl-cacheable | +| `913bc99` | `--hush-load` / `SHEN_HUSH_LOAD=1` | silence load echo only; golden runners usable (issue #46 item 3) | +| `d3ecc91` | codegen exact-arity `(cons A B)` inline | prng wall best-of-5 **3.20s → 1.43s (−55%)**; 0 blacklisted traces | +| `082e872` | default GC pause 400 (`SHEN_GC`) | ~−15% prng / −20% world CPU; peak RSS ~30→60 MB | +| `b396fed` | native `fn` fast path | prng CPU **~−27%**, world **~−12%** (interleaved A/B) | +| *(this)* | native iterative `append` | prng CPU med **~−3%**, mean **~−4 to −8%**; world med **~−19%** (serial warm) | + +## Current tip vs shen-cl (warm, this session) + +| Suite | shen-lua (user+sys, serial warm med) | shen-cl (`script`, wall) | +|-------|--------------------------------------|---------------------------| +| `shen/tests/prng` | ~0.87–1.0 s CPU (tip after append) | ~1.1–1.3 s wall / ~1.0 s CPU | +| `shen/tests/world` | ~2–3 s CPU (high variance) | ~1.75–1.85 s wall / ~1.6 s CPU | + +Gap to CL is now roughly **1–2×** on prng (was ~12× at branch start with cold +fasl / uncached StLib). Remaining cost is dominated by pure-Shen bit-list +SHA-256 / bigint arithmetic, not load or `fn`. + +## Profiling snapshot (prng warm, after `fn` fix, before append) + +`jit.p` modes (suite load only, post-boot): + +| mode | top signals | +|------|-------------| +| `v` | ~38% Compiled / ~29% Interpreted / ~28% GC / ~5% JIT | +| `Fl` (leaf) | `byte.bits` 22%, `byte.bits.weights` 13%, `list.take` 11%, **`append` 10%**, EQ 4%, APP 4%, `bits.xor` 4%, `cons?` 3%, `hd` 3% | + +`fn` is no longer in the leaf top (prior fix). After native `append`, leaf +share of append drops modestly; remaining time is user list recursion + GC +cons churn (bits as cons lists). + +## Tried / not shipped this session + +1. **Relax `pure_tail_self` for bare `(shen.f-error name)`** + Multi-clause `define` ends in `(shen.f-error )`; treating the bare + name as residual recursion blocked loop lowering on pure-tail helpers + (`list.drop`, `list.nth`, …). Fix correctly emits `LOOP` for those. On + this SHA suite (short lists, N≈8–32) serial warm prng **mean regressed + ~+14%** — short tail-call chains already JIT well; the loop form did not + pay. Left unmerged. Still a good lever for deep-iteration code; re-open + with a depth heuristic or workload-specific gate if needed. + +2. **Codegen inline `hd` / `tl` / `cons?`** + Same late-binding tradeoff as `cons`. Combined package was within noise / + slightly mean-regressive on prng; not shipped alone. + +3. **Default GC pause 800** + Med ~−12% prng CPU vs 400, but peak RSS ~60→106 MB (world ~110→216 MB). + Keep `SHEN_GC=800` as documented batch override; default stays 400. + +## Remaining headroom / risks + +- **Structural**: urdr represents words as 32-element bit lists; `list.take`, + `bits.xor`, `byte.bits.weights` dominate. Port opts cannot change digest + semantics or golden outputs. Host-SHA primitives are explicitly out of + scope unless vector-gated and trivial. +- **Tail-recursion modulo cons** for `(cons H (self …))` (append / take / + map-like) would target the remaining non-tail list constructors without + host SHA. +- **Interpreter share still ~25–30%** on prng — further trace-friendly + inlining of tiny prims may help, but measure carefully (short-list loops + can regress). +- **Thermal noise**: single-run wall times vary ~2×; always A/B with dual + `SHEN_FASL_DIR` / `SHEN_KERNEL_CACHE` and report med/mean of N≥6. +- **Issue #46**: cold-start and hush-load items addressed; suite compute gap + largely structural. + +## Gates + +- `make test`: **500 pass / 0 fail** +- `luajit run-kernel-tests.lua`: **134 passed / 0 failed, ok** +- urdr prng / world: **ALL PASS**; stdout byte-identical to pre-change warm + golden (modulo `run time:` banners) diff --git a/prims.lua b/prims.lua index cf02485..f12f4b0 100644 --- a/prims.lua +++ b/prims.lua @@ -701,6 +701,38 @@ function P.install_native_stdlib() end local function reverse(lst) return reverse_help(lst, NIL) end + -- append (sys.kl): + -- (cond ((= () A) B) + -- ((cons? A) (cons (hd A) (append (tl A) B))) + -- (true (simple-error "attempt to append a non-list"))) + -- The KL is NON-tail-recursive: every element of A is a stack frame that + -- re-looks-up F["hd"]/F["tl"]/F["append"] and builds one cons on the way + -- back. On urdr's software-SHA-256 suite (word.rotr concatenates 32-bit + -- bit-lists via append) that is ~10% of leaf samples. Native: walk A once + -- building a fresh spine iteratively (exactly |A| new cons cells — same + -- allocation as the recursive form, no intermediate reverse list), then + -- splice B onto the last cell. Fresh cells are not shared until we return, + -- so in-place tail assignment is unobservable. Improper / non-list inputs + -- delegate to the original so the simple-error message stays byte-identical. + local orig_append = F["append"] + local function append(a, b) + if a == NIL then return b end + if not is_cons(a) then return orig_append(a, b) end + local orig = a + local head = cons(a[1], NIL) + local last = head + a = a[2] + while is_cons(a) do + local cell = cons(a[1], NIL) + last[2] = cell + last = cell + a = a[2] + end + if a ~= NIL then return orig_append(orig, b) end + last[2] = b + return head + end + -- shen.map-h (and map) : map a function over a list, building the result -- reversed then reversing (exactly as the KL does). The mapper is applied via -- APP (it may be a closure, a partial, or a symbol). @@ -863,6 +895,7 @@ function P.install_native_stdlib() install("shen.incinfs", incinfs, 0) install("element?", element_q, 2) install("assoc", assoc, 2) + install("append", append, 2) install("shen.reverse-help", reverse_help, 2) install("reverse", reverse, 1) install("shen.map-h", map_h, 3) From 03f67b0472350850b5df3628f22c77c338c84a48 Mon Sep 17 00:00:00 2001 From: Reuben Brooks Date: Mon, 27 Jul 2026 18:58:59 -0500 Subject: [PATCH 7/7] feat: shen.x.sha256 host via OpenSSL libcrypto Binds shen.x.sha256-octets-host from install_native_stdlib when libcrypto loads. Disable with SHEN_X_SHA256=pure. See pyrex41/shen-extensions. --- prims.lua | 55 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/prims.lua b/prims.lua index f12f4b0..6f41c61 100644 --- a/prims.lua +++ b/prims.lua @@ -900,6 +900,61 @@ function P.install_native_stdlib() install("reverse", reverse, 1) install("shen.map-h", map_h, 3) install("map", map, 2) + + -- shen.x host SHA-256 (OpenSSL libcrypto). See pyrex41/shen-extensions. + -- Disable with SHEN_X_SHA256=pure. + if os.getenv("SHEN_X_SHA256") ~= "pure" then + local ok_ffi, ffi = pcall(require, "ffi") + if ok_ffi then + local function load_crypto() + local candidates = { + "crypto", "libcrypto", + "/Users/reuben/.local/Homebrew/opt/openssl@3/lib/libcrypto.dylib", + "/opt/homebrew/opt/openssl@3/lib/libcrypto.dylib", + "/usr/local/opt/openssl@3/lib/libcrypto.dylib", + "libcrypto.so.3", "libcrypto.so.1.1", "libcrypto.so", + } + for _, name in ipairs(candidates) do + local ok, lib = pcall(ffi.load, name) + if ok then return lib end + end + return nil + end + local crypto = load_crypto() + if crypto then + pcall(function() + ffi.cdef[[ + unsigned char *SHA256(const unsigned char *d, size_t n, unsigned char *md); + ]] + end) + local function sha256_octets_host(lst) + local buf = {} + local cur = lst + while cur ~= NIL do + if not is_cons(cur) then + ERR("shen.x.sha256-octets-host: expected list of bytes 0..255") + end + local b = cur[1] + if type(b) ~= "number" or b < 0 or b > 255 or b ~= math.floor(b) then + ERR("shen.x.sha256-octets-host: expected list of bytes 0..255") + end + buf[#buf+1] = string.char(b) + cur = cur[2] + end + local data = table.concat(buf) + local md = ffi.new("unsigned char[32]") + crypto.SHA256(data, #data, md) + local acc = NIL + for i = 31, 0, -1 do + acc = cons(tonumber(md[i]), acc) + end + return acc + end + install("shen.x.sha256-octets-host", sha256_octets_host, 1) + GLOBALS["shen.x.*sha256-backend*"] = intern("host") + end + end + end end -- ---- loader / eval -------------------------------------------------------