feat: shen.x.sha256 host (OpenSSL)

bin/shen

@@ -21,21 +21,29 @@ end
 local shen = require("shen")
 
 local USAGE = [[
-usage: shen [-q] [-e EXPR | FILE] ...
+usage: shen [-q] [--hush-load] [-e EXPR | FILE] ...
   no arguments      start the interactive Shen REPL
   FILE              (load) the file (full Shen syntax), then exit
   -e, --eval EXPR   evaluate EXPR and print its value
   -q, --quiet       (set *hush* true) — silence load echo and hushable output
+                    (on the 41.2 kernel *hush* gates pr itself, so -q also
+                    silences the program's own (output ...) lines)
+  --hush-load       silence only load's chatter — the per-form value/type
+                    echo and "run time"/"typechecked" banners; (output ...)
+                    from the loaded program still prints. Env: SHEN_HUSH_LOAD=1
 ]]
 
 -- parse: a list of {kind="eval"|"load", arg=...} in command-line order
 local actions = {}
 local quiet = false
+local hush_load = false
 local i = 1
 while i <= #arg do
   local a = arg[i]
   if a == "-q" or a == "--quiet" then
     quiet = true
+  elseif a == "--hush-load" then
+    hush_load = true
   elseif a == "-e" or a == "--eval" then
     i = i + 1
     if arg[i] == nil then
@@ -60,6 +68,7 @@ if #actions == 0 then
   -- the kernel's own toplevel loop.
   shen.boot{quiet = quiet}
   if quiet then shen.prims.GLOBALS["*hush*"] = true end
+  if hush_load then shen.prims.HUSH_LOAD_ECHO = true end
   local ok, repl = pcall(require, "repl")
   if ok and type(repl) == "table" and type(repl.run) == "function" then
     repl.run{ P = shen.prims, quiet = quiet }
@@ -87,6 +96,7 @@ end
 
 shen.boot{quiet=true}
 if quiet then shen.prims.GLOBALS["*hush*"] = true end
+if hush_load then shen.prims.HUSH_LOAD_ECHO = true end
 for _, act in ipairs(actions) do
   local ok, err = pcall(function()
     if act.kind == "load" then

boot.lua

@@ -56,6 +56,22 @@ do
       end
     end
   end
+  -- GC tuning. Compiled-KL workloads are cons-churn-heavy (jit.p on urdr's
+  -- software SHA-256 suite: ~27% of wall time in the GC at LuaJIT's default
+  -- pause=200), and most of that churn is short-lived list cells. Raising
+  -- the pause to 400 (heap may grow to 4x live before a full cycle) cuts
+  -- suite CPU ~15-20% for about 2x peak RSS (30MB -> 60MB on that suite).
+  -- SHEN_GC=off keeps the host's defaults (embedders that manage the GC
+  -- themselves should set it); SHEN_GC="pause[,stepmul]" sets explicit
+  -- values (e.g. SHEN_GC=800,100 buys another ~10% on batch runs at ~110MB;
+  -- SHEN_GC=200 is LuaJIT's default pause).
+  local gc = os.getenv("SHEN_GC")
+  if gc ~= "off" then
+    local pause, stepmul
+    if gc and gc ~= "" then pause, stepmul = gc:match("^(%d+),?(%d*)$") end
+    collectgarbage("setpause", tonumber(pause) or 400)
+    if stepmul and stepmul ~= "" then collectgarbage("setstepmul", tonumber(stepmul)) end
+  end
 end
 
 local function find_kldir()
@@ -520,7 +536,8 @@ end
 -- Known: (destroy ...) at the REPL between loads is not in the key.
 -- SHEN_FASL=off disables; SHEN_FASL_DIR overrides ~/.cache/shen-lua-fasl;
 -- SHEN_FASL_DEBUG=1 logs hits/misses to stderr.
-local FASL_FORMAT = "SHENFASL4"  -- 4: added "e" (per-form value/type echo)
+local FASL_FORMAT = "SHENFASL5"  -- 5: "lt" (shen.*lambdatable* delta by name);
+                                 -- 4: "e" (per-form value/type echo)
                                  -- records; 3: "pc" (shen.compile-prolog)
 local FASL_STACK = {}
 local FASL_ROLL = 2166136261
@@ -569,6 +586,8 @@ end
 --   | M\n <ser name>                    shen.record-macro (fn rebuilt by name)
 --   | P\n <ser x><ser ptr><ser y>       (put ... *property-vector*)
 --   | E\n #bytes\n bytes                per-form value/type echo (stoutput)
+--   | A\n <ser names>                   shen.*lambdatable* delta (entries
+--                                       rebuilt by name via shen.lambda-entry)
 --   | G\n <ser name><ser val> }*        (set ...) outside any chunk
 --   narity\n {ar SP name\n}*  kbase\n nkdata\n entries  gensym\n
 local function fasl_write(path, rec, arity0)
@@ -588,6 +607,9 @@ local function fasl_write(path, rec, arity0)
     elseif r.k == "lf" then
       parts[#parts+1] = "L\n"
       kdata_ser(r.name, parts)
+    elseif r.k == "lt" then
+      parts[#parts+1] = "A\n"
+      kdata_ser(r.names, parts)
     elseif r.k == "dt" then
       parts[#parts+1] = "T\n"
       kdata_ser(r.name, parts)
@@ -670,6 +692,9 @@ local function fasl_read(path)
     elseif k == "L" then
       local v = de_n(1); if not v then return nil end
       recs[i] = { k = "lf", name = v[1] }
+    elseif k == "A" then
+      local v = de_n(1); if not v then return nil end
+      recs[i] = { k = "lt", names = v[1] }
     elseif k == "T" then
       local v = de_n(2); if not v then return nil end
       recs[i] = { k = "dt", name = v[1], rules = v[2] }
@@ -729,6 +754,23 @@ local function fasl_replay(cached)
       local val = R.is_cons(entry) and entry[2] or entry
       P.F["put"](r.name, R.intern("shen.lambda-form"), val,
                  P.GLOBALS["*property-vector*"])
+    elseif r.k == "lt" then
+      -- 41.2 kernel path: (set shen.*lambdatable* ...) carries live curried
+      -- lambdas, so the recording stored only the NAMES whose entries the set
+      -- added/replaced. Rebuild each entry from the live defun exactly the way
+      -- shen.update-lambdatable does (shen.lambda-entry reads the arity
+      -- property, replayed by the preceding "p" record in stream order).
+      local names = r.names
+      while R.is_cons(names) do
+        local name = names[1]
+        local entry = P.F["shen.lambda-entry"](name)
+        if R.is_cons(entry) then
+          P.F["set"](R.intern("shen.*lambdatable*"),
+                     P.F["shen.assoc->"](name, entry[2],
+                                         P.GLOBALS["shen.*lambdatable*"]))
+        end
+        names = names[2]
+      end
     elseif r.k == "dt" then
       P.F["shen.process-datatype"](r.name, r.rules)
     elseif r.k == "sy" then
@@ -746,7 +788,12 @@ local function fasl_replay(cached)
     elseif r.k == "e" then
       -- re-emit the per-form echo through the live `pr` so replay-time *hush*
       -- still gates it (a -q hit stays silent, exactly as a -q cold load).
-      P.F["pr"](r.bytes, P.GLOBALS["*stoutput*"])
+      -- hush-load likewise gates it here: on a replay `load`'s wrapper never
+      -- runs (orig_load is skipped entirely), so the depth-based pr gate
+      -- can't see these; check the flag directly for the same effect.
+      if not P.HUSH_LOAD_ECHO then
+        P.F["pr"](r.bytes, P.GLOBALS["*stoutput*"])
+      end
     else -- "g"
       P.F["set"](r.name, r.val)
     end
@@ -759,6 +806,43 @@ local function fasl_replay(cached)
   end
 end
 
+-- ---- hush-load quiet mode (issue #46 item 3) -------------------------------
+-- --hush-load / SHEN_HUSH_LOAD=1 silences ONLY what `load` itself prints to
+-- standard output — the per-form value/type echo (shen.eval-and-print on the
+-- raw path, shen.work-through on the tc path) and the "run time: N secs" /
+-- "typechecked in N inferences" banners — while user (output ...)/pr from the
+-- loaded forms still writes. That makes `bin/shen --hush-load FILE` output
+-- deterministic and cross-port comparable (only the program's own output),
+-- which -q cannot do: on the 41.2 kernel *hush* gates pr itself, so -q
+-- silences the tests' (output ...) lines too.
+--
+-- Mechanism: load's own chatter is pr'd to (stoutput) OUTSIDE any eval-kl
+-- chunk, i.e. at the same chunk-nesting depth `load` was entered at, whereas
+-- everything a loaded form prints runs INSIDE its chunk (depth+1) — the same
+-- distinction the fasl "e" capture uses. Wrap `load` to publish that entry
+-- depth in P.LOADPR_DEPTH (saved/restored, so nested loads work: a nested
+-- load sits inside its caller's chunk, one level deeper); the native pr
+-- (prims.lua) drops stdout writes at exactly that depth while the flag is on.
+-- P.CHUNK_DEPTH is maintained by prims.compile_and_load only when recording
+-- or when the flag is on — default-mode behavior and codepaths are unchanged.
+-- Installed BEFORE install_fasl so the fasl wrappers stay outermost: a fasl
+-- miss still records the echo "e" bytes (its pr wrapper runs before the
+-- native pr drops the write), so a cache written under --hush-load replays
+-- the full echo to a later non-hushed run; a fasl hit never reaches this
+-- wrapper and is gated at the "e" replay site instead.
+local function install_hush_load()
+  local orig_load = P.F["load"]
+  P.F["load"] = function(fname)
+    if not P.HUSH_LOAD_ECHO then return orig_load(fname) end
+    local saved = P.LOADPR_DEPTH
+    P.LOADPR_DEPTH = P.CHUNK_DEPTH
+    local ok, res = pcall(orig_load, fname)
+    P.LOADPR_DEPTH = saved
+    if not ok then error(res, 0) end
+    return res
+  end
+end
+
 local function install_fasl()
   local dir = fasl_dir()
   if not dir then return end
@@ -848,6 +932,32 @@ local function install_fasl()
     -- the gensym counter churns on every expansion-time gensym; the replay
     -- fast-forward (max) covers it without hundreds of noise records
     if nm == "shen.*gensym*" then return nil end
+    if nm == "shen.*lambdatable*" then
+      -- 41.2 kernel: shen.update-lambdatable / update-lambda-table set the
+      -- whole assoc list, whose entries are (name . live-curried-lambda) —
+      -- unserializable, and the reason every stdlib/user load used to be
+      -- fasl-uncacheable. The recorder runs BEFORE the original set, so the
+      -- global still holds the old table: diff it against `val` and record
+      -- only the names whose entry was added/replaced (an "lt" record; the
+      -- fns are rebuilt on replay via shen.lambda-entry).
+      local old = {}
+      local t = P.GLOBALS["shen.*lambdatable*"]
+      while R.is_cons(t) do
+        local e = t[1]
+        if R.is_cons(e) and R.is_symbol(e[1]) then old[e[1].name] = e[2] end
+        t = t[2]
+      end
+      local names = R.NIL
+      t = val
+      while R.is_cons(t) do
+        local e = t[1]
+        if R.is_cons(e) and R.is_symbol(e[1]) and old[e[1].name] ~= e[2] then
+          names = R.cons(e[1], names)
+        end
+        t = t[2]
+      end
+      return { k = "lt", names = names }
+    end
     return { k = "g", name = name, val = val }
   end)
 
@@ -1041,6 +1151,7 @@ end
 
 P.load_kernel = function(verbose)
   load_kernel(verbose)
+  install_hush_load()   -- before install_fasl: fasl wrappers stay outermost
   install_fasl()   -- after native overrides so the declare wrapper composes
   -- Lua<->Shen interop surface (lua_interop.lua). Installed LAST so the
   -- typed bridge (lua.function) sees the fully composed F["declare"]

compiler.lua

@@ -509,6 +509,21 @@ local function ccall(form, env)
         if helper and ar == 2 then
           return helper .. "(" .. argstr .. ")"
         end
+        -- Inline exact-arity (cons A B) as a bare table+metatable construction
+        -- (ENV.CMT is the runtime Cons metatable). Rationale: cons is the
+        -- single hottest call in list-heavy workloads (urdr's software
+        -- SHA-256: bits are cons lists), and routing it through the F["cons"]
+        -- wrapper proto makes that tiny function a JIT trace hotspot in its
+        -- own right — on LuaJIT it ends up blacklisted (repeated aborts of
+        -- traces rooted in it), after which EVERY caller's trace that reaches
+        -- it aborts too ("blacklisted at prims.lua"), demoting whole suites to
+        -- the interpreter. Emitted inline there is no proto to blacklist, and
+        -- hot traces can sink the allocation. Same late-binding tradeoff as
+        -- the ARITH2 fast paths above: a runtime redefinition of `cons` is
+        -- not seen by already-compiled exact-arity call sites.
+        if name == "cons" and ar == 2 then
+          return "setmetatable({" .. argstr .. "}, CMT)"
+        end
         return ftab_ref(name) .. "(" .. argstr .. ")"
       else
         -- Arity mismatch at compile time: route through APP so dispatch uses

doc/PERF-URDR-RESULTS.md

@@ -0,0 +1,99 @@
+# Urdr pure-Shen SHA workload — port-side performance results
+
+Branch: `perf/urdr-workload`  
+Host: macOS arm64, LuaJIT 2.1 rolling (`/Users/reuben/.local/Homebrew/bin/luajit`)  
+Pins: urdr suites under `shen/tests/{prng,world,search}`; shen-cl baseline
+`urdr-shen-cl-41.2/bin/sbcl/shen script …`.
+
+Invocation (warm fasl; no `script` subcommand on shen-lua):
+
+```bash
+export PATH="/Users/reuben/.local/Homebrew/bin:$PATH"
+export SHEN_LUA=/Users/reuben/projects/shen-lua/bin/shen
+export SHEN_CL=/Users/reuben/projects/urdr-shen-cl-41.2/bin/sbcl/shen
+cd /Users/reuben/projects/urdr
+/usr/bin/time -lp $SHEN_LUA shen/tests/prng/run-tests.shen
+/usr/bin/time -lp $SHEN_CL script shen/tests/prng/run-tests.shen
+```
+
+Wall times are thermally noisy on this host; prefer **user+sys CPU**,
+**min-of-N / median of serial warm runs**, and dual-cache interleaved A/B
+when comparing two SHAs.
+
+## Series on this branch (vs `main` @ c85ad7e)
+
+| Commit | Change | Primary measured effect |
+|--------|--------|-------------------------|
+| `58c3d9c` | fasl lambdatable-by-name (SHENFASL5) | trivial startup **1.5s → 0.34s** warm; 21/21 StLib fasl-cacheable |
+| `913bc99` | `--hush-load` / `SHEN_HUSH_LOAD=1` | silence load echo only; golden runners usable (issue #46 item 3) |
+| `d3ecc91` | codegen exact-arity `(cons A B)` inline | prng wall best-of-5 **3.20s → 1.43s (−55%)**; 0 blacklisted traces |
+| `082e872` | default GC pause 400 (`SHEN_GC`) | ~−15% prng / −20% world CPU; peak RSS ~30→60 MB |
+| `b396fed` | native `fn` fast path | prng CPU **~−27%**, world **~−12%** (interleaved A/B) |
+| *(this)* | native iterative `append` | prng CPU med **~−3%**, mean **~−4 to −8%**; world med **~−19%** (serial warm) |
+
+## Current tip vs shen-cl (warm, this session)
+
+| Suite | shen-lua (user+sys, serial warm med) | shen-cl (`script`, wall) |
+|-------|--------------------------------------|---------------------------|
+| `shen/tests/prng` | ~0.87–1.0 s CPU (tip after append) | ~1.1–1.3 s wall / ~1.0 s CPU |
+| `shen/tests/world` | ~2–3 s CPU (high variance) | ~1.75–1.85 s wall / ~1.6 s CPU |
+
+Gap to CL is now roughly **1–2×** on prng (was ~12× at branch start with cold
+fasl / uncached StLib). Remaining cost is dominated by pure-Shen bit-list
+SHA-256 / bigint arithmetic, not load or `fn`.
+
+## Profiling snapshot (prng warm, after `fn` fix, before append)
+
+`jit.p` modes (suite load only, post-boot):
+
+| mode | top signals |
+|------|-------------|
+| `v` | ~38% Compiled / ~29% Interpreted / ~28% GC / ~5% JIT |
+| `Fl` (leaf) | `byte.bits` 22%, `byte.bits.weights` 13%, `list.take` 11%, **`append` 10%**, EQ 4%, APP 4%, `bits.xor` 4%, `cons?` 3%, `hd` 3% |
+
+`fn` is no longer in the leaf top (prior fix). After native `append`, leaf
+share of append drops modestly; remaining time is user list recursion + GC
+cons churn (bits as cons lists).
+
+## Tried / not shipped this session
+
+1. **Relax `pure_tail_self` for bare `(shen.f-error name)`**  
+   Multi-clause `define` ends in `(shen.f-error <name>)`; treating the bare
+   name as residual recursion blocked loop lowering on pure-tail helpers
+   (`list.drop`, `list.nth`, …). Fix correctly emits `LOOP` for those. On
+   this SHA suite (short lists, N≈8–32) serial warm prng **mean regressed
+   ~+14%** — short tail-call chains already JIT well; the loop form did not
+   pay. Left unmerged. Still a good lever for deep-iteration code; re-open
+   with a depth heuristic or workload-specific gate if needed.
+
+2. **Codegen inline `hd` / `tl` / `cons?`**  
+   Same late-binding tradeoff as `cons`. Combined package was within noise /
+   slightly mean-regressive on prng; not shipped alone.
+
+3. **Default GC pause 800**  
+   Med ~−12% prng CPU vs 400, but peak RSS ~60→106 MB (world ~110→216 MB).
+   Keep `SHEN_GC=800` as documented batch override; default stays 400.
+
+## Remaining headroom / risks
+
+- **Structural**: urdr represents words as 32-element bit lists; `list.take`,
+  `bits.xor`, `byte.bits.weights` dominate. Port opts cannot change digest
+  semantics or golden outputs. Host-SHA primitives are explicitly out of
+  scope unless vector-gated and trivial.
+- **Tail-recursion modulo cons** for `(cons H (self …))` (append / take /
+  map-like) would target the remaining non-tail list constructors without
+  host SHA.
+- **Interpreter share still ~25–30%** on prng — further trace-friendly
+  inlining of tiny prims may help, but measure carefully (short-list loops
+  can regress).
+- **Thermal noise**: single-run wall times vary ~2×; always A/B with dual
+  `SHEN_FASL_DIR` / `SHEN_KERNEL_CACHE` and report med/mean of N≥6.
+- **Issue #46**: cold-start and hush-load items addressed; suite compute gap
+  largely structural.
+
+## Gates
+
+- `make test`: **500 pass / 0 fail**
+- `luajit run-kernel-tests.lua`: **134 passed / 0 failed, ok**
+- urdr prng / world: **ALL PASS**; stdout byte-identical to pre-change warm
+  golden (modulo `run time:` banners)