Name tests by family and number, after z390

CHANGELOG.md

@@ -23,6 +23,35 @@ Booth — Changelog
   layout `bir.h` claims is checked rather than assumed
   (Zane Hambly, 2026-08-09)
 
+- tests are named for their family and position, `rvi01` and `tdf39` rather
+  than `rv_isel_max_frame_slots_in_range`, after z390's `TESTDCB1`. The family
+  is the file stem, the old descriptive name became a description the runner
+  prints, and `fam_order` in `tests/tmain.c` is the one place a family is
+  declared (Zane Hambly, 2026-08-10)
+
+- the runner refuses to start if a test registers an unknown family, a name
+  that disagrees with its number, or a number already taken. 278 of the 380
+  tests were registering under families `cat_order` did not list, so they ran
+  unheaded in link order, `--list` showed 102 of them and `--cat rv_enc` ran
+  none of them while exiting 0 (Zane Hambly, 2026-08-10)
+
+- `--fam` replaces `--cat`, which still works, and `--families` lists the
+  families with their files and counts (Zane Hambly, 2026-08-10)
+
+- the keyword table `lookup_keyword` binary searches is checked for ordering,
+  and every keyword is checked to still lex as a keyword. A misfiled entry lexed
+  as an identifier and surfaced as a parse error somewhere else entirely
+  (Zane Hambly, 2026-08-10)
+
+- `make repro` reads a sidecar `tests/NAME.opt` per fixture instead of counting
+  every refusal as a silent skip. Eight refusals were being hidden, one of them
+  a live `v_mfma` verifier failure, and an `xfail` that starts passing is now
+  reported too (Zane Hambly, 2026-08-10)
+
+- `tests/trpi.c` collects regressions for bugs that shipped, seeded with #160's
+  line-number corruption, which had four sites fixed and no test
+  (Zane Hambly, 2026-08-10)
+
 ## 2026-08-07
 
 Version 0.5.2.

CONTRIBUTING.md

@@ -159,14 +159,52 @@ Tenstorrent additionally sits above BIR through TDF (Tile DataFlow), a small IR
 # Build
 make
 
-# Run the test suite (currently 274 tests across the frontends,
+# Run the test suite (currently 390 tests across the frontends,
 # IR, backends, runtime, and SYSPRINT)
 make test
 
+# List the families, or run just one of them
+./trunner --families
+./trunner --fam rvi
+./trunner --list
+
 # Run the emulator test suite (RDNA3, requires tinygrad mockgpu in WSL)
 python tests/emu/run_emu.py
 ```
 
+### Test naming
+
+Tests are named for their family and their position in it, `rvi01`, `tdf39`,
+`smk04`, which is lifted straight off z390's `rt\test` directory where the
+members run `TESTDCB1` through `TESTDCB9`. It is a bit cargo-culted from the
+mainframe world and the eight character member limit it comes from stopped
+mattering decades ago, but an alphabetical listing groups and orders itself,
+grep agrees with the runner about what comes first, and nothing drifts.
+
+The family is the file stem and there is one family per file. `trv_isel.c`
+carries `rvi`, `ttdf.c` carries `tdf`. Add a test with:
+
+```c
+static void rvi66(void) { ...; PASS(); }
+TH_REG("rvi", 66, "what it checks, in a phrase", rvi66)
+```
+
+The name says nothing on its own, so the description carries it and shows up in
+`--list` and in the run. Keep it inside `TH_DESCW`. Every family has to appear
+in `fam_order` in `tests/tmain.c`, which is also where the running order lives,
+and `th_check` refuses to run if a name, number or family does not line up.
+That check exists because 278 of the tests used to register under families the
+runner had never heard of, where `--list` could not see them and `--cat` could
+not reach them.
+
+Regression tests for bugs that actually shipped go in `tests/trpi.c` with the
+issue number leading the description, z390's `RPI1540` habit with our numbering.
+
+`.cu` fixtures may carry a sidecar `tests/NAME.opt` declaring modes they are
+known not to survive, one `xfail MODE reason` per line. `make repro` requires
+every other mode to compile twice to identical bytes, and it reports an `xfail`
+that starts passing as well, so a stale exception does not hide a fix.
+
 Verify your changes don't introduce encoding regressions:
 ```bash
 llvm-objdump -d --mcpu=gfx1100 output.hsaco

Makefile

@@ -110,7 +110,9 @@ TSRC    = tests/tmain.c tests/tsmoke.c tests/tcomp.c tests/tenc.c \
           tests/tcbsync.c \
           tests/tsoft_fp.c \
           tests/tsysprint.c \
-          tests/tbackend.c
+          tests/tbackend.c \
+          tests/tordr.c \
+          tests/trpi.c
 
 TOBJS   = $(TSRC:%.c=$(OBJDIR)/%.o)
 COBJS   = $(OBJDIR)/src/kauri_impl.o $(OBJDIR)/src/ir/bir.o $(OBJDIR)/src/ir/bir_print.o $(OBJDIR)/src/ir/bir_lower.o $(OBJDIR)/src/ir/bir_mem2reg.o $(OBJDIR)/src/ir/bir_cfold.o $(OBJDIR)/src/ir/bir_dce.o $(OBJDIR)/src/ir/bir_struct.o $(OBJDIR)/src/ir/bir_insert.o $(OBJDIR)/src/ir/bir_sroa.o $(OBJDIR)/src/ir/bir_inline.o \

src/fe/lexer.c

@@ -115,6 +115,15 @@ static int lookup_keyword(const char *src, uint32_t len)
     return TOK_IDENT;
 }
 
+/* The table stays static; the suite only needs to read the order. */
+int lexer_kw_count(void) { return NUM_KEYWORDS; }
+
+const char *lexer_kw_at(int i)
+{
+    if (i < 0 || i >= NUM_KEYWORDS) return NULL;
+    return keywords[i].name;
+}
+
 static const char *tok_names[] = {
     [TOK_INT_LIT]       = "INT_LIT",
     [TOK_FLOAT_LIT]     = "FLOAT_LIT",

src/fe/lexer.h

@@ -26,4 +26,12 @@ int  lexer_tokenize(lexer_t *L);
 int  lexer_token_text(const lexer_t *L, const token_t *tok,
                       char *buf, int bufsize);
 
+/* lookup_keyword binary searches the keyword table, so keeping that table in
+ * order is VERY IMPORTANT. Put an entry in the wrong place and the keyword
+ * quietly lexes as an identifier, then blows up as a parse error somewhere
+ * else entirely. Handed out so the suite can check the order instead of
+ * trusting the joke above the table. */
+int         lexer_kw_count(void);
+const char *lexer_kw_at(int i);
+
 #endif /* BARRACUDA_LEXER_H */

tests/reprocheck.sh

@@ -1,6 +1,13 @@
 #!/bin/sh
 # Same input twice, same bytes out. Separate processes, so ASLR and any
 # pointer-ordered container shows up as a mismatch.
+#
+# Each fixture may carry a sidecar tests/NAME.opt listing the modes it is known
+# not to survive, one "xfail MODE reason" per line. That is z390 keeping
+# TESTDCB1.OPT next to TESTDCB1.MLC rather than parking a table of special
+# cases off in the runner. This script used to read any refusal as a skip and
+# say nothing about it, so eight of them sat quiet, and a backend that started
+# refusing every fixture would still have come out green.
 set -u
 
 root=$(git rev-parse --show-toplevel)
@@ -13,13 +20,47 @@ kath=./kath.exe
 modes="--amdgpu --nvidia-ptx --ir"
 fail=0
 checked=0
-skipped=0
+declared=0
+
+EMPTY=$(printf '' | sha256sum | cut -d' ' -f1)
+
+# Prints the declared reason when fixture $1 is xfail for mode $2, nothing
+# otherwise. Always prints something on a match, so callers can test for it.
+xfail_text() {
+  opt="${1%.cu}.opt"
+  [ -f "$opt" ] || return 0
+  awk -v m="$2" '
+    { sub(/#.*/, "") }
+    $1 == "xfail" && ($2 == "all" || $2 == m) {
+      $1 = ""; $2 = ""; sub(/^ +/, "")
+      print ($0 == "" ? "(no reason given)" : $0)
+      exit
+    }' "$opt"
+}
 
 for f in tests/*.cu; do
   for m in $modes; do
+    why=$(xfail_text "$f" "$m")
+
+    if [ -n "$why" ]; then
+      # Declared broken. Confirm it still is, so a fix does not slip past.
+      if "$kath" $m "$f" >/dev/null 2>&1; then
+        fail=$((fail + 1))
+        echo "XPASS: $m $f succeeds now, drop its xfail from ${f%.cu}.opt" >&2
+      else
+        declared=$((declared + 1))
+      fi
+      continue
+    fi
+
     a=$("$kath" $m "$f" 2>/dev/null | sha256sum 2>/dev/null | cut -d' ' -f1)
-    # A file the backend refuses is not a reproducibility failure.
-    if [ -z "$a" ]; then skipped=$((skipped + 1)); continue; fi
+    if [ -z "$a" ] || [ "$a" = "$EMPTY" ]; then
+      fail=$((fail + 1))
+      echo "NO OUTPUT: $m $f emitted nothing and declares no xfail" >&2
+      echo "  add one to ${f%.cu}.opt if that is deliberate" >&2
+      continue
+    fi
+
     b=$("$kath" $m "$f" 2>/dev/null | sha256sum 2>/dev/null | cut -d' ' -f1)
     checked=$((checked + 1))
     if [ "$a" != "$b" ]; then
@@ -33,8 +74,8 @@ done
 
 if [ "$fail" -ne 0 ]; then
   echo >&2
-  echo "reprocheck: $fail of $checked runs differed" >&2
+  echo "reprocheck: $fail problem(s) alongside $checked reproducible runs" >&2
   exit 1
 fi
 
-echo "reprocheck: $checked runs reproducible ($skipped skipped)"
+echo "reprocheck: $checked runs reproducible, $declared declared xfail"