diff --git a/Makefile b/Makefile index 148aaccc1..73fcb5eb2 100644 --- a/Makefile +++ b/Makefile @@ -141,7 +141,7 @@ OBJECTS := \ firmware.o \ gxf.o gxf_asm.o \ heapblock.o \ - hv.o hv_vm.o hv_exc.o hv_vuart.o hv_wdt.o hv_asm.o hv_aic.o hv_virtio.o \ + hv.o hv_vm.o hv_exc.o hv_vuart.o hv_wdt.o hv_asm.o hv_aic.o hv_t8140.o hv_virtio.o \ i2c.o \ iodev.o \ iova.o \ diff --git a/proxyclient/m1n1/hv/__init__.py b/proxyclient/m1n1/hv/__init__.py index fd8595698..d877303cd 100644 --- a/proxyclient/m1n1/hv/__init__.py +++ b/proxyclient/m1n1/hv/__init__.py @@ -109,6 +109,10 @@ def __init__(self, iface, proxy, utils): self.hook_exceptions = False self.started_cpus = {} self.started = False + self.start_entry = None + self.start_args = None + self.secondary_start_entry = None + self.secondary_start_args = None self.ctx = None self.hvcall_handlers = {} self.switching_context = False @@ -714,7 +718,10 @@ def handle_msr(self, ctx, iss=None): sys.stdout.flush() if enc in xlate: value = self.p.hv_translate(value, True, False) - self.u.msr(enc2, value, call=self.p.gl2_call) + call=None + if self.u.cpu_features.apple_sysregs_unlocked: + call=self.p.gl2_call + self.u.msr(enc2, value, call=call) self.log(f"Pass: msr {name}, x{iss.Rt} = {value:x} (OK) ({sysreg_name(enc2)})") ctx.elr += 4 @@ -1429,15 +1436,16 @@ def init(self): self.u.msr(MDCR_EL2, mdcr.value) self.u.msr(MDSCR_EL1, MDSCR(MDE=1).value) - # Enable AMX - amx_ctl = AMX_CONFIG(self.u.mrs(AMX_CONFIG_EL1)) - amx_ctl.EN_EL1 = 1 - self.u.msr(AMX_CONFIG_EL1, amx_ctl.value) + if self.u.cpu_features.apple_sysregs_unlocked: + # Enable AMX + amx_ctl = AMX_CONFIG(self.u.mrs(AMX_CONFIG_EL1)) + amx_ctl.EN_EL1 = 1 + self.u.msr(AMX_CONFIG_EL1, amx_ctl.value) - # Set guest AP keys - self.u.msr(VMKEYLO_EL2, 0x4E7672476F6E6147) - self.u.msr(VMKEYHI_EL2, 0x697665596F755570) - self.u.msr(APSTS_EL12, 1) + # Set guest AP keys + self.u.msr(VMKEYLO_EL2, 0x4E7672476F6E6147) + self.u.msr(VMKEYHI_EL2, 0x697665596F755570) + self.u.msr(APSTS_EL12, 1) self.map_vuart() @@ -1475,10 +1483,12 @@ def rh(base, off, width): self.log(f"PMGR R {base:x}+{off:x}:{width} = 0x{data:x} -> 0x{ret:x}") return ret - atc = f"ATC{self.iodev - IODEV.USB0}_USB" - atc_aon = f"ATC{self.iodev - IODEV.USB0}_USB_AON" - - hook_devs = ["UART0", atc, atc_aon] + # When the proxy is on the debugusb link there is no ATC PHY to protect + # and the index would go negative (ie, ATC-3_USB) + hook_devs = ["UART0"] + if self.iodev >= IODEV.USB0: + idx = self.iodev - IODEV.USB0 + hook_devs += [f"ATC{idx}_USB", f"ATC{idx}_USB_AON"] pmgr = self.adt["/arm-io/pmgr"] dev_by_name = {dev.name: dev for dev in pmgr.devices} @@ -1487,13 +1497,11 @@ def rh(base, off, width): pmgr_hooks = [] def hook_pmgr_dev(dev): - ps = pmgr.ps_regs[dev.psreg] - if dev.psidx or dev.psreg: - addr = pmgr.get_reg(ps.reg)[0] + ps.offset + dev.psidx * 8 - pmgr_hooks.append(addr) - for idx in self.adt.pmgr_dev_get_parents(dev): - if idx in dev_by_id: - hook_pmgr_dev(dev_by_id[idx]) + addr = self.adt.pmgr_dev_get_addr(dev) + pmgr_hooks.append(addr) + for idx in self.adt.pmgr_dev_get_parents(dev): + if idx in dev_by_id: + hook_pmgr_dev(dev_by_id[idx]) for name in hook_devs: dev = dev_by_name[name] @@ -1562,11 +1570,11 @@ def cpustart_wh(base, off, data, width): chip_id = self.u.adt["/chosen"].chip_id if chip_id in (0x8103, 0x6000, 0x6001, 0x6002): cpu_start = 0x54000 + die * 0x20_0000_0000 - elif chip_id in (0x8112, 0x8122, 0x6030): + elif chip_id in (0x8112, 0x8122, 0x8132, 0x8140, 0x6030): cpu_start = 0x34000 + die * 0x20_0000_0000 elif chip_id in (0x6020, 0x6021, 0x6022): cpu_start = 0x28000 + die * 0x20_0000_0000 - elif chip_id in (0x6031,): + elif chip_id in (0x6031, 0x6034, 0x6040): cpu_start = 0x88000 + die * 0x20_0000_0000 else: self.log("CPUSTART unknown for this SoC!") @@ -1576,6 +1584,19 @@ def cpustart_wh(base, off, data, width): self.map_hook(pmgr0_start + cpu_start, 0x20, write=cpustart_wh) self.add_tracer(zone, "CPU_START", TraceMode.RESERVED) + if not self.u.cpu_features.apple_sysregs_unlocked: + + def rvbar_rh(base, off, width): + ret = self.entry & ~0xfff | 1 + self.log(f"RVBAR R {base:x}+{off:x}:{width} -> 0x{ret:x}") + return ret + + for cpu in self.adt["cpus"]: + addr, _ = cpu.cpu_impl_reg + zone = irange(addr, 4) + self.map_hook(addr, 4, read=rvbar_rh) + self.add_tracer(zone, "RVBAR", TraceMode.RESERVED) + def start_secondary(self, die, cluster, cpu): self.log(f"Starting guest secondary {die}:{cluster}:{cpu}") @@ -1586,13 +1607,24 @@ def start_secondary(self, die, cluster, cpu): self.log("CPU not found!") return - entry = self.p.read64(node.cpu_impl_reg[0]) & 0xfffffffffff + secondary_entry = self.secondary_start_entry + secondary_args = self.secondary_start_args + + if secondary_entry is not None: + entry = secondary_entry + elif self.u.cpu_features.apple_sysregs_unlocked: + entry = self.p.read64(node.cpu_impl_reg[0]) & 0xfffffffffff + else: + entry = self.entry & ~0xfff index = node.cpu_id self.log(f" CPU #{index}: RVBAR = {entry:#x}") self.sysreg[index] = {} self.started_cpus[index] = (die, cluster, cpu) - self.p.hv_start_secondary(index, entry) + if secondary_args is not None: + self.p.hv_start_secondary(index, entry, *secondary_args) + else: + self.p.hv_start_secondary(index, entry) def setup_adt(self): self.adt["product"].product_name += " on m1n1 hypervisor" @@ -1779,14 +1811,15 @@ def remove_oslog(node): elif self.tba.revision == 3: self.iface.writemem(guest_base + self.bootargs_off, BootArgs_r3.build(self.tba)) - print("Setting secondary CPU RVBARs...") - rvbar = self.entry & ~0xfff - for cpu in self.adt["cpus"]: - if cpu.state == "running": - continue - addr, size = cpu.cpu_impl_reg - print(f" {cpu.name}: [0x{addr:x}] = 0x{rvbar:x}") - self.p.write64(addr, rvbar) + if self.u.cpu_features.apple_sysregs_unlocked: + print("Setting secondary CPU RVBARs...") + rvbar = self.entry & ~0xfff + for cpu in self.adt["cpus"]: + if cpu.state == "running": + continue + addr, size = cpu.cpu_impl_reg + print(f" {cpu.name}: [0x{addr:x}] = 0x{rvbar:x}") + self.p.write64(addr, rvbar) def _load_macho_symbols(self): self.symbol_dict = self.macho.symbols @@ -1882,10 +1915,84 @@ def load_hook_m3(data, segname, size, fileoff, dest): print("Done.") return a.tobytes() - #image = macho.prepare_image(load_hook) + def load_hook_m4(data, segname, size, fileoff, dest): + if segname not in ("__TEXT_EXEC", "__TEXT_BOOT_EXEC"): + return data + + nop = 0xd503201f + print(f"Patching segment {segname}...") + + a = array.array("I", data) + seg_vmaddr = macho.vmin + dest + + # genter -> HVC to allow us to catch SPTM entry calls + p = 0 + while (p := data.find(b"\x20\x00", p)) != -1: + if (p & 3) != 2: + p += 1 + continue + opcode = a[p // 4] + if 0x00201420 <= opcode <= 0x00201424: # genter #imm (imm 0..4) + a[p // 4] = self.hvc(0) + p += 4 + + # sysreg neutralization: NOP writes to guarded impdef regs and + # SME control register + nop_msr = [ + 0xd51cf100, # msr KERNKEYLO_EL1 (s3_4_c15_c1_0) + 0xd51cf120, # msr KERNKEYHI_EL1 (s3_4_c15_c1_1) + 0xd51cfdc0, # msr s3_4_c15_c13_6 (Apple PAC key) + 0xd51cfde0, # msr s3_4_c15_c13_7 (Apple PAC key) + 0xd51c12a0, # msr SMPRIMAP_EL2 (s3_4_c1_c2_5) + ] + p = 0 + while (p := data.find(b"\x1c\xd5", p)) != -1: + if (p & 3) != 2: + p += 1 + continue + opcode = a[p // 4] & ~0x1f + if opcode in nop_msr: + if opcode == 0xd51c12a0: + print(f" 0x{seg_vmaddr + (p & ~3):x}: SMPRIMAP_EL2 -> noop") + a[p // 4] = nop + p += 4 + + # commpage: force XNU to publish VM-safe userspace feature policy + # bytes. The Apple timebase path and HW-TPRO/SPRR path use + # userspace-inaccessible Apple registers under this EL1 guest. + # + # hard coded kernelcache addresses is a hack we should find a way + # to avoid + off = 0xfffffe000b5a3330 - seg_vmaddr + if 0 <= off < size: + if a[off // 4] != 0x52800068: + raise RuntimeError("_COMM_PAGE_USER_TIMEBASE patch target mismatch") + a[off // 4] = 0x52800008 + print(f" 0xfffffe000b5a3330: _COMM_PAGE_USER_TIMEBASE=0") + + off = 0xfffffe000b5a3388 - seg_vmaddr + if 0 <= off < size: + if a[off // 4] != 0x52800028: + raise RuntimeError("_COMM_PAGE_CONT_HWCLOCK patch target mismatch") + a[off // 4] = 0x52800008 + print(f" 0xfffffe000b5a3388: _COMM_PAGE_CONT_HWCLOCK=0") + + off = 0xfffffe000b5a358c - seg_vmaddr + if 0 <= off < size: + if a[off // 4] != 0x39043128: + raise RuntimeError("_COMM_PAGE_HW_TPRO patch target mismatch") + a[off // 4] = 0x3904313f + print(f" 0xfffffe000b5a358c: _COMM_PAGE_HW_TPRO=0") + + print("Done.") + return a.tobytes() + chip_id = self.u.adt["/chosen"].chip_id if chip_id in (0x8122, 0x6030, 0x6031, 0x6032, 0x6034): image = macho.prepare_image(load_hook_m3) + elif chip_id in (0x8132, 0x8140, 0x6040, 0x6041): + # M4 / A18 Pro class: T8132, T8140, T6040 (M4 Pro), T6041 (M4 Max) + image = macho.prepare_image(load_hook_m4) else: image = macho.prepare_image() self.load_raw(image, entryoffset=(macho.entry - macho.vmin), use_xnu_symbols=self.xnu_mode and symfile is not None, vmin=macho.vmin) @@ -1968,13 +2075,19 @@ def start(self): print("Shutting down framebuffer...") self.p.fb_shutdown(True) - print("Enabling SPRR...") - self.u.msr(SPRR_CONFIG_EL1, 1) + if self.u.cpu_features.apple_sysregs_unlocked: + print("Enabling SPRR...") + self.u.msr(SPRR_CONFIG_EL1, 1) + + print("Enabling GXF...") + self.u.msr(GXF_CONFIG_EL1, 1) - print("Enabling GXF...") - self.u.msr(GXF_CONFIG_EL1, 1) + entry = self.start_entry if self.start_entry is not None else self.entry + args = self.start_args + if args is None: + args = (self.guest_base + self.bootargs_off,) - print(f"Jumping to entrypoint at 0x{self.entry:x}") + print(f"Jumping to entrypoint at 0x{entry:x}") self.iface.dev.timeout = None self.default_sigint = signal.signal(signal.SIGINT, self._handle_sigint) @@ -1991,6 +2104,6 @@ def start(self): break self.started_cpus[cpu_node.cpu_id] = (getattr(cpu_node, "die_id", 0), cpu_node.cluster_id, cpu_node.cpu_id) self.sysreg[cpu_node.cpu_id] = {} - self.p.hv_start(self.entry, self.guest_base + self.bootargs_off) + self.p.hv_start(entry, *args) from .. import trace diff --git a/proxyclient/m1n1/proxy.py b/proxyclient/m1n1/proxy.py index ad9faf9a1..01cd22cbc 100644 --- a/proxyclient/m1n1/proxy.py +++ b/proxyclient/m1n1/proxy.py @@ -507,7 +507,7 @@ class GUARD(IntFlag): "amx" / bool_, "actlr_el2" / bool_, "counter_redirect" / bool_, - "padding" / Bytes(1), + "sme_enabled" / bool_, ) # Uses UartInterface.proxyreq() to send requests to M1N1 and process @@ -1189,8 +1189,8 @@ def display_is_external(self): def dapf_init_all(self): return self.request(self.P_DAPF_INIT_ALL) - def dapf_init(self, path): - return self.request(self.P_DAPF_INIT, path) + def dapf_init(self, path, index=1): + return self.request(self.P_DAPF_INIT, path, index) def cpufreq_init(self): return self.request(self.P_CPUFREQ_INIT) diff --git a/proxyclient/m1n1/xnutools.py b/proxyclient/m1n1/xnutools.py index f7fb59fa8..4c04027b1 100644 --- a/proxyclient/m1n1/xnutools.py +++ b/proxyclient/m1n1/xnutools.py @@ -1,7 +1,12 @@ # SPDX-License-Identifier: MIT +import pathlib import re +import struct +import time from construct import * +from .macho import MachOLoadCmdType + __all__ = [] DebuggerState = Struct( @@ -113,5 +118,184 @@ def format_arg(match): format_arg, string) print(string + "\n", end="") +_msgbuf_va_cache = {} +_msgbuf_dump_seq = 0 +MAX_MSGBUF_DUMP_SIZE = 8 * 1024 * 1024 + + +def _macho_bytes(macho): + pos = macho.io.tell() + try: + macho.io.seek(macho.off) + return macho.io.read(macho.size) + finally: + macho.io.seek(pos) + + +def _macho_fileoff_to_va(macho, fileoff): + for image in [macho] + list(getattr(macho, "subfiles", {}).values()): + for cmd in image.get_cmds(MachOLoadCmdType.SEGMENT_64): + start = image.off + cmd.args.fileoff + end = start + cmd.args.filesize + if start <= fileoff < end: + return cmd.args.vmaddr + (fileoff - start) + return None + + +def find_msgbuf_va(macho): + cached = _msgbuf_va_cache.get(id(macho), False) + if cached is not False: + return cached + + data = _macho_bytes(macho) + magic = struct.pack("> 56) == 0xff + + +def _strip_pac(va): + va = int(va) + if (va & 0xffffff00_00000000) == 0xfffffe00_00000000: + return va + return (va & 0x000000ff_ffffffff) | 0xfffffe00_00000000 + + +def _decode_msgbuf(raw, msg_bufx, msg_bufr): + size = len(raw) + if size == 0: + return b"" + + def printable_score(blob): + if not blob: + return 0.0 + keep = sum(1 for b in blob if b in (9, 10, 13) or 0x20 <= b < 0x7f) + return keep / max(1, len(blob)) + + ordered = raw[:msg_bufx] if 0 <= msg_bufx <= size else raw + if 0 <= msg_bufx < size and any(raw[msg_bufx:]): + ring = raw[msg_bufx:] + raw[:msg_bufx] + if printable_score(ring.replace(b"\x00", b"")) >= 0.70: + ordered = ring + if 0 <= msg_bufr < size and 0 <= msg_bufx < size and msg_bufr != msg_bufx: + unread = raw[msg_bufr:msg_bufx] if msg_bufr < msg_bufx else raw[msg_bufr:] + raw[:msg_bufx] + if printable_score(unread.replace(b"\x00", b"")) >= 0.70: + ordered = unread + + return ordered.replace(b"\x00", b"") + + +def dump_msgbuf(u, macho, dump_dir="logs", tail_lines=80): + global _msgbuf_dump_seq + + msgbuf_va = find_msgbuf_va(macho) + if msgbuf_va is None: + return None + + hdr = _read_kernel_va(u, msgbuf_va, 24) + if hdr is None or len(hdr) < 24: + print(f"[host] XNU-MSGBUF: header read failed at 0x{msgbuf_va:x}", + flush=True) + return None + + magic, msg_size, msg_bufx, msg_bufr, msg_bufc = struct.unpack(" MAX_MSGBUF_DUMP_SIZE: + print(f"[host] XNU-MSGBUF: invalid header magic=0x{magic:x} " + f"size={msg_size} bufx={msg_bufx} bufr={msg_bufr} " + f"bufc=0x{msg_bufc:x}", flush=True) + return None + + if not _is_kernel_va(msg_bufc): + msg_bufc = _strip_pac(msg_bufc) + if not _is_kernel_va(msg_bufc): + print(f"[host] XNU-MSGBUF: non-kernel msg_bufc=0x{msg_bufc:x}", + flush=True) + return None + + raw = _read_kernel_va(u, msg_bufc, msg_size) + if raw is None: + print(f"[host] XNU-MSGBUF: buffer read failed at 0x{msg_bufc:x} " + f"size=0x{msg_size:x}", flush=True) + return None + + text_bytes = _decode_msgbuf(raw, msg_bufx, msg_bufr) + text = text_bytes.decode("utf-8", errors="replace") + if text and not text.endswith("\n"): + text += "\n" + + dump_dir = pathlib.Path(dump_dir) + dump_dir.mkdir(parents=True, exist_ok=True) + stamp = time.strftime("%Y%m%d_%H%M%S") + _msgbuf_dump_seq += 1 + stem = f"xnu_msgbuf_{stamp}_{_msgbuf_dump_seq:02d}" + raw_path = dump_dir / f"{stem}.raw" + txt_path = dump_dir / f"{stem}.txt" + raw_path.write_bytes(raw) + txt_path.write_text(text, encoding="utf-8", errors="replace") + + print(f"[host] XNU-MSGBUF: dumped {len(text_bytes)} text bytes " + f"(raw {msg_size} bytes) msgbuf=0x{msgbuf_va:x} " + f"bufc=0x{msg_bufc:x} bufx={msg_bufx} bufr={msg_bufr} " + f"path={txt_path}", flush=True) + tail = [line for line in text.splitlines() if line.strip()] + if tail and tail_lines: + print(f"[host] XNU-MSGBUF tail ({min(tail_lines, len(tail))}/{len(tail)} lines):", + flush=True) + for line in tail[-tail_lines:]: + print(f"[xnu-log] {line}", flush=True) + return txt_path + + __all__.extend(k for k, v in globals().items() if (callable(v) or isinstance(v, type)) and v.__module__ == __name__) diff --git a/proxyclient/tools/chainload.py b/proxyclient/tools/chainload.py index 414b548cf..d01d706d1 100755 --- a/proxyclient/tools/chainload.py +++ b/proxyclient/tools/chainload.py @@ -99,7 +99,12 @@ def remove_oslog(node): for cpu in u.adt["cpus"]: if cpu.state == "running": continue + addr, size = cpu.cpu_impl_reg + if p.read64(addr) & 1: + print(f" {cpu.name}: [0x{addr:x}] RVBAR locked, skipping") + continue + print(f" {cpu.name}: [0x{addr:x}] = 0x{rvbar:x}") p.write64(addr, rvbar) diff --git a/proxyclient/tools/run_guest.py b/proxyclient/tools/run_guest.py index bb10e3cac..6ac199187 100755 --- a/proxyclient/tools/run_guest.py +++ b/proxyclient/tools/run_guest.py @@ -9,6 +9,26 @@ def volumespec(s): return tuple(s.split(":", 2)) + +def sptm_hv_boot_args(extra=()): + def boot_arg_key(arg): + return arg.split("=", 1)[0] + + words = list(extra) + defaults = [ + "-nobsdmgroot", # dodges a panic we otherwise hit on this boot path + "wdt=-1", # disables some internal XNU watchdog + "sprr_tpro=0", # disable XNU's TPRO boot policy; the commpage bit is patched separately + "sprr_tpro_pagers=0", # ... same, for pager mappings + "-v", # Optional: verbose boot + f"msgbuf={1024 * 1024}", # Optional: enlarge the kernel msgbuf + ] + for arg in defaults: + key = boot_arg_key(arg) + if not any(boot_arg_key(w) == key for w in words): + words.append(arg) + return " ".join(words) + parser = argparse.ArgumentParser(description='Run a Mach-O payload under the hypervisor') parser.add_argument('-s', '--symbols', type=pathlib.Path) parser.add_argument('-m', '--script', type=pathlib.Path, action='append', default=[]) @@ -71,6 +91,24 @@ def volumespec(s): if args.debug_xnu: hv.adt["chosen"].debug_enabled = 1 +# Exclaves are not yet supported +if not args.raw and u.adt["/chosen"].chip_id in (0x8132, 0x8140, 0x6040, 0x6041): + for name in ("/arm-io/exdisplaypipe", "/arm-io/exdisplaypipe-s-proxy", + "/arm-io/dcp-exclave-ioreporting", "/arm-io/dcp-exclave-mailbox"): + try: + del hv.adt[name] + except KeyError: + pass + # The internal DCP's iop-dcp-nub binds its RTKit transport to the (now-gone) + # dcp-exclave-mailbox via routes=206; clearing routes makes the DCP firmware + # fall back to the plain ASC mailbox like the routeless external dcpext. + try: + nub = hv.adt["/arm-io/dcp/iop-dcp-nub"] + if getattr(nub, "routes", None) is not None: + del nub.routes + except KeyError: + pass + if args.volume: for path, tag in args.volume: hv.attach_virtio(Virtio9PTransport(root=path, tag=tag)) @@ -78,7 +116,10 @@ def volumespec(s): if args.logfile: hv.set_logfile(args.logfile.open("w")) -if len(args.boot_args) > 0: +# macOS-under-HV needs a specific boot-arg set on the M4/macOS chip_ids +if not args.raw and u.adt["/chosen"].chip_id in (0x8132, 0x8140, 0x6040, 0x6041): + hv.set_bootargs(sptm_hv_boot_args(args.boot_args)) +elif len(args.boot_args) > 0: boot_args = " ".join(args.boot_args) hv.set_bootargs(boot_args) diff --git a/src/arm_cpu_regs.h b/src/arm_cpu_regs.h index a45868ec6..8cd45d88a 100644 --- a/src/arm_cpu_regs.h +++ b/src/arm_cpu_regs.h @@ -188,6 +188,21 @@ #define ID_AA64MMFR0_TGran4_2 GENMASK(43, 40) #define ID_AA64MMFR0_TGran64_2 GENMASK(39, 36) #define ID_AA64MMFR0_TGran16_2 GENMASK(35, 32) + +#define SYS_CPTR_EL2 sys_reg(3, 4, 1, 1, 2) +#define CPTR_EL2_SMEN GENMASK(25, 24) +#define CPTR_EL2_SMEN_NONE (0x3UL << 24) +#define CPTR_EL2_FPEN GENMASK(21, 20) +#define CPTR_EL2_FPEN_NONE (0x3UL << 20) +#define CPTR_EL2_ZEN GENMASK(17, 16) +#define CPTR_EL2_ZEN_NONE (0x3UL << 16) + +#define SYS_HFGRTR_EL2 sys_reg(3, 4, 1, 1, 4) +#define SYS_HFGWTR_EL2 sys_reg(3, 4, 1, 1, 5) +#define HFGxTR_EL2_nTPIDR2_EL0 BIT(55) +#define HFGxTR_EL2_nSMPRI_EL1 BIT(54) + +#define SYS_SMPRIMAP_EL2 sys_reg(3, 4, 1, 2, 5) #define ID_AA64MMFR0_TGran4 GENMASK(31, 28) #define ID_AA64MMFR0_TGran64 GENMASK(27, 24) #define ID_AA64MMFR0_TGran16 GENMASK(23, 20) diff --git a/src/chickens.c b/src/chickens.c index 0cf98c48d..0041c9c22 100644 --- a/src/chickens.c +++ b/src/chickens.c @@ -113,6 +113,7 @@ const struct midr_part_features features_m4 = { .sleep_mode = SLEEP_NONE, // XXX probably new mode required .fast_ipi = true, .actlr_el2 = true, + .sme_enabled = true, }; /* diff --git a/src/cpu_regs.h b/src/cpu_regs.h index f40cc8a9e..9169fb374 100644 --- a/src/cpu_regs.h +++ b/src/cpu_regs.h @@ -14,11 +14,20 @@ #define SYS_IMP_APL_AMX_CTL_EL1 sys_reg(3, 4, 15, 1, 4) #define SYS_IMP_APL_AMX_CTL_EL2 sys_reg(3, 4, 15, 4, 7) #define SYS_IMP_APL_AMX_CTL_EL12 sys_reg(3, 4, 15, 4, 6) +/* AMX/AIDR2 version-ID probe; XNU boot-panics if it reads an unknown value. */ +#define SYS_IMP_APL_AMXIDR_EL1 sys_reg(3, 6, 15, 2, 7) #define AMX_CTL_EN BIT(63) #define AMX_CTL_EN_EL1 BIT(62) #define SYS_IMP_APL_CNTVCT_ALIAS_EL0 sys_reg(3, 4, 15, 10, 6) +/* Apple kernel deadline timer (XNU's local scheduler timer on Apple Silicon). + * CNTVCTSS aliases CNTVCT_ALIAS above (same encoding). */ +#define SYS_IMP_APL_KERNEL_CNTV_CTL_EL0 sys_reg(3, 1, 15, 0, 5) +#define SYS_IMP_APL_KERNEL_CNTV_TVAL_EL0 sys_reg(3, 1, 15, 15, 4) +#define SYS_IMP_APL_KERNEL_CNTVCT_EL0 sys_reg(3, 4, 15, 11, 7) +#define SYS_IMP_APL_KERNEL_CNTKCTL_EL1 sys_reg(3, 4, 15, 9, 6) +#define SYS_IMP_APL_JCTL_EL0 sys_reg(3, 4, 15, 15, 6) /* HID registers */ #define SYS_IMP_APL_HID0 sys_reg(3, 0, 15, 0, 0) @@ -462,6 +471,30 @@ #define AGTCNTRDIR_DISABLE BIT(1) #define AGTCNTRDIR_EL0_TRAP_CTL BIT(0) +/* Impdef state XNU round-trips, locked on M4+ */ +#define SYS_IMP_APL_WATCHDOGDIAG0 sys_reg(3, 5, 15, 2, 6) +#define SYS_IMP_APL_S3_1_C15_C13_4 sys_reg(3, 1, 15, 13, 4) +#define SYS_IMP_APL_S3_6_C15_C0_4 sys_reg(3, 6, 15, 0, 4) +#define SYS_IMP_APL_S3_6_C15_C0_5 sys_reg(3, 6, 15, 0, 5) +#define SYS_IMP_APL_S3_4_C15_C12_0 sys_reg(3, 4, 15, 12, 0) + +/* T8140 CLPC sampler controls/counters. Live EL2 access faults in the SPTM path. */ +#define SYS_IMP_APL_CLPC_CORE_PERF0 sys_reg(3, 7, 15, 0, 3) +#define SYS_IMP_APL_CLPC_CORE_PERF1 sys_reg(3, 7, 15, 2, 3) +#define SYS_IMP_APL_CLPC_CORE_PERF2 sys_reg(3, 7, 15, 4, 3) +#define SYS_IMP_APL_CLPC_CORE_PERF_ENABLE0 sys_reg(3, 7, 15, 1, 3) +#define SYS_IMP_APL_CLPC_CORE_PERF_ENABLE1 sys_reg(3, 7, 15, 3, 3) +#define SYS_IMP_APL_CLPC_CORE_PERF_ENABLE2 sys_reg(3, 7, 15, 5, 3) +#define SYS_IMP_APL_CLPC_CORE_PERF_CTRL2 sys_reg(3, 7, 15, 5, 0) +#define SYS_IMP_APL_CLPC_CORE_ACC0 sys_reg(3, 1, 15, 8, 3) +#define SYS_IMP_APL_CLPC_CORE_ACC1 sys_reg(3, 1, 15, 9, 3) +#define SYS_IMP_APL_CLPC_CORE_ACC2 sys_reg(3, 1, 15, 10, 3) +#define SYS_IMP_APL_CLPC_CORE_ACC3 sys_reg(3, 1, 15, 11, 3) +#define SYS_IMP_APL_CLPC_CORE_ACC4 sys_reg(3, 1, 15, 12, 3) +#define SYS_IMP_APL_CLPC_MEM_STALL_CFG0 sys_reg(3, 1, 15, 0, 3) +#define SYS_IMP_APL_CLPC_MEM_STALL_CFG1 sys_reg(3, 1, 15, 1, 3) +#define SYS_IMP_APL_CLPC_MEM_STALL_CFG2 sys_reg(3, 1, 15, 2, 3) + // Performance monitor registers #define SYS_IMP_APL_PMCR0 sys_reg(3, 1, 15, 0, 0) #define PMCR0_CNT_EN_MASK (MASK(8) | GENMASK(33, 32)) @@ -714,6 +747,7 @@ #define SYS_IMP_APL_SPRR_PERM_EL1 sys_reg(3, 6, 15, 1, 6) #define SYS_IMP_APL_SPRR_PERM_EL02 sys_reg(3, 4, 15, 5, 2) #define SYS_IMP_APL_SPRR_PERM_EL12 sys_reg(3, 6, 15, 15, 7) +#define SYS_IMP_APL_SPRR_UMPRR_EL1 sys_reg(3, 6, 15, 3, 0) #define SYS_IMP_APL_TPIDR_GL1 sys_reg(3, 6, 15, 10, 1) #define SYS_IMP_APL_VBAR_GL1 sys_reg(3, 6, 15, 10, 2) diff --git a/src/dapf.c b/src/dapf.c index 3b4b773dc..a39edba9b 100644 --- a/src/dapf.c +++ b/src/dapf.c @@ -21,7 +21,7 @@ struct dapf_t8020_config { static int dapf_init_t8020(const char *path, u64 base, int node) { - u32 length; + u32 length = 0; const char *prop = "filter-data-instance-0"; const struct dapf_t8020_config *config = adt_getprop(adt, node, prop, &length); @@ -101,12 +101,22 @@ static int dapf_init_t8110b(u64 base, struct dapf_t8110b_config *config, u32 len static int dapf_init_t8110(const char *path, u64 base, int node) { - u32 length; - const char *prop = "dapf-instance-0"; - const void *config = adt_getprop(adt, node, prop, &length); + u32 length = 0; + char prop[32]; + const void *config = NULL; + + // The dapf config is not always under instance 0; find the first present + // dapf-instance-N. + for (int i = 0; i < 8; i++) { + snprintf(prop, sizeof(prop), "dapf-instance-%d", i); + length = 0; + config = adt_getprop(adt, node, prop, &length); + if (config && length) + break; + } if (!config || !length) { - printf("dapf: Error getting ADT node %s property %s.\n", path, prop); + printf("dapf: Error getting ADT node %s dapf-instance property.\n", path); return -1; } @@ -171,8 +181,11 @@ struct entry { }; struct entry dapf_entries[] = { - {"/arm-io/dart-aop", 1}, {"/arm-io/dart-mtp", 1}, {"/arm-io/dart-pmp", 1}, - {"/arm-io/dart-isp", 5}, {"/arm-io/dart-isp0", 5}, {NULL, -1}, + {"/arm-io/dart-aop", 1}, {"/arm-io/dart-mtp", 1}, + {"/arm-io/dart-pmp", 1}, {"/arm-io/dart-dcp", 1}, + {"/arm-io/dart-dcpext0", 1}, {"/arm-io/dart-isp", 5}, + {"/arm-io/dart-isp0", 5}, {"/arm-io/dart-ane", 3}, + {"/arm-io/dart-ave", 3}, {NULL, -1}, }; int dapf_init_all(void) diff --git a/src/hv.c b/src/hv.c index 3500f2509..4135a7b40 100644 --- a/src/hv.c +++ b/src/hv.c @@ -49,10 +49,72 @@ struct hv_secondary_info_t { uint64_t gxf_config; uint64_t agt_cnt_rdir_el1; uint64_t agt_cnt_rdir_el12; + uint64_t guest_ttbr0; + uint64_t guest_ttbr1; + uint64_t guest_tcr; + uint64_t guest_mair; + uint64_t guest_amair; + uint64_t guest_contextidr; + uint64_t guest_cpacr; + uint64_t guest_vbar; + uint64_t guest_sctlr; }; static struct hv_secondary_info_t hv_secondary_info; +static void hv_capture_guest_el12_state(struct hv_secondary_info_t *info) +{ + info->guest_ttbr0 = mrs(TTBR0_EL12); + info->guest_ttbr1 = mrs(TTBR1_EL12); + info->guest_tcr = mrs(TCR_EL12); + info->guest_mair = mrs(MAIR_EL12); + info->guest_amair = mrs(AMAIR_EL12); + info->guest_contextidr = mrs(CONTEXTIDR_EL12); + info->guest_cpacr = mrs(CPACR_EL12); + info->guest_vbar = mrs(VBAR_EL12); + info->guest_sctlr = mrs(SCTLR_EL12); +} + +static void hv_restore_guest_el12_state(const struct hv_secondary_info_t *info) +{ + /* + * On SPTM-enabled SoCs XNU boots with the mmu already enabled and expect + * seeded state (previous SoCs would enable the mmu and create the state + * themselves). Leaving stale m1n1 EL2 state causes the secondaries to + * fault; program them with the running guest EL1 state instead. + */ + msr(TTBR0_EL12, info->guest_ttbr0); + msr(TTBR1_EL12, info->guest_ttbr1); + msr(TCR_EL12, info->guest_tcr); + msr(MAIR_EL12, info->guest_mair); + msr(AMAIR_EL12, info->guest_amair); + msr(CONTEXTIDR_EL12, info->guest_contextidr); + msr(CPACR_EL12, info->guest_cpacr); + msr(VBAR_EL12, info->guest_vbar); + sysop("isb"); + msr(SCTLR_EL12, info->guest_sctlr); + sysop("isb"); +} + +static void hv_config_sme(bool verbose) +{ + if (!cpu_features->sme_enabled) + return; + + reg_mask(SYS_CPTR_EL2, CPTR_EL2_SMEN | CPTR_EL2_FPEN | CPTR_EL2_ZEN, + CPTR_EL2_SMEN_NONE | CPTR_EL2_FPEN_NONE | CPTR_EL2_ZEN_NONE); + + if (FIELD_GET(ID_AA64MMFR0_FGT, mrs(ID_AA64MMFR0_EL1))) { + reg_set(SYS_HFGRTR_EL2, HFGxTR_EL2_nTPIDR2_EL0 | HFGxTR_EL2_nSMPRI_EL1); + reg_set(SYS_HFGWTR_EL2, HFGxTR_EL2_nTPIDR2_EL0 | HFGxTR_EL2_nSMPRI_EL1); + } + + msr(SYS_SMPRIMAP_EL2, 0); + + if (verbose) + printf("HV: SME enabled (CPTR_EL2=%lx)\n", mrs(SYS_CPTR_EL2)); +} + void hv_init(void) { pcie_shutdown(); @@ -68,13 +130,26 @@ void hv_init(void) hv_pt_init(); // Configure hypervisor defaults - hv_write_hcr(HCR_API | // Allow PAuth instructions - HCR_APK | // Allow PAuth key registers - HCR_TEA | // Trap external aborts - HCR_E2H | // VHE mode (forced) - HCR_RW | // AArch64 guest - HCR_AMO | // Trap SError exceptions - HCR_VM); // Enable stage 2 translation + u64 hcr = HCR_API | // Allow PAuth instructions + HCR_APK | // Allow PAuth key registers + HCR_TEA | // Trap external aborts + HCR_E2H | // VHE mode (forced) + HCR_RW | // AArch64 guest + HCR_AMO | // Trap SError exceptions + HCR_VM; // Enable stage 2 translation + + /* + * On guarded (M4-class) SoCs the Apple impdef sysregs are GL2-only and fault + * when executed from EL2, so trap the guest's EL1 accesses to us and soft- + * model/shadow them. Do NOT set TIDCP on unlocked SoCs: there the guest + * accesses these registers directly at EL1 and trapping them would regress + * existing (M1-M3) virtualization. + */ + if (!cpu_features->apple_sysregs_unlocked) + hcr |= HCR_TIDCP; + + hv_write_hcr(hcr); + hv_config_sme(true); // No guest vectors initially msr(VBAR_EL12, 0); @@ -124,6 +199,10 @@ void hv_start(void *entry, u64 regs[4]) hv_started_cpus[boot_cpu_idx] = true; + // Install the T8140 CPU/ACC/CPM write guard before the guest runs (no-op on + // other SoCs). Must be after the guest's device mappings are established. + hv_t8140_map_accumulators(); + msr(VBAR_EL1, _hv_vectors_start); if (gxf_enabled()) @@ -151,11 +230,14 @@ void hv_start(void *entry, u64 regs[4]) hv_secondary_info.agt_cnt_rdir_el1 = mrs(SYS_IMP_APL_AGTCNTRDIR_EL1); hv_secondary_info.agt_cnt_rdir_el12 = mrs(SYS_IMP_APL_AGTCNTRDIR_EL12); } + if (!cpu_features->apple_sysregs_unlocked) + hv_capture_guest_el12_state(&hv_secondary_info); hv_arm_tick(false); hv_pinned_cpu = -1; hv_want_cpu = -1; hv_cpus_in_guest = BIT(smp_id()); + hv_apply_time_stealing_offset(); hv_enter_guest(regs[0], regs[1], regs[2], regs[3], entry); @@ -193,33 +275,45 @@ void hv_start(void *entry, u64 regs[4]) static void hv_init_secondary(struct hv_secondary_info_t *info) { - gxf_init(); + if (cpu_features->apple_sysregs_unlocked) + gxf_init(); msr(VBAR_EL1, _hv_vectors_start); msr(HCR_EL2, info->hcr); + hv_config_sme(false); msr(HACR_EL2, info->hacr); msr(VTCR_EL2, info->vtcr); msr(VTTBR_EL2, info->vttbr); msr(MDCR_EL2, info->mdcr); msr(MDSCR_EL1, info->mdscr); - msr(SYS_IMP_APL_AMX_CTL_EL2, info->amx_ctl); - msr(SYS_IMP_APL_APVMKEYLO_EL2, info->apvmkeylo); - msr(SYS_IMP_APL_APVMKEYHI_EL2, info->apvmkeyhi); - msr(SYS_IMP_APL_APSTS_EL12, info->apsts); + + if (cpu_features->apple_sysregs_unlocked) { + msr(SYS_IMP_APL_AMX_CTL_EL2, info->amx_ctl); + msr(SYS_IMP_APL_APVMKEYLO_EL2, info->apvmkeylo); + msr(SYS_IMP_APL_APVMKEYHI_EL2, info->apvmkeyhi); + msr(SYS_IMP_APL_APSTS_EL12, info->apsts); + } msr(ACTLR_EL2, info->actlr_el2); - if (cpu_features->actlr_el2) - msr(SYS_ACTLR_EL12, info->actlr_el1); - else - msr(SYS_IMP_APL_ACTLR_EL12, info->actlr_el1); + if (cpu_features->apple_sysregs_unlocked) { + if (cpu_features->actlr_el2) + msr(SYS_ACTLR_EL12, info->actlr_el1); + else + msr(SYS_IMP_APL_ACTLR_EL12, info->actlr_el1); + } msr(CNTHCTL_EL2, info->cnthctl); - msr(SYS_IMP_APL_SPRR_CONFIG_EL1, info->sprr_config); - msr(SYS_IMP_APL_GXF_CONFIG_EL1, info->gxf_config); + if (cpu_features->apple_sysregs_unlocked) { + msr(SYS_IMP_APL_SPRR_CONFIG_EL1, info->sprr_config); + msr(SYS_IMP_APL_GXF_CONFIG_EL1, info->gxf_config); + } if (cpu_features->counter_redirect) { msr(SYS_IMP_APL_AGTCNTRDIR_EL1, info->agt_cnt_rdir_el1); msr(SYS_IMP_APL_AGTCNTRDIR_EL12, info->agt_cnt_rdir_el12); } + if (!cpu_features->apple_sysregs_unlocked) + hv_restore_guest_el12_state(info); + if (cpu_features->apple_sysregs_unlocked) reg_mask(SYS_IMP_APL_CYC_OVRD, CYC_OVRD_WFI_MODE_MASK, CYC_OVRD_WFI_MODE(0)); @@ -231,6 +325,7 @@ static void hv_init_secondary(struct hv_secondary_info_t *info) static void hv_enter_secondary(void *entry, u64 regs[4]) { + hv_apply_time_stealing_offset(); hv_enter_guest(regs[0], regs[1], regs[2], regs[3], entry); spin_lock(&bhl); @@ -248,6 +343,8 @@ void hv_start_secondary(int cpu, void *entry, u64 regs[4]) printf("HV: Initializing secondary %d\n", cpu); iodev_console_flush(); + if (!cpu_features->apple_sysregs_unlocked) + hv_capture_guest_el12_state(&hv_secondary_info); mmu_init_secondary(cpu); iodev_console_flush(); smp_call4(cpu, hv_init_secondary, (u64)&hv_secondary_info, 0, 0, 0); diff --git a/src/hv.h b/src/hv.h index 783fad86c..291ec8a3d 100644 --- a/src/hv.h +++ b/src/hv.h @@ -58,6 +58,9 @@ int hv_unmap(u64 from, u64 size); int hv_map_hw(u64 from, u64 to, u64 size); int hv_map_sw(u64 from, u64 to, u64 size); int hv_map_hook(u64 from, hv_hook_t *hook, u64 size); + +/* T8140 CPU/ACC/CPM write guard (no-op on other SoCs); see hv_t8140.c. */ +int hv_t8140_map_accumulators(void); u64 hv_translate(u64 addr, bool s1only, bool w, u64 *par_out); u64 hv_pt_walk(u64 addr); bool hv_handle_dabort(struct exc_info *ctx); @@ -78,6 +81,7 @@ void virtio_put_buffer(u64 base, int qu, u32 id, u32 len); /* Exceptions */ void hv_exc_proxy(struct exc_info *ctx, uartproxy_boot_reason_t reason, u32 type, void *extra); void hv_set_time_stealing(bool enabled, bool reset); +void hv_apply_time_stealing_offset(void); void hv_add_time(s64 time); /* WDT */ diff --git a/src/hv_exc.c b/src/hv_exc.c index e56c7f5f5..b835cb513 100644 --- a/src/hv_exc.c +++ b/src/hv_exc.c @@ -27,13 +27,148 @@ struct hv_pcpu_data { u32 pmc_pending; u64 pmc_irq_mode; u64 exc_entry_pmcr0_cnt; + u64 kernel_cntv_cval; + u64 kernel_cntv_ctl; + u64 kernel_cntkctl_el1; + bool kernel_cntv_valid; + /* Soft-cached guarded impdef registers (see SYSREG_SHADOW users below). */ + u64 agtcntrdir_el1; + u64 agtcntrdir_el12; + u64 siq_cfg_el1; + u64 jctl_el0; + u64 watchdogdiag0; + u64 impdef_c13_4; + u64 impdef_c15_c0_4; + u64 impdef_c15_c0_5; + u64 impdef_c12_0; + u64 amx_ctx_el1; + u64 amx_ctl_el1; + u64 amx_state_t; } ALIGNED(64); struct hv_pcpu_data pcpu[MAX_CPUS]; +static u64 hv_clpc_core_perf_ctrl2[MAX_CPUS]; +static u64 hv_clpc_mem_stall_cfg[MAX_CPUS][3]; +static s64 hv_clpc_counter_bias[MAX_CPUS][16]; + +/* + * SPRR permission-remap table, soft-cached on M4+ SoCs where the real register + * faults at EL2. Values taken from Sven's blog post which appear to work. + */ +static u64 hv_sprr_perm_el0 = 0x2010002030100000; +static u64 hv_sprr_perm_el1 = 0x2020a506f020f0e0; +static u64 hv_sprr_umprr_el1; + +/* + * Track EL2/proxy time hidden from the guest. The architectural counter and + * Apple CNTVCT alias run in different clock domains, so each needs its own + * accumulator. + */ +static u64 stolen_time = 0; +static u64 stolen_time_kernel_cntv = 0; + +/* + * M4-class SoCs schedule based off an Apple impdef timer that is locked in raw + * boot mode. We soft-model it: keep the deadline in a per-CPU shadow, read + * "now" from a counter (KERNEL_CNTVCTSS == CNTVCT_ALIAS, readable and + * already passed through), and deliver the timer as a virtual FIQ from + * hv_update_fiq() once the soft deadline elapses. + */ +static inline u64 hv_kernel_cntv_now(void) +{ + /* + * CNTVOFF_EL2 only offsets the architectural virtual counter. XNU's Apple + * kernel deadline timer reads CNTVCT_ALIAS instead, so track stolen time for + * that timer separately in CNTVCT_ALIAS units and subtract it here. + */ + return mrs(SYS_IMP_APL_CNTVCT_ALIAS_EL0) - stolen_time_kernel_cntv; +} + +static inline s64 hv_kernel_cntv_delta(void) +{ + return (s64)(PERCPU(kernel_cntv_cval) - hv_kernel_cntv_now()); +} + +static void hv_kernel_cntv_init(void) +{ + if (PERCPU(kernel_cntv_valid)) + return; + + /* CTL/TVAL are readable at EL2; capture XNU's initial deadline once. */ + u32 tval = (u32)mrs(SYS_IMP_APL_KERNEL_CNTV_TVAL_EL0); + PERCPU(kernel_cntv_ctl) = + mrs(SYS_IMP_APL_KERNEL_CNTV_CTL_EL0) & (CNTx_CTL_ENABLE | CNTx_CTL_IMASK); + PERCPU(kernel_cntv_cval) = hv_kernel_cntv_now() + (s64)(s32)tval; + PERCPU(kernel_cntv_valid) = true; +} + +static u64 hv_kernel_cntv_read_ctl(void) +{ + hv_kernel_cntv_init(); + + u64 ctl = PERCPU(kernel_cntv_ctl) & (CNTx_CTL_ENABLE | CNTx_CTL_IMASK); + if ((ctl & CNTx_CTL_ENABLE) && hv_kernel_cntv_delta() <= 0) + ctl |= CNTx_CTL_ISTATUS; + return ctl; +} + +static u64 hv_kernel_cntv_read_tval(void) +{ + hv_kernel_cntv_init(); + return (u32)hv_kernel_cntv_delta(); +} + +static void hv_kernel_cntv_write_ctl(u64 val) +{ + hv_kernel_cntv_init(); + PERCPU(kernel_cntv_ctl) = val & (CNTx_CTL_ENABLE | CNTx_CTL_IMASK); +} + +static void hv_kernel_cntv_write_tval(u64 val) +{ + hv_kernel_cntv_init(); + PERCPU(kernel_cntv_cval) = hv_kernel_cntv_now() + (s64)(s32)(u32)val; +} + +static bool hv_kernel_cntv_pending(void) +{ + if (!PERCPU(kernel_cntv_valid)) + return false; + + u64 ctl = hv_kernel_cntv_read_ctl(); + return (ctl & (CNTx_CTL_ISTATUS | CNTx_CTL_IMASK | CNTx_CTL_ENABLE)) == + (CNTx_CTL_ISTATUS | CNTx_CTL_ENABLE); +} + +#define KERNEL_CNTKCTL_CNTHCTL_MASK \ + (CNTHCTL_EL0PCTEN | CNTHCTL_EL0VCTEN | CNTHCTL_EVNTI | CNTHCTL_EVNTDIR | CNTHCTL_EVNTEN) + +static bool hv_handle_kernel_cntkctl(struct exc_info *ctx, u64 rt, bool is_read) +{ + if (is_read) { + if (rt < 31) + ctx->regs[rt] = PERCPU(kernel_cntkctl_el1); + return true; + } + + u64 val = rt < 31 ? ctx->regs[rt] : 0; + PERCPU(kernel_cntkctl_el1) = val; + + /* + * XNU programs WFE timeout events and EL0 timebase access through this + * Apple alias. On guarded SoCs, direct EL2 writes to the Apple register can + * fault, so mirror only the architectural CNTHCTL_EL2 bits that correspond + * to the guest-visible Apple control state. + */ + msr(SYS_CNTHCTL_EL2, (mrs(SYS_CNTHCTL_EL2) & ~KERNEL_CNTKCTL_CNTHCTL_MASK) | + (val & KERNEL_CNTKCTL_CNTHCTL_MASK)); + sysop("isb"); + return true; +} + void hv_exit_guest(void) __attribute__((noreturn)); -static u64 stolen_time = 0; static u64 exc_entry_time; extern u64 hv_cpus_in_guest; @@ -57,6 +192,7 @@ static void _hv_exc_proxy(struct exc_info *ctx, uartproxy_boot_reason_t reason, hv_rendezvous(); u64 entry_time = mrs(CNTPCT_EL0); + u64 kernel_cntv_entry_time = mrs(SYS_IMP_APL_CNTVCT_ALIAS_EL0); ctx->elr_phys = hv_translate(ctx->elr, false, false, NULL); ctx->far_phys = hv_translate(ctx->far, false, false, NULL); @@ -77,8 +213,9 @@ static void _hv_exc_proxy(struct exc_info *ctx, uartproxy_boot_reason_t reason, case EXC_RET_HANDLED: hv_wdt_breadcrumb('p'); if (time_stealing) { - u64 lost = mrs(CNTPCT_EL0) - entry_time; - stolen_time += lost; + stolen_time += mrs(CNTPCT_EL0) - entry_time; + stolen_time_kernel_cntv += mrs(SYS_IMP_APL_CNTVCT_ALIAS_EL0) - + kernel_cntv_entry_time; } break; case EXC_EXIT_GUEST: @@ -142,13 +279,28 @@ void hv_exc_proxy(struct exc_info *ctx, uartproxy_boot_reason_t reason, u32 type void hv_set_time_stealing(bool enabled, bool reset) { time_stealing = enabled; - if (reset) + if (reset) { stolen_time = 0; + stolen_time_kernel_cntv = 0; + } + hv_apply_time_stealing_offset(); +} + +void hv_apply_time_stealing_offset(void) +{ + msr(CNTVOFF_EL2, stolen_time); } void hv_add_time(s64 time) { + u64 ticks = time < 0 ? (u64)-time : (u64)time; + u64 kernel_ticks = (u64)((__uint128_t)ticks * 24000000ULL / mrs(CNTFRQ_EL0)); + stolen_time -= (u64)time; + if (time < 0) + stolen_time_kernel_cntv += kernel_ticks; + else + stolen_time_kernel_cntv -= kernel_ticks; } static void hv_update_fiq(void) @@ -171,6 +323,7 @@ static void hv_update_fiq(void) } fiq_pending |= PERCPU(ipi_pending) || PERCPU(pmc_pending); + fiq_pending |= hv_kernel_cntv_pending(); sysop("isb"); @@ -197,6 +350,87 @@ static void hv_update_fiq(void) _msr(sr_tkn(sr), regs[rt]); \ return true; +/* + * Soft-cache a guarded Apple impdef register: return the last value the guest + * wrote (or the seeded reset default on first read) and swallow the write. + * This drops the register's hardware side effect entirely. `store` is any + * lvalue: PERCPU(field) for per-CPU state, or a static global seeded with the + * reset value the guest expects to read back. + */ +#define SYSREG_SHADOW(sr, store) \ + case SYSREG_ISS(sr): \ + if (is_read) \ + regs[rt] = (store); \ + else \ + (store) = regs[rt]; \ + return true; + +static bool hv_handle_clpc_counter(struct exc_info *ctx, u64 rt, bool is_read, unsigned int counter) +{ + /* + * Apple's PMGR kext consumes these as perf/power deltas. Live EL2 access + * is unsafe in the current T8140/SPTM context, so provide coherent + * monotonic guest-visible counters. + */ + if (counter >= 16) + return false; + + u64 cpu = mrs(TPIDR_EL2); + u64 base = mrs(CNTPCT_EL0) + (counter * 0x1000000ULL); + + if (is_read) { + if (rt < 31) + ctx->regs[rt] = (u64)((s64)base + hv_clpc_counter_bias[cpu][counter]); + } else { + u64 val = rt < 31 ? ctx->regs[rt] : 0; + hv_clpc_counter_bias[cpu][counter] = (s64)val - (s64)base; + } + + return true; +} + +static bool hv_handle_clpc_mem_stall_config(struct exc_info *ctx, u64 rt, bool is_read, + unsigned int regno) +{ + /* + * Apple's PMGR kext programs these CoreMemStallSampler controls with + * read/modify/write. Keep readback coherent, but do not touch live EL2 + * hardware registers. + */ + if (regno >= 3) + return false; + + u64 cpu = mrs(TPIDR_EL2); + u64 *cache = &hv_clpc_mem_stall_cfg[cpu][regno]; + + if (is_read) { + if (rt < 31) + ctx->regs[rt] = *cache; + } else { + *cache = rt < 31 ? ctx->regs[rt] : 0; + } + + return true; +} + +static bool hv_handle_clpc_core_perf_ctrl2(struct exc_info *ctx, u64 rt, bool is_read) +{ + /* + * Another sysreg used by Apple's PMGR. XNU only needs coherent state here; + * live execution faults in this raw guest path. + */ + u64 cpu = mrs(TPIDR_EL2); + + if (is_read) { + if (rt < 31) + ctx->regs[rt] = hv_clpc_core_perf_ctrl2[cpu]; + } else { + hv_clpc_core_perf_ctrl2[cpu] = rt < 31 ? ctx->regs[rt] : 0; + } + + return true; +} + static bool hv_handle_msr_unlocked(struct exc_info *ctx, u64 iss) { u64 reg = iss & (ESR_ISS_MSR_OP0 | ESR_ISS_MSR_OP2 | ESR_ISS_MSR_OP1 | ESR_ISS_MSR_CRn | @@ -218,6 +452,23 @@ static bool hv_handle_msr_unlocked(struct exc_info *ctx, u64 iss) SYSREG_MAP(SYS_CNTP_CTL_EL0, SYS_CNTP_CTL_EL02) SYSREG_MAP(SYS_CNTP_CVAL_EL0, SYS_CNTP_CVAL_EL02) SYSREG_MAP(SYS_CNTP_TVAL_EL0, SYS_CNTP_TVAL_EL02) + /* Apple kernel deadline timer that we shadow soft-modeled, CTL/TVAL + * are write-locked at EL2; the counters read through. */ + case SYSREG_ISS(SYS_IMP_APL_KERNEL_CNTV_CTL_EL0): + if (is_read) + regs[rt] = hv_kernel_cntv_read_ctl(); + else + hv_kernel_cntv_write_ctl(regs[rt]); + return true; + case SYSREG_ISS(SYS_IMP_APL_KERNEL_CNTV_TVAL_EL0): + if (is_read) + regs[rt] = hv_kernel_cntv_read_tval(); + else + hv_kernel_cntv_write_tval(regs[rt]); + return true; + case SYSREG_ISS(SYS_IMP_APL_KERNEL_CNTKCTL_EL1): + return hv_handle_kernel_cntkctl(ctx, rt, is_read); + SYSREG_PASS(SYS_IMP_APL_KERNEL_CNTVCT_EL0) /* Spammy stuff seen on t600x p-cores */ /* These are PMU/PMC registers */ SYSREG_PASS(sys_reg(3, 2, 15, 12, 0)); @@ -239,11 +490,119 @@ static bool hv_handle_msr_unlocked(struct exc_info *ctx, u64 iss) SYSREG_MAP(SYS_IMP_APL_ASPSR_GL1, SYS_IMP_APL_ASPSR_GL12) SYSREG_MAP(SYS_IMP_APL_ELR_GL1, SYS_IMP_APL_ELR_GL12) SYSREG_MAP(SYS_IMP_APL_ESR_GL1, SYS_IMP_APL_ESR_GL12) - SYSREG_MAP(SYS_IMP_APL_SPRR_PERM_EL1, SYS_IMP_APL_SPRR_PERM_EL12) + /* + * SPRR permission remapping. On unlocked SoCs, redirect to the EL12 + * alias as before; on M4+ SoCs the register faults at EL2, so + * soft-cache it + */ + case SYSREG_ISS(SYS_IMP_APL_SPRR_PERM_EL0): + if (is_read) + regs[rt] = hv_sprr_perm_el0; + else + hv_sprr_perm_el0 = regs[rt]; + return true; + case SYSREG_ISS(SYS_IMP_APL_SPRR_PERM_EL1): + if (cpu_features->apple_sysregs_unlocked) { + if (is_read) + regs[rt] = _mrs(sr_tkn(SYS_IMP_APL_SPRR_PERM_EL12)); + else + _msr(sr_tkn(SYS_IMP_APL_SPRR_PERM_EL12), regs[rt]); + } else if (is_read) { + regs[rt] = hv_sprr_perm_el1; + } else { + hv_sprr_perm_el1 = regs[rt]; + } + return true; + case SYSREG_ISS(SYS_IMP_APL_SPRR_UMPRR_EL1): + if (is_read) + regs[rt] = hv_sprr_umprr_el1 & 0xffffffff; + else + hv_sprr_umprr_el1 = regs[rt] & 0xffffffff; + return true; + /* + * Impdef sysregs locked on M4+ that are required by XNU + */ + SYSREG_SHADOW(SYS_IMP_APL_AGTCNTRDIR_EL1, PERCPU(agtcntrdir_el1)) + SYSREG_SHADOW(SYS_IMP_APL_AGTCNTRDIR_EL12, PERCPU(agtcntrdir_el12)) + SYSREG_SHADOW(SYS_IMP_APL_SIQ_CFG_EL1, PERCPU(siq_cfg_el1)) + SYSREG_SHADOW(SYS_IMP_APL_JCTL_EL0, PERCPU(jctl_el0)) + SYSREG_SHADOW(SYS_IMP_APL_WATCHDOGDIAG0, PERCPU(watchdogdiag0)) + SYSREG_SHADOW(SYS_IMP_APL_S3_1_C15_C13_4, PERCPU(impdef_c13_4)) + SYSREG_SHADOW(SYS_IMP_APL_S3_6_C15_C0_4, PERCPU(impdef_c15_c0_4)) + SYSREG_SHADOW(SYS_IMP_APL_S3_6_C15_C0_5, PERCPU(impdef_c15_c0_5)) + SYSREG_SHADOW(SYS_IMP_APL_S3_4_C15_C12_0, PERCPU(impdef_c12_0)) + case SYSREG_ISS(SYS_IMP_APL_CLPC_CORE_PERF0): + return hv_handle_clpc_counter(ctx, rt, is_read, 0); + case SYSREG_ISS(SYS_IMP_APL_CLPC_CORE_PERF1): + return hv_handle_clpc_counter(ctx, rt, is_read, 1); + case SYSREG_ISS(SYS_IMP_APL_CLPC_CORE_PERF2): + return hv_handle_clpc_counter(ctx, rt, is_read, 2); + case SYSREG_ISS(SYS_IMP_APL_CLPC_CORE_ACC0): + return hv_handle_clpc_counter(ctx, rt, is_read, 0); + case SYSREG_ISS(SYS_IMP_APL_CLPC_CORE_ACC1): + return hv_handle_clpc_counter(ctx, rt, is_read, 1); + case SYSREG_ISS(SYS_IMP_APL_CLPC_CORE_ACC2): + return hv_handle_clpc_counter(ctx, rt, is_read, 2); + case SYSREG_ISS(SYS_IMP_APL_CLPC_CORE_ACC3): + return hv_handle_clpc_counter(ctx, rt, is_read, 3); + case SYSREG_ISS(SYS_IMP_APL_CLPC_CORE_ACC4): + return hv_handle_clpc_counter(ctx, rt, is_read, 4); + case SYSREG_ISS(SYS_IMP_APL_CLPC_CORE_PERF_ENABLE0): + case SYSREG_ISS(SYS_IMP_APL_CLPC_CORE_PERF_ENABLE1): + case SYSREG_ISS(SYS_IMP_APL_CLPC_CORE_PERF_ENABLE2): + if (is_read) + regs[rt] = 0; + return true; + case SYSREG_ISS(SYS_IMP_APL_CLPC_CORE_PERF_CTRL2): + return hv_handle_clpc_core_perf_ctrl2(ctx, rt, is_read); + case SYSREG_ISS(SYS_IMP_APL_CLPC_MEM_STALL_CFG0): + return hv_handle_clpc_mem_stall_config(ctx, rt, is_read, 0); + case SYSREG_ISS(SYS_IMP_APL_CLPC_MEM_STALL_CFG1): + return hv_handle_clpc_mem_stall_config(ctx, rt, is_read, 1); + case SYSREG_ISS(SYS_IMP_APL_CLPC_MEM_STALL_CFG2): + return hv_handle_clpc_mem_stall_config(ctx, rt, is_read, 2); SYSREG_MAP(SYS_IMP_APL_APCTL_EL1, SYS_IMP_APL_APCTL_EL12) - SYSREG_MAP(SYS_IMP_APL_AMX_CTL_EL1, SYS_IMP_APL_AMX_CTL_EL12) - /* FIXME:Might be wrong */ - SYSREG_PASS(SYS_IMP_APL_AMX_STATE_T) + /* + * AMX. On M4 userspace uses SME, so AMX is inert, but XNU's per-CPU + * init and context switch still touch these regs, which fault at EL2 on + * guarded SoCs. Advertise a valid version (AMXIDR/AIDR2, else boot + * panics) and soft-cache the control/state on guarded SoCs so XNU's + * read-after-write is coherent; the real AMX unit is never used. Unlocked + * SoCs keep the original map/pass behavior. + */ + case SYSREG_ISS(SYS_IMP_APL_AMXIDR_EL1): + if (!is_read) + return false; + if (cpu_features->apple_sysregs_unlocked) + regs[rt] = _mrs(sr_tkn(SYS_IMP_APL_AMXIDR_EL1)); + else + regs[rt] = BIT(5); /* AMX v6 */ + return true; + SYSREG_SHADOW(SYS_IMP_APL_AMX_CTX_EL1, PERCPU(amx_ctx_el1)) + case SYSREG_ISS(SYS_IMP_APL_AMX_CTL_EL1): + if (cpu_features->apple_sysregs_unlocked) { + if (is_read) + regs[rt] = _mrs(sr_tkn(SYS_IMP_APL_AMX_CTL_EL12)); + else + _msr(sr_tkn(SYS_IMP_APL_AMX_CTL_EL12), regs[rt]); + } else if (is_read) { + regs[rt] = PERCPU(amx_ctl_el1); + } else { + PERCPU(amx_ctl_el1) = regs[rt]; + } + return true; + case SYSREG_ISS(SYS_IMP_APL_AMX_STATE_T): + if (cpu_features->apple_sysregs_unlocked) { + if (is_read) + regs[rt] = _mrs(sr_tkn(SYS_IMP_APL_AMX_STATE_T)); + else + _msr(sr_tkn(SYS_IMP_APL_AMX_STATE_T), regs[rt]); + } else if (is_read) { + regs[rt] = PERCPU(amx_state_t); + } else { + PERCPU(amx_state_t) = regs[rt]; + } + return true; /* pass through PMU handling */ SYSREG_PASS(SYS_IMP_APL_PMCR1) SYSREG_PASS(SYS_IMP_APL_PMCR2) @@ -420,7 +779,7 @@ static void hv_exc_exit(struct exc_info *ctx) hv_update_fiq(); /* reenable PMU counters */ reg_set(SYS_IMP_APL_PMCR0, PERCPU(exc_entry_pmcr0_cnt)); - msr(CNTVOFF_EL2, stolen_time); + hv_apply_time_stealing_offset(); spin_unlock(&bhl); hv_maybe_exit(); __atomic_or_fetch(&hv_cpus_in_guest, BIT(smp_id()), __ATOMIC_ACQUIRE); diff --git a/src/hv_t8140.c b/src/hv_t8140.c new file mode 100644 index 000000000..d6f0c8092 --- /dev/null +++ b/src/hv_t8140.c @@ -0,0 +1,108 @@ +/* SPDX-License-Identifier: MIT */ +/* + * This T8140 CPU/ACC/CPM hook is a horrible hack that we don't know how to + * get rid of. We spent almost two weeks experimenting and ultimately + * converged to this file. + * + * Known failed edits: + * + * - Deleting this hook or forwarding all writes causes PEH/DPE bringup + * failures. + * - Dropping all writes produced SPMI errors and hard-rebooted the machine, + * and did not fix SPMI. + * + */ + +#include "hv.h" +#include "cpu_regs.h" +#include "memory.h" +#include "smp.h" +#include "soc.h" +#include "string.h" +#include "utils.h" + +#define T8140_ACC_PAGE_SIZE 0x4000UL +#define T8140_ACC_GUARD_FIRST_PAGE 2 + +static const u64 t8140_acc_pages[] = { + 0x210074000UL, 0x211074000UL, 0x210e44000UL, 0x211e44000UL, + 0x210e48000UL, 0x211e48000UL, 0x210058000UL, 0x210158000UL, + 0x210258000UL, 0x210358000UL, 0x211058000UL, 0x211158000UL, +}; + +static void t8140_flush_stage2_hooks(void) +{ + sysop("dsb ishst"); + sysop("tlbi vmalls12e1is"); + sysop("dsb ish"); + sysop("isb"); +} + +static int t8140_acc_page_index(u64 ipa) +{ + u64 page = ipa & ~(T8140_ACC_PAGE_SIZE - 1); + + for (u32 i = 0; i < ARRAY_SIZE(t8140_acc_pages); i++) + if (page == t8140_acc_pages[i]) + return i; + + return -1; +} + +#define T8140_ACC_PC_DPE_LOOP_LO 0xfffffe00097b7b00UL +#define T8140_ACC_PC_DPE_LOOP_HI 0xfffffe00097b8d00UL + +static bool t8140_acc_hook(struct exc_info *ctx, u64 ipa, u64 *val, bool write, int width) +{ + if (width < 0 || width > 6) + return false; + + int idx = t8140_acc_page_index(ipa); + if (idx < 0) + return false; + + u64 bytes = 1UL << width; + u64 off = ipa & (T8140_ACC_PAGE_SIZE - 1); + if (off + bytes > T8140_ACC_PAGE_SIZE) + return false; + + if (write) { + u64 pc = ctx->elr; + if (pc >= T8140_ACC_PC_DPE_LOOP_LO && pc < T8140_ACC_PC_DPE_LOOP_HI) { + return hv_pa_rw(ctx, ipa, val, true, width); + } + if (idx >= 6 && idx <= 9 && (off == 0x1000 || off == 0x1008)) { + return hv_pa_rw(ctx, ipa, val, true, width); + } + /* + * For E-core CPUs (idx 6..9 -> pages 0x21x058000), forward so the + * silicon power state matches XNU's view; otherwise cpu0 re-enters + * peh.c with a stale SError that never clears. P-core CPUs (idx + * 10..11) are HV-managed; drop those. + */ + if (idx >= 6 && idx <= 9) { + return hv_pa_rw(ctx, ipa, val, true, width); + } + return true; + } + + return hv_pa_rw(ctx, ipa, val, false, width); +} + +int hv_t8140_map_accumulators(void) +{ + if (chip_id != T8140) + return 0; + + int failures = 0; + for (u32 i = T8140_ACC_GUARD_FIRST_PAGE; i < ARRAY_SIZE(t8140_acc_pages); i++) + failures += hv_map_hook(t8140_acc_pages[i], t8140_acc_hook, + T8140_ACC_PAGE_SIZE) != 0; + + u32 requested = ARRAY_SIZE(t8140_acc_pages) - T8140_ACC_GUARD_FIRST_PAGE; + u32 installed = requested - failures; + printf("HV: T8140 CPU/ACC/CPM write guard for %lu/%lu pages\n", + (u64)installed, (u64)requested); + t8140_flush_stage2_hooks(); + return failures ? -failures : 0; +} diff --git a/src/mcc.c b/src/mcc.c index 375388399..9d75f79bd 100644 --- a/src/mcc.c +++ b/src/mcc.c @@ -29,6 +29,8 @@ static bool mcc_initialized = false; #define T6031_DCS_OFFSET 0x400000 #define T6031_DCS_STRIDE 0x200000 +#define T8140_PLANE_STRIDE 0x40000 + #define PLANE_TZ_MAX_REGS 4 struct tz_regs { @@ -124,6 +126,7 @@ struct mcc_regs { u32 cache_status_mask; u32 cache_status_val; u32 cache_disable; + bool has_cache_ctrl; struct tz_regs *tz; }; @@ -154,6 +157,11 @@ int mcc_enable_cache(void) if (!mcc_initialized) return -1; + if (!mcc_regs[0].has_cache_ctrl) { + printf("MCC: cache enable skipped (unsupported)\n"); + return 0; + } + /* The 6030 memory controller supports setting a waymask, but the desktop chips do not appear to use it */ for (int mcc = 0; mcc < mcc_count; mcc++) { @@ -176,6 +184,48 @@ int mcc_enable_cache(void) return ret; } +static void mcc_unmap_carveout(const char *name, u64 start, u64 size) +{ + if (!start || !size) + return; + + if (mcc_carveout_count >= ARRAY_SIZE(mcc_carveouts) - 1) { + printf("MMU: carveout list full, cannot record %s at 0x%lx..0x%lx\n", name, start, + start + size); + return; + } + + printf("MMU: Unmapping %s carveout at 0x%lx..0x%lx\n", name, start, start + size); + mmu_rm_mapping(start, size); + mmu_rm_mapping(start | REGION_RWX_EL0, size); + mmu_rm_mapping(start | REGION_RW_EL0, size); + mmu_rm_mapping(start | REGION_RX_EL1, size); + mcc_carveouts[mcc_carveout_count].base = start; + mcc_carveouts[mcc_carveout_count].size = size; + mcc_carveout_count++; +} + +static void mcc_unmap_adt_carveout(int node, const char *name) +{ + u64 range[2]; + + if (ADT_GETPROP_ARRAY(adt, node, name, range) != sizeof(range)) + return; + + mcc_unmap_carveout(name, range[0], range[1]); +} + +static void mcc_unmap_adt_tz_carveouts(void) +{ + int node = adt_path_offset(adt, "/chosen/carveout-memory-map"); + + if (node < 0) + return; + + mcc_unmap_adt_carveout(node, "region-id-4"); + mcc_unmap_adt_carveout(node, "region-id-2"); +} + int mcc_unmap_carveouts(void) { if (!mcc_initialized) @@ -184,39 +234,37 @@ int mcc_unmap_carveouts(void) mcc_carveout_count = 0; memset(mcc_carveouts, 0, sizeof mcc_carveouts); - // All MCCs and planes should have identical configs - // Note: For unhandled machines, the TZ regions can be found (on m1, m2, m3) by looking at - // region-id-2 and region-id-4 on a booted macos, in the /chosen/carveout-memory-map DT node. - // This can be used along with dumping the mcc reg space to find the correct start/end/enable - // above. - for (u32 i = 0; i < mcc_regs[0].tz->count; i++) { - uint64_t off = mcc_regs[0].tz->stride * i; - uint64_t start = plane_read32(0, 0, mcc_regs[0].tz->start + off); - uint64_t end = plane_read32(0, 0, mcc_regs[0].tz->end + off); - bool enabled = plane_read32(0, 0, mcc_regs[0].tz->enable + off); - - if (enabled) { - if (!start || start == end) { - printf("MMU: TZ%d region has bad bounds 0x%lx..0x%lx (iBoot bug?)\n", i, start, - end); - continue; + if (mcc_regs[0].tz) { + // All MCCs and planes should have identical configs + // Note: For unhandled machines, the TZ regions can be found (on m1, m2, m3) by looking at + // region-id-2 and region-id-4 on a booted macos, in the /chosen/carveout-memory-map DT node. + // This can be used along with dumping the mcc reg space to find the correct start/end/enable + // above. + for (u32 i = 0; i < mcc_regs[0].tz->count; i++) { + uint64_t off = mcc_regs[0].tz->stride * i; + uint64_t start = plane_read32(0, 0, mcc_regs[0].tz->start + off); + uint64_t end = plane_read32(0, 0, mcc_regs[0].tz->end + off); + bool enabled = plane_read32(0, 0, mcc_regs[0].tz->enable + off); + + if (enabled) { + if (!start || start == end) { + printf("MMU: TZ%d region has bad bounds 0x%lx..0x%lx (iBoot bug?)\n", i, + start, end); + continue; + } + + start = start << 12; + end = (end + 1) << 12; + start |= ram_base; + end |= ram_base; + mcc_unmap_carveout("TZ", start, end - start); } - - start = start << 12; - end = (end + 1) << 12; - start |= ram_base; - end |= ram_base; - printf("MMU: Unmapping TZ%d region at 0x%lx..0x%lx\n", i, start, end); - mmu_rm_mapping(start, end - start); - mmu_rm_mapping(start | REGION_RWX_EL0, end - start); - mmu_rm_mapping(start | REGION_RW_EL0, end - start); - mmu_rm_mapping(start | REGION_RX_EL1, end - start); - mcc_carveouts[mcc_carveout_count].base = start; - mcc_carveouts[mcc_carveout_count].size = end - start; - mcc_carveout_count++; } } + if (!mcc_carveout_count) + mcc_unmap_adt_tz_carveouts(); + return 0; } @@ -256,6 +304,7 @@ int mcc_init_t8103(int node, int *path, bool t8112) mcc_regs[0].cache_status_mask = T8103_CACHE_STATUS_MASK; mcc_regs[0].cache_status_val = T8103_CACHE_STATUS_VAL; mcc_regs[0].cache_disable = t8112 ? T8112_CACHE_DISABLE : 0; + mcc_regs[0].has_cache_ctrl = true; mcc_regs[0].tz = &t8103_tz_regs; printf("MCC: Initialized T8103 MCC (%d channels)\n", val); @@ -306,6 +355,7 @@ int mcc_init_t6000(int node, int *path, bool t602x) mcc_regs[i].cache_status_mask = T6000_CACHE_STATUS_MASK; mcc_regs[i].cache_status_val = T6000_CACHE_STATUS_VAL; mcc_regs[i].cache_disable = 0; + mcc_regs[i].has_cache_ctrl = true; mcc_regs[i].tz = t602x ? &t602x_tz_regs : &t8103_tz_regs; } @@ -372,6 +422,7 @@ int mcc_init_t6031(int node, int *path) mcc_regs[i].cache_status_mask = T6031_CACHE_STATUS_MASK; mcc_regs[i].cache_status_val = T6031_CACHE_STATUS_VAL; mcc_regs[i].cache_disable = 0; + mcc_regs[i].has_cache_ctrl = true; mcc_regs[i].tz = &t603x_tz_regs; } @@ -384,6 +435,66 @@ int mcc_init_t6031(int node, int *path) return 0; } +int mcc_init_t8140(int node, int *path) +{ + u32 amcc_count = 1; + u32 amcc_reg_idx = 0; + u32 dcs_count = 0; + u32 plane_count = 0; + u32 plane_stride = T8140_PLANE_STRIDE; + int lock_node; + + printf("MCC: Initializing T8140 AMCCs...\n"); + + if (ADT_GETPROP(adt, node, "amcc-reg-idx", &amcc_reg_idx) < 0) { + printf("MCC: Failed to get amcc-reg-idx property!\n"); + return -1; + } + + ADT_GETPROP(adt, node, "amcc-count", &amcc_count); + ADT_GETPROP(adt, node, "dcs-count-per-amcc", &dcs_count); + + lock_node = adt_path_offset(adt, "/chosen/lock-regs/amcc"); + if (lock_node >= 0) { + ADT_GETPROP(adt, lock_node, "plane-count", &plane_count); + ADT_GETPROP(adt, lock_node, "plane-stride", &plane_stride); + } + if (!plane_count) { + u32 plane_mask = 0; + + if (ADT_GETPROP(adt, node, "amcc-plane-enable-mask", &plane_mask) > 0) + plane_count = __builtin_popcount(plane_mask); + } + if (!plane_count) + plane_count = 4; + + mcc_count = amcc_count; + if (mcc_count > MAX_MCC_INSTANCES) { + printf("MCC: Too many instances, increase MAX_MCC_INSTANCES!\n"); + mcc_count = MAX_MCC_INSTANCES; + } + + for (int i = 0; i < mcc_count; i++) { + if (adt_get_reg(adt, path, "reg", amcc_reg_idx + i, &mcc_regs[i].plane_base, NULL)) { + printf("MCC: Failed to get AMCC reg index %d!\n", amcc_reg_idx + i); + return -1; + } + + mcc_regs[i].plane_stride = plane_stride; + mcc_regs[i].plane_count = plane_count; + mcc_regs[i].dcs_count = dcs_count; + mcc_regs[i].has_cache_ctrl = false; + mcc_regs[i].tz = NULL; + } + + printf("MCC: Initialized T8140 AMCCs (%d instances, %d planes, %d channels)\n", mcc_count, + mcc_regs[0].plane_count, mcc_regs[0].dcs_count); + + mcc_initialized = true; + + return 0; +} + int mcc_init(void) { int path[8]; @@ -404,6 +515,8 @@ int mcc_init(void) return mcc_init_t6000(node, path, true); } else if (adt_is_compatible(adt, node, "mcc,t6031")) { return mcc_init_t6031(node, path); + } else if (adt_is_compatible(adt, node, "mcc,t8140")) { + return mcc_init_t8140(node, path); } else { printf("MCC: Unsupported version:%s\n", adt_get_property(adt, node, "compatible")->value); return -1; diff --git a/src/memory.c b/src/memory.c index e6e1abaec..f25ba52c6 100644 --- a/src/memory.c +++ b/src/memory.c @@ -429,11 +429,103 @@ void mmu_map_framebuffer(u64 addr, size_t size) mmu_add_mapping(addr, addr, size, MAIR_IDX_NORMAL_NC, PERM_RW_EL0); } +int display_get_vram(u64 *paddr, u64 *size); + +static int mmu_get_iboot_handoff_range(u64 *start, u64 *size) +{ + u64 range[2]; + int chosen = adt_path_offset(adt, "/chosen"); + + if (chosen >= 0 && + ADT_GETPROP_ARRAY(adt, chosen, "iboot-handoff", range) == sizeof(range)) + goto found; + + int carveouts = adt_path_offset(adt, "/chosen/carveout-memory-map"); + if (carveouts >= 0 && + ADT_GETPROP_ARRAY(adt, carveouts, "region-id-1", range) == sizeof(range)) + goto found; + + return -1; + +found: + if (!range[0] || !range[1]) + return -1; + + *start = ALIGN_DOWN(range[0], PAGE_SIZE); + *size = ALIGN_UP(range[0] + range[1], PAGE_SIZE) - *start; + return 0; +} + +static void mmu_add_ram_mapping_handoff_nc(u64 from, u64 to, size_t size, + u8 attribute_index, u64 perms, + u64 handoff_start, + u64 handoff_size) +{ + u64 end = to + size; + u64 handoff_end = handoff_start + handoff_size; + + if (!handoff_start || !handoff_size || handoff_end <= to || + handoff_start >= end) { + mmu_add_mapping(from, to, size, attribute_index, perms); + return; + } + + u64 cur_to = to; + if (handoff_start > cur_to) { + u64 before_size = handoff_start - cur_to; + mmu_add_mapping(from, cur_to, before_size, attribute_index, perms); + from += before_size; + cur_to += before_size; + } + + u64 nc_end = min(handoff_end, end); + u64 nc_size = nc_end - cur_to; + mmu_add_mapping(from, cur_to, nc_size, MAIR_IDX_NORMAL_NC, perms); + from += nc_size; + cur_to += nc_size; + + if (cur_to < end) + mmu_add_mapping(from, cur_to, end - cur_to, attribute_index, perms); +} + +static void mmu_map_nc_aliases(u64 addr, size_t size, const char *name) +{ + u64 start = ALIGN_DOWN(addr, PAGE_SIZE); + u64 end = ALIGN_UP(addr + size, PAGE_SIZE); + size_t map_size = end - start; + + if (!addr || !size) + return; + + printf("MMU: Adding Normal-NC mappings at 0x%lx..0x%lx for %s\n", + start, end, name); + mmu_add_mapping(start, start, map_size, MAIR_IDX_NORMAL_NC, PERM_RWX); + mmu_add_mapping(start | REGION_RWX_EL0, start, map_size, + MAIR_IDX_NORMAL_NC, PERM_RWX_EL0); + mmu_add_mapping(start | REGION_RW_EL0, start, map_size, + MAIR_IDX_NORMAL_NC, PERM_RW_EL0); + mmu_add_mapping(start | REGION_RX_EL1, start, map_size, + MAIR_IDX_NORMAL_NC, PERM_RX_EL0); +} + +static void mmu_map_iboot_handoff_nc(void) +{ + u64 start, size; + + if (mmu_get_iboot_handoff_range(&start, &size) < 0) + return; + + mmu_map_nc_aliases(start, size, "/chosen/iboot-handoff"); + +} + static void mmu_add_default_mappings(void) { ram_base = ALIGN_DOWN(cur_boot_args.phys_base, BIT(32)); uint64_t ram_size = cur_boot_args.mem_size + cur_boot_args.phys_base - ram_base; ram_size = ALIGN_DOWN(ram_size, PAGE_SIZE); + u64 handoff_start = 0, handoff_size = 0; + mmu_get_iboot_handoff_range(&handoff_start, &handoff_size); printf("MMU: RAM base: 0x%lx\n", ram_base); printf("MMU: Top of normal RAM: 0x%lx\n", ram_base + ram_size); @@ -445,11 +537,25 @@ static void mmu_add_default_mappings(void) * With SPRR enabled, this becomes RW. * This range includes all real RAM, including carveouts */ - mmu_add_mapping(ram_base, ram_base, mem_size_actual, MAIR_IDX_NORMAL, PERM_RWX); + mmu_add_ram_mapping_handoff_nc(ram_base, ram_base, mem_size_actual, + MAIR_IDX_NORMAL, PERM_RWX, + handoff_start, handoff_size); /* Unmap carveout regions */ mcc_unmap_carveouts(); + /* + * Re-map the boot framebuffer (/vram) Normal-NC. The Normal-WB identity map + * lets m1n1/iBoot cacheable accesses allocate AMCC SLC directory tags + * for /vram; after handoff XNU's non-coherent framebuffer writes hit those + * stale tags and the AMCC raises UNEXP_RT_HIT_DIR -> PEH panic. + */ + { + u64 fb_pa = 0, fb_size = 0; + if (display_get_vram(&fb_pa, &fb_size) == 0 && fb_pa && fb_size) + mmu_map_framebuffer(fb_pa, fb_size); + } + /* * Remap m1n1 executable code as RX. */ @@ -466,19 +572,32 @@ static void mmu_add_default_mappings(void) * read/writable/exec by EL0 (but not executable by EL1) * With SPRR enabled, this becomes RX_EL0. */ - mmu_add_mapping(ram_base | REGION_RWX_EL0, ram_base, ram_size, MAIR_IDX_NORMAL, PERM_RWX_EL0); + mmu_add_ram_mapping_handoff_nc(ram_base | REGION_RWX_EL0, ram_base, + ram_size, MAIR_IDX_NORMAL, PERM_RWX_EL0, + handoff_start, handoff_size); /* * Create mapping for RAM from 0x98_0000_0000, * read/writable by EL0 (but not executable by EL1) * With SPRR enabled, this becomes RW_EL0. */ - mmu_add_mapping(ram_base | REGION_RW_EL0, ram_base, ram_size, MAIR_IDX_NORMAL, PERM_RW_EL0); + mmu_add_ram_mapping_handoff_nc(ram_base | REGION_RW_EL0, ram_base, + ram_size, MAIR_IDX_NORMAL, PERM_RW_EL0, + handoff_start, handoff_size); /* * Create mapping for RAM from 0xa8_0000_0000, * read/executable by EL1 * This allows executing from dynamic regions in EL1 */ - mmu_add_mapping(ram_base | REGION_RX_EL1, ram_base, ram_size, MAIR_IDX_NORMAL, PERM_RX_EL0); + mmu_add_ram_mapping_handoff_nc(ram_base | REGION_RX_EL1, ram_base, + ram_size, MAIR_IDX_NORMAL, PERM_RX_EL0, + handoff_start, handoff_size); + + /* + * The iBoot handoff page is a protected low carveout on T8140 + * (/chosen/iboot-handoff, region-id-1). Keep m1n1's aliases uncached so + * we do not add any new WB state for this page. + */ + mmu_map_iboot_handoff_nc(); /* * Create four separate full mappings of MMIO space, with different access types diff --git a/src/proxy.c b/src/proxy.c index 0c1501662..a30e60e44 100644 --- a/src/proxy.c +++ b/src/proxy.c @@ -614,7 +614,7 @@ int proxy_process(ProxyRequest *request, ProxyReply *reply) reply->retval = dapf_init_all(); break; case P_DAPF_INIT: - reply->retval = dapf_init((const char *)request->args[0], 1); + reply->retval = dapf_init((const char *)request->args[0], request->args[1]); break; case P_CPUFREQ_INIT: diff --git a/src/sptm/README.md b/src/sptm/README.md new file mode 100644 index 000000000..eb3228073 --- /dev/null +++ b/src/sptm/README.md @@ -0,0 +1,7 @@ +# SPTM Emulator + +This directory is tainted bringup code. Keep reverse-engineered SPTM endpoint +behavior here while the generic hypervisor and loader plumbing stay separate. + +The emulator is linked into the streamed stage2 image when `SPTM_EMUL=1`; the +stage1 boot object does not include it. diff --git a/src/sptm/dart.c b/src/sptm/dart.c new file mode 100644 index 000000000..38f9b57f2 --- /dev/null +++ b/src/sptm/dart.c @@ -0,0 +1,2348 @@ +/* SPDX-License-Identifier: MIT */ +#include "private.h" + +int debug_printf(const char *fmt, ...); + +static inline bool_t dart_validate_xnu_object(u64 obj, u32 *gapf_count_out, + u32 *inst_count_out, + u64 *state_out) { + u32 mode, gapf_count, inst_count; + u64 state; + + if (!is_kernel_va(obj)) + return 0; + if (!guest_read32(obj + DART_OBJ_MODE_OFF, &mode) || mode != 2) + return 0; + if (!guest_read32(obj + DART_OBJ_GAPF_COUNT_OFF, &gapf_count)) + return 0; + if (gapf_count == 0 || gapf_count > 128) + return 0; + if (!guest_read32(obj + DART_OBJ_INST_COUNT_OFF, &inst_count)) + return 0; + if (inst_count == 0 || inst_count > 16) + return 0; + if (!guest_read64(obj + DART_OBJ_STATE_OFF, &state) || !is_kernel_va(state)) + return 0; + + *gapf_count_out = gapf_count; + *inst_count_out = inst_count; + *state_out = state; + return 1; +} + +static inline void dart_record_xnu_object(u32 table, u32 endpoint, u64 dart_id, + u64 obj, u64 source, + u32 gapf_count, u32 inst_count, + u64 state) { + u32 slot = DART_OBJ_CACHE_SLOTS; + + for (u32 i = 0; i < DART_OBJ_CACHE_SLOTS; i++) { + if (g_state.dart_obj_cache_va[i] != 0 && + g_state.dart_obj_cache_id[i] == dart_id) { + slot = i; + break; + } + } + if (slot == DART_OBJ_CACHE_SLOTS) { + slot = (u32)(g_state.dart_obj_cache_idx & (DART_OBJ_CACHE_SLOTS - 1)); + g_state.dart_obj_cache_idx++; + } + + g_state.dart_obj_cache_id[slot] = dart_id; + g_state.dart_obj_cache_va[slot] = obj; + g_state.dart_obj_last[0] = table; + g_state.dart_obj_last[1] = endpoint; + g_state.dart_obj_last[2] = dart_id; + g_state.dart_obj_last[3] = obj; + g_state.dart_obj_last[4] = source; + g_state.dart_obj_last[5] = gapf_count; + g_state.dart_obj_last[6] = inst_count; + g_state.dart_obj_last[7] = state; + for (u32 i = 0; i < 16; i++) { + g_state.dart_obj_last_base_va[i] = 0; + g_state.dart_obj_last_base_pa[i] = 0; + } + for (u32 i = 0; i < inst_count && i < 16; i++) { + u64 mmio_va, mmio_pa; + if (!guest_read64(state + 0x20UL + (u64)i * 0x70UL, &mmio_va)) + continue; + g_state.dart_obj_last_base_va[i] = mmio_va; + if (is_kernel_va(mmio_va) && guest_va_to_pa(mmio_va, 1, &mmio_pa)) { + g_state.dart_obj_last_base_pa[i] = mmio_pa; + } else if (mmio_va >= 0x100000000UL && + (mmio_va & (DART_C_PAGE_SIZE - 1)) == 0) { + g_state.dart_obj_last_base_pa[i] = mmio_va; + } + } + g_state.dart_obj_cache_hits++; +} + +static inline bool_t dart_try_candidate(u32 table, u32 endpoint, u64 dart_id, + u64 candidate, u64 source) { + u32 gapf_count, inst_count; + u64 state; + + if (!dart_validate_xnu_object(candidate, &gapf_count, &inst_count, &state)) + return 0; + dart_record_xnu_object(table, endpoint, dart_id, candidate, source, + gapf_count, inst_count, state); + return 1; +} + +void dart_cache_xnu_object_from_regs(u32 table, u32 endpoint, + u64 *regs) { + u64 dart_id = regs[0]; + + if (table != SPTM_TABLE_T8110_DART_XNU && + table != SPTM_TABLE_T8110_DART_SK && + table != SPTM_TABLE_GEN3_DART_XNU && + table != SPTM_TABLE_GEN3_DART_SK) + return; + + /* Endpoint 6 (init) also needs the recovered AppleT8110DART object so + * Python can import locked live TTBR roots without doing UART-backed + * object/frame probing. Map/unmap endpoints still stay out of this path. */ + if (endpoint != 4 && endpoint != 5 && endpoint != 6) + return; + + if (dart_try_candidate(table, endpoint, dart_id, regs[19], 0x19)) + return; + + u64 fp = regs[29]; + for (u32 depth = 0; depth < 16; depth++) { + u64 saved_fp, saved_lr, candidate; + + if (!is_kernel_va(fp)) + break; + if (!guest_read64(fp, &saved_fp) || !guest_read64(fp + 8, &saved_lr)) + break; + + if (guest_read64(fp - 8, &candidate) && + dart_try_candidate(table, endpoint, dart_id, candidate, + 0x100000UL | ((u64)depth << 8) | 19)) + return; + if (guest_read64(fp - 16, &candidate) && + dart_try_candidate(table, endpoint, dart_id, candidate, + 0x100000UL | ((u64)depth << 8) | 20)) + return; + if (guest_read64(fp - 24, &candidate) && + dart_try_candidate(table, endpoint, dart_id, candidate, + 0x100000UL | ((u64)depth << 8) | 21)) + return; + if (guest_read64(fp - 32, &candidate) && + dart_try_candidate(table, endpoint, dart_id, candidate, + 0x100000UL | ((u64)depth << 8) | 22)) + return; + + if (saved_fp == fp) + break; + fp = saved_fp; + } + + g_state.dart_obj_cache_misses++; + g_state.dart_obj_last[0] = table; + g_state.dart_obj_last[1] = endpoint; + g_state.dart_obj_last[2] = dart_id; + g_state.dart_obj_last[3] = 0; + g_state.dart_obj_last[4] = 0; + g_state.dart_obj_last[5] = 0; + g_state.dart_obj_last[6] = 0; + g_state.dart_obj_last[7] = 0; +} + +static inline bool_t dart_c_looks_like_mmio(u64 pa) { + return !is_kernel_va(pa) && + pa >= 0x100000000UL && + (pa & (DART_C_PAGE_SIZE - 1)) == 0; +} + +static inline u32 dart_c_read32(u64 addr) { + return *(volatile u32 *)addr; +} + +static inline void dart_c_write32(u64 addr, u32 value) { + *(volatile u32 *)addr = value; +} + +static inline u64 dart_c_pte_from_pa(u64 paddr) { + return (((paddr >> 14) & 0x0fffffffUL) << 10) | 1UL; +} + +static inline u64 dart_c_pa_from_pte(u64 pte) { + return ((pte >> 10) & 0x0fffffffUL) << 14; +} + +static inline u32 dart_c_ttbr_from_pa(u64 paddr) { + return (u32)((((paddr >> 14) << 2) & 0xfffffffcU) | DART_T8110_TTBR_VALID); +} + +static inline u64 dart_c_pa_from_ttbr(u32 ttbr) { + return (u64)((ttbr & 0xfffffffcU) >> 2) << 14; +} + +static inline u32 dart_c_idx(u64 dva, u32 level) { + u64 page = dva >> 14; + if (level == 0) return (u32)((page >> 33) & 0x7ff); + if (level == 1) return (u32)((page >> 22) & 0x7ff); + if (level == 2) return (u32)((page >> 11) & 0x7ff); + return (u32)(page & 0x7ff); +} + +static inline u64 dart_c_effective_dva(struct dart_c_instance *inst, u32 sid, + struct dart_c_sid_state *st, u64 dva) { + (void)inst; + (void)sid; + (void)st; + + /* + * The firmware validates the full DVA, but a non-FOUR_LEVELS stream starts + * its TTBR root at the DVA[25:35] table. DVA[36:46] is not part of the page + * table walk for that mode; root_level encodes that below. + */ + return dva; +} + +static inline u8 dart_c_root_level_from_tcr(u32 tcr) { + return (tcr & DART_T8110_TCR_FOUR_LEVEL) ? 1 : 2; +} + +static inline u32 dart_c_tcr_from_root_level(u8 root_level) { + u32 tcr = DART_T8110_TCR_TRANSLATE_EN; + if (root_level <= 1) + tcr |= DART_T8110_TCR_FOUR_LEVEL; + return tcr; +} + +static inline void dart_c_clean64(u64 addr) { + clean_pt_range_poc((volatile u64 *)addr, 8); +} + +static inline u64 dart_c_leaf_pte(u64 paddr, u64 dva, u64 bytes, u64 bits) { + u64 start = (dva & (DART_C_PAGE_SIZE - 1)) >> 2; + u64 end = ((dva & (DART_C_PAGE_SIZE - 1)) + bytes - 1) >> 2; + if (end > 0xfff) + end = 0xfff; + return dart_c_pte_from_pa(paddr) | + (bits & 0xfUL) | + (start << 52) | + (end << 40); +} + +static inline u64 dart_c_perm_bits(u64 flags) { + /* + * 26.6b2 SPTM no longer uses the older 4-bit reversal here. The valid bit + * is supplied by dart_c_pte_from_pa(); only prot bits 0/1 contribute. + */ + return ((flags << 1) & 0x4UL) | ((flags & 1UL) << 3); +} + +static inline u64 dart_c_leaf_perm_bits(struct dart_c_instance *inst, + u64 paddr, u64 flags) { + (void)paddr; + + /* + * 26.6 SPTM records relaxed-rw-protections and then consults an internal + * frame-type policy table before changing PTE permission bits. Keep the + * flag in state, but leave PTE permissions unchanged until that policy is + * recovered instead of guessing a write-protect set. + */ + (void)inst->relaxed_rw_protections; + return dart_c_perm_bits(flags); +} + +static inline bool_t dart_c_map_needs_tlbi(struct dart_c_instance *inst, + u64 old_pte, u64 new_pte) { + if (old_pte == new_pte) + return 0; + if (!inst->avoid_tlbi_in_map) + return 1; + return (old_pte & 1UL) != 0; +} + +static inline u64 dart_c_validate_dva_range(u64 dva, u64 size) { + if (!size) + return SPTM_SUCCESS; + if (size > 0x2000000UL) + return DART_C_STATUS_VIOLATION; + if (dva >> 42) + return DART_C_STATUS_VIOLATION; + if ((size - 1UL) > (((1UL << 42) - 1UL) - dva)) + return DART_C_STATUS_VIOLATION; + + u64 pages = ((dva & (DART_C_PAGE_SIZE - 1UL)) + size + + DART_C_PAGE_SIZE - 1UL) / DART_C_PAGE_SIZE; + if (pages > 0x800UL) + return DART_C_STATUS_VIOLATION; + return SPTM_SUCCESS; +} + +static inline struct dart_c_instance *dart_c_get(u64 dart_id) { + u32 empty = DART_C_MAX_DARTS; + + for (u32 i = 0; i < DART_C_MAX_DARTS; i++) { + if (g_state.dart.inst[i].used && g_state.dart.inst[i].dart_id == dart_id) + return &g_state.dart.inst[i]; + if (!g_state.dart.inst[i].used && empty == DART_C_MAX_DARTS) + empty = i; + } + + if (empty == DART_C_MAX_DARTS) + empty = (u32)(dart_id & (DART_C_MAX_DARTS - 1)); + + struct dart_c_instance *inst = &g_state.dart.inst[empty]; + for (u32 i = 0; i < sizeof(*inst); i++) + ((volatile u8 *)inst)[i] = 0; + inst->dart_id = dart_id; + inst->used = 1; + return inst; +} + +static inline struct dart_c_sid_state *dart_c_sid(struct dart_c_instance *inst, + u64 sid) { + if (sid >= DART_C_MAX_SIDS) + return (struct dart_c_sid_state *)0; + struct dart_c_sid_state *st = &inst->sid[sid]; + st->valid = 1; + return st; +} + +static inline struct dart_c_sid_state * +dart_c_sid_lookup(struct dart_c_instance *inst, u64 sid) { + if (sid >= DART_C_MAX_SIDS) + return (struct dart_c_sid_state *)0; + struct dart_c_sid_state *st = &inst->sid[sid]; + return st->valid ? st : (struct dart_c_sid_state *)0; +} + +static inline bool_t dart_c_sid_exclave(struct dart_c_sid_state *st) { + return st && st->exclave; +} + +static inline bool_t dart_c_sid_translates(struct dart_c_sid_state *st) { + return st && ((st->tcr & DART_T8110_TCR_TRANSLATE_EN) != 0); +} + +static inline u32 dart_c_num_streams(u64 base) { + u32 num = dart_c_read32(base + DART_T8110_PARAMS4) & 0x1ffU; + if (num == 0 || num > DART_C_MAX_STREAMS) + num = DART_C_MAX_STREAMS; + return num; +} + +static inline u32 dart_c_stream_words(u32 num) { + return (num + 31U) >> 5; +} + +static inline u64 dart_c_first_mmio_base(struct dart_c_instance *inst) { + for (u32 i = 0; i < inst->base_count && i < DART_C_MAX_BASES; i++) { + u64 base = inst->base_pa[i]; + if (dart_c_looks_like_mmio(base)) + return base; + } + return 0; +} + +static inline u64 dart_c_pack_hw_sid_regs(struct dart_c_instance *inst, + u32 sid) { + u64 base = dart_c_first_mmio_base(inst); + if (!base || sid >= DART_C_MAX_SIDS) + return 0; + return ((u64)dart_c_read32(base + DART_T8110_TCR + sid * 4) << 32) | + dart_c_read32(base + DART_T8110_TTBR + sid * 4); +} + +static inline u64 dart_c_pack_hw_stream_error(struct dart_c_instance *inst, + u32 sid) { + u64 base = dart_c_first_mmio_base(inst); + if (!base) + return 0; + u32 stream_word = 0; + u32 num = dart_c_num_streams(base); + if (sid < num) { + stream_word = dart_c_read32(base + DART_T8110_ENABLE_STREAMS + + ((sid >> 5) * 4)); + } + return ((u64)dart_c_read32(base + DART_T8110_ERROR) << 32) | stream_word; +} + +static inline bool_t dart_c_base_needs_new_tlb_cmd(u64 base) { + return (dart_c_read32(base + DART_T8110_PARAMS3) & 0xffffU) >= 0x0202U; +} + +static inline u64 dart_c_tlb_raw_command(u64 base, u32 command) { + dart_c_write32(base + DART_T8110_TLB_CMD, command); + for (u32 i = 0; i < 100; i++) { + u32 status = dart_c_read32(base + DART_T8110_TLB_CMD); + if (!(status & DART_T8110_TLB_CMD_BUSY)) + return ((u64)status << 32); + } + return DART_C_ERR_TIMEOUT | ((u64)dart_c_read32(base + DART_T8110_TLB_CMD) << 32); +} + +static inline u64 dart_c_tlb_command(u64 base, u32 op, u32 sid) { + u32 command = ((op & 7U) << 8) | (sid & 0xffU); + + if (op == DART_T8110_TLB_OP_FLUSH_SID && + dart_c_base_needs_new_tlb_cmd(base)) + command |= DART_T8110_TLB_CMD_NEW_DART; + + return dart_c_tlb_raw_command(base, command); +} + +static inline u64 dart_c_flush_sid_base(u64 base, u32 sid) { + return dart_c_tlb_command(base, DART_T8110_TLB_OP_FLUSH_SID, sid); +} + +static inline u64 dart_c_flush_sid_range_base(u64 base, u32 sid, + u64 start_dva, u64 end_dva) { + u32 start_page = (u32)((start_dva >> 14) & 0x0fffffffU); + u32 end_page = (u32)((end_dva >> 14) & 0x0fffffffU); + u32 command = DART_T8110_TLB_CMD_VA_RANGE | + DART_T8110_TLB_CMD_STT_FLUSH | + ((DART_T8110_TLB_OP_FLUSH_SID & 7U) << 8) | + (sid & 0xffU); + + if (dart_c_base_needs_new_tlb_cmd(base)) + command |= DART_T8110_TLB_CMD_NEW_DART; + + dart_c_write32(base + DART_T8110_TLB_START_DVA_PAGE, start_page << 2); + __asm__ volatile("dsb sy" ::: "memory"); + dart_c_write32(base + DART_T8110_TLB_END_DVA_PAGE, end_page << 2); + __asm__ volatile("dsb sy" ::: "memory"); + return dart_c_tlb_raw_command(base, command); +} + +static inline void dart_c_clear_errors_base(u64 base) { + u32 num = dart_c_num_streams(base); + u32 words = dart_c_stream_words(num); + u32 error = dart_c_read32(base + DART_T8110_ERROR); + dart_c_write32(base + DART_T8110_ERROR, error); + for (u32 i = 0; i < words; i++) + dart_c_write32(base + DART_T8110_ERROR_STREAMS + i * 4, 0xffffffffU); + dart_c_write32(base + DART_T8110_ERROR_MASK, 0xffffffffU); +} + +static inline u32 dart_c_import_live_base(struct dart_c_instance *inst, u64 base, + bool_t *preserve_out) { + u32 num = dart_c_num_streams(base); + if (num > DART_C_MAX_SIDS) + num = DART_C_MAX_SIDS; + + u32 protect = dart_c_read32(base + DART_T8110_PROTECT); + u32 protect_lock = dart_c_read32(base + DART_T8110_PROTECT_LOCK); + bool_t preserve = ((protect | protect_lock) & DART_T8110_PROTECT_TTBR_TCR) ? 1 : 0; + u32 enable_words[(DART_C_MAX_STREAMS + 31U) >> 5]; + u32 words = dart_c_stream_words(num); + if (words > ((DART_C_MAX_STREAMS + 31U) >> 5)) + words = ((DART_C_MAX_STREAMS + 31U) >> 5); + + for (u32 i = 0; i < words; i++) + enable_words[i] = dart_c_read32(base + DART_T8110_ENABLE_STREAMS + i * 4); + + for (u32 sid = 0; sid < num; sid++) { + u32 tcr = dart_c_read32(base + DART_T8110_TCR + sid * 4); + u32 ttbr = dart_c_read32(base + DART_T8110_TTBR + sid * 4); + if (ttbr & DART_T8110_TTBR_VALID) + preserve = 1; + if (tcr & DART_T8110_TCR_REMAP_EN) + preserve = 1; + if ((tcr & DART_T8110_TCR_TRANSLATE_EN) && ttbr) + preserve = 1; + } + + *preserve_out = preserve; + if (!preserve) + return 0; + + inst->locked = 1; + u32 imported = 0; + for (u32 sid = 0; sid < num; sid++) { + u32 tcr = dart_c_read32(base + DART_T8110_TCR + sid * 4); + u32 ttbr = dart_c_read32(base + DART_T8110_TTBR + sid * 4); + u32 word = (sid >> 5) < words ? enable_words[sid >> 5] : 0; + u8 stream_enabled = (word & (1U << (sid & 31))) ? 1 : 0; + if (!tcr && !ttbr && !stream_enabled) + continue; + if ((tcr & DART_T8110_TCR_TRANSLATE_EN) && + !(ttbr & DART_T8110_TTBR_VALID) && !ttbr) { + /* + * Several T8110-class DART register files contain non-reset-looking + * stale values in unused SID slots at raw boot. Treating TE with a + * non-valid TTBR as live state later replays a translated stream + * with no root, which is not a coherent DART configuration. + */ + if (!stream_enabled) + continue; + } + + struct dart_c_sid_state *st = dart_c_sid_lookup(inst, sid); + if (dart_c_sid_exclave(st)) { + if (stream_enabled) + st->stream_enabled = 1; + imported++; + continue; + } + if (st && (st->tcr & (DART_T8110_TCR_REMAP_EN | + DART_T8110_TCR_BYPASS_DART | + DART_T8110_TCR_BYPASS_DAPF))) { + imported++; + continue; + } + if (st && st->root_valid && !(ttbr & DART_T8110_TTBR_VALID)) { + if (stream_enabled) + st->stream_enabled = 1; + imported++; + continue; + } + st = dart_c_sid(inst, sid); + if (!st) + continue; + st->tcr = tcr; + st->ttbr = ttbr; + /* + * SPTM keeps ADT initial streams as state and replays them during + * finalize. A pre-finalize ENABLE_STREAMS read may still be clear, so + * live import may set this bit but must not erase seeded state. + */ + if (stream_enabled) + st->stream_enabled = 1; + st->translation_enabled = (tcr & DART_T8110_TCR_TRANSLATE_EN) ? 1 : 0; + st->root_level = dart_c_root_level_from_tcr(tcr); + if (ttbr & DART_T8110_TTBR_VALID) { + st->root_valid = 1; + st->root_pt_pa = dart_c_pa_from_ttbr(ttbr); + st->raw_replay = 0; + } else { + st->root_valid = 0; + st->root_pt_pa = 0; + st->raw_replay = + ((tcr & DART_T8110_TCR_TRANSLATE_EN) && ttbr) ? 1 : 0; + } + imported++; + } + + inst->last[10] = imported; + inst->last[11] = num; + inst->last[12] = protect; + inst->last[13] = protect_lock; + g_state.dart.last[10] = imported; + g_state.dart.last[11] = num; + g_state.dart.last[12] = protect; + g_state.dart.last[13] = protect_lock; + return imported; +} + +static inline bool_t dart_c_has_seeded_live_sid(struct dart_c_instance *inst) { + for (u32 sid = 0; sid < DART_C_MAX_SIDS; sid++) { + struct dart_c_sid_state *st = &inst->sid[sid]; + if (!st->valid) + continue; + if (st->ttbr || st->root_valid || st->stream_enabled) + return 1; + if (st->tcr && !(st->tcr & DART_T8110_TCR_REMAP_EN)) + return 1; + } + return 0; +} + +static inline void dart_c_for_each_base_flush_sid(struct dart_c_instance *inst, + u32 sid) { + for (u32 i = 0; i < inst->base_count && i < DART_C_MAX_BASES; i++) { + if (dart_c_looks_like_mmio(inst->base_pa[i])) + dart_c_flush_sid_base(inst->base_pa[i], sid); + } +} + +static inline void dart_c_for_each_base_flush_sid_range(struct dart_c_instance *inst, + u32 sid, + u64 start_dva, + u64 end_dva) { + for (u32 i = 0; i < inst->base_count && i < DART_C_MAX_BASES; i++) { + if (dart_c_looks_like_mmio(inst->base_pa[i])) + dart_c_flush_sid_range_base(inst->base_pa[i], sid, start_dva, + end_dva); + } +} + +static inline void dart_c_for_each_base_flush_map_change(struct dart_c_instance *inst, + u32 sid, + u64 start_dva, + u64 end_dva) { + if (inst->flush_by_dva) + dart_c_for_each_base_flush_sid_range(inst, sid, start_dva, end_dva); + else + dart_c_for_each_base_flush_sid(inst, sid); +} + +static inline void dart_c_flush_known_sids_base(struct dart_c_instance *inst, + u64 base) { + u32 touched[DART_C_MAX_SIDS / 32]; + + if (!dart_c_looks_like_mmio(base)) + return; + + for (u32 i = 0; i < DART_C_MAX_SIDS / 32; i++) + touched[i] = 0; + + u32 num = dart_c_num_streams(base); + if (num > DART_C_MAX_SIDS) + num = DART_C_MAX_SIDS; + u32 words = dart_c_stream_words(num); + if (words > DART_C_MAX_SIDS / 32) + words = DART_C_MAX_SIDS / 32; + + for (u32 word = 0; word < words; word++) { + u32 enabled = dart_c_read32(base + DART_T8110_ENABLE_STREAMS + + (u64)word * 4UL); + touched[word] |= enabled; + } + + for (u32 sid = 0; sid < DART_C_MAX_SIDS; sid++) { + struct dart_c_sid_state *st = &inst->sid[sid]; + if (!st->valid && !st->root_valid && !st->stream_enabled && + !st->ttbr) + continue; + touched[sid >> 5] |= 1U << (sid & 31); + } + + for (u32 sid = 0; sid < num; sid++) + if (touched[sid >> 5] & (1U << (sid & 31))) + dart_c_flush_sid_base(base, sid); +} + +static inline void dart_c_add_base(struct dart_c_instance *inst, u64 base) { + if (!dart_c_looks_like_mmio(base)) + return; + for (u32 i = 0; i < inst->base_count && i < DART_C_MAX_BASES; i++) { + if (inst->base_pa[i] == base) + return; + } + if (inst->base_count < DART_C_MAX_BASES) { + inst->base_pa[inst->base_count++] = base; + /* + * Real SPTM maps through an already-recovered DART object and flushes + * after leaf updates. If our object recovery learns the MMIO base after + * a software PTE update, catch the hardware up once the base is known. + */ + dart_c_flush_known_sids_base(inst, base); + } +} + +static inline u64 dart_c_range_sid(struct dart_c_identity_range *range) { + return range->sid & DART_C_RANGE_SID_MASK; +} + +static inline u64 dart_c_range_flags(struct dart_c_identity_range *range) { + return range->sid & ~DART_C_RANGE_SID_MASK; +} + +static inline bool_t dart_c_range_matches_inst(struct dart_c_instance *inst, + struct dart_c_identity_range *range) { + for (u32 bi = 0; bi < inst->base_count && bi < DART_C_MAX_BASES; bi++) { + if (inst->base_pa[bi] == range->base_pa) + return 1; + } + return 0; +} + +static inline bool_t dart_c_restore_boot_premap_page(struct dart_c_instance *inst, + u32 sid, u64 dva, + u64 slot) { + u64 page = dva & ~(DART_C_PAGE_SIZE - 1UL); + + for (u32 ri = 0; ri < g_state.dart.identity_count && + ri < DART_C_MAX_IDENTITY_RANGES; ri++) { + struct dart_c_identity_range *range = &g_state.dart.identity[ri]; + u64 flags = dart_c_range_flags(range); + u64 start, end; + + if (!(flags & DART_C_RANGE_FLAG_BOOT_PREMAP) || + (flags & DART_C_RANGE_FLAG_REMAP) || + !dart_c_range_matches_inst(inst, range) || + dart_c_range_sid(range) != sid || + range->end <= range->start) + continue; + + start = range->start & ~(DART_C_PAGE_SIZE - 1UL); + end = (range->end + DART_C_PAGE_SIZE - 1UL) & + ~(DART_C_PAGE_SIZE - 1UL); + if (page < start || page >= end) + continue; + + *(volatile u64 *)slot = + dart_c_leaf_pte(page, page, DART_C_PAGE_SIZE, 0); + dart_c_clean64(slot); + return 1; + } + + return 0; +} + +static inline void dart_c_program_sid_tcr_ttbr(struct dart_c_instance *inst, + u32 sid, u32 tcr, u32 ttbr) { + u32 writes = 0; + for (u32 i = 0; i < inst->base_count && i < DART_C_MAX_BASES; i++) { + u64 base = inst->base_pa[i]; + if (!dart_c_looks_like_mmio(base)) + continue; + u64 tcr_reg = base + DART_T8110_TCR + sid * 4; + u64 ttbr_reg = base + DART_T8110_TTBR + sid * 4; + if (dart_c_read32(tcr_reg) != tcr) { + dart_c_write32(tcr_reg, tcr); + writes++; + } + if (dart_c_read32(ttbr_reg) != ttbr) { + dart_c_write32(ttbr_reg, ttbr); + writes++; + } + } + if (writes) { + __asm__ volatile("dsb sy" ::: "memory"); + dart_c_for_each_base_flush_sid(inst, sid); + } +} + +static inline void dart_c_configure_sid_remap(struct dart_c_instance *inst, + u32 sid, u32 target) { + struct dart_c_sid_state *st = dart_c_sid_lookup(inst, sid); + if (!st) + return; + + st->tcr = DART_T8110_TCR_REMAP_EN | DART_T8110_TCR_REMAP_TARGET(target); + st->ttbr = 0; + st->root_pt_pa = 0; + st->root_valid = 0; + st->root_level = 2; + st->translation_enabled = 0; + st->raw_replay = 0; + dart_c_program_sid_tcr_ttbr(inst, sid, st->tcr, 0); +} + +static inline u64 dart_c_walk_leaf_slot(struct dart_c_instance *inst, u32 sid, + struct dart_c_sid_state *st, u64 dva, + u64 *slot_out); +static inline u64 dart_c_ensure_leaf_table(struct dart_c_instance *inst, u32 sid, + struct dart_c_sid_state *st, u64 dva, + u64 *table_out, u64 *allocs_out, + u64 *imports_out); + +static inline bool_t dart_c_import_live_sid(struct dart_c_instance *inst, u32 sid) { + if (sid >= DART_C_MAX_SIDS) + return 0; + + for (u32 i = 0; i < inst->base_count && i < DART_C_MAX_BASES; i++) { + u64 base = inst->base_pa[i]; + if (!dart_c_looks_like_mmio(base)) + continue; + + u32 tcr = dart_c_read32(base + DART_T8110_TCR + sid * 4); + u32 ttbr = dart_c_read32(base + DART_T8110_TTBR + sid * 4); + if (!(ttbr & DART_T8110_TTBR_VALID) && + !(ttbr && (tcr & DART_T8110_TCR_TRANSLATE_EN))) + continue; + + struct dart_c_sid_state *st = dart_c_sid(inst, sid); + if (!st) + return 0; + + u32 enable_word = dart_c_read32(base + DART_T8110_ENABLE_STREAMS + + ((sid >> 5) * 4)); + struct dart_c_sid_state *existing = dart_c_sid_lookup(inst, sid); + if (dart_c_sid_exclave(existing)) { + if (enable_word & (1U << (sid & 31))) + existing->stream_enabled = 1; + return 1; + } + + st->tcr = tcr; + st->ttbr = ttbr; + st->valid = 1; + st->root_level = dart_c_root_level_from_tcr(tcr); + if (enable_word & (1U << (sid & 31))) + st->stream_enabled = 1; + st->translation_enabled = (tcr & DART_T8110_TCR_TRANSLATE_EN) ? 1 : 0; + if (ttbr & DART_T8110_TTBR_VALID) { + st->root_valid = 1; + st->root_pt_pa = dart_c_pa_from_ttbr(ttbr); + st->raw_replay = 0; + } else { + st->root_valid = 0; + st->root_pt_pa = 0; + st->raw_replay = + ((tcr & DART_T8110_TCR_TRANSLATE_EN) && ttbr) ? 1 : 0; + } + inst->locked = 1; + return 1; + } + + return 0; +} + +static inline bool_t dart_c_pt_region_page_referenced(u64 start, u64 end, + u64 pa) { + start &= ~(DART_C_PAGE_SIZE - 1UL); + end = (end + DART_C_PAGE_SIZE - 1UL) & ~(DART_C_PAGE_SIZE - 1UL); + for (u64 table = start; table < end; table += DART_C_PAGE_SIZE) { + volatile u64 *entries = (volatile u64 *)table; + for (u32 i = 0; i < DART_C_TT_ENTRIES; i++) { + u64 pte = entries[i]; + if ((pte & 1UL) && dart_c_pa_from_pte(pte) == pa) + return 1; + } + } + return 0; +} + +static inline u64 dart_c_alloc_pt_region_table(struct dart_c_instance *inst, + u32 sid, + struct dart_c_sid_state *st) { + if (!inst || !st) + return 0; + + for (u32 ri = 0; ri < g_state.dart.identity_count && + ri < DART_C_MAX_IDENTITY_RANGES; ri++) { + struct dart_c_identity_range *range = &g_state.dart.identity[ri]; + u64 flags = dart_c_range_flags(range); + if (!(flags & DART_C_RANGE_FLAG_PT_REGION) || + !dart_c_range_matches_inst(inst, range) || + dart_c_range_sid(range) != sid || + range->end <= range->start) + continue; + + u64 scan_start = st->root_pt_pa; + if (scan_start < range->start || scan_start >= range->end) + scan_start = range->start; + scan_start &= ~(DART_C_PAGE_SIZE - 1UL); + + u64 cur = (range->start + DART_C_PAGE_SIZE - 1UL) & + ~(DART_C_PAGE_SIZE - 1UL); + u64 end = range->end & ~(DART_C_PAGE_SIZE - 1UL); + while (cur < end) { + if (cur == st->root_pt_pa || + dart_c_pt_region_page_referenced(scan_start, range->end, cur)) { + cur += DART_C_PAGE_SIZE; + continue; + } + + volatile u64 *entries = (volatile u64 *)cur; + for (u32 i = 0; i < DART_C_TT_ENTRIES; i++) + entries[i] = 0; + clean_range_poc((void *)cur, DART_C_PAGE_SIZE); + sptm_coproc_cache_maint_range((const void *)cur, DART_C_PAGE_SIZE); + g_state.dart.pt_pool_allocs++; + return cur; + } + } + + g_state.dart.pt_pool_failures++; + return 0; +} + +static inline bool_t dart_c_pa_in_pt_region(struct dart_c_instance *inst, + u32 sid, u64 pa) { + pa &= ~(DART_C_PAGE_SIZE - 1UL); + for (u32 ri = 0; ri < g_state.dart.identity_count && + ri < DART_C_MAX_IDENTITY_RANGES; ri++) { + struct dart_c_identity_range *range = &g_state.dart.identity[ri]; + u64 flags = dart_c_range_flags(range); + if (!(flags & DART_C_RANGE_FLAG_PT_REGION) || + !dart_c_range_matches_inst(inst, range) || + dart_c_range_sid(range) != sid) + continue; + if (range->start <= pa && pa < range->end) + return 1; + } + return 0; +} + +static inline u64 dart_c_shadow_child_table_if_needed( + struct dart_c_instance *inst, u32 sid, struct dart_c_sid_state *st, + u64 pt_pa) { + if (!inst || !st || !st->root_valid || !st->root_pt_pa) + return pt_pa; + + if (dart_c_pa_in_pt_region(inst, sid, pt_pa)) + return pt_pa; + + u64 shadow = dart_c_alloc_pt_region_table(inst, sid, st); + if (!shadow) + return pt_pa; + + g_state.dart.last[8] = inst->dart_id; + g_state.dart.last[9] = sid; + g_state.dart.last[10] = pt_pa; + g_state.dart.last[11] = shadow; + g_state.dart.last[12] = 0x53484457UL; /* "SHDW" */ + return shadow; +} + +static inline u32 dart_c_map_identity_range_page(struct dart_c_instance *inst, + u32 sid, u64 dva, u64 flags) { + struct dart_c_sid_state *st = dart_c_sid(inst, sid); + u64 table, slot; + (void)flags; + + if (!st || !st->root_valid || !st->root_pt_pa) + return 0; + + dva &= ~(DART_C_PAGE_SIZE - 1); + + if (dart_c_ensure_leaf_table(inst, sid, st, dva, &table, 0, 0)) + return 0; + + slot = table + (u64)dart_c_idx(dva, 3) * 8UL; + *(volatile u64 *)slot = dart_c_leaf_pte(dva, dva, DART_C_PAGE_SIZE, 0); + dart_c_clean64(slot); + + inst->last[8] = 0x44415046UL; /* "DAPF" */ + inst->last[9] = sid; + inst->last[10] = dva; + inst->last[11] = slot; + inst->last[12] = 0; + inst->last[13] = 0; + g_state.dart.last[8] = inst->dart_id; + g_state.dart.last[9] = sid; + g_state.dart.last[10] = dva; + g_state.dart.last[11] = slot; + return 1; +} + +static inline u32 dart_c_apply_identity_ranges(struct dart_c_instance *inst) { + u32 pages = 0; + + for (u32 ri = 0; ri < g_state.dart.identity_count && + ri < DART_C_MAX_IDENTITY_RANGES; ri++) { + struct dart_c_identity_range *range = &g_state.dart.identity[ri]; + u64 flags = dart_c_range_flags(range); + u64 sid = dart_c_range_sid(range); + u64 flush_start = 0; + u64 flush_end = 0; + if (!(flags & DART_C_RANGE_FLAG_BOOT_PREMAP) || + (flags & DART_C_RANGE_FLAG_REMAP) || + !dart_c_range_matches_inst(inst, range) || + sid >= DART_C_MAX_SIDS || + range->end <= range->start) + continue; + + u64 start = range->start & ~(DART_C_PAGE_SIZE - 1); + u64 end = (range->end + DART_C_PAGE_SIZE - 1) & + ~(DART_C_PAGE_SIZE - 1); + for (u64 dva = start; dva < end && pages < 4096; dva += DART_C_PAGE_SIZE) { + if (dart_c_map_identity_range_page(inst, (u32)sid, dva, flags)) { + if (!flush_end) + flush_start = dva; + flush_end = dva + DART_C_PAGE_SIZE; + pages++; + } + } + + if (flush_end > flush_start) { + /* + * These leaves are written by the emulator into persistent + * bootloader-owned DART roots. Real t8110dart_map flushes hardware + * by DVA range immediately after leaf updates; do the same for + * bootstrap leaves so stale negative display translations cannot + * survive until scanout. + */ + dart_c_for_each_base_flush_map_change(inst, (u32)sid, + flush_start, + flush_end - 1UL); + } + } + + if (pages) { + inst->last[14] = pages; + g_state.dart.last[12] = pages; + g_state.dart.last[13] = g_state.dart.identity_count; + } + return pages; +} + +static inline u32 dart_c_apply_bootstrap_remaps(struct dart_c_instance *inst) { + u32 count = 0; + + for (u32 ri = 0; ri < g_state.dart.identity_count && + ri < DART_C_MAX_IDENTITY_RANGES; ri++) { + struct dart_c_identity_range *range = &g_state.dart.identity[ri]; + if (!(dart_c_range_flags(range) & DART_C_RANGE_FLAG_REMAP)) + continue; + if (!dart_c_range_matches_inst(inst, range)) + continue; + + u32 sid = (u32)dart_c_range_sid(range); + u32 target = (u32)(range->start & DART_C_RANGE_SID_MASK); + if (sid >= DART_C_MAX_SIDS || target >= DART_C_MAX_SIDS) + continue; + dart_c_configure_sid_remap(inst, sid, target); + count++; + } + + if (count) { + inst->last[13] = 0x524d5444UL; /* "DTMR" */ + inst->last[14] = count; + g_state.dart.last[13] = inst->dart_id; + g_state.dart.last[14] = count; + } + return count; +} + +static inline bool_t dart_c_import_object_details(struct dart_c_instance *inst, + u64 obj) { + u32 gapf_count, inst_count; + u64 state; + + if (!dart_validate_xnu_object(obj, &gapf_count, &inst_count, &state)) + return 0; + + inst->obj_va = obj; + inst->state_va = state; + for (u32 i = 0; i < inst_count && i < 16; i++) { + u64 mmio_va, mmio_pa; + if (!guest_read64(state + 0x20UL + (u64)i * 0x70UL, &mmio_va)) + continue; + if (is_kernel_va(mmio_va) && guest_va_to_pa(mmio_va, 1, &mmio_pa)) + dart_c_add_base(inst, mmio_pa); + else if (dart_c_looks_like_mmio(mmio_va)) + dart_c_add_base(inst, mmio_va); + } + return 1; +} + +static inline void dart_c_import_cached_bases(struct dart_c_instance *inst) { + if (g_state.dart_obj_last[2] == inst->dart_id && + g_state.dart_obj_last[3] != 0) { + inst->obj_va = g_state.dart_obj_last[3]; + inst->state_va = g_state.dart_obj_last[7]; + for (u32 i = 0; i < 16; i++) + dart_c_add_base(inst, g_state.dart_obj_last_base_pa[i]); + return; + } + + /* + * Endpoint stack layouts differ enough that a later recovery attempt can + * miss even though an earlier endpoint already validated this DART object. + * Real SPTM keeps a per-DART state object; mirror that by falling back to + * the per-ID cache instead of letting a transient miss erase live state. + */ + for (u32 i = 0; i < DART_OBJ_CACHE_SLOTS; i++) { + if (g_state.dart_obj_cache_id[i] == inst->dart_id && + g_state.dart_obj_cache_va[i] != 0 && + dart_c_import_object_details(inst, g_state.dart_obj_cache_va[i])) + return; + } +} + +static inline u64 dart_c_walk_to_table(struct dart_c_instance *inst, u32 sid, + struct dart_c_sid_state *st, u64 dva, + u32 table_level, u64 *table_out, + u32 *idx_out) { + if (!st || !st->root_valid || !st->root_pt_pa) + return DART_C_ERR_NO_TTBR; + + dva = dart_c_effective_dva(inst, sid, st, dva); + + if (table_level < st->root_level || table_level > 3) + return DART_C_ERR_NO_TABLE; + if (table_level == st->root_level) { + *table_out = st->root_pt_pa; + *idx_out = dart_c_idx(dva, table_level); + return 0; + } + + u64 table = st->root_pt_pa; + for (u32 level = st->root_level; level < table_level; level++) { + u32 idx = dart_c_idx(dva, level); + u64 pte = *(volatile u64 *)(table + (u64)idx * 8UL); + if (!(pte & 1UL)) + return DART_C_ERR_NO_TABLE; + table = dart_c_pa_from_pte(pte); + } + *table_out = table; + *idx_out = dart_c_idx(dva, table_level); + return 0; +} + +static inline u64 dart_c_walk_leaf_slot(struct dart_c_instance *inst, u32 sid, + struct dart_c_sid_state *st, u64 dva, + u64 *slot_out) { + u64 table; + u32 idx; + u64 rc = dart_c_walk_to_table(inst, sid, st, dva, 3, &table, &idx); + if (rc) + return rc; + *slot_out = table + (u64)idx * 8UL; + return 0; +} + +static inline u64 dart_c_ensure_leaf_table(struct dart_c_instance *inst, u32 sid, + struct dart_c_sid_state *st, u64 dva, + u64 *table_out, u64 *allocs_out, + u64 *imports_out) { + if (!st || !st->root_valid || !st->root_pt_pa) + return DART_C_ERR_NO_TTBR; + + dva = dart_c_effective_dva(inst, sid, st, dva); + + u64 table = st->root_pt_pa; + u64 allocs = 0; + u64 imports = 0; + + for (u32 level = st->root_level; level < 3; level++) { + u32 idx = dart_c_idx(dva, level); + u64 slot = table + (u64)idx * 8UL; + u64 pte = *(volatile u64 *)slot; + if (pte & 1UL) { + table = dart_c_pa_from_pte(pte); + imports++; + continue; + } + + u64 child = dart_c_alloc_pt_region_table(inst, sid, st); + if (!child) + return DART_C_ERR_NO_TABLE; + *(volatile u64 *)slot = dart_c_pte_from_pa(child); + dart_c_clean64(slot); + table = child; + allocs++; + } + + *table_out = table; + if (allocs_out) + *allocs_out = allocs; + if (imports_out) + *imports_out = imports; + return 0; +} + +static inline void dart_c_snapshot_leaf(struct dart_c_instance *inst, u64 sid, + u64 dva, u64 *slot_out, + u64 *pte_out) { + *slot_out = 0; + *pte_out = 0; + struct dart_c_sid_state *st = dart_c_sid_lookup(inst, sid); + if (!st) + return; + u64 slot; + if (dart_c_walk_leaf_slot(inst, (u32)sid, st, dva, &slot)) + return; + *slot_out = slot; + *pte_out = *(volatile u64 *)slot; +} + +static inline void dart_c_record_recent(struct dart_c_instance *inst, + u32 table, u32 endpoint, u64 *regs, + u64 rc, u64 sid, u64 before_slot, + u64 before_pte, u64 after_slot, + u64 after_pte) { + u64 seq = ++g_state.dart.recent_idx; + struct dart_c_recent_event *ev = + &g_state.dart.recent[(seq - 1) & (DART_C_RECENT_SLOTS - 1)]; + + ev->seq = seq; + ev->table = table; + ev->endpoint = endpoint; + ev->dart_id = inst ? inst->dart_id : 0; + ev->sid = sid; + for (u32 i = 0; i < 6; i++) + ev->regs[i] = regs[i]; + ev->before_slot = before_slot; + ev->before_pte = before_pte; + ev->after_slot = after_slot; + ev->after_pte = after_pte; + ev->rc = rc; +} + +#define DART_C_SID0_TRACE_DVA 0x300704048UL +#define DART_C_SID0_REASON_DIRECT (1UL << 0) +#define DART_C_SID0_REASON_GLOBAL (1UL << 2) +#define DART_C_SID0_REASON_FLUSH (1UL << 3) + +static inline u64 dart_c_sid0_pack_tcr_ttbr(struct dart_c_sid_state *st) { + if (!st) + return 0; + return ((u64)st->ttbr << 32) | st->tcr; +} + +static inline u64 dart_c_sid0_pack_root_flags(struct dart_c_sid_state *st) { + if (!st) + return 0; + return (st->root_pt_pa & 0x00ffffffffffffffUL) | + ((u64)(st->valid & 1U) << 56) | + ((u64)(st->root_valid & 1U) << 57) | + ((u64)(st->root_level & 3U) << 58) | + ((u64)(st->stream_enabled & 1U) << 60) | + ((u64)(dart_c_sid_translates(st) & 1U) << 61) | + ((u64)(st->raw_replay & 1U) << 62) | + ((u64)(st->initial_stream_enabled & 1U) << 63); +} + +static inline void dart_c_snapshot_sid0(struct dart_c_instance *inst, u64 dva, + u64 *tcr_ttbr_out, + u64 *root_flags_out, + u64 *slot_out, u64 *pte_out) { + struct dart_c_sid_state *st = dart_c_sid_lookup(inst, 0); + *tcr_ttbr_out = dart_c_sid0_pack_tcr_ttbr(st); + *root_flags_out = dart_c_sid0_pack_root_flags(st); + dart_c_snapshot_leaf(inst, 0, dva, slot_out, pte_out); +} + +static inline bool_t dart_c_sid0_trace_reason(struct dart_c_instance *inst, + bool_t xnu_table, + bool_t sk_table, + u32 endpoint, u64 *regs, + u64 *reason_out) { + u64 reason = 0; + (void)inst; + + if (sk_table) { + if (endpoint == 0 && regs[1] == 0) + reason = DART_C_SID0_REASON_DIRECT; + if (endpoint == 1 || endpoint == 2 || endpoint == 3) + reason = DART_C_SID0_REASON_GLOBAL; + } else if (xnu_table) { + switch (endpoint) { + case 0: + case 1: + case 8: + case 9: + case 13: + if (regs[1] == 0) + reason = DART_C_SID0_REASON_DIRECT; + break; + case 2: + case 3: + if (regs[1] == 0) + reason = DART_C_SID0_REASON_DIRECT; + break; + case 4: + case 5: + case 6: + reason = DART_C_SID0_REASON_GLOBAL; + break; + case 7: + if (regs[1] == 0) + reason = DART_C_SID0_REASON_FLUSH; + break; + case 11: + case 16: + reason = DART_C_SID0_REASON_FLUSH; + break; + default: + break; + } + } + + *reason_out = reason; + return reason != 0; +} + +static inline void dart_c_record_sid0_trace(struct dart_c_instance *inst, + u32 table, u32 endpoint, u64 *regs, + u64 rc, u64 reason, + u64 before_tcr_ttbr, + u64 before_root_flags, + u64 before_pte, + u64 after_tcr_ttbr, + u64 after_root_flags, + u64 after_pte) { + u64 seq = ++g_state.dart.sid0_trace_idx; + struct dart_c_sid0_event *ev = + &g_state.dart.sid0_trace[(seq - 1) & (DART_C_SID0_TRACE_SLOTS - 1)]; + + ev->seq = seq; + ev->meta = ((u64)(table & 0xffffU)) | + ((u64)(endpoint & 0xffffU) << 16) | + ((u64)((inst ? inst->dart_id : 0) & 0xffffU) << 32) | + ((u64)(reason & 0xffffU) << 48); + for (u32 i = 0; i < 4; i++) + ev->regs[i] = regs[i]; + ev->before_tcr_ttbr = before_tcr_ttbr; + ev->before_root_flags = before_root_flags; + ev->before_pte = before_pte; + ev->after_tcr_ttbr = after_tcr_ttbr; + ev->after_root_flags = after_root_flags; + ev->after_pte = after_pte; + ev->rc = rc; +} + +static inline u64 dart_c_ep0_install_table(struct dart_c_instance *inst, u64 *regs) { + u64 sid = regs[1]; + u64 dva = regs[2]; + u32 level = (u32)regs[3]; + u64 pt_pa = regs[4]; + struct dart_c_sid_state *st = dart_c_sid(inst, sid); + if (!st || (pt_pa & (DART_C_PAGE_SIZE - 1)) || level > 2) + return DART_C_STATUS_VIOLATION; + if (dart_c_sid_exclave(st)) + return DART_C_STATUS_VIOLATION; + + /* + * Firmware map_table uses level 0 as the root install for FOUR_LEVELS + * streams and level 1 as the root install for normal streams. Once a + * level-0 root is installed, later level-1 and level-2 requests install + * child tables below that root; they must not replace it. + */ + bool_t root_install = (!st->root_valid || level < st->root_level); + u32 root_install_level = (level == 0) ? 1U : 2U; + + g_state.dart.last[8] = sid; + g_state.dart.last[9] = level; + g_state.dart.last[10] = pt_pa; + g_state.dart.last[11] = 0; + g_state.dart.last[12] = 0; + g_state.dart.last[13] = st->root_level; + g_state.dart.last[14] = st->root_valid; + g_state.dart.last[15] = st->root_pt_pa; + + if (root_install && level > 1) + return DART_C_STATUS_VIOLATION; + + if (root_install && st->root_valid) + dart_c_import_live_sid(inst, (u32)sid); + + if (root_install) { + st->root_valid = 1; + st->root_level = (u8)root_install_level; + st->root_pt_pa = pt_pa; + st->ttbr = dart_c_ttbr_from_pa(pt_pa); + st->tcr = dart_c_tcr_from_root_level(st->root_level); + st->raw_replay = 0; + st->translation_enabled = 1; + dart_c_program_sid_tcr_ttbr(inst, (u32)sid, st->tcr, st->ttbr); + return SPTM_SUCCESS; + } + + if (level < st->root_level || level > 2) + return DART_C_STATUS_VIOLATION; + + u64 parent; + u32 idx; + u64 rc = dart_c_walk_to_table(inst, (u32)sid, st, dva, level, + &parent, &idx); + if (rc) + return SPTM_SUCCESS; /* Adapter glue: missing imported parent state is non-fatal. */ + u64 live_pt_pa = dart_c_shadow_child_table_if_needed( + inst, (u32)sid, st, pt_pa); + *(volatile u64 *)(parent + (u64)idx * 8UL) = dart_c_pte_from_pa(live_pt_pa); + dart_c_clean64(parent + (u64)idx * 8UL); + g_state.dart.last[11] = parent; + g_state.dart.last[12] = idx; + g_state.dart.last[13] = st->root_level; + g_state.dart.last[14] = st->root_valid; + g_state.dart.last[15] = st->root_pt_pa; + dart_c_for_each_base_flush_sid(inst, (u32)sid); + return SPTM_SUCCESS; +} + +static inline u64 dart_c_ep1_unmap_table(struct dart_c_instance *inst, u64 *regs) { + u64 sid = regs[1]; + u64 dva = regs[2]; + u32 level = (u32)regs[3]; + struct dart_c_sid_state *st = dart_c_sid(inst, sid); + if (!st) + return DART_C_STATUS_VIOLATION; + if (dart_c_sid_exclave(st)) + return DART_C_STATUS_VIOLATION; + if (!st->root_valid) + return SPTM_SUCCESS; + + if ((st->root_level == 1 && level == 0) || + (st->root_level == 2 && level == 1)) { + st->root_valid = 0; + st->root_pt_pa = 0; + st->ttbr = 0; + st->tcr &= ~DART_T8110_TCR_TRANSLATE_EN; + st->translation_enabled = 0; + st->raw_replay = 0; + for (u32 i = 0; i < inst->base_count && i < DART_C_MAX_BASES; i++) { + u64 base = inst->base_pa[i]; + if (dart_c_looks_like_mmio(base)) + dart_c_write32(base + DART_T8110_TTBR + sid * 4, 0); + } + dart_c_for_each_base_flush_sid(inst, (u32)sid); + return SPTM_SUCCESS; + } + + if (level < st->root_level || level > 2) + return DART_C_STATUS_VIOLATION; + + u64 parent; + u32 idx; + u64 rc = dart_c_walk_to_table(inst, (u32)sid, st, dva, level, + &parent, &idx); + if (!rc) { + *(volatile u64 *)(parent + (u64)idx * 8UL) = 0; + dart_c_clean64(parent + (u64)idx * 8UL); + } + dart_c_for_each_base_flush_sid(inst, (u32)sid); + return SPTM_SUCCESS; +} + +static inline bool_t dart_c_is_txm_secure_channel_dva(u64 dva) { + if (!g_state.txm.secure_chan_dva) + return 0; + return (dva & ~(DART_C_PAGE_SIZE - 1UL)) == + (g_state.txm.secure_chan_dva & ~(DART_C_PAGE_SIZE - 1UL)); +} + +static inline bool_t dart_c_is_txm_secure_channel_pa(u64 pa) { + if (!pa) + return 0; + return ft_get_type(pa & ~(DART_C_PAGE_SIZE - 1UL)) == + TXM_SEP_SECURE_CHANNEL; +} + +static inline bool_t dart_c_is_txm_secure_channel_dart(struct dart_c_instance *inst) { + /* + * txm-secure-channel-base is a /arm-io/dart-sep property. DCP and other + * DARTs use the same high DVA numbering and can legitimately have ordinary + * mappings at the same numeric DVA, so keep this reservation scoped to the + * SEP DART that owns the property. + */ + return inst && inst->dart_id == 3; +} + +static inline bool_t dart_c_validate_txm_secure_channel_map(struct dart_c_instance *inst, + u64 dva, u64 pa) { + if (!g_state.txm.secure_chan_dva) + return 1; + if (!dart_c_is_txm_secure_channel_dart(inst)) + return 1; + + /* + * Real sptm_t8110dart_map treats /arm-io/dart-sep's + * txm-secure-channel-base/size as a reserved DVA window. With that DT + * state present, the reserve window may only map frame type 0x3d, and + * frame type 0x3d may only be mapped inside that window. + */ + return dart_c_is_txm_secure_channel_dva(dva) == + dart_c_is_txm_secure_channel_pa(pa); +} + +static inline bool_t dart_c_leaf_slot_is_txm_secure_channel(u64 slot) { + u64 pte = *(volatile u64 *)slot; + if ((pte & 1UL) == 0) + return 0; + return ft_get_type(dart_c_pa_from_pte(pte)) == TXM_SEP_SECURE_CHANNEL; +} + +static inline u64 dart_c_ep2_map_from_pa_list(struct dart_c_instance *inst, + u64 *regs) { + u64 sid = regs[1]; + u64 dva = regs[2]; + u64 scratch_pa = regs[3]; + u64 size = regs[4]; + u64 flags = regs[5]; + struct dart_c_sid_state *st = dart_c_sid(inst, sid); + u64 first_old_pa = 0; + u64 first_new_pa = 0; + bool_t flush_sid = 0; + if (dart_c_sid_exclave(st)) + return DART_C_STATUS_VIOLATION; + if (!st || !size || (scratch_pa & 7UL)) + return SPTM_SUCCESS; + u64 range_rc = dart_c_validate_dva_range(dva, size); + if (range_rc) + return range_rc; + + u64 pages = ((dva & (DART_C_PAGE_SIZE - 1)) + size + DART_C_PAGE_SIZE - 1) / + DART_C_PAGE_SIZE; + + u64 cur = dva; + u64 remaining = size; + for (u64 i = 0; i < pages; i++) { + u64 table, slot; + u64 page_bytes = DART_C_PAGE_SIZE - (cur & (DART_C_PAGE_SIZE - 1)); + if (page_bytes > remaining) + page_bytes = remaining; + if (dart_c_ensure_leaf_table(inst, (u32)sid, st, cur, &table, 0, 0)) + break; + slot = table + (u64)dart_c_idx(cur, 3) * 8UL; + u64 scratch_slot = scratch_pa + i * 8UL; + u64 old_pte = *(volatile u64 *)slot; + u64 old_pa = (old_pte & 1UL) ? dart_c_pa_from_pte(old_pte) : 0; + u64 new_pa = *(volatile u64 *)scratch_slot; + if (i == 0) { + first_old_pa = old_pa; + first_new_pa = new_pa; + } + if (!dart_c_validate_txm_secure_channel_map(inst, cur, new_pa)) { + inst->last[8] = 0x54584d56UL; /* "TXMV" */ + inst->last[9] = sid; + inst->last[10] = cur; + inst->last[11] = new_pa; + inst->last[12] = ft_get_type(new_pa); + inst->last[13] = old_pa; + g_state.dart.last[8] = 0x54584d56UL; + g_state.dart.last[9] = sid; + g_state.dart.last[10] = cur; + g_state.dart.last[11] = new_pa; + g_state.dart.last[12] = ft_get_type(new_pa); + g_state.dart.last[13] = old_pa; + return DART_C_STATUS_VIOLATION; + } + /* + * Real t8110dart_map copies the caller's PA-list page into SPTM-owned + * scratch, consumes the new PAs from that copy, and keeps old PAs only + * for internal release bookkeeping. Do not write old mappings back into + * XNU's supplied PA-list page. + */ + u64 new_pte = new_pa ? + dart_c_leaf_pte(new_pa, cur, page_bytes, + dart_c_leaf_perm_bits(inst, new_pa, flags)) : 0; + *(volatile u64 *)slot = new_pte; + dart_c_clean64(slot); + if (dart_c_map_needs_tlbi(inst, old_pte, new_pte)) + flush_sid = 1; + remaining -= page_bytes; + cur += page_bytes; + } + if (flush_sid) { + u64 flush_start = dva & ~(DART_C_PAGE_SIZE - 1UL); + u64 flush_end = (dva + size + DART_C_PAGE_SIZE - 1UL) & + ~(DART_C_PAGE_SIZE - 1UL); + if (flush_end > flush_start) + dart_c_for_each_base_flush_map_change(inst, (u32)sid, + flush_start, + flush_end - 1UL); + } + inst->last[8] = 0x4d41504cUL; /* "MAPL" */ + inst->last[9] = sid; + inst->last[10] = dva; + inst->last[11] = size; + inst->last[12] = pages; + inst->last[13] = first_old_pa; + inst->last[14] = first_new_pa; + inst->last[15] = flags; + g_state.dart.last[8] = 0x4d41504cUL; + g_state.dart.last[9] = first_old_pa; + g_state.dart.last[10] = first_new_pa; + g_state.dart.last[11] = pages; + g_state.dart.last[12] = flags; + return SPTM_SUCCESS; +} + +static inline u64 dart_c_ep3_unmap_range(struct dart_c_instance *inst, u64 *regs, + bool_t flush) { + u64 sid = regs[1]; + u64 start = regs[2]; + u64 size_or_end = regs[3]; + u64 end = flush ? size_or_end : start + size_or_end; + struct dart_c_sid_state *st = dart_c_sid(inst, sid); + u64 unmapped = 0; + if (dart_c_sid_exclave(st)) + return DART_C_STATUS_VIOLATION; + if (!st || end < start) + return SPTM_SUCCESS; + if (!flush) { + u64 range_rc = dart_c_validate_dva_range(start, size_or_end); + if (range_rc) + return range_rc; + } + start &= ~(DART_C_PAGE_SIZE - 1); + end = (end + DART_C_PAGE_SIZE - 1) & ~(DART_C_PAGE_SIZE - 1); + for (u64 dva = start; dva < end; dva += DART_C_PAGE_SIZE) { + u64 slot; + if (!dart_c_walk_leaf_slot(inst, (u32)sid, st, dva, &slot)) { + if (dart_c_leaf_slot_is_txm_secure_channel(slot)) + return DART_C_STATUS_VIOLATION; + if (!dart_c_restore_boot_premap_page(inst, (u32)sid, dva, slot)) { + *(volatile u64 *)slot = 0; + dart_c_clean64(slot); + } + unmapped++; + } + } + if (end > start) + dart_c_for_each_base_flush_map_change(inst, (u32)sid, start, + end - 1UL); + inst->last[8] = 0x554e4d50UL; /* "UNMP" */ + inst->last[9] = sid; + inst->last[10] = start; + inst->last[11] = end - start; + inst->last[12] = unmapped; + g_state.dart.last[8] = 0x554e4d50UL; + g_state.dart.last[9] = sid; + g_state.dart.last[10] = start; + g_state.dart.last[11] = end; + g_state.dart.last[12] = unmapped; + return SPTM_SUCCESS; +} + +static inline u64 dart_c_ep12_translate(struct dart_c_instance *inst, u64 *regs) { + u64 sid = regs[1]; + u64 dva = regs[2]; + struct dart_c_sid_state *st = dart_c_sid_lookup(inst, sid); + if (!st && sid < DART_C_MAX_SIDS && dart_c_import_live_sid(inst, (u32)sid)) + st = dart_c_sid_lookup(inst, sid); + u64 slot; + if (!st || dart_c_walk_leaf_slot(inst, (u32)sid, st, dva, &slot)) { + regs[1] = 0; + return SPTM_SUCCESS; + } + u64 pte = *(volatile u64 *)slot; + regs[1] = (pte & 1UL) ? (dart_c_pa_from_pte(pte) | (dva & (DART_C_PAGE_SIZE - 1))) : 0; + return SPTM_SUCCESS; +} + +#define DART_DAPF_SLICE_STRIDE 0x40UL +#define DART_DAPF_CTRL 0x00UL +#define DART_DAPF_START_LO 0x08UL +#define DART_DAPF_START_HI 0x0cUL +#define DART_DAPF_END_LO 0x10UL +#define DART_DAPF_END_HI 0x14UL +#define DART_DAPF_WORD_BASE 0x20UL + +static inline u32 dart_c_dapf_count(struct dart_c_instance *inst) { + u32 count = inst->dapf.count; + if (count > DART_C_MAX_DAPF_SLICES) + count = DART_C_MAX_DAPF_SLICES; + return count; +} + +static inline u32 dart_c_dapf_word_count(struct dart_c_instance *inst) { + u32 sid_count = inst->dapf.sid_count; + if (sid_count == 0 || sid_count > DART_C_MAX_STREAMS) + sid_count = DART_C_MAX_STREAMS; + u32 words = (sid_count + 31U) >> 5; + if (words == 0) + words = 1; + if (words > 8) + words = 8; + return words; +} + +static inline u32 dart_c_program_dapf_power_up(struct dart_c_instance *inst) { + u64 base = inst->dapf.base_pa; + u32 count = dart_c_dapf_count(inst); + u32 words = dart_c_dapf_word_count(inst); + u32 writes = 0; + + if (!base || !count || !dart_c_looks_like_mmio(base)) + return 0; + + for (u32 i = 0; i < count; i++) { + struct dart_c_dapf_slice *desc = &inst->dapf.slice[i]; + u64 off = base + (u64)i * DART_DAPF_SLICE_STRIDE; + dart_c_write32(off + DART_DAPF_START_LO, (u32)desc->start); + dart_c_write32(off + DART_DAPF_START_HI, (u32)(desc->start >> 32)); + dart_c_write32(off + DART_DAPF_END_LO, (u32)desc->end); + dart_c_write32(off + DART_DAPF_END_HI, (u32)(desc->end >> 32)); + writes += 4; + + for (u32 word = 0; word < words; word++) { + dart_c_write32(off + DART_DAPF_WORD_BASE + (u64)word * 4UL, + desc->words[word]); + writes++; + } + + dart_c_write32(off + DART_DAPF_CTRL, desc->ctrl); + writes++; + } + + __asm__ volatile("dsb sy" ::: "memory"); + return writes; +} + +#define DART_PIOGW_PROT_BASE 0x1120UL +#define DART_PIOGW_PROT_STRIDE 0x20UL +#define DART_PIOGW_PROT_CTRL 0x00UL +#define DART_PIOGW_PROT_START_LO 0x04UL +#define DART_PIOGW_PROT_START_HI 0x08UL +#define DART_PIOGW_PROT_END_LO 0x0cUL +#define DART_PIOGW_PROT_END_HI 0x10UL + +static inline u32 dart_c_piogw_desc_count(struct dart_c_instance *inst) { + u32 count = inst->piogw_desc_count; + if (count > DART_C_MAX_PIOGW_DESCS) + count = DART_C_MAX_PIOGW_DESCS; + return count; +} + +static inline u32 dart_c_piogw_count(struct dart_c_instance *inst) { + u32 count = inst->piogw_count; + if (count > DART_C_MAX_PIOGW_RANGES) + count = DART_C_MAX_PIOGW_RANGES; + return count; +} + +static inline u32 dart_c_program_piogw_power_up(struct dart_c_instance *inst) { + u32 desc_count = dart_c_piogw_desc_count(inst); + u32 piogw_count = dart_c_piogw_count(inst); + u32 writes = 0; + + if (!piogw_count || !desc_count) + return 0; + + for (u32 i = 0; i < piogw_count; i++) { + u64 base = inst->piogw_base_pa[i]; + if (!dart_c_looks_like_mmio(base)) + continue; + if (dart_c_read32(base + 0x10UL) != 0 || + dart_c_read32(base + 0x0cUL) != 0) + continue; + + dart_c_write32(base + DART_PIOGW_PROT_BASE + + DART_PIOGW_PROT_START_LO, 0); + dart_c_write32(base + DART_PIOGW_PROT_BASE + + DART_PIOGW_PROT_START_HI, 0); + dart_c_write32(base + DART_PIOGW_PROT_BASE + + DART_PIOGW_PROT_END_LO, 0xffffffffU); + dart_c_write32(base + DART_PIOGW_PROT_BASE + + DART_PIOGW_PROT_END_HI, 0xffffffffU); + dart_c_write32(base + DART_PIOGW_PROT_BASE + + DART_PIOGW_PROT_CTRL, 3); + writes += 5; + + for (u32 j = 0; j < desc_count; j++) { + struct dart_c_piogw_desc *desc = &inst->piogw_desc[j]; + u32 slice = desc->slice; + u64 addr = desc->addr; + if (slice < 1 || slice > 9 || addr == 0) + continue; + u64 off = DART_PIOGW_PROT_BASE + (u64)slice * DART_PIOGW_PROT_STRIDE; + dart_c_write32(base + off + DART_PIOGW_PROT_START_LO, (u32)addr); + dart_c_write32(base + off + DART_PIOGW_PROT_START_HI, + (u32)(addr >> 32)); + dart_c_write32(base + off + DART_PIOGW_PROT_END_LO, (u32)(addr + 3)); + dart_c_write32(base + off + DART_PIOGW_PROT_END_HI, + (u32)((addr + 3) >> 32)); + dart_c_write32(base + off + DART_PIOGW_PROT_CTRL, 6); + writes += 5; + } + + dart_c_write32(base + 4UL, dart_c_read32(base + 4UL) | 1U); + writes++; + } + + return writes; +} + +static inline u32 dart_c_program_piogw_power_down(struct dart_c_instance *inst) { + u32 desc_count = dart_c_piogw_desc_count(inst); + u32 piogw_count = dart_c_piogw_count(inst); + u32 writes = 0; + + if (!piogw_count || !desc_count) + return 0; + + for (u32 i = 0; i < piogw_count; i++) { + u64 base = inst->piogw_base_pa[i]; + if (!dart_c_looks_like_mmio(base)) + continue; + if (dart_c_read32(base + 0x10UL) != 0 || + dart_c_read32(base + 0x0cUL) != 0) + continue; + + for (u32 j = 0; j < desc_count; j++) { + struct dart_c_piogw_desc *desc = &inst->piogw_desc[j]; + u32 slice = desc->slice; + if (slice < 1 || slice > 9) + continue; + u64 off = DART_PIOGW_PROT_BASE + (u64)slice * DART_PIOGW_PROT_STRIDE; + dart_c_write32(base + off + DART_PIOGW_PROT_CTRL, 0); + writes++; + } + } + + return writes; +} + +static inline u32 dart_c_snapshot_live_streams(struct dart_c_instance *inst) { + u32 seen = 0; + + for (u32 sid = 0; sid < DART_C_MAX_SIDS; sid++) { + struct dart_c_sid_state *st = &inst->sid[sid]; + if (st->valid) + st->stream_enabled = 0; + } + + for (u32 i = 0; i < inst->base_count && i < DART_C_MAX_BASES; i++) { + u64 base = inst->base_pa[i]; + if (!dart_c_looks_like_mmio(base)) + continue; + u32 num = dart_c_num_streams(base); + if (num > DART_C_MAX_SIDS) + num = DART_C_MAX_SIDS; + u32 words = dart_c_stream_words(num); + for (u32 word = 0; word < words; word++) { + u32 bits = dart_c_read32(base + DART_T8110_ENABLE_STREAMS + word * 4); + while (bits) { + u32 bit = __builtin_ctz(bits); + u32 sid = word * 32 + bit; + bits &= bits - 1; + if (sid >= num) + continue; + struct dart_c_sid_state *st = dart_c_sid(inst, sid); + if (st) + st->stream_enabled = 1; + seen++; + } + } + } + + return seen; +} + +static inline u32 dart_c_disable_translated_streams(struct dart_c_instance *inst) { + u32 writes = 0; + + for (u32 sid = 0; sid < DART_C_MAX_SIDS; sid++) { + struct dart_c_sid_state *st = dart_c_sid_lookup(inst, sid); + if (dart_c_sid_exclave(st) || !dart_c_sid_translates(st)) + continue; + for (u32 i = 0; i < inst->base_count && i < DART_C_MAX_BASES; i++) { + u64 base = inst->base_pa[i]; + if (!dart_c_looks_like_mmio(base)) + continue; + dart_c_write32(base + DART_T8110_DISABLE_STREAMS + + ((sid >> 5) * 4), 1U << (sid & 31)); + writes++; + } + } + + return writes; +} + +static inline u32 dart_c_finalize_replay(struct dart_c_instance *inst) { + u32 writes = 0; + u32 touched[DART_C_MAX_SIDS / 32]; + + for (u32 i = 0; i < DART_C_MAX_SIDS / 32; i++) + touched[i] = 0; + + for (u32 i = 0; i < inst->base_count && i < DART_C_MAX_BASES; i++) { + u64 base = inst->base_pa[i]; + if (!dart_c_looks_like_mmio(base)) + continue; + + for (u32 sid = 0; sid < DART_C_MAX_SIDS; sid++) { + struct dart_c_sid_state *st = &inst->sid[sid]; + if (!st->valid) + continue; + bool_t replay_regs = + !dart_c_sid_exclave(st) && + (st->root_valid || st->raw_replay || + (st->tcr & (DART_T8110_TCR_REMAP_EN | + DART_T8110_TCR_BYPASS_DART | + DART_T8110_TCR_BYPASS_DAPF))); + if (replay_regs) { + u64 tcr_reg = base + DART_T8110_TCR + sid * 4; + u64 ttbr_reg = base + DART_T8110_TTBR + sid * 4; + touched[sid >> 5] |= 1U << (sid & 31); + if (dart_c_read32(tcr_reg) != st->tcr) { + dart_c_write32(tcr_reg, st->tcr); + writes++; + } + if (dart_c_read32(ttbr_reg) != st->ttbr) { + dart_c_write32(ttbr_reg, st->ttbr); + writes++; + } + } + } + + u32 num = dart_c_num_streams(base); + if (num > DART_C_MAX_SIDS) + num = DART_C_MAX_SIDS; + u32 words = dart_c_stream_words(num); + for (u32 word = 0; word < words; word++) { + u32 bits = 0; + for (u32 bit = 0; bit < 32; bit++) { + u32 sid = word * 32 + bit; + if (sid >= num) + break; + struct dart_c_sid_state *st = &inst->sid[sid]; + if (st->valid && + (st->stream_enabled || st->initial_stream_enabled)) { + bits |= 1U << bit; + touched[sid >> 5] |= 1U << (sid & 31); + } + } + dart_c_write32(base + DART_T8110_ENABLE_STREAMS + word * 4, bits); + writes++; + } + } + + /* + * Persistent display/DCP roots can contain valid leaves before any map call + * touches them in this boot. Once finalize replays that root or enables a + * stream, force the hardware translation cache to forget older negative + * entries before firmware starts reading through the DART. + */ + __asm__ volatile("dsb sy" ::: "memory"); + for (u32 sid = 0; sid < DART_C_MAX_SIDS; sid++) + if (touched[sid >> 5] & (1U << (sid & 31))) + dart_c_for_each_base_flush_sid(inst, sid); + + return writes; +} + +static inline u32 dart_c_count_raw_replay_state(struct dart_c_instance *inst) { + u32 count = 0; + + for (u32 sid = 0; sid < DART_C_MAX_SIDS; sid++) { + struct dart_c_sid_state *st = &inst->sid[sid]; + if (st->valid && st->raw_replay) + count++; + } + + return count; +} + +static inline u32 dart_c_xnu_clock_protection(struct dart_c_instance *inst, + bool_t enable) { + u64 obj = inst->obj_va; + u32 gapf_count, inst_count; + u64 state; + + if (!dart_validate_xnu_object(obj, &gapf_count, &inst_count, &state)) { + dart_c_import_cached_bases(inst); + obj = inst->obj_va; + } + if (!dart_validate_xnu_object(obj, &gapf_count, &inst_count, &state)) + return 0; + inst->state_va = state; + + u32 writes = 0; + for (u32 gapf_i = 0; gapf_i < gapf_count; gapf_i++) { + u64 base_va; + if (!guest_read64(obj + 0x17d0UL + (u64)gapf_i * 0x18UL, &base_va)) + continue; + if (!is_kernel_va(base_va)) + continue; + + for (u32 inst_i = 0; inst_i < inst_count; inst_i++) { + u32 slice_idx; + u64 entry = obj + 0x14f8UL + (u64)inst_i * 0x70UL + + (u64)gapf_i * 4UL; + if (!guest_read32(entry, &slice_idx) || slice_idx > 0x3ffU) + continue; + + u64 reg_va = base_va + 0x100UL + (u64)slice_idx * 0x40UL; + u64 reg_pa; + if (!guest_va_to_pa(reg_va, 1, ®_pa)) + continue; + + u32 old = dart_c_read32(reg_pa); + u32 new_value = enable ? (old | 0x10U) : (old & ~0x10U); + dart_c_write32(reg_pa, new_value); + writes++; + } + } + + inst->clock_protection = enable ? (writes != 0) : 0; + inst->last[8] = enable; + inst->last[9] = writes; + g_state.dart.last[8] = enable; + g_state.dart.last[9] = writes; + return writes; +} + +bool_t dart_handle_table(u32 table, u32 endpoint, u64 *regs) { + bool_t t8110_xnu = table == SPTM_TABLE_T8110_DART_XNU; + bool_t t8110_sk = table == SPTM_TABLE_T8110_DART_SK; + bool_t gen3_xnu = table == SPTM_TABLE_GEN3_DART_XNU; + bool_t gen3_sk = table == SPTM_TABLE_GEN3_DART_SK; + bool_t xnu_table = t8110_xnu || gen3_xnu; + bool_t sk_table = t8110_sk || gen3_sk; + + if (!xnu_table && !sk_table) + return 0; + + struct dart_c_instance *inst = dart_c_get(regs[0]); + if (!inst) + return 0; + + g_state.dart.enabled = 1; + g_state.dart.total_calls++; + inst->total_calls++; + if (endpoint < DART_C_MAX_ENDPOINTS) { + inst->ep_count[endpoint]++; + } + g_state.dart.last[0] = table; + g_state.dart.last[1] = endpoint; + g_state.dart.last[2] = regs[0]; + g_state.dart.last[3] = regs[1]; + g_state.dart.last[4] = regs[2]; + g_state.dart.last[5] = regs[3]; + for (u32 i = 0; i < 16; i++) + inst->last[i] = (i < 8) ? regs[i] : 0; + + u64 rc = SPTM_SUCCESS; + bool_t record_dart_lifetime = 0; + u64 record_sid = 0; + u64 record_dva = 0; + u64 before_slot = 0; + u64 before_pte = 0; + u64 after_slot = 0; + u64 after_pte = 0; + bool_t record_sid0_trace = 0; + u64 sid0_reason = 0; + u64 sid0_before_tcr_ttbr = 0; + u64 sid0_before_root_flags = 0; + u64 sid0_before_slot = 0; + u64 sid0_before_pte = 0; + u64 sid0_after_tcr_ttbr = 0; + u64 sid0_after_root_flags = 0; + u64 sid0_after_slot = 0; + u64 sid0_after_pte = 0; + + if (xnu_table && (endpoint == 2 || endpoint == 3)) { + record_dart_lifetime = 1; + record_sid = regs[1]; + record_dva = regs[2]; + if (record_dart_lifetime) + dart_c_snapshot_leaf(inst, record_sid, record_dva, + &before_slot, &before_pte); + } else if (xnu_table && + (endpoint == 0 || endpoint == 1 || + endpoint == 4 || endpoint == 5 || endpoint == 6 || + endpoint == 7 || endpoint == 8 || endpoint == 9 || + endpoint == 10 || endpoint == 11 || endpoint == 16)) { + record_dart_lifetime = 1; + record_sid = (endpoint == 7 || endpoint == 8 || endpoint == 14) ? + regs[1] : 0; + record_dva = regs[2]; + if (endpoint == 0 || endpoint == 1) { + dart_c_snapshot_leaf(inst, record_sid, record_dva, + &before_slot, &before_pte); + } else { + before_slot = dart_c_pack_hw_sid_regs(inst, (u32)record_sid); + before_pte = dart_c_pack_hw_stream_error(inst, (u32)record_sid); + } + } + + record_sid0_trace = dart_c_sid0_trace_reason(inst, xnu_table, sk_table, + endpoint, regs, &sid0_reason); + if (record_sid0_trace) { + dart_c_snapshot_sid0(inst, DART_C_SID0_TRACE_DVA, + &sid0_before_tcr_ttbr, + &sid0_before_root_flags, + &sid0_before_slot, + &sid0_before_pte); + } + + if (sk_table) { + switch (endpoint) { + case 0: + rc = dart_c_ep3_unmap_range(inst, regs, 1); + break; + case 1: + inst->locked = 0; + rc = SPTM_SUCCESS; + break; + case 2: + inst->clock_protection++; + rc = SPTM_SUCCESS; + break; + case 3: + regs[1] = inst->clock_protection; + rc = SPTM_SUCCESS; + break; + default: + rc = SPTM_SUCCESS; + break; + } + } else { + if (endpoint == 4 || endpoint == 5 || endpoint == 6) { + dart_cache_xnu_object_from_regs(table, endpoint, regs); + dart_c_import_cached_bases(inst); + } + if (endpoint == 6 && dart_c_looks_like_mmio(regs[1])) + dart_c_add_base(inst, regs[1]); + + switch (endpoint) { + case 0: + rc = dart_c_ep0_install_table(inst, regs); + if (rc == SPTM_SUCCESS) + dart_c_apply_identity_ranges(inst); + break; + case 1: + rc = dart_c_ep1_unmap_table(inst, regs); + break; + case 2: + rc = dart_c_ep2_map_from_pa_list(inst, regs); + break; + case 3: + rc = dart_c_ep3_unmap_range(inst, regs, 0); + break; + case 4: + if (t8110_xnu || gen3_xnu) { + u32 stream_seen = dart_c_snapshot_live_streams(inst); + u32 stream_disables = dart_c_disable_translated_streams(inst); + u32 piogw_writes = dart_c_program_piogw_power_down(inst); + inst->init_state = 1; + dart_c_xnu_clock_protection(inst, 0); + inst->last[13] = stream_seen; + inst->last[14] = stream_disables; + inst->last[15] = piogw_writes; + g_state.dart.last[13] = stream_seen; + g_state.dart.last[14] = stream_disables; + g_state.dart.last[15] = piogw_writes; + } else { + inst->last[8] = 0x494f4354UL; /* "IOCT" */ + inst->last[9] = endpoint; + } + rc = SPTM_SUCCESS; + break; + case 5: { + u32 piogw_writes = dart_c_program_piogw_power_up(inst); + u32 dapf_writes = dart_c_program_dapf_power_up(inst); + u32 raw_replay_sids = dart_c_count_raw_replay_state(inst); + inst->init_state = 2; + inst->locked = 1; + dart_c_apply_bootstrap_remaps(inst); + dart_c_apply_identity_ranges(inst); + u32 replay_writes = dart_c_finalize_replay(inst); + dart_c_xnu_clock_protection(inst, 1); + inst->last[10] = raw_replay_sids; + inst->last[11] = replay_writes; + inst->last[14] = dapf_writes; + inst->last[15] = piogw_writes; + g_state.dart.last[10] = raw_replay_sids; + g_state.dart.last[11] = replay_writes; + g_state.dart.last[14] = dapf_writes; + g_state.dart.last[15] = piogw_writes; + rc = SPTM_SUCCESS; + break; + } + case 6: { + inst->init_state = 1; + u32 preserved = 0; + u32 reset = 0; + u32 imported = 0; + for (u32 i = 0; i < inst->base_count && i < DART_C_MAX_BASES; i++) { + u64 base = inst->base_pa[i]; + if (!dart_c_looks_like_mmio(base)) + continue; + bool_t preserve = 0; + imported += dart_c_import_live_base(inst, base, &preserve); + if (!preserve && dart_c_has_seeded_live_sid(inst)) + preserve = 1; + if (preserve) { + /* + * Bootloader-owned display/AOP/DCP DARTs can have + * protected TTBR/TCR registers. On those DARTs, resetting + * TCR/TTBR is ineffective, but enabling every stream is + * very effective and can expose stale DMA as a real XNU + * DART panic. Preserve the live root and only clear stale + * interrupt state. + */ + dart_c_clear_errors_base(base); + preserved++; + continue; + } + dart_c_clear_errors_base(base); + reset++; + } + inst->last[10] = imported; + inst->last[11] = preserved; + inst->last[12] = reset; + g_state.dart.last[10] = imported; + g_state.dart.last[11] = preserved; + g_state.dart.last[12] = reset; + dart_c_apply_bootstrap_remaps(inst); + dart_c_apply_identity_ranges(inst); + rc = SPTM_SUCCESS; + break; + } + case 7: { + u32 sid = (u32)regs[1]; + struct dart_c_sid_state *st = dart_c_sid_lookup(inst, sid); + if (!st || dart_c_sid_exclave(st)) { + rc = DART_C_STATUS_VIOLATION; + break; + } + if (dart_c_sid_translates(st)) { + st->stream_enabled = 0; + for (u32 i = 0; i < inst->base_count && i < DART_C_MAX_BASES; i++) { + u64 base = inst->base_pa[i]; + if (!dart_c_looks_like_mmio(base)) + continue; + dart_c_write32(base + DART_T8110_DISABLE_STREAMS + + ((sid >> 5) * 4), 1U << (sid & 31)); + } + } + rc = SPTM_SUCCESS; + break; + } + case 8: { + u32 sid = (u32)regs[1]; + struct dart_c_sid_state *st = dart_c_sid_lookup(inst, sid); + if (!st || dart_c_sid_exclave(st)) { + rc = DART_C_STATUS_VIOLATION; + break; + } + if (dart_c_sid_translates(st)) { + st->stream_enabled = 1; + for (u32 i = 0; i < inst->base_count && i < DART_C_MAX_BASES; i++) { + u64 base = inst->base_pa[i]; + if (!dart_c_looks_like_mmio(base)) + continue; + dart_c_write32(base + DART_T8110_ENABLE_STREAMS + + ((sid >> 5) * 4), 1U << (sid & 31)); + } + __asm__ volatile("dsb sy" ::: "memory"); + dart_c_for_each_base_flush_sid(inst, sid); + } + rc = SPTM_SUCCESS; + break; + } + case 9: { + u32 instance = (u32)regs[1]; + u32 mask = (u32)regs[2] & 0x7fffU; + if (instance >= inst->base_count || + instance >= DART_C_MAX_BASES || + !dart_c_looks_like_mmio(inst->base_pa[instance]) || + !mask || (mask & (mask - 1U))) { + rc = DART_C_STATUS_VIOLATION; + break; + } + dart_c_write32(inst->base_pa[instance] + DART_T8110_ERROR, mask); + inst->last[8] = 0x45525243UL; /* "ERRC" */ + inst->last[9] = instance; + inst->last[10] = mask; + rc = SPTM_SUCCESS; + break; + } + case 10: { + u32 instance = (u32)regs[1]; + u32 mask = (u32)regs[2] & 0x777U; + if (instance >= inst->base_count || + instance >= DART_C_MAX_BASES || + !dart_c_looks_like_mmio(inst->base_pa[instance])) { + rc = DART_C_STATUS_VIOLATION; + break; + } + if (mask) + dart_c_write32(inst->base_pa[instance] + + DART_T8110_ERROR_MASK, mask); + inst->last[8] = 0x4552524dUL; /* "ERRM" */ + inst->last[9] = instance; + inst->last[10] = mask; + rc = SPTM_SUCCESS; + break; + } + case 11: + for (u32 i = 0; i < inst->base_count && i < DART_C_MAX_BASES; i++) { + u64 base = inst->base_pa[i]; + if (!dart_c_looks_like_mmio(base)) + continue; + dart_c_clear_errors_base(base); + } + rc = SPTM_SUCCESS; + break; + case 12: { + u32 instance = (u32)regs[1]; + if (instance >= inst->base_count || + instance >= DART_C_MAX_BASES || + !dart_c_looks_like_mmio(inst->base_pa[instance])) { + rc = DART_C_STATUS_VIOLATION; + break; + } + u32 command = (((u32)regs[3] & 0xfU) << 4) | + (((u32)regs[2] & 0x3fffU) << 8) | + ((u32)regs[4] & 7U); + u64 base = inst->base_pa[instance]; + dart_c_write32(base + DART_T8110_TLB_CMD, command); + for (u32 poll = 0; poll < 100; poll++) { + if (!(dart_c_read32(base + DART_T8110_TLB_CMD) & + DART_T8110_TLB_CMD_BUSY)) + break; + } + regs[1] = dart_c_read32(base + 0x88UL); + regs[2] = dart_c_read32(base + 0x90UL); + inst->last[8] = 0x544c4243UL; /* "TLBC" */ + inst->last[9] = instance; + inst->last[10] = command; + break; + } + case 13: + inst->last[8] = 0x41504653UL; /* "APFS" */ + inst->last[9] = regs[1]; + inst->last[10] = regs[2]; + rc = SPTM_SUCCESS; + break; + case 14: { + struct dart_c_sid_state *st = dart_c_sid_lookup(inst, regs[1]); + regs[1] = st ? st->status : 0; + rc = SPTM_SUCCESS; + break; + } + case 15: { + inst->last[8] = 0x494f4354UL; /* "IOCT" */ + inst->last[9] = endpoint; + inst->last[10] = regs[1]; + inst->last[11] = regs[2]; + rc = SPTM_SUCCESS; + break; + } + case 16: + for (u32 i = 0; i < inst->base_count && i < DART_C_MAX_BASES; i++) + if (dart_c_looks_like_mmio(inst->base_pa[i])) + dart_c_clear_errors_base(inst->base_pa[i]); + rc = SPTM_SUCCESS; + break; + case 17: + rc = DART_C_STATUS_VIOLATION; + break; + case 18: + for (u32 i = 0; i < inst->base_count && i < DART_C_MAX_BASES; i++) + if (dart_c_looks_like_mmio(inst->base_pa[i])) + dart_c_clear_errors_base(inst->base_pa[i]); + inst->last[8] = 0x45525252UL; /* "ERRR" */ + inst->last[9] = endpoint; + rc = SPTM_SUCCESS; + break; + default: + rc = DART_C_STATUS_VIOLATION; + break; + } + } + + if (record_dart_lifetime) { + if (endpoint == 0 || endpoint == 1 || endpoint == 2 || endpoint == 3) { + dart_c_snapshot_leaf(inst, record_sid, record_dva, + &after_slot, &after_pte); + } else { + after_slot = dart_c_pack_hw_sid_regs(inst, (u32)record_sid); + after_pte = dart_c_pack_hw_stream_error(inst, (u32)record_sid); + } + dart_c_record_recent(inst, table, endpoint, regs, rc, record_sid, + before_slot, before_pte, after_slot, after_pte); + } + if (record_sid0_trace) { + dart_c_snapshot_sid0(inst, DART_C_SID0_TRACE_DVA, + &sid0_after_tcr_ttbr, + &sid0_after_root_flags, + &sid0_after_slot, + &sid0_after_pte); + bool_t changed = + sid0_before_tcr_ttbr != sid0_after_tcr_ttbr || + sid0_before_root_flags != sid0_after_root_flags || + sid0_before_slot != sid0_after_slot || + sid0_before_pte != sid0_after_pte; + if (changed || !(sid0_reason & DART_C_SID0_REASON_FLUSH)) { + dart_c_record_sid0_trace(inst, table, endpoint, regs, rc, sid0_reason, + sid0_before_tcr_ttbr, + sid0_before_root_flags, + sid0_before_pte, + sid0_after_tcr_ttbr, + sid0_after_root_flags, + sid0_after_pte); + } + } + + if (rc == DART_C_STATUS_VIOLATION) { + g_state.dart.violations++; + inst->violations++; + } else { + g_state.dart.fast_path_calls++; + inst->fast_path_calls++; + } + g_state.dart.last[6] = rc; + g_state.dart.last[7] = inst->base_count; + regs[0] = rc; + sev(); + return 1; +} diff --git a/src/sptm/emul.c b/src/sptm/emul.c new file mode 100644 index 000000000..04a3d18a1 --- /dev/null +++ b/src/sptm/emul.c @@ -0,0 +1,326 @@ +/* SPDX-License-Identifier: MIT */ +/* + * DO NOT MERGE: tainted SPTM endpoint emulator imported from the bringup tree. + * + * This is intentionally isolated under src/sptm while we separate the clean + * hypervisor plumbing from the reverse-engineered SPTM behavior. The old tree + * built this as a hot-loaded C object; this branch links it directly into the + * streamed stage2 hypervisor so endpoint state has one C source of truth. + */ + +#include "emul.h" +#include "private.h" + +#include "../memory.h" +#include "../utils.h" +#include "../xnuboot.h" + +int debug_printf(const char *fmt, ...); + +/* The single state instance exported for Python-side boot seeding. Keep this + * 1MiB even though struct sptm_emul_state is smaller: a few emulated SPTM + * devices use the tail as private scratch, matching the old bringup layout. */ +u8 g_sptm_state[0x100000] __attribute__((aligned(0x4000))); + +bool_t sptm_handler(void *ctx) { + /* ctx points at struct exc_info. regs[] is at offset 0. */ + u64 *regs = (u64 *)((unsigned char *)ctx + REGS_OFFSET); + u64 esr = *(u64 *)((unsigned char *)ctx + ESR_OFFSET); + u64 *ctx_spsr_ptr = (u64 *)((unsigned char *)ctx + 0x100); + + /* XNU's kernel-mode invariant is DAIF.D == 1. Most SPTM calls are + * direct in-kernel HVCs, so returning with the trapped SPSR unchanged can + * propagate a transient D-clear state until ml_set_interrupts_enabled() + * panics with "debug exceptions enabled in kernel mode". Enforce only D + * here; do not touch A/I/F, since the timer/FIQ path depends on preserving + * XNU's interrupt mask state exactly. */ + *ctx_spsr_ptr |= 0x200; + + /* DEBUG: snapshot ALL guest GP regs at trap entry — BEFORE any C + * code can touch them. Lets us bisect register-clobber bugs + * introduced by new C handlers. Cheap (31 u64 stores). */ + { + u64 idx = g_state.sptm_in_idx & 0x7f; + for (int r = 0; r < 31; r++) { + g_state.sptm_in_regs[idx][r] = regs[r]; + } + g_state.sptm_in_idx = (g_state.sptm_in_idx + 1) & 0x7f; + } + + /* --------------------------------------------------------------- + * SPRR sysreg fast-path. EL1's vbar slot 0 has been patched to + * HVC #0, so when xnu (e.g. _pmap_ro_zone_memcpy_internal in + * libT8140) writes SPRR_PERM_EL1 from EL1, the UNDEF lands here. + * SPRR is GL2-gated and m1n1 EL2 cannot complete the write — we + * soft-cache it. Without this fast path each trap pays a UART + * round-trip (~hundreds of ms × thousands of RO-zone init writes). + * + * Detect by inspecting the trapping instruction at ELR_EL12. If + * it's any MSR/MRS to S3_6_C15_C1_x (op1=6, CRn=15, CRm=1): + * - MSR: cache write value (no actual sysreg touch) + * - MRS: return cached/seeded value into Rt + * Then advance ELR_EL12 by 4 by setting ctx->elr to elr12+4 and + * let m1n1 ERET past xnu's vector handler back to user code at + * elr12+4. + * --------------------------------------------------------------- */ + if (g_state.kc_vmin != 0) { + u64 esr12, elr12, spsr12, elr2; + /* ESR_EL12 = S3_5_C5_C2_0; ELR_EL12 = S3_5_C4_C0_1; SPSR_EL12 = S3_5_C4_C0_0 */ + __asm__ volatile("mrs %0, S3_5_C5_C2_0" : "=r"(esr12)); + __asm__ volatile("mrs %0, S3_5_C4_C0_1" : "=r"(elr12)); + __asm__ volatile("mrs %0, S3_5_C4_C0_0" : "=r"(spsr12)); + __asm__ volatile("mrs %0, elr_el2" : "=r"(elr2)); + u32 ec12 = (u32)((esr12 >> 26) & 0x3f); + /* CRITICAL: distinguish vector-induced HVC (real SPRR sysreg trap) + * from in-kernel HVC dispatch (e.g. SPTM trampoline calls + * `hvc #0` directly with x16=selector). For the latter, ELR_EL12 + * and ESR_EL12 are STALE from the prior EL1 vector entry — they + * retain the LAST real SPRR trap's state. If we don't filter, + * we mis-fast-path SPTM HVCs as if they were SPRR sysreg traps, + * ERET-ing to a random PC inside pmap_ro_zone. ELR_EL2 is always + * fresh: for vector-induced traps it points at the patched HVC + * inside vbar_el1; for in-kernel HVCs it points at the dispatch + * site in xnu code (outside vbar_el1). + * + * vbar_el1 in the current 26.5 setup = 0xfffffe000b388000 (vector table is + * 0x800 bytes). Only treat as SPRR if ELR_EL2 falls in this range. */ + bool_t in_vbar = (elr2 >= 0xfffffe000b388000UL && + elr2 < 0xfffffe000b388800UL); + if (ec12 == 0 && in_vbar && + elr12 >= g_state.kc_vmin && elr12 < g_state.kc_vmax) { + u64 insn_pa = g_state.kc_guest_base + (elr12 - g_state.kc_vmin); + /* DEFENSIVE GUARD (2026-05-09): ensure insn_pa is a low PA, not a + * high VA. Symptom: m1n1 takes EL2 sync exception with FAR in + * 0xfffffe000c282xxx range when reading insn at "insn_pa" — that's + * a guest VA, not a PA, so EL2 stage-1 doesn't have it mapped. + * Cause: elr12 can be stale/bogus from a previous EL1 vector entry + * (the in_vbar+ec12==0 filter isn't perfect across all paths), so + * the computed insn_pa can land outside our PA range. Guard: + * - insn_pa must NOT have any high-VA bits set (PAs are < 4 TB) + * - insn_pa must be >= kc_guest_base (no arith underflow) + * - insn_pa must be < kc_guest_base + kc_size (kept in bounds) + * On failure: return unhandled so the host can stop with full + * diagnostics instead of crashing EL2. */ + u64 kc_size_va = g_state.kc_vmax - g_state.kc_vmin; + bool_t insn_pa_ok = (insn_pa >= g_state.kc_guest_base) && + ((insn_pa - g_state.kc_guest_base) < kc_size_va) && + ((insn_pa & 0xfffffe0000000000UL) == 0); + if (!insn_pa_ok) { + g_state.fallthrough_calls++; + return 0; /* unsafe insn_pa — hard-fail in host diagnostics */ + } + u32 insn = *(volatile u32 *)insn_pa; + /* MSR S3_6_C15_C1_x, xN → 0xd51ef1 ?? */ + /* MRS xN, S3_6_C15_C1_x → 0xd53ef1 ?? */ + u32 base = insn & 0xffffff00u; + if (base == 0xd51ef100u || base == 0xd53ef100u) { + u32 is_read = (insn >> 21) & 1; + u32 op2 = (insn >> 5) & 7; + u32 Rt = insn & 0x1f; + if (is_read) { + u64 val; + if (op2 == 6) val = g_state.sprr_perm_el1_cache; + else if (op2 == 5) val = g_state.sprr_perm_el0_cache; + else if (op2 == 0) val = g_state.sprr_config_el1_cache; + else val = 0; + if (Rt < 31) regs[Rt] = val; + } else { + u64 val = (Rt < 31) ? regs[Rt] : 0; + if (op2 == 6) g_state.sprr_perm_el1_cache = val; + else if (op2 == 5) g_state.sprr_perm_el0_cache = val; + else if (op2 == 0) g_state.sprr_config_el1_cache = val; + } + /* If we're inside pmap_ro_zone_memcpy_internal, capture + * its args into the ring buffer. By the time the first + * SPRR MSR fires (~+0x90 from function start), the + * prologue has stashed args into x19..x24 — they're + * stable for the rest of the call. Records: (elr, x22, + * x24, x23, x20, x19) = (pc, zid, va, offset, src, size). */ + if (g_state.pmap_rzm_lo != 0 && + elr12 >= g_state.pmap_rzm_lo && + elr12 < g_state.pmap_rzm_hi) { + u64 idx = g_state.pmap_rzm_idx & 0x7f; + g_state.pmap_rzm_ring[idx][0] = elr12; + g_state.pmap_rzm_ring[idx][1] = regs[22]; /* zid */ + g_state.pmap_rzm_ring[idx][2] = regs[24]; /* va */ + g_state.pmap_rzm_ring[idx][3] = regs[23]; /* offset */ + g_state.pmap_rzm_ring[idx][4] = regs[20]; /* new_data */ + g_state.pmap_rzm_ring[idx][5] = regs[19]; /* new_data_size */ + /* Also capture full register snapshot in parallel ring. + * Helps tell whether it's args that are wrong or the whole + * machine state that's corrupted. */ + for (int r = 0; r < 31; r++) { + g_state.pmap_rzm_full[idx][r] = regs[r]; + } + g_state.pmap_rzm_full[idx][31] = spsr12; + g_state.pmap_rzm_full[idx][32] = elr12; + g_state.pmap_rzm_idx = (g_state.pmap_rzm_idx + 1) & 0x7f; + g_state.pmap_rzm_count++; + } + + /* Skip xnu's vector handler entirely. ctx->elr currently + * points at the HVC in vbar_el1; redirect ERET straight to + * elr12+4, the original post-fault instruction boundary. + * + * CRITICAL: also restore ctx->spsr from SPSR_EL12. Without + * this, m1n1 ERETs with SPSR_EL2 which is EL1h (the vector + * handler's mode after entry), NOT the EL1t/EL1h state the + * function was actually running in. Result: SPSel switches + * mid-function, ldp/retab use the wrong stack — guaranteed + * PAC failure on signed-return functions. We mask DAIF for + * good measure (matches the Python EL1 vector handler). */ + u64 *ctx_elr = (u64 *)((unsigned char *)ctx + 0x108); + *ctx_elr = elr12 + 4; + /* Only mask DAIF.D (bit 9) to avoid xnu's + * "debug exceptions enabled in kernel mode" panic in + * ml_set_interrupts_enabled_with_debug. Preserve + * DAIF.I so vIRQ injection can actually fire. */ + *ctx_spsr_ptr = spsr12 | 0x200; + g_state.sprr_fast_count++; + g_state.fast_path_calls++; + sev(); + return 1; + } + } + } + + u64 x16 = regs[16]; + if (!is_dispatch_hvc(esr, x16)) { + u64 elr; + __asm__ volatile("mrs %0, elr_el2" : "=r"(elr)); + if ((esr & ESR_ISS_MASK) == SPTM_GENTER_DISPATCH_CALL && + (elr == XNU_265_GENTER_NODISPATCH_RET_PC || + elr == XNU_266_GENTER_NODISPATCH_RET_PC)) { + regs[0] = SPTM_SUCCESS; + g_state.fast_path_calls++; + sev(); + return 1; + } + g_state.fallthrough_calls++; + return 0; + } + u32 domain = (u32)((x16 >> 48) & 0xff); + u32 table = (u32)((x16 >> 32) & 0xff); + u32 endpoint = (u32)(x16 & 0xffffffff); + + if (domain == TXM_DOMAIN && table == TXM_TABLE_XNU) { + if (txm_handle_selector(endpoint, x16, regs)) + return 1; + g_state.fallthrough_calls++; + return 0; + } + + /* Only handle domain=0 (SPTM/XNU) here. Other domains have different + * endpoint numbering and must never be decoded as XNU_BOOTSTRAP calls. */ + if (domain != 0) { + g_state.fallthrough_calls++; + return 0; + } + + g_state.total_calls++; + g_state.last_x16 = x16; + g_state.last_endpoint = endpoint; + /* Record into ring buffer (last 128 calls). Also read ESR_EL2 + ELR_EL2 + * so we can distinguish SPTM HVC #0 from vector-trap HVC #0xFEEx and + * see which xnu code issued the HVC. */ + { + u64 idx = g_state.ring_idx & 0x7f; + u64 esr, elr; + __asm__ volatile("mrs %0, esr_el2" : "=r"(esr)); + __asm__ volatile("mrs %0, elr_el2" : "=r"(elr)); + g_state.ring[idx][0] = x16; + g_state.ring[idx][1] = regs[0]; + g_state.ring[idx][2] = regs[1]; + g_state.ring[idx][3] = regs[2]; + g_state.ring[idx][4] = regs[3]; + g_state.ring[idx][5] = esr; + g_state.ring[idx][6] = elr; + g_state.ring[idx][7] = regs[30]; /* x30 = LR = caller of trampoline */ + g_state.ring_idx = (g_state.ring_idx + 1) & 0x7f; + } + + /* Do not force periodic Python fall-throughs. UART round trips are too + * expensive for boot progress; visibility comes from the shared rings and + * explicit panic/watchdog dumps. */ + + if (table == SPTM_TABLE_NVME) { + if (nvme_handle_table6(endpoint, regs)) + return 1; + g_state.fallthrough_calls++; + return 0; + } + + if (table == SPTM_TABLE_T8110_DART_XNU || + table == SPTM_TABLE_T8110_DART_SK || + table == SPTM_TABLE_GEN3_DART_XNU || + table == SPTM_TABLE_GEN3_DART_SK) { + if (dart_handle_table(table, endpoint, regs)) + return 1; + g_state.fallthrough_calls++; + return 0; + } + + if (table == SPTM_TABLE_SART) { + if (sart_handle_table5(endpoint, regs)) + return 1; + g_state.fallthrough_calls++; + return 0; + } + + if (table == SPTM_TABLE_UAT) { + if (uat_handle_table7(endpoint, regs)) + return 1; + g_state.fallthrough_calls++; + return 0; + } + + if (table == SPTM_TABLE_CPUTRACE) { + /* CPUTRACE is a diagnostic/performance facility, not a boot-critical + * memory-management endpoint. The firmware exposes 13 known selectors + * on this table; acknowledge them here instead of forcing a UART + * round-trip to the removed Python fallback. regs[0]=0 is both the + * normal success code and the conservative "unsupported/no carveout" + * answer for query-shaped helpers. */ + if (endpoint <= 12) { + return finish_hvc_success(ctx, regs); + } + g_state.fallthrough_calls++; + return 0; + } + + if (compat_void_non_xnu_table(table)) { + return finish_hvc_success(ctx, regs); + } + + /* Fast-path the XNU_BOOTSTRAP table. Other non-XNU tables still fall + * through to host, except NVMe table 6 above which is intentionally in C + * to avoid Python/C split state on the ANS boot path. */ + if (table != SPTM_TABLE_XNU_BOOTSTRAP) { + dart_cache_xnu_object_from_regs(table, endpoint, regs); + g_state.fallthrough_calls++; + return 0; /* not handled */ + } + + + return xnu_handle_table0(ctx, endpoint, regs); +} + +extern void *hv_hvc_handler; + +void sptm_emul_init(void) +{ + hv_hvc_handler = (void *)sptm_handler; + + /* + * Map the RW_EL0 RAM alias Normal-NC. The emulator places the coprocessor / + * IOMMU page tables it manages in this alias, and they must be uncached so + * m1n1 does not allocate AMCC SLC directory tags on pages that AMCC/PMP + * agents later read non-coherently (otherwise: UNEXP_RT_HIT_DIR -> panic). + * This lives here, not in memory.c, so upstream memory.c stays generic and + * carries no SPTM layout knowledge. + */ + u64 ram_base = ALIGN_DOWN(cur_boot_args.phys_base, BIT(32)); + u64 ram_size = ALIGN_DOWN(cur_boot_args.mem_size + cur_boot_args.phys_base - ram_base, + get_page_size()); + mmu_add_mapping(ram_base | REGION_RW_EL0, ram_base, ram_size, MAIR_IDX_NORMAL_NC, PERM_RW_EL0); +} diff --git a/src/sptm/emul.h b/src/sptm/emul.h new file mode 100644 index 000000000..8882b87ab --- /dev/null +++ b/src/sptm/emul.h @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef SPTM_EMUL_H +#define SPTM_EMUL_H + +void sptm_emul_init(void); + +#endif diff --git a/src/sptm/nvme.c b/src/sptm/nvme.c new file mode 100644 index 000000000..1d9d8403a --- /dev/null +++ b/src/sptm/nvme.c @@ -0,0 +1,1252 @@ +/* SPDX-License-Identifier: MIT */ +#include "private.h" +#include "../memory.h" + +int debug_printf(const char *fmt, ...); +extern u64 hv_translate(u64 addr, _Bool s1only, _Bool w, u64 *par_out); + +#define NVME_NORMAL_BAR_FROM_NVMMU(nvmmu) ((nvmmu) - 0x28000UL) +#define NVME_NORMAL_BAR_UNKNOWN_CTRL 0x24008U +#define NVME_NORMAL_BAR_LINEAR_SQ_CTRL 0x24908U +#define NVME_NORMAL_BAR_MODESEL 0x1304U + +static inline void nvme_record_last(u32 endpoint, u64 rc, u64 *regs) { + g_state.nvme.last[0] = endpoint; + g_state.nvme.last[1] = rc; + g_state.nvme.last[2] = regs[0]; + g_state.nvme.last[3] = regs[1]; + g_state.nvme.last[4] = regs[2]; + g_state.nvme.last[5] = regs[3]; + g_state.nvme.last[6] = regs[4]; + g_state.nvme.last[7] = regs[5]; + g_state.nvme.last_rc = (u32)rc; +} + +static inline u64 nvme_violation(u32 endpoint, u64 *regs, u32 reason) { + g_state.nvme.violations++; + g_state.nvme.last[8] = reason; + nvme_record_last(endpoint, NVME_STATUS_VIOLATION, regs); + return NVME_STATUS_VIOLATION; +} + +static inline bool_t nvme_mmio_base_valid(u64 base) { + return base != 0 && base != ~0UL; +} + +static inline void nvme_mmio_write32(u64 base, u32 off, u32 val) { + *(volatile u32 *)(base + off) = val; +} + +static inline u32 nvme_mmio_read32(u64 base, u32 off) { + return *(volatile u32 *)(base + off); +} + +static inline u64 nvme_mmio_read64(u64 base, u32 off) { + u64 lo = nvme_mmio_read32(base, off); + u64 hi = nvme_mmio_read32(base, off + 4); + return lo | (hi << 32); +} + +static inline void nvme_bar_write32(u32 off, u32 val) { + nvme_mmio_write32(g_state.nvme.bar_pa, off, val); +} + +static inline void nvme_bar_write64(u32 off, u64 val) { + nvme_bar_write32(off, (u32)(val & 0xffffffffUL)); + nvme_bar_write32(off + 4, (u32)(val >> 32)); +} + +static inline void nvme_nvmmu_write32(u32 off, u32 val) { + nvme_mmio_write32(g_state.nvme.nvmmu_pa, off, val); +} + +static inline void nvme_nvmmu_write64(u32 off, u64 val) { + nvme_nvmmu_write32(off, (u32)(val & 0xffffffffUL)); + nvme_nvmmu_write32(off + 4, (u32)(val >> 32)); +} + +static inline u64 nvme_cntpct(void) { + u64 val; + __asm__ volatile("mrs %0, CNTPCT_EL0" : "=r"(val)); + return val; +} + +static inline u64 nvme_cntfrq(void) { + u64 val; + __asm__ volatile("mrs %0, CNTFRQ_EL0" : "=r"(val)); + return val; +} + +static inline void nvme_wfe(void) { + __asm__ volatile("wfe" ::: "memory"); +} + +static inline void nvme_dsb_sy(void) { + sysop("dsb sy"); +} + +static void nvme_diag_sart_live_entry(u32 idx, u64 *paddr, u64 *size, + u32 *flags) { + u32 cfg; + u32 paddr_raw; + u32 size_raw; + + *paddr = 0; + *size = 0; + *flags = 0; + + if (!g_state.sart.base_pa || idx >= SART_C_MAX_ENTRIES) + return; + + cfg = *(volatile u32 *)(g_state.sart.base_pa + idx * 4U); + paddr_raw = *(volatile u32 *)(g_state.sart.base_pa + 0x40U + idx * 4U); + + if (g_state.sart.version == 3) { + *flags = cfg & 0xffU; + size_raw = *(volatile u32 *)(g_state.sart.base_pa + 0x80U + idx * 4U); + } else if (g_state.sart.version == 2) { + *flags = (cfg >> 24) & 0xffU; + size_raw = cfg & ((1U << 24) - 1U); + } else { + *flags = (cfg >> 24) & 0x1fU; + size_raw = cfg & ((1U << 19) - 1U); + } + + if (*flags && !g_state.sart.exclusive_bounds) + size_raw++; + + *paddr = (u64)paddr_raw << g_state.sart.paddr_shift; + *size = (u64)size_raw << g_state.sart.size_shift; +} + +static void nvme_diag_sart_readback_once(void) { + static u32 printed; + + if (printed) + return; + printed = 1; + + if (!(g_state.nvme.debug_flags & NVME_DBG_PRINT_CALLS)) + return; + + debug_printf("C-NVME sart enabled=%u active=%u base=0x%lx protected=0x%x total=%lu last_rc=0x%x\n", + g_state.sart.enabled, g_state.sart.active, + g_state.sart.base_pa, g_state.sart.protected_mask, + g_state.sart.total_calls, g_state.sart.last_rc); + debug_printf("C-NVME ranges trusted=0x%lx..0x%lx secondary=0x%lx..0x%lx asq=0x%lx acq=0x%lx admin_kva=0x%lx/0x%lx\n", + g_state.nvme.trusted_io_base, + g_state.nvme.trusted_io_base + g_state.nvme.trusted_io_size, + g_state.nvme.secondary_io_base, + g_state.nvme.secondary_io_base + g_state.nvme.secondary_io_size, + g_state.nvme.asq_pa_cached, g_state.nvme.acq_pa_cached, + g_state.nvme.admin_asq_kva, g_state.nvme.admin_acq_kva); + + for (u32 i = 0; i < SART_C_MAX_ENTRIES; i++) { + u64 live_pa, live_size; + u32 live_flags; + struct sart_c_entry shadow = g_state.sart.shadow[i]; + + nvme_diag_sart_live_entry(i, &live_pa, &live_size, &live_flags); + if (!live_flags && !shadow.flags) + continue; + + debug_printf("C-NVME sart[%u] live=0x%lx+0x%lx f=0x%x shadow=0x%lx+0x%lx f=0x%x g=%u prot=%u\n", + i, live_pa, live_size, live_flags, + shadow.paddr, shadow.size, shadow.flags, shadow.guard, + (g_state.sart.protected_mask & (1U << i)) ? 1U : 0U); + } +} + +static bool_t nvme_guest_read64(u64 va, u64 *out) { + u64 pa; + + if ((va >> 48) != 0xffff || (va & 7)) + return 0; + + pa = hv_translate(va, 0, 0, 0); + if (!pa) + return 0; + + *out = *(volatile u64 *)pa; + return 1; +} + +static inline u16 nvme_load_le16(u8 *p) { + return (u16)p[0] | ((u16)p[1] << 8); +} + +static inline u32 nvme_load_le32(u8 *p) { + return (u32)p[0] | ((u32)p[1] << 8) | ((u32)p[2] << 16) | + ((u32)p[3] << 24); +} + +static void nvme_print_backtrace(u64 fp) { + debug_printf(" fp=0x%lx bt=", fp); + + for (u32 depth = 0; depth < 8; depth++) { + u64 saved_fp; + u64 saved_lr; + + if (!nvme_guest_read64(fp, &saved_fp) || + !nvme_guest_read64(fp + 8, &saved_lr)) + break; + + debug_printf("%s0x%lx", depth ? "," : "", saved_lr); + + if (saved_fp == fp || saved_fp < fp) + break; + fp = saved_fp; + } +} + +static void nvme_diag_ep5_readback_once(void) { + static u32 printed; + + if (printed) + return; + printed = 1; + + if (!nvme_mmio_base_valid(g_state.nvme.bar_pa) || + !nvme_mmio_base_valid(g_state.nvme.nvmmu_pa)) { + debug_printf("C-NVME ep5rb missing bar=0x%lx nvmmu=0x%lx\n", + g_state.nvme.bar_pa, g_state.nvme.nvmmu_pa); + return; + } + + nvme_dsb_sy(); + u32 aqa = nvme_mmio_read32(g_state.nvme.bar_pa, NVME_BAR_AQA); + u64 asq = nvme_mmio_read64(g_state.nvme.bar_pa, NVME_BAR_ASQ); + u64 acq = nvme_mmio_read64(g_state.nvme.bar_pa, NVME_BAR_ACQ); + u32 ioqa = nvme_mmio_read32(g_state.nvme.bar_pa, NVME_BAR_IOQA); + u32 cg = nvme_mmio_read32(g_state.nvme.nvmmu_pa, NVME_BAR_COASTGUARD); + u64 cgasq = nvme_mmio_read64(g_state.nvme.nvmmu_pa, NVME_BAR_CG_ASQ); + u64 cgacq = nvme_mmio_read64(g_state.nvme.nvmmu_pa, NVME_BAR_CG_ACQ); + u64 normal_bar = NVME_NORMAL_BAR_FROM_NVMMU(g_state.nvme.nvmmu_pa); + u32 unknown_ctrl = nvme_mmio_read32(normal_bar, + NVME_NORMAL_BAR_UNKNOWN_CTRL); + u32 linear_sq_ctrl = nvme_mmio_read32(normal_bar, + NVME_NORMAL_BAR_LINEAR_SQ_CTRL); + u32 modesel = nvme_mmio_read32(normal_bar, NVME_NORMAL_BAR_MODESEL); + + g_state.nvme.last[9] = 0xe5; + g_state.nvme.last[10] = ((u64)aqa << 32) | ioqa; + g_state.nvme.last[11] = asq; + g_state.nvme.last[12] = acq; + g_state.nvme.last[13] = ((u64)cg << 32) | unknown_ctrl; + g_state.nvme.last[14] = cgasq; + g_state.nvme.last[15] = cgacq; + + if (!(g_state.nvme.debug_flags & + (NVME_DBG_PRINT_CALLS | NVME_DBG_PRINT_MINIMAL))) + return; + + if (g_state.nvme.debug_flags & NVME_DBG_PRINT_MINIMAL) { + debug_printf("C-NVME ep5min allowed=0x%x aqa=0x%x asq=0x%lx acq=0x%lx ioqa=0x%x csts=0x%x\n", + g_state.nvme.allowed_functions, aqa, asq, acq, ioqa, + nvme_mmio_read32(normal_bar, 0x1c)); + } + + if (g_state.nvme.debug_flags & NVME_DBG_PRINT_CALLS) { + debug_printf("C-NVME ep5rb bar=0x%lx nvmmu=0x%lx allowed=0x%x\n", + g_state.nvme.bar_pa, g_state.nvme.nvmmu_pa, + g_state.nvme.allowed_functions); + debug_printf("C-NVME ep5rb aqa=0x%x asq=0x%lx acq=0x%lx ioqa=0x%x\n", + aqa, asq, acq, ioqa); + debug_printf("C-NVME ep5rb cg=0x%x cgasq=0x%lx cgacq=0x%lx cached aqa=0x%x ioqa=0x%x\n", + cg, cgasq, cgacq, g_state.nvme.aqa_cached, + g_state.nvme.ioqa_cached); + debug_printf("C-NVME ep5rb normal unknown=0x%x linear=0x%x modesel=0x%x\n", + unknown_ctrl, linear_sq_ctrl, modesel); + nvme_diag_sart_readback_once(); + } +} + +static void nvme_diag_ep1_readback(u32 qid, u32 cid, u32 count, u16 nlb, + u16 dir_bits, u8 mode, u64 first, + u64 second_or_list) { + u64 tcb_base = 0; + u16 word0 = 0; + u16 word1 = 0; + u16 word2 = 0; + u16 word3 = 0; + + if (qid < 2 && g_state.nvme.tcb_kva[qid] != 0) { + tcb_base = g_state.nvme.tcb_kva[qid] + (u64)cid * 0x80UL; + word0 = *(volatile u16 *)(tcb_base + 0); + word1 = *(volatile u16 *)(tcb_base + 2); + word2 = *(volatile u16 *)(tcb_base + 4); + word3 = *(volatile u16 *)(tcb_base + 6); + } + + g_state.nvme.last[9] = 0xe1; + g_state.nvme.last[10] = ((u64)qid << 48) | ((u64)cid << 32) | + ((u64)count << 16) | nlb; + g_state.nvme.last[11] = ((u64)dir_bits << 32) | mode; + g_state.nvme.last[12] = first; + g_state.nvme.last[13] = second_or_list; + g_state.nvme.last[14] = tcb_base; + g_state.nvme.last[15] = ((u64)word0 << 48) | ((u64)word1 << 32) | + ((u64)word2 << 16) | word3; + + if (!(g_state.nvme.debug_flags & NVME_DBG_PRINT_CALLS)) + return; + + debug_printf("C-NVME ep1rb qid=%u cid=%u count=%u nlb=0x%x dir=0x%x mode=%u\n", + qid, cid, count, nlb, dir_bits, mode); + debug_printf("C-NVME ep1rb first=0x%lx second_or_list=0x%lx tcb=0x%lx words=%x,%x,%x,%x\n", + first, second_or_list, tcb_base, word0, word1, word2, word3); +} + +static void nvme_diag_cmd_map(u32 qid, u32 cid, u32 count, u16 nlb, + u16 dir_bits, u8 mode, u64 first, + u64 second_or_list, u64 template_pa, + u64 seglist_pa, u8 *tmpl) { + if (!(g_state.nvme.debug_flags & NVME_DBG_CMD_TRACE)) + return; + + debug_printf("C-NVME cmd map qid=%u cid=%u count=%u nlb=0x%x dir=0x%x mode=%u tpl=0x%lx seg=0x%lx prp1=0x%lx prp2=0x%lx\n", + qid, cid, count, nlb, dir_bits, mode, template_pa, seglist_pa, + first, second_or_list); + debug_printf("C-NVME cmd map words=%x,%x,%x,%x,%x,%x,%x,%x cdw10=%x cdw11=%x cdw12=%x cdw13=%x\n", + nvme_load_le16(&tmpl[0x00]), nvme_load_le16(&tmpl[0x02]), + nvme_load_le16(&tmpl[0x04]), nvme_load_le16(&tmpl[0x06]), + nvme_load_le16(&tmpl[0x08]), nvme_load_le16(&tmpl[0x0a]), + nvme_load_le16(&tmpl[0x0c]), nvme_load_le16(&tmpl[0x0e]), + nvme_load_le32(&tmpl[0x28]), nvme_load_le32(&tmpl[0x2c]), + nvme_load_le32(&tmpl[0x30]), nvme_load_le32(&tmpl[0x34])); +} + +static void nvme_diag_ep2_prp_readback(u32 qid, u32 cid, u32 count, + u16 dir_bits, u8 mode, u64 first) { + static u32 printed; + u64 q0 = 0; + u64 q1 = 0; + u64 q2 = 0; + u64 q3 = 0; + u64 q178 = 0; + u64 q180 = 0; + u64 q188 = 0; + u8 b180 = 0; + + if (!(g_state.nvme.debug_flags & NVME_DBG_PRINT_CALLS) || + printed >= 8 || count == 0 || first == 0) + return; + printed++; + + q0 = *(volatile u64 *)(first + 0x00); + q1 = *(volatile u64 *)(first + 0x08); + q2 = *(volatile u64 *)(first + 0x10); + q3 = *(volatile u64 *)(first + 0x18); + if (g_state.nvme.debug_flags & NVME_DBG_PRP_WINDOW) { + q178 = *(volatile u64 *)(first + 0x178); + q180 = *(volatile u64 *)(first + 0x180); + q188 = *(volatile u64 *)(first + 0x188); + b180 = *(volatile u8 *)(first + 0x180); + } + + g_state.nvme.last[9] = 0xe2; + g_state.nvme.last[10] = ((u64)qid << 48) | ((u64)cid << 32) | + ((u64)count << 16) | dir_bits; + g_state.nvme.last[11] = ((u64)mode << 56) | first; + g_state.nvme.last[12] = q0; + g_state.nvme.last[13] = q1; + g_state.nvme.last[14] = q2; + g_state.nvme.last[15] = q3; + + debug_printf("C-NVME ep2rb qid=%u cid=%u count=%u dir=0x%x mode=%u prp1=0x%lx q=%lx,%lx,%lx,%lx\n", + qid, cid, count, dir_bits, mode, first, q0, q1, q2, q3); + if (g_state.nvme.debug_flags & NVME_DBG_PRP_WINDOW) { + debug_printf("C-NVME ep2rb180 qid=%u cid=%u prp1=0x%lx b180=0x%x q178=%lx q180=%lx q188=%lx\n", + qid, cid, first, b180, q178, q180, q188); + } +} + +static void nvme_diag_cmd_unmap(u32 qid, u32 cid, u32 retry, u32 count, + u16 dir_bits, u8 mode, u64 first, + u64 second_or_list) { + if (!(g_state.nvme.debug_flags & NVME_DBG_CMD_TRACE)) + return; + + debug_printf("C-NVME cmd unmap qid=%u cid=%u retry=%u count=%u dir=0x%x mode=%u prp1=0x%lx prp2=0x%lx\n", + qid, cid, retry, count, dir_bits, mode, first, + second_or_list); +} + +static inline bool_t nvme_range_contains(u64 base, u64 size, u64 pa, u64 len) { + if (base == 0 || size == 0) + return 0; + if (len == 0) + len = 1; + if (pa < base) + return 0; + if ((pa - base) > size) + return 0; + if (len > (size - (pa - base))) + return 0; + return 1; +} + +static inline bool_t nvme_in_trusted_io(u64 pa, u64 len) { + if (nvme_range_contains(g_state.nvme.trusted_io_base, + g_state.nvme.trusted_io_size, pa, len)) + return 1; + if (nvme_range_contains(g_state.nvme.secondary_io_base, + g_state.nvme.secondary_io_size, pa, len)) + return 1; + if (g_state.nvme.trusted_io_base == 0 && g_state.nvme.trusted_io_size == 0 && + g_state.nvme.secondary_io_base == 0 && g_state.nvme.secondary_io_size == 0) + return 1; + return 0; +} + +static inline bool_t nvme_validate_page_pa(u64 pa, u64 page_size) { + if (pa == 0) + return 0; + if (pa & (page_size - 1)) + return 0; + if (!nvme_in_trusted_io(pa, page_size)) + return 0; + return 1; +} + +static inline bool_t nvme_validate_ans_sha_page_pa(u64 pa) { + if (!nvme_validate_page_pa(pa, NVME_PRP_PAGE_SIZE)) + return 0; + if (pa & 0x3000UL) + return 0; + return 1; +} + +static inline bool_t nvme_validate_buffer(u64 pa, u64 len) { + if (pa == 0) + return 0; + return nvme_in_trusted_io(pa, len ? len : 1); +} + +static inline bool_t nvme_in_main_ram(u64 pa, u64 len) { + return nvme_range_contains(g_state.nvme.secondary_io_base, + g_state.nvme.secondary_io_size, + pa, len ? len : 1); +} + +static inline bool_t nvme_validate_guest_copy_buffer(u64 pa, u64 len) { + if (pa == 0 || len == 0) + return 0; + if (!nvme_in_main_ram(pa, len)) + return 0; + if (((pa & (NVME_PAGE_SIZE - 1)) + len) > NVME_PAGE_SIZE) + return 0; + return 1; +} + +static inline bool_t nvme_validate_prp(u64 pa, bool_t allow_offset) { + if (!nvme_validate_buffer(pa, 1)) + return 0; + if (!allow_offset && (pa & (NVME_PRP_PAGE_SIZE - 1))) + return 0; + return 1; +} + +static inline bool_t nvme_func_enter(u32 idx) { + if (idx >= 16) + return 0; + if (g_state.nvme.func_state[idx] == 1) + return 0; + g_state.nvme.func_state[idx] = 1; + return 1; +} + +static inline void nvme_func_exit(u32 idx) { + if (idx < 16) + g_state.nvme.func_state[idx] = 0; +} + +static inline bool_t nvme_cid_enter(u32 cid) { + if (cid >= g_state.nvme.max_cid || cid >= NVME_C_MAX_CID) + return 0; + if (g_state.nvme.cid_mode[cid] != NVME_CID_AVAILABLE) + return 0; + g_state.nvme.cid_mode[cid] = NVME_CID_BUSY; + return 1; +} + +static inline void nvme_cid_clear(u32 cid) { + if (cid < g_state.nvme.max_cid && cid < NVME_C_MAX_CID) + g_state.nvme.cid_mode[cid] = NVME_CID_AVAILABLE; +} + +static void nvme_diag_queue_summary(u32 event, u32 qid, u32 cid, u64 rc) { + static u64 last_map1; + static u64 last_unmap1; + u32 zero = 0; + u32 busy = 0; + u32 active0 = 0; + u32 active1 = 0; + u32 retry0 = 0; + u32 retry1 = 0; + u32 other = 0; + u32 max_cid = g_state.nvme.max_cid; + bool_t force; + + if (!(g_state.nvme.debug_flags & NVME_DBG_SUMMARY)) + return; + + if (max_cid > NVME_C_MAX_CID) + max_cid = NVME_C_MAX_CID; + + force = (event >= 3U); + if (!force) { + if (event == 1U && qid == 1) { + u64 n = g_state.nvme.ep_count[10]; + if ((n & 31UL) != 0 || n == last_map1) + return; + last_map1 = n; + } else if (event == 2U && qid == 1) { + u64 n = g_state.nvme.ep_count[12]; + if ((n & 31UL) != 0 || n == last_unmap1) + return; + last_unmap1 = n; + } else { + return; + } + } + + for (u32 i = 0; i < max_cid; i++) { + switch (g_state.nvme.cid_mode[i]) { + case NVME_CID_AVAILABLE: zero++; break; + case NVME_CID_BUSY: busy++; break; + case 0x03: active0++; break; + case 0x13: active1++; break; + case 0x04: retry0++; break; + case 0x14: retry1++; break; + default: other++; break; + } + } + + debug_printf("C-NVME qsum ev=%u qid=%u cid=%u rc=0x%lx " + "map0=%lu map1=%lu unmap0=%lu unmap1=%lu " + "retry=%lu mapfail=%lu unmapfail=%lu " + "zero=%u busy=%u act0=%u act1=%u retry0=%u retry1=%u other=%u " + "tl=%u viol=%u\n", + event, qid, cid, rc, + g_state.nvme.ep_count[9], g_state.nvme.ep_count[10], + g_state.nvme.ep_count[11], g_state.nvme.ep_count[12], + g_state.nvme.ep_count[13], g_state.nvme.ep_count[14], + g_state.nvme.ep_count[15], + zero, busy, active0, active1, retry0, retry1, other, + g_state.nvme.tl_timeouts, g_state.nvme.violations); +} + +static inline u32 nvme_ft_read_u32(u64 pa, u64 off) { + u64 idx; + if (!ft_get_idx(pa, &idx)) + return 0; + volatile u32 *cnt = + (volatile u32 *)(g_state.frame_table_base + idx * 16 + off); + return *cnt; +} + +static void nvme_diag_iommu_ref(const char *op, u64 pa, u8 mode, + u32 before, u32 after) { + static u32 printed; + u64 idx = 0; + + if (!(g_state.nvme.debug_flags & NVME_DBG_IOMMU)) + return; + if (printed >= 96) + return; + printed++; + + bool_t indexed = ft_get_idx(pa, &idx); + debug_printf("C-NVME iommu %s pa=0x%lx mode=%u indexed=%u idx=0x%lx " + "type=0x%x page_ref=%u table_ref=%u ro=%u rw=%u " + "before=%u after=%u\n", + op, pa, mode, indexed, idx, ft_get_type(pa), + ft_read_u16(pa, 4), ft_read_u16(pa, 8), + nvme_ft_read_u32(pa, 8), nvme_ft_read_u32(pa, 12), + before, after); +} + +static inline bool_t nvme_iommu_acquire(u64 pa, u8 mode) { + u64 off = (mode == NVME_IOMMU_RO) ? 8 : 12; + u32 before = nvme_ft_read_u32(pa, off); + if (mode == NVME_IOMMU_RO) + ft_inc_u32(pa, 8); + else + ft_inc_u32(pa, 12); + nvme_diag_iommu_ref("acq", pa, mode, before, nvme_ft_read_u32(pa, off)); + return 1; +} + +static inline void nvme_iommu_release_mode(u64 pa, u8 mode) { + if (pa == 0) + return; + u64 off = (mode == NVME_IOMMU_RO) ? 8 : 12; + u32 before = nvme_ft_read_u32(pa, off); + if (mode == NVME_IOMMU_RO) + ft_dec_u32(pa, 8); + else + ft_dec_u32(pa, 12); + nvme_diag_iommu_ref("rel", pa, mode, before, nvme_ft_read_u32(pa, off)); +} + +static inline void nvme_store_le64(u8 *p, u64 v) { + for (u32 i = 0; i < 8; i++) + p[i] = (u8)(v >> (i * 8)); +} + +static bool_t nvme_tl_wa_wait(u32 qid, u32 cid, u16 dir_bits, bool_t retry) { + if (!(g_state.nvme.quirks & NVME_QUIRK_TL_WA) || !(dir_bits & 0x300U)) + return 1; + if (!nvme_mmio_base_valid(g_state.nvme.tl_status_pa) || + !nvme_mmio_base_valid(g_state.nvme.tl_mask_pa) || + !nvme_mmio_base_valid(g_state.nvme.tl_ctrl_pa)) + return 0; + if (g_state.nvme.tl_num_sl > 16) + return 0; + + if (!retry) { + bool_t h2f = (dir_bits & 0x100U) != 0; + u32 mask = h2f ? 0x800000U : 0x80U; + u32 ctrl = h2f ? 0x400000U : 0x40U; + + nvme_mmio_write32(g_state.nvme.tl_mask_pa, 0x4, mask); + nvme_mmio_write32(g_state.nvme.tl_mask_pa, 0xc, mask); + nvme_mmio_write32(g_state.nvme.tl_ctrl_pa, 0x4, ctrl); + g_state.nvme.tl_slot_done[cid] = 0; + } + + nvme_dsb_sy(); + u64 start = nvme_cntpct(); + u64 freq = nvme_cntfrq(); + u64 timeout = (freq & ~0x1fUL) / 4000; + u64 nap_threshold = (freq & ~0x1fUL) / 100000; + u64 nap_period = (freq & ~0x7fUL) / 10000000; + if (timeout == 0) + timeout = 1; + if (nap_threshold == 0) + nap_threshold = 1; + if (nap_period == 0) + nap_period = 1; + + for (;;) { + u16 done = g_state.nvme.tl_slot_done[cid]; + u32 complete = 0; + + for (u32 slot = 0; slot < g_state.nvme.tl_num_sl; slot++) { + u16 bit = (u16)(1U << slot); + + if (done & bit) { + complete++; + continue; + } + + u32 status = nvme_mmio_read32(g_state.nvme.tl_status_pa, + slot * 0x10000U); + u32 pending; + if (dir_bits & 0x100U) + pending = (~status) & 0xffU; + else + pending = (~status) & 0x7f0000U; + if (pending == 0) { + done |= bit; + complete++; + } + } + g_state.nvme.tl_slot_done[cid] = done; + + if (complete == g_state.nvme.tl_num_sl) { + nvme_mmio_write32(g_state.nvme.tl_mask_pa, 0x4, 0x800080U); + nvme_mmio_write32(g_state.nvme.tl_mask_pa, 0xc, 0x800080U); + nvme_mmio_write32(g_state.nvme.tl_ctrl_pa, 0x4, 0x400040U); + g_state.nvme.tl_slot_done[cid] = 0; + return 1; + } + + nvme_dsb_sy(); + u64 elapsed = nvme_cntpct() - start; + if (elapsed >= timeout) + break; + if (elapsed > nap_threshold) { + nvme_dsb_sy(); + u64 nap = nvme_cntpct(); + while ((nvme_cntpct() - nap) < nap_period) { + nvme_dsb_sy(); + nvme_wfe(); + } + } + } + + g_state.nvme.tl_timeouts++; + g_state.nvme.cid_mode[cid] = (u8)((qid << 4) | 0x4U); + return 0; +} + +static u64 nvme_ep3_validate(u64 *regs) { + u32 queue_entries = (u32)regs[0]; + u32 proto = (u32)regs[1]; + u32 expected_proto = g_state.nvme.linear_sq ? 2U : 1U; + + if (!(g_state.nvme.allowed_functions & NVME_AF_QUEUE_VALIDATE)) + return nvme_violation(NVME_FN_VALIDATE_QENTRIES, regs, 1); + if (queue_entries != g_state.nvme.queue_count || proto != expected_proto) + return nvme_violation(NVME_FN_VALIDATE_QENTRIES, regs, 3); + return SPTM_SUCCESS; +} + +static u64 nvme_ep0_enable_coastguard(u64 *regs) { + if (!(g_state.nvme.allowed_functions & NVME_AF_ENABLE_COASTGUARD)) + return nvme_violation(NVME_FN_ENABLE_COASTGUARD, regs, 1); + if (g_state.nvme.admin_asq_kva == 0 || g_state.nvme.admin_acq_kva == 0) + return nvme_violation(NVME_FN_ENABLE_COASTGUARD, regs, 2); + if (!nvme_mmio_base_valid(g_state.nvme.nvmmu_pa)) + return nvme_violation(NVME_FN_ENABLE_COASTGUARD, regs, 3); + + nvme_nvmmu_write64(NVME_BAR_CG_ASQ, g_state.nvme.admin_asq_kva); + nvme_nvmmu_write64(NVME_BAR_CG_ACQ, g_state.nvme.admin_acq_kva); + nvme_nvmmu_write32(NVME_BAR_COASTGUARD, NVME_COASTGUARD_VALUE); + nvme_dsb_sy(); + + if (g_state.nvme.packed_writes_present) + g_state.nvme.allowed_functions |= NVME_AF_BAR_ADMIN_Q; + else + g_state.nvme.allowed_functions |= (NVME_AF_TCB_REGISTER | + NVME_AF_TCB_INVALIDATE); + return SPTM_SUCCESS; +} + +static u64 nvme_ep4_admin_queue(u64 *regs) { + u64 asq_pa = regs[0]; + u32 asq_size = (u32)regs[1]; + u64 acq_pa = regs[2]; + u32 acq_size = (u32)regs[3]; + u32 aqa; + u64 rc = NVME_STATUS_VIOLATION; + bool_t new_asq = 0; + bool_t new_acq = 0; + + if (!(g_state.nvme.allowed_functions & NVME_AF_BAR_ADMIN_Q)) + return nvme_violation(NVME_FN_BAR_ADMIN_QUEUE_REGS, regs, 1); + if (!nvme_func_enter(4)) + return nvme_violation(NVME_FN_BAR_ADMIN_QUEUE_REGS, regs, 2); + if (!nvme_mmio_base_valid(g_state.nvme.bar_pa)) + goto out; + + if (asq_size > NVME_QSIZE_MAX || acq_size > NVME_QSIZE_MAX) + goto out; + aqa = (asq_size & 0xfffU) | ((acq_size & 0xfffU) << 16); + if (g_state.nvme.aqa_cached != ~0U && + g_state.nvme.aqa_cached != aqa) + goto out; + if (g_state.nvme.asq_pa_cached != ~0UL && + g_state.nvme.asq_pa_cached != asq_pa) + goto out; + if (g_state.nvme.acq_pa_cached != ~0UL && + g_state.nvme.acq_pa_cached != acq_pa) + goto out; + if (!nvme_validate_page_pa(asq_pa, NVME_PRP_PAGE_SIZE) || + !nvme_validate_page_pa(acq_pa, NVME_PRP_PAGE_SIZE)) + goto out; + new_asq = g_state.nvme.asq_pa_cached == ~0UL; + new_acq = g_state.nvme.acq_pa_cached == ~0UL; + if (new_asq && !nvme_iommu_acquire(asq_pa, NVME_IOMMU_RW)) + goto out; + if (new_acq && !nvme_iommu_acquire(acq_pa, NVME_IOMMU_RW)) { + if (new_asq) + nvme_iommu_release_mode(asq_pa, NVME_IOMMU_RW); + goto out; + } + nvme_bar_write32(NVME_BAR_AQA, aqa); + nvme_bar_write64(NVME_BAR_ASQ, asq_pa); + nvme_bar_write64(NVME_BAR_ACQ, acq_pa); + barrier(); + + g_state.nvme.aqa_cached = aqa; + g_state.nvme.asq_pa_cached = asq_pa; + g_state.nvme.acq_pa_cached = acq_pa; + g_state.nvme.allowed_functions |= (NVME_AF_TCB_REGISTER | + NVME_AF_TCB_INVALIDATE | + NVME_AF_BAR_IOQA); + rc = SPTM_SUCCESS; +out: + nvme_func_exit(4); + if (rc != SPTM_SUCCESS) + return nvme_violation(NVME_FN_BAR_ADMIN_QUEUE_REGS, regs, 3); + return rc; +} + +static u64 nvme_ep5_ioqa(u64 *regs) { + u32 iosq_size = (u32)regs[0]; + u32 iocq_size = (u32)regs[1]; + u32 ioqa; + u64 rc = NVME_STATUS_VIOLATION; + + if (!(g_state.nvme.allowed_functions & NVME_AF_BAR_IOQA)) + return nvme_violation(NVME_FN_BAR_IOQA_REG, regs, 1); + if (!nvme_func_enter(5)) + return nvme_violation(NVME_FN_BAR_IOQA_REG, regs, 2); + if (!nvme_mmio_base_valid(g_state.nvme.bar_pa)) + goto out; + + if (iosq_size > g_state.nvme.queue_count || + iocq_size > g_state.nvme.queue_count) + goto out; + ioqa = (iosq_size & 0xffffU) | ((iocq_size & 0xffffU) << 16); + if (g_state.nvme.ioqa_cached != ~0U && g_state.nvme.ioqa_cached != ioqa) + goto out; + nvme_bar_write32(NVME_BAR_IOQA, ioqa); + barrier(); + g_state.nvme.ioqa_cached = ioqa; + if (g_state.nvme.packed_writes_present) + g_state.nvme.allowed_functions |= NVME_AF_BAR_IOCQ; + rc = SPTM_SUCCESS; +out: + nvme_func_exit(5); + if (rc != SPTM_SUCCESS) + return nvme_violation(NVME_FN_BAR_IOQA_REG, regs, 3); + return rc; +} + +static u64 nvme_ep6_iosq(u64 *regs) { + u64 iosq_pa = regs[0]; + u64 rc = NVME_STATUS_VIOLATION; + + if (!(g_state.nvme.allowed_functions & NVME_AF_BAR_IOSQ)) + return nvme_violation(NVME_FN_BAR_IOSQ_REG, regs, 1); + if (!nvme_func_enter(6)) + return nvme_violation(NVME_FN_BAR_IOSQ_REG, regs, 2); + if (!nvme_mmio_base_valid(g_state.nvme.bar_pa)) + goto out; + if (!nvme_validate_page_pa(iosq_pa, NVME_PRP_PAGE_SIZE)) + goto out; + if (g_state.nvme.iosq_pa_cached != ~0UL && + g_state.nvme.iosq_pa_cached != iosq_pa) + goto out; + if (g_state.nvme.iosq_pa_cached == ~0UL && + !nvme_iommu_acquire(iosq_pa, NVME_IOMMU_RW)) + goto out; + nvme_bar_write64(NVME_BAR_IOSQ_BASE, iosq_pa); + barrier(); + g_state.nvme.iosq_pa_cached = iosq_pa; + rc = SPTM_SUCCESS; +out: + nvme_func_exit(6); + if (rc != SPTM_SUCCESS) + return nvme_violation(NVME_FN_BAR_IOSQ_REG, regs, 3); + return rc; +} + +static u64 nvme_ep7_iocq(u64 *regs) { + u64 iocq_pa = regs[0]; + u64 rc = NVME_STATUS_VIOLATION; + + if (!(g_state.nvme.allowed_functions & NVME_AF_BAR_IOCQ)) + return nvme_violation(NVME_FN_BAR_IOCQ_REG, regs, 1); + if (!nvme_func_enter(7)) + return nvme_violation(NVME_FN_BAR_IOCQ_REG, regs, 2); + if (!nvme_mmio_base_valid(g_state.nvme.bar_pa)) + goto out; + if (!nvme_validate_page_pa(iocq_pa, NVME_PRP_PAGE_SIZE)) + goto out; + if (g_state.nvme.iocq_pa_cached != ~0UL && + g_state.nvme.iocq_pa_cached != iocq_pa) + goto out; + if (g_state.nvme.iocq_pa_cached == ~0UL && + !nvme_iommu_acquire(iocq_pa, NVME_IOMMU_RW)) + goto out; + nvme_bar_write64(NVME_BAR_IOCQ_BASE, iocq_pa); + barrier(); + g_state.nvme.iocq_pa_cached = iocq_pa; + if (g_state.nvme.packed_writes_present) + g_state.nvme.allowed_functions |= NVME_AF_BAR_IOSQ; + rc = SPTM_SUCCESS; +out: + nvme_func_exit(7); + if (rc != SPTM_SUCCESS) + return nvme_violation(NVME_FN_BAR_IOCQ_REG, regs, 3); + return rc; +} + +static u64 nvme_ep8_ans_sha(u64 *regs) { + u64 ans_sha_pa = regs[0]; + u64 ans_sha_size = regs[1]; + u32 pwc = (u32)regs[2]; + u32 pwc_mask = pwc & 3U; + u64 rc = NVME_STATUS_VIOLATION; + + if (!(g_state.nvme.allowed_functions & NVME_AF_ANS_SHA)) + return nvme_violation(NVME_FN_ANS_SHA_REG, regs, 1); + if (!g_state.nvme.ans_sha_present) + return nvme_violation(NVME_FN_ANS_SHA_REG, regs, 2); + if (!nvme_func_enter(8)) + return nvme_violation(NVME_FN_ANS_SHA_REG, regs, 3); + if (ans_sha_size != (u64)g_state.nvme.queue_count * NVME_PAGE_SIZE) + goto out; + if (g_state.nvme.ans_sha_pa_cached != ~0UL && + (g_state.nvme.ans_sha_pa_cached != ans_sha_pa || + g_state.nvme.ans_sha_pwc_cached != pwc_mask)) + goto out; + if (g_state.nvme.ans_sha_pwc_cached != ~0U && + g_state.nvme.ans_sha_pwc_cached != pwc_mask) + goto out; + for (u64 off = 0; off < ans_sha_size; off += NVME_PAGE_SIZE) { + if (!nvme_validate_ans_sha_page_pa(ans_sha_pa + off)) + goto out; + } + if (g_state.nvme.ans_sha_pa_cached == ~0UL) { + for (u64 off = 0; off < ans_sha_size; off += NVME_PAGE_SIZE) { + if (!nvme_iommu_acquire(ans_sha_pa + off, NVME_IOMMU_RW)) + goto out; + } + } + if (g_state.nvme.ans_sha_kva != 0) { + if (g_state.nvme.ans_sha_pwc_cached == ~0U) + g_state.nvme.ans_sha_pwc_cached = pwc_mask; + *(volatile u32 *)(g_state.nvme.ans_sha_kva + 0x0) = + (u32)(ans_sha_pa >> 14); + *(volatile u32 *)(g_state.nvme.ans_sha_kva + 0x4) = + g_state.nvme.ans_sha_pwc_cached; + } else { + goto out; + } + barrier(); + g_state.nvme.ans_sha_pa_cached = ans_sha_pa; + rc = SPTM_SUCCESS; +out: + nvme_func_exit(8); + if (rc != SPTM_SUCCESS) + return nvme_violation(NVME_FN_ANS_SHA_REG, regs, 4); + return rc; +} + +static u64 nvme_ep1_map_pages(u64 *regs) { + u32 qid = (u32)regs[0]; + u32 cid = (u32)regs[1]; + u64 template_pa = regs[2]; + u64 seglist_pa = regs[3]; + u32 count = (u32)regs[4]; + u8 tmpl[0x80]; + u16 nlb; + u16 dir_bits; + u8 mode; + u64 first = 0, second = 0, second_or_list = 0; + u32 acquired_count = 0; + bool_t cid_entered = 0; + + if (!(g_state.nvme.allowed_functions & NVME_AF_TCB_REGISTER)) + return nvme_violation(NVME_FN_MAP_PAGES, regs, 1); + if (qid > 1 || cid >= g_state.nvme.queue_count || + cid >= g_state.nvme.max_cid || cid >= NVME_C_MAX_CID || + count > 0x101U) + return nvme_violation(NVME_FN_MAP_PAGES, regs, 2); + if (!nvme_cid_enter(cid)) + return nvme_violation(NVME_FN_MAP_PAGES, regs, 3); + cid_entered = 1; + + if (!nvme_validate_guest_copy_buffer(template_pa, 0x80)) + goto fail; + for (u32 i = 0; i < 0x80; i++) + tmpl[i] = *(volatile u8 *)(template_pa + i); + + nlb = nvme_load_le16(&tmpl[4]); + dir_bits = nvme_load_le16(&tmpl[0]) & 0x1f00U; + if ((g_state.nvme.quirks & NVME_QUIRK_TL_WA) && + (dir_bits & 0x300U) == 0x300U) + goto fail; + mode = (dir_bits & 0x100U) ? NVME_IOMMU_RW : NVME_IOMMU_RO; + g_state.nvme.tcb_dir[cid] = dir_bits; + g_state.nvme.tcb_nlb[cid] = nlb; + for (u32 i = 0x18; i < 0x28; i++) + tmpl[i] = 0; + + if (count == 0) { + if (nlb != 0) + goto fail; + } else { + bool_t first_aligned; + u32 expected_nlb; + + if (!nvme_validate_guest_copy_buffer(seglist_pa, (u64)count * 8UL)) + goto fail; + first = *(volatile u64 *)seglist_pa; + first_aligned = ((first & (NVME_PRP_PAGE_SIZE - 1)) == 0); + if (!first_aligned && count == 1) + goto fail; + expected_nlb = count - (first_aligned ? 1U : 2U); + if (expected_nlb != nlb) + goto fail; + if (!nvme_validate_prp(first, 1) || + !nvme_iommu_acquire(first, mode)) + goto fail; + acquired_count = 1; + nvme_store_le64(&tmpl[0x18], first); + + if (count >= 2) { + second = *(volatile u64 *)(seglist_pa + 8); + if (!nvme_validate_prp(second, 0) || + !nvme_iommu_acquire(second, mode)) + goto fail; + acquired_count = 2; + if (count == 2) { + second_or_list = second; + nvme_store_le64(&tmpl[0x20], second); + } else { + u64 prp_list = g_state.nvme.prp_list_kva + (u64)cid * 0x800UL; + if (g_state.nvme.prp_list_kva == 0) + goto fail; + second_or_list = prp_list; + nvme_store_le64(&tmpl[0x20], prp_list); + for (u32 i = 1; i < count; i++) { + u64 pa = *(volatile u64 *)(seglist_pa + (u64)i * 8UL); + if (!nvme_validate_prp(pa, 0)) + goto fail; + if (i > 1) { + if (!nvme_iommu_acquire(pa, mode)) + goto fail; + acquired_count++; + } + *(volatile u64 *)(prp_list + (u64)(i - 1) * 8UL) = pa; + } + } + } + } + + tmpl[0] = 0; + tmpl[1] = 0; + + if (g_state.nvme.tcb_kva[qid] != 0) { + u64 tcb_base = g_state.nvme.tcb_kva[qid] + (u64)cid * 0x80UL; + if (*(volatile u16 *)tcb_base != 0 || + *(volatile u16 *)(tcb_base + 4) != 0) + goto fail; + for (u32 i = 0; i < 0x80; i++) + *(volatile u8 *)(tcb_base + i) = tmpl[i]; + *(volatile u16 *)(tcb_base + 2) = (u16)cid; + if (g_state.nvme.linear_sq) { + barrier(); + *(volatile u16 *)tcb_base = dir_bits; + } + } + + if ((g_state.nvme.quirks & NVME_QUIRK_PRP_FLUSH_WA) && count > 0x10U) { + u64 prp_list = g_state.nvme.prp_list_kva + (u64)cid * 0x800UL; + u32 flush_len = 0x1000U; + + if (count < 0x100U) + flush_len = ((count * 8U) + 0x7fU) & 0x1f80U; + sysop("dsb sy"); + if (flush_len != 0) + dc_civac_range((void *)prp_list, flush_len); + } + + g_state.nvme.tcb_count[cid] = count; + g_state.nvme.tcb_mode[cid] = mode; + g_state.nvme.tcb_qid[cid] = (u8)qid; + g_state.nvme.tcb_prp1[cid] = first; + g_state.nvme.tcb_prp2[cid] = second_or_list; + g_state.nvme.cid_mode[cid] = + (u8)((qid << 4) | (g_state.nvme.linear_sq ? 0x3U : 0x2U)); + barrier(); + if (qid < 2) + g_state.nvme.ep_count[9 + qid]++; + nvme_diag_queue_summary(1, qid, cid, SPTM_SUCCESS); + nvme_diag_cmd_map(qid, cid, count, nlb, dir_bits, mode, first, + second_or_list, template_pa, seglist_pa, tmpl); + nvme_diag_ep1_readback(qid, cid, count, nlb, dir_bits, mode, first, + second_or_list); + return SPTM_SUCCESS; + +fail: + if (cid_entered) + nvme_cid_clear(cid); + if (acquired_count >= 1) + nvme_iommu_release_mode(first, mode); + if (acquired_count >= 2) + nvme_iommu_release_mode(second, mode); + for (u32 i = 2; i < acquired_count; i++) { + u64 pa = *(volatile u64 *)(seglist_pa + (u64)i * 8UL); + nvme_iommu_release_mode(pa, mode); + } + g_state.nvme.ep_count[14]++; + nvme_diag_queue_summary(3, qid, cid, NVME_STATUS_VIOLATION); + return nvme_violation(NVME_FN_MAP_PAGES, regs, 4); +} + +static u64 nvme_ep2_violation(u64 *regs, u32 qid, u32 cid, u32 reason) { + g_state.nvme.ep_count[15]++; + nvme_diag_queue_summary(4, qid, cid, NVME_STATUS_VIOLATION); + return nvme_violation(NVME_FN_UNMAP_PAGES, regs, reason); +} + +static u64 nvme_ep2_unmap_pages(u64 *regs) { + u32 qid = (u32)regs[0]; + u32 cid = (u32)regs[1]; + u32 retry = (u32)regs[2] & 1U; + u8 cur; + u8 expected; + u32 count; + u16 dir_bits; + + if (!(g_state.nvme.allowed_functions & NVME_AF_TCB_INVALIDATE)) + return nvme_ep2_violation(regs, qid, cid, 1); + if (qid > 1 || cid >= g_state.nvme.queue_count || + cid >= g_state.nvme.max_cid || cid >= NVME_C_MAX_CID) + return nvme_ep2_violation(regs, qid, cid, 2); + if (retry) + g_state.nvme.ep_count[13]++; + cur = g_state.nvme.cid_mode[cid]; + expected = (u8)((qid << 4) | (retry ? 0x4U : 0x3U)); + if (cur != expected) + return nvme_ep2_violation(regs, qid, cid, 3); + g_state.nvme.cid_mode[cid] = NVME_CID_BUSY; + + count = g_state.nvme.tcb_count[cid]; + dir_bits = g_state.nvme.tcb_dir[cid]; + + if (!retry) { + if (g_state.nvme.tcb_kva[qid] != 0) { + u64 tcb_base = g_state.nvme.tcb_kva[qid] + (u64)cid * 0x80UL; + *(volatile u16 *)tcb_base = 0; + } + if ((g_state.nvme.quirks & NVME_QUIRK_VDMA_WA) && + (!nvme_mmio_base_valid(g_state.nvme.vdma_status_pa) || + (*(volatile u32 *)(g_state.nvme.vdma_status_pa + + (u64)cid * 0x20UL) & 0x300U) != 0)) + return nvme_ep2_violation(regs, qid, cid, 5); + if (!nvme_mmio_base_valid(g_state.nvme.nvmmu_pa)) + return nvme_ep2_violation(regs, qid, cid, 6); + barrier(); + nvme_nvmmu_write32(NVME_BAR_TCB_INVALIDATE, cid); + u32 stat = nvme_mmio_read32(g_state.nvme.nvmmu_pa, + ((cid >> 2) & 0x3fffU) * 4U); + u32 shift = (cid & 3U) | ((cid & 3U) << 2); + if (((stat >> shift) & 0xfU) != 0) + return nvme_ep2_violation(regs, qid, cid, 7); + } + + if (!nvme_tl_wa_wait(qid, cid, dir_bits, retry)) { + g_state.nvme.ep_count[15]++; + nvme_diag_queue_summary(5, qid, cid, 0); + return 0; + } + + nvme_diag_cmd_unmap(qid, cid, retry, count, dir_bits, + g_state.nvme.tcb_mode[cid], + g_state.nvme.tcb_prp1[cid], + g_state.nvme.tcb_prp2[cid]); + nvme_diag_ep2_prp_readback(qid, cid, count, dir_bits, + g_state.nvme.tcb_mode[cid], + g_state.nvme.tcb_prp1[cid]); + + if (count >= 1) + nvme_iommu_release_mode(g_state.nvme.tcb_prp1[cid], + g_state.nvme.tcb_mode[cid]); + if (count == 2) { + nvme_iommu_release_mode(g_state.nvme.tcb_prp2[cid], + g_state.nvme.tcb_mode[cid]); + } else if (count > 2) { + u64 prp_list = g_state.nvme.prp_list_kva ? + g_state.nvme.prp_list_kva + (u64)cid * 0x800UL : + g_state.nvme.tcb_prp2[cid]; + for (u32 i = 0; i < count - 1; i++) { + u64 pa = *(volatile u64 *)(prp_list + (u64)i * 8UL); + nvme_iommu_release_mode(pa, g_state.nvme.tcb_mode[cid]); + } + for (u32 i = 0; i < count - 1; i++) + *(volatile u64 *)(prp_list + (u64)i * 8UL) = 0; + } + + if (g_state.nvme.tcb_kva[qid] != 0) { + u64 tcb_base = g_state.nvme.tcb_kva[qid] + (u64)cid * 0x80UL; + for (u32 i = 0; i < 0x80; i++) + *(volatile u8 *)(tcb_base + i) = 0; + } + if ((g_state.nvme.quirks & NVME_QUIRK_PRP_FLUSH_WA) && count > 0xffU) { + u64 prp_list = g_state.nvme.tcb_prp2[cid]; + + sysop("dsb sy"); + if (prp_list != 0) + dc_civac_range((void *)(prp_list + 0x800UL), 0x800); + } + + g_state.nvme.tcb_count[cid] = 0; + g_state.nvme.tcb_prp1[cid] = 0; + g_state.nvme.tcb_prp2[cid] = 0; + g_state.nvme.tcb_dir[cid] = 0; + nvme_cid_clear(cid); + barrier(); + if (qid < 2) + g_state.nvme.ep_count[11 + qid]++; + nvme_diag_queue_summary(2, qid, cid, 1); + /* sptm_nvme_unmap_pages is the odd table-6 endpoint whose RE'd return is + * boolean-like: 1 means success, 0 is the retry-path failure value. Keep + * this as 1 even though most SPTM endpoints use SPTM_SUCCESS == 0. */ + return 1; +} + +bool_t nvme_handle_table6(u32 endpoint, u64 *regs) { + u64 rc = NVME_STATUS_VIOLATION; + u64 orig0 = regs[0]; + u64 orig1 = regs[1]; + u64 orig2 = regs[2]; + u64 orig3 = regs[3]; + + if (!g_state.nvme.enabled) + return 0; + + g_state.nvme.total_calls++; + if (endpoint < 16) + g_state.nvme.ep_count[endpoint]++; + + switch (endpoint) { + case NVME_FN_ENABLE_COASTGUARD: rc = nvme_ep0_enable_coastguard(regs); break; + case NVME_FN_MAP_PAGES: rc = nvme_ep1_map_pages(regs); break; + case NVME_FN_UNMAP_PAGES: rc = nvme_ep2_unmap_pages(regs); break; + case NVME_FN_VALIDATE_QENTRIES: rc = nvme_ep3_validate(regs); break; + case NVME_FN_BAR_ADMIN_QUEUE_REGS: rc = nvme_ep4_admin_queue(regs); break; + case NVME_FN_BAR_IOQA_REG: rc = nvme_ep5_ioqa(regs); break; + case NVME_FN_BAR_IOSQ_REG: rc = nvme_ep6_iosq(regs); break; + case NVME_FN_BAR_IOCQ_REG: rc = nvme_ep7_iocq(regs); break; + case NVME_FN_ANS_SHA_REG: rc = nvme_ep8_ans_sha(regs); break; + default: + rc = nvme_violation(endpoint, regs, 0xff); + break; + } + + nvme_record_last(endpoint, rc, regs); + if (endpoint == NVME_FN_BAR_IOQA_REG && rc == SPTM_SUCCESS) + nvme_diag_ep5_readback_once(); + if (g_state.nvme.debug_flags & NVME_DBG_PRINT_MINIMAL) { + if (endpoint == NVME_FN_ENABLE_COASTGUARD || + endpoint == NVME_FN_MAP_PAGES || + endpoint == NVME_FN_UNMAP_PAGES || + endpoint == NVME_FN_VALIDATE_QENTRIES || + endpoint == NVME_FN_BAR_ADMIN_QUEUE_REGS || + endpoint == NVME_FN_BAR_IOQA_REG || + endpoint == NVME_FN_BAR_IOSQ_REG || + endpoint == NVME_FN_BAR_IOCQ_REG) { + debug_printf("C-NVME min ep=%u rc=0x%lx allowed=0x%x args=%lx,%lx,%lx,%lx\n", + endpoint, rc, g_state.nvme.allowed_functions, + orig0, orig1, orig2, orig3); + } + } + if ((g_state.nvme.debug_flags & NVME_DBG_PRINT_CALLS) && + (g_state.nvme.total_calls <= 128 || + endpoint == NVME_FN_BAR_IOSQ_REG || + endpoint == NVME_FN_BAR_IOCQ_REG || + (rc != SPTM_SUCCESS && + !(endpoint == NVME_FN_UNMAP_PAGES && rc == 1)))) { + u64 elr; + __asm__ volatile("mrs %0, elr_el2" : "=r"(elr)); + debug_printf("C-NVME ep=%u rc=0x%lx allowed=0x%x pc=0x%lx lr=0x%lx args=%lx,%lx,%lx,%lx", + endpoint, rc, g_state.nvme.allowed_functions, + elr, regs[30], orig0, orig1, orig2, orig3); + nvme_print_backtrace(regs[29]); + debug_printf("\n"); + } + regs[0] = rc; + g_state.nvme.fast_path_calls++; + g_state.fast_path_calls++; + sev(); + return 1; +} diff --git a/src/sptm/private.h b/src/sptm/private.h new file mode 100644 index 000000000..3185745ca --- /dev/null +++ b/src/sptm/private.h @@ -0,0 +1,2472 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef SPTM_PRIVATE_H +#define SPTM_PRIVATE_H + +/* + * DO NOT MERGE: tainted SPTM endpoint emulator imported from the bringup tree. + * + * This is intentionally isolated under src/sptm while we separate the clean + * hypervisor plumbing from the reverse-engineered SPTM behavior. The old tree + * built this as a hot-loaded C object; this branch links it directly into the + * streamed stage2 hypervisor so endpoint state has one C source of truth. + */ + +typedef unsigned long u64; +typedef unsigned int u32; +typedef unsigned short u16; +typedef unsigned char u8; +typedef int bool_t; /* avoid in freestanding */ + +void dc_civac_range(void *addr, u64 length); +void mmu_add_mapping(u64 from, u64 to, u64 size, u8 attribute_index, u64 perms); +int hv_map(u64 from, u64 to, u64 size, u64 incr); +u64 hv_pt_walk(u64 addr); + +/* SPTM constants — match the proxyclient/tools/sptm Python helpers. */ +#define SPTM_SUCCESS 0 +#define SPTM_MAP_VALID 1 +#define SPTM_UPDATE_DELAYED_TLBI 5 +#define SPTM_MAP_PADDR_CONFLICT 6 +#define SPTM_TABLE_NOT_PRESENT 7 +#define SPTM_TABLE_ALREADY_PRESENT 8 + +#define SPTM_UPDATE_SW_WIRED (1U << 0) +#define SPTM_UPDATE_PERMS_AND_WAS_WRITABLE (1U << 1) +#define SPTM_UPDATE_NG (1U << 2) +#define SPTM_UPDATE_AF (1U << 3) +#define SPTM_UPDATE_SH (1U << 4) +#define SPTM_UPDATE_MAIR (1U << 5) +#define SPTM_UPDATE_MASK 0x3fU +#define SPTM_UPDATE_DEFER_TLBI (1U << 8) +#define SPTM_UPDATE_SKIP_PAPT (1U << 9) + +#define HV_S2_PTE_ACCESS (1UL << 10) +#define HV_S2_PTE_SH_NS (3UL << 8) +#define HV_S2_PTE_S2AP_RW (3UL << 6) +#define HV_S2_MEMATTR_WB (0xfUL << 2) +/* With HCR_EL2.FWB set, stage-2 MemAttr 5 is Normal Non-cacheable. */ +#define HV_S2_MEMATTR_NORMAL_NC (0x5UL << 2) +/* + * GPU-shared coherency type: Outer-WB, Inner-NC (matches XNU's MAIR 0xf4 "Shared" + * and the AGX UAT AttrIndex-2). Inner-NC makes CPU writes skip L1/L2 and reach the + * SLC (Point of Coherency) the GPU reads, so the GPU sees ongoing CPU writes with + * no per-write flush. Under FWB=1: bits[3:2]=outer,[1:0]=inner,{01=NC,11=WB} -> 0b1101. + */ +#define HV_S2_MEMATTR_SHARED (0xdUL << 2) +#define HV_S2_PTE_VALID 1UL +#define HV_S2_PTE_NORMAL_WB (HV_S2_PTE_ACCESS | HV_S2_PTE_SH_NS | \ + HV_S2_PTE_S2AP_RW | HV_S2_MEMATTR_WB | \ + HV_S2_PTE_VALID) +#define HV_S2_PTE_NORMAL_NC (HV_S2_PTE_ACCESS | HV_S2_PTE_SH_NS | \ + HV_S2_PTE_S2AP_RW | \ + HV_S2_MEMATTR_NORMAL_NC | \ + HV_S2_PTE_VALID) +#define HV_S2_PTE_SHARED (HV_S2_PTE_ACCESS | HV_S2_PTE_SH_NS | \ + HV_S2_PTE_S2AP_RW | \ + HV_S2_MEMATTR_SHARED | \ + HV_S2_PTE_VALID) + +#define SPTM_EL2_MAIR_NORMAL 0U +#define SPTM_EL2_MAIR_NORMAL_NC 1U +#define SPTM_EL2_PERM_RWX 0UL +#define SPTM_PAPT_ATTR_RT_NC (1UL << 2) /* AttrIdx=1, SH=0 */ +#define SPTM_PAPT_ATTR_NORMAL_WB (2UL << 8) /* AttrIdx=0, inner-shareable */ +#define SPTM_PAPT_OPT_ALT_TO_NC 0x80000000U +#define SPTM_PAPT_OPT_ALT_TO_WB 0x40000000U + +#define PAPT_UPDATE_RING_SLOTS 64 +#define PAPT_UPDATE_RC_WRITE 1 +#define PAPT_UPDATE_RC_SKIP_PAPT 2 +#define PAPT_UPDATE_RC_NO_ATTR 3 +#define PAPT_UPDATE_RC_UNMANAGED 4 +#define PAPT_UPDATE_RC_BAD_TEMPLATE 5 +#define PAPT_UPDATE_RC_BAD_ROOT 6 +#define PAPT_UPDATE_RC_BAD_WALK 7 +#define PAPT_UPDATE_RC_BAD_EXISTING 8 +#define PAPT_UPDATE_RC_SAME_ATTRS 9 +#define ALT_PTD_TRACE_RING_SLOTS 4 +#define PAPT_NVME_CLASS_COUNT 8 +#define PAPT_NVME_SEED_ASQ 0 +#define PAPT_NVME_SEED_ACQ 1 +#define PAPT_NVME_PRP_LIST 2 +#define PAPT_NVME_XNU_ASQ 3 +#define PAPT_NVME_XNU_ACQ 4 +#define PAPT_NVME_XNU_IOSQ 5 +#define PAPT_NVME_XNU_IOCQ 6 +#define PAPT_NVME_ACTIVE_PRP 7 + +#define SPTM_TABLE_XNU_BOOTSTRAP 0 +#define SPTM_TABLE_TXM_BOOTSTRAP 1 +#define SPTM_TABLE_SK_BOOTSTRAP 2 +#define SPTM_TABLE_T8110_DART_XNU 3 +#define SPTM_TABLE_T8110_DART_SK 4 +#define SPTM_TABLE_SART 5 +#define SPTM_TABLE_NVME 6 +#define SPTM_TABLE_UAT 7 +#define SPTM_TABLE_SHART 8 +/* Public SPTM tables call 9 "reserved", but the t8140 SPTM firmware uses it + * for CPU trace/perf-trace helpers. Boot does not need a real trace buffer: + * report unsupported/no-carveout/success and keep moving. */ +#define SPTM_TABLE_CPUTRACE 9 +#define SPTM_TABLE_HIB 10 +#define SPTM_TABLE_GEN3_DART_XNU 11 +#define SPTM_TABLE_GEN3_DART_SK 12 +#define SPTM_TABLE_T6000_DART_XNU 13 +#define SPTM_TABLE_INVALID 14 +#define DART_OBJ_CACHE_SLOTS 64 +#define NVME_C_MAX_CID 0x101 +#define SART_C_MAX_ENTRIES 16 +#define SART_C_MAX_ENDPOINTS 3 + +#define SPTM_GENTER_DISPATCH_CALL 0 +#define SPTM_DISPATCH_RESERVED_MASK 0xff00ff0000000000UL +#define ESR_ISS_MASK 0x1ffffffUL + +/* t8140 top-level non-dispatch GENTER return sites. Mechanical + * genter->HVC patching erases the original immediate, so GENTER #1..#4 + * reach us as HVC #0 with arbitrary/poison x16. Only the observed returning + * stub is fast-pathed here; other invalid HVC #0 sites still fall through + * loudly instead of hiding a corrupted dispatch call. */ +#define XNU_265_GENTER_NODISPATCH_RET_PC 0xfffffe000bcf4d9cUL +#define XNU_266_GENTER_NODISPATCH_RET_PC 0xfffffe000bcfac08UL + +#define SPTM_FN_LOCKDOWN 0 +#define SPTM_FN_RETYPE 1 +#define SPTM_FN_MAP_PAGE 2 +#define SPTM_FN_MAP_TABLE 3 +#define SPTM_FN_UNMAP_TABLE 4 +#define SPTM_FN_UPDATE_REGION 5 +#define SPTM_FN_UPDATE_DISJOINT 6 +#define SPTM_FN_UNMAP_REGION 7 +#define SPTM_FN_UNMAP_DISJOINT 8 +#define SPTM_FN_CONFIGURE_SHAREDREGION 9 +#define SPTM_FN_NEST_REGION 10 +#define SPTM_FN_UNNEST_REGION 11 +#define SPTM_FN_CONFIGURE_ROOT 12 +#define SPTM_FN_SWITCH_ROOT 13 +#define SPTM_FN_REGISTER_CPU 14 +#define SPTM_FN_FIXUPS_COMPLETE 15 +#define SPTM_FN_SIGN_USER_POINTER 16 +#define SPTM_FN_AUTH_USER_POINTER 17 +#define SPTM_FN_REGISTER_EXC_RETURN 18 +#define SPTM_FN_CPU_ID 19 +#define SPTM_FN_SLIDE_REGION 20 +#define SPTM_FN_UPDATE_DISJOINT_MULTIPAGE 21 +#define SPTM_FN_REG_READ 22 +#define SPTM_FN_REG_WRITE 23 +#define SPTM_FN_GUEST_VA_TO_IPA 24 +#define SPTM_FN_GUEST_STAGE1_TLBOP 25 +#define SPTM_FN_GUEST_STAGE2_TLBOP 26 +#define SPTM_FN_GUEST_DISPATCH 27 +#define SPTM_FN_GUEST_EXIT 28 +#define SPTM_FN_MAP_SK_DOMAIN 29 +#define SPTM_FN_HIB_BEGIN 30 +#define SPTM_FN_HIB_VERIFY_HASH_NON_WIRED 31 +#define SPTM_FN_HIB_FINALIZE_NON_WIRED 32 +#define SPTM_FN_IOFILTER_PROTECTED_WRITE 33 +#define SPTM_FN_SPTM_SYSCTL 37 +#define SPTM_FN_DISABLE_KERNEL_MODE_CPA2 38 +#define SPTM_FN_SET_SHARED_REGION 39 +#define SPTM_FN_BATCH_SIGN_USER_POINTER 40 +#define SPTM_FN_SURT_ALLOC 41 +#define SPTM_FN_SURT_FREE 42 +#define SPTM_FN_CONDEMN_LEAF_TABLE 43 +#define SPTM_FN_UNCONDEMN_LEAF_TABLE 44 +#define SPTM_FN_SERIAL_PUTC 45 +#define SPTM_FN_SERIAL_DISABLE 46 +#define SPTM_FN_PROGRAM_IRGKEY 48 +#define SPTM_FN_REG_SNAPSHOT 49 + +#define SPTM_MAX_CPUS_EMUL 16 + +#define SPTM_CPU_PIO_F_SEEDED (1ULL << 0) +#define SPTM_CPU_PIO_F_REGISTERED (1ULL << 1) +#define SPTM_CPU_PIO_F_MISSING (1ULL << 2) + +/* NVMe table 6 endpoint IDs and state-machine constants. Endpoint semantics + * live here; Python only seeds ADT-derived configuration and SPTM-owned + * buffers before guest start. Endpoint 1 receives a TCB template PA and a + * segment-list PA; it does not receive direct PRP1/PRP2 arguments. */ +#define NVME_FN_ENABLE_COASTGUARD 0 +#define NVME_FN_MAP_PAGES 1 +#define NVME_FN_UNMAP_PAGES 2 +#define NVME_FN_VALIDATE_QENTRIES 3 +#define NVME_FN_BAR_ADMIN_QUEUE_REGS 4 +#define NVME_FN_BAR_IOQA_REG 5 +#define NVME_FN_BAR_IOSQ_REG 6 +#define NVME_FN_BAR_IOCQ_REG 7 +#define NVME_FN_ANS_SHA_REG 8 + +#define NVME_AF_ENABLE_COASTGUARD (1u << 0) +#define NVME_AF_TCB_REGISTER (1u << 1) +#define NVME_AF_TCB_INVALIDATE (1u << 2) +#define NVME_AF_QUEUE_VALIDATE (1u << 3) +#define NVME_AF_BAR_ADMIN_Q (1u << 4) +#define NVME_AF_BAR_IOQA (1u << 5) +#define NVME_AF_BAR_IOSQ (1u << 6) +#define NVME_AF_BAR_IOCQ (1u << 7) +#define NVME_AF_ANS_SHA (1u << 8) + +#define NVME_STATUS_VIOLATION 0x100UL +#define NVME_PAGE_SIZE 0x4000UL +#define NVME_PRP_PAGE_SIZE 0x1000UL +#define NVME_QSIZE_MAX 0xffeU +#define NVME_COASTGUARD_VALUE 0x3fU + +#define NVME_BAR_AQA 0x24U +#define NVME_BAR_ASQ 0x28U +#define NVME_BAR_ACQ 0x30U +#define NVME_BAR_COASTGUARD 0x100U +#define NVME_BAR_CG_ASQ 0x108U +#define NVME_BAR_CG_ACQ 0x110U +#define NVME_BAR_IOSQ_BASE 0x1200U +#define NVME_BAR_IOCQ_BASE 0x1208U +#define NVME_BAR_IOQA 0x1210U +#define NVME_BAR_TCB_INVALIDATE 0x118U + +#define NVME_CID_AVAILABLE 0 +#define NVME_CID_BUSY 1 +#define NVME_IOMMU_RO 1 +#define NVME_IOMMU_RW 2 + +#define NVME_QUIRK_PRP_FLUSH_WA (1u << 0) +#define NVME_QUIRK_TL_WA (1u << 1) +#define NVME_QUIRK_VDMA_WA (1u << 2) + +#define NVME_DBG_PRINT_CALLS (1u << 0) +#define NVME_DBG_PRP_WINDOW (1u << 1) +#define NVME_DBG_PRINT_MINIMAL (1u << 2) +#define NVME_DBG_SUMMARY (1u << 3) +#define NVME_DBG_IOMMU (1u << 4) +#define NVME_DBG_CMD_TRACE (1u << 5) + +/* + * T8110 DART C emulator. + * + * The register constants and TLB/error sequencing below intentionally follow + * the public Asahi Linux apple-dart.c T8110 model: + * archive/references/asahi_linux/apple-dart.c + * + * This is SPTM endpoint emulation, not Linux's ownership driver. Endpoint 6 + * preserves bootloader-owned state instead of doing apple_dart_hw_reset()'s + * destructive TCR/TTBR clear. Later endpoints install roots, stream enables, + * and TLB operations explicitly, matching the SPTM ABI observed in the real + * t8140 firmware. + */ +#define DART_C_MAX_DARTS 32 +#define DART_C_MAX_BASES 16 +#define DART_C_MAX_SIDS 256 +#define DART_C_MAX_IDENTITY_RANGES 1024 +#define DART_C_MAX_ENDPOINTS 19 +#define DART_C_RECENT_SLOTS 1024 +#define DART_C_SID0_TRACE_SLOTS 4096 +#define DART_C_MAX_DAPF_SLICES 32 +#define DART_C_MAX_PIOGW_RANGES 2 +#define DART_C_MAX_PIOGW_DESCS 2 + +#define UAT_C_MAX_ENDPOINTS 13 +#define UAT_C_MAX_STATES 32 +#define UAT_C_RECENT_SLOTS 32 +#define UAT_C_PAGE_SIZE 0x4000UL +#define UAT_C_CTX_NONE 0xffffU + +#define UAT_FN_INIT_STATE 0 +#define UAT_FN_DESTROY_STATE 1 +#define UAT_FN_MAP_TABLE 2 +#define UAT_FN_UNMAP_TABLE 3 +#define UAT_FN_MAP_BEGIN 4 +#define UAT_FN_MAP_CONTINUE 5 +#define UAT_FN_PREP_FW_UNMAP_BEGIN 6 +#define UAT_FN_PREP_FW_UNMAP_CONTINUE 7 +#define UAT_FN_UNMAP_BEGIN 8 +#define UAT_FN_UNMAP_CONTINUE 9 +#define UAT_FN_SET_CTX_ID 10 +#define UAT_FN_REMOVE_CTX_ID 11 +#define UAT_FN_GET_INFO 12 + +#define DART_C_STATUS_VIOLATION 0x100UL +#define DART_C_PAGE_SIZE 0x4000UL +#define DART_C_TT_ENTRIES 2048 +#define DART_C_MAX_STREAMS 256 +#define DART_C_PTE_NO_CACHE (1UL << 1) + +#define DART_C_RANGE_SID_MASK 0xffffUL +#define DART_C_RANGE_FLAG_REMAP (1UL << 63) +#define DART_C_RANGE_FLAG_BOOT_PREMAP (1UL << 60) +#define DART_C_RANGE_FLAG_VM_WINDOW (1UL << 59) +#define DART_C_RANGE_FLAG_PT_REGION (1UL << 58) +#define DART_C_RANGE_FLAG_BYPASS (1UL << 57) +#define DART_C_RANGE_FLAG_APF_BYPASS (1UL << 56) +#define DART_C_RANGE_FLAG_REAL_TIME (1UL << 55) + +/* TXM peer domain (domain=2/table=0) constants. XNU 26.5 passes the TXM + * selector in x16's endpoint field, x0 is the physical TXM thread-stack page, + * and TXM writes TXMSharedContextData_t at PAGE_SIZE - 1024 inside that page. */ +#define TXM_DOMAIN 2 +#define TXM_TABLE_XNU 0 +#define TXM_C_MAX_ENDPOINTS 64 +#define TXM_C_TRACK_SLOTS 32 +#define TXM_C_SYNTH_ALIGN 0x40UL +#define TXM_THREAD_STACK_SHARED_OFF 0x3c00UL +#define TXM_SHARED_RETURN_CODE_OFF 0x08UL +#define TXM_SHARED_RETURN_TYPE_OFF 0x10UL +#define TXM_SHARED_NUM_WORDS_OFF 0x18UL +#define TXM_SHARED_WORDS_OFF 0x20UL +#define TXM_STACK_RETURN_WORDS 6 + +#define TXM_SUCCESS 0UL +#define TXM_RETURN_GENERIC 1UL +#define TXM_RETURN_NOT_FOUND 8UL +#define TXM_RETURN_NOT_PERMITTED 38UL +#define TXM_RETURN_NOT_SUPPORTED 41UL + +#define TXM_SEL_GET_LOG_INFO 1U +#define TXM_SEL_GET_CODE_SIGNING_INFO 2U +#define TXM_SEL_GET_TRUST_CACHE_INFO 3U +#define TXM_SEL_GET_SECURE_CHANNEL 7U +#define TXM_SEL_ADD_FREE_LIST_PAGE 10U +#define TXM_SEL_GET_FREE_LIST_PAGE 11U +#define TXM_SEL_LOAD_TRUST_CACHE 12U +#define TXM_SEL_QUERY_TRUST_CACHE 14U +#define TXM_SEL_QUERY_TC_REM 15U +#define TXM_SEL_CHECK_TC_UUID 16U +#define TXM_SEL_REGISTER_PROFILE 17U +#define TXM_SEL_UNREGISTER_PROFILE 19U +#define TXM_SEL_REGISTER_SIGNATURE 22U +#define TXM_SEL_UNREGISTER_SIGNATURE 23U +#define TXM_SEL_RECONSTITUTE_SIG 25U +#define TXM_SEL_GET_LOCAL_PUBKEY 27U +#define TXM_SEL_MATCH_COMPILATION 30U +#define TXM_SEL_ACQUIRE_SIGNING_ID 32U +#define TXM_SEL_ACCEL_ENTITLEMENTS 34U +#define TXM_SEL_REGISTER_ASPACE 35U +#define TXM_SEL_GET_ENTITLEMENTS_CTX 43U +#define TXM_SEL_RESOLVE_KENT_ASPACE 44U +#define TXM_SEL_IMAGE4_DISPATCH 45U +#define TXM_SEL_IMAGE4_GET_EXPORTS 46U +#define TXM_SEL_IMAGE4_GET_NONCE 51U + +#define DART_T8110_PARAMS3 0x008U +#define DART_T8110_PARAMS4 0x00cU +#define DART_T8110_TLB_CMD 0x080U +#define DART_T8110_TLB_CMD_BUSY (1U << 31) +#define DART_T8110_TLB_CMD_NEW_DART (1U << 15) +#define DART_T8110_TLB_CMD_VA_RANGE (1U << 14) +#define DART_T8110_TLB_CMD_STT_FLUSH (1U << 13) +#define DART_T8110_TLB_CMD_NO_STC_FLUSH (1U << 12) +#define DART_T8110_TLB_OP_FLUSH_ALL 0U +#define DART_T8110_TLB_OP_FLUSH_SID 1U +#define DART_T8110_TLB_START_DVA_PAGE 0x098U +#define DART_T8110_TLB_END_DVA_PAGE 0x0a0U +#define DART_T8110_ERROR 0x100U +#define DART_T8110_ERROR_MASK 0x104U +#define DART_T8110_ERROR_STREAMS 0x1c0U +#define DART_T8110_PROTECT 0x200U +#define DART_T8110_PROTECT_TTBR_TCR (1U << 0) +#define DART_T8110_PROTECT_LOCK 0x208U +#define DART_T8110_ENABLE_STREAMS 0xc00U +#define DART_T8110_DISABLE_STREAMS 0xc20U +#define DART_T8110_TCR 0x1000U +#define DART_T8110_TCR_TRANSLATE_EN (1U << 0) +#define DART_T8110_TCR_BYPASS_DART (1U << 1) +#define DART_T8110_TCR_BYPASS_DAPF (1U << 2) +#define DART_T8110_TCR_FOUR_LEVEL (1U << 3) +#define DART_T8110_TCR_REMAP_EN (1U << 7) +#define DART_T8110_TCR_REMAP_TARGET(x) (((x) & 0xffU) << 8) +#define DART_T8110_TCR_DISABLED (DART_T8110_TCR_BYPASS_DART | \ + DART_T8110_TCR_BYPASS_DAPF) +#define DART_T8110_TTBR 0x1400U +#define DART_T8110_TTBR_VALID (1U << 0) + +#define DART_C_HOT_OK 0U +#define DART_C_ERR_LOCKED (1U << 0) +#define DART_C_ERR_TIMEOUT (1U << 1) +#define DART_C_ERR_ALIGN (1U << 2) +#define DART_C_ERR_NO_TTBR (1U << 3) +#define DART_C_ERR_NO_TABLE (1U << 4) + +struct dart_c_sid_state { + u32 tcr; + u32 ttbr; + u32 protection; + u32 status; + u64 root_pt_pa; + u8 valid; + u8 root_valid; + u8 root_level; + u8 stream_enabled; + u8 translation_enabled; + u8 exclave; + u8 raw_replay; + u8 initial_stream_enabled; +}; + +struct dart_c_piogw_desc { + u32 slice; + u32 pad; + u64 addr; +}; + +struct dart_c_dapf_slice { + u64 start; + u64 end; + u32 ctrl; + u32 words[8]; +}; + +struct dart_c_dapf_state { + u64 base_pa; + u32 count; + u32 sid_count; + struct dart_c_dapf_slice slice[DART_C_MAX_DAPF_SLICES]; + u8 reserved[0x800 - 0x10 - DART_C_MAX_DAPF_SLICES * 0x38]; +}; + +struct dart_c_instance { + u64 dart_id; + u64 obj_va; + u64 state_va; + u64 base_pa[DART_C_MAX_BASES]; + u32 base_count; + u32 init_state; + u32 clock_protection; + u32 locked; + u32 used; + u8 avoid_tlbi_in_map; + u8 relaxed_rw_protections; + u8 flush_by_dva; + u8 pad0[1]; + u64 total_calls; + u64 fast_path_calls; + u64 violations; + u64 last[16]; + u64 ep_count[DART_C_MAX_ENDPOINTS]; + struct dart_c_dapf_state dapf; + struct dart_c_sid_state sid[DART_C_MAX_SIDS]; + u64 piogw_base_pa[DART_C_MAX_PIOGW_RANGES]; + u32 piogw_count; + u32 piogw_desc_count; + struct dart_c_piogw_desc piogw_desc[DART_C_MAX_PIOGW_DESCS]; +}; + +struct dart_c_recent_event { + u64 seq; + u64 table; + u64 endpoint; + u64 dart_id; + u64 sid; + u64 regs[6]; + u64 before_slot; + u64 before_pte; + u64 after_slot; + u64 after_pte; + u64 rc; +}; + +struct dart_c_sid0_event { + u64 seq; + u64 meta; /* table, endpoint, dart_id, reason: 16 bits each */ + u64 regs[4]; + u64 before_tcr_ttbr; + u64 before_root_flags; + u64 before_pte; + u64 after_tcr_ttbr; + u64 after_root_flags; + u64 after_pte; + u64 rc; +}; + +struct dart_c_identity_range { + u64 base_pa; + u64 sid; + u64 start; + u64 end; +}; + +struct dart_c_state { + u32 enabled; + u32 pad0; + u32 violations; + u32 pad1; + u64 total_calls; + u64 fast_path_calls; + u64 fallthrough_calls; + u64 last[16]; + struct dart_c_instance inst[DART_C_MAX_DARTS]; + u64 pt_pool_base; + u64 pt_pool_end; + u64 pt_pool_next; + u64 pt_pool_allocs; + u64 pt_pool_failures; + u64 identity_count; + struct dart_c_identity_range identity[DART_C_MAX_IDENTITY_RANGES]; + u64 recent_idx; + struct dart_c_recent_event recent[DART_C_RECENT_SLOTS]; + u64 sid0_trace_idx; + struct dart_c_sid0_event sid0_trace[DART_C_SID0_TRACE_SLOTS]; +}; + +struct nvme_c_state { + u32 enabled; + u32 queue_count; + u32 allowed_functions; + u8 secure_reg_layout; + u8 packed_writes_present; + u8 linear_sq; + u8 ans_sha_present; + u8 debug_flags; + u8 quirks; + u16 max_cid; + u32 violations; + u64 total_calls; + u64 fast_path_calls; + u64 fallthrough_calls; + u64 bar_pa; + u64 trusted_io_base; + u64 trusted_io_size; + u64 secondary_io_base; + u64 secondary_io_size; + u64 admin_asq_kva; + u64 admin_acq_kva; + u64 prp_list_kva; + u64 ans_sha_kva; + u64 tcb_kva[2]; + u32 aqa_cached; + u32 ioqa_cached; + u32 ans_sha_pwc_cached; + u32 last_rc; + u64 asq_pa_cached; + u64 acq_pa_cached; + u64 iosq_pa_cached; + u64 iocq_pa_cached; + u64 ans_sha_pa_cached; + u64 last[16]; + u64 ep_count[16]; + u8 cid_mode[NVME_C_MAX_CID]; + u8 func_state[16]; + u16 tcb_dir[NVME_C_MAX_CID]; + u16 tcb_nlb[NVME_C_MAX_CID]; + u32 tcb_count[NVME_C_MAX_CID]; + u8 tcb_mode[NVME_C_MAX_CID]; + u8 tcb_qid[NVME_C_MAX_CID]; + u16 pad0[NVME_C_MAX_CID]; + u64 tcb_prp1[NVME_C_MAX_CID]; + u64 tcb_prp2[NVME_C_MAX_CID]; + u64 nvmmu_pa; + u64 tl_status_pa; + u64 tl_mask_pa; + u64 tl_ctrl_pa; + u32 tl_num_sl; + u32 tl_timeouts; + u16 tl_slot_done[NVME_C_MAX_CID]; + u64 vdma_status_pa; +}; +_Static_assert(sizeof(struct nvme_c_state) == 0x2128, "nvme_c_state layout drift"); + +struct sart_c_entry { + u64 paddr; + u64 size; + u32 flags; + u32 guard; +}; + +struct sart_c_state { + u32 enabled; + u32 active; + u32 version; + u32 flags_allow; + u32 size_shift; + u32 paddr_shift; + u32 size_max; + u32 protected_mask; + u32 violations; + u32 last_rc; + u64 total_calls; + u64 fast_path_calls; + u64 fallthrough_calls; + u64 base_pa; + u64 power_canary_pa; + u32 power_canary_offset; + u32 power_canary_count; + u32 exclusive_bounds; + u32 reserved; + u64 last[8]; + u64 ep_count[SART_C_MAX_ENDPOINTS]; + struct sart_c_entry shadow[SART_C_MAX_ENTRIES]; +}; +_Static_assert(sizeof(struct sart_c_entry) == 0x18, "sart_c_entry layout drift"); +_Static_assert(sizeof(struct sart_c_state) == 0x238, "sart_c_state layout drift"); + +struct uat_c_root_state { + u64 handle; + u64 root0_pa; + u64 root1_pa; + u64 current_va; + u64 seglist_pa; + u64 num_segs; + u64 current_seg; + u64 seg_offset; + u64 options; + u64 last[8]; + u16 ctx_id; + u8 used; + u8 state; + u8 type; + u8 pad[19]; +}; + +struct uat_c_recent_event { + u64 seq; + u64 endpoint; + u64 handle; + u64 regs[5]; + u64 root_pa; + u64 slot_pa; + u64 before; + u64 after; + u64 rc; +}; + +struct uat_c_state { + u32 enabled; + u32 violations; + u64 total_calls; + u64 fast_path_calls; + u64 fallthrough_calls; + u64 last_rc; + u64 mode; + u64 vaddr_shift; + u64 l1_index_mask; + u64 segment_limit; + u64 mapping_limit; + u64 state_object_size; + u64 gpu_region_pa; + u64 gpu_region_size; + u64 gfx_shared_region_pa; + u64 gfx_shared_region_size; + u64 gfx_shared_l2_pa; + u64 gfx_shared_l2_size; + u64 gfx_handoff_pa; + u64 gfx_handoff_size; + u64 last[16]; + u64 ep_count[UAT_C_MAX_ENDPOINTS]; + struct uat_c_root_state roots[UAT_C_MAX_STATES]; + u64 recent_idx; + struct uat_c_recent_event recent[UAT_C_RECENT_SLOTS]; + u64 selector2_pa; + u64 selector9_pa; + u64 raw_gfx_shared_l2_pa; + u64 selector2_root_pa; + u64 sapt_base; + u64 sapt_entries; + u64 sapt_dram_base; + u64 sapt_dram_end; + u64 sapt_policy; +}; +_Static_assert(sizeof(struct uat_c_root_state) == 0xa0, "uat_c_root_state layout drift"); +_Static_assert(sizeof(struct uat_c_recent_event) == 0x68, "uat_c_recent_event layout drift"); +_Static_assert(sizeof(struct uat_c_state) == 0x22d0, "uat_c_state layout drift"); + +struct sptm_cpu_pio_entry { + u64 phys_id; + u64 cpu_base; + u64 cpu_size; + u64 acc_base; + u64 acc_size; + u64 cpm_base; + u64 cpm_size; + u64 cpu_pages; + u64 acc_pages; + u64 cpm_pages; + u64 registered; + u64 logical_id; + u64 flags; + u64 reserved[3]; +}; + +struct sptm_cpu_pio_state { + u64 seeded_count; + u64 registered_count; + u64 last_phys; + u64 last_slot; + u64 last_flags; + u64 reserved[3]; + struct sptm_cpu_pio_entry entries[SPTM_MAX_CPUS_EMUL]; +}; + +struct papt_update_event { + u64 rc; + u64 pa; + u64 options; + u64 papt_va; + u64 root_pa; + u64 template_pte; + u64 existing_pte; + u64 new_pte; + u64 caller_elr; + u64 inner_root_pa; + u64 inner_va; + u64 inner_template_pte; +}; + +struct papt_update_state { + u64 calls; + u64 candidates; + u64 writes; + u64 skip_no_attr; + u64 skip_skip_papt; + u64 skip_unmanaged; + u64 skip_bad_template; + u64 skip_bad_root; + u64 skip_bad_walk; + u64 skip_bad_existing; + u64 skip_same_attrs; + u64 ring_idx; + struct papt_update_event ring[PAPT_UPDATE_RING_SLOTS]; + u64 nvme_hits[PAPT_NVME_CLASS_COUNT]; + u64 nvme_writes[PAPT_NVME_CLASS_COUNT]; +}; +_Static_assert(sizeof(struct papt_update_event) == 0x60, "papt_update_event layout drift"); +_Static_assert(sizeof(struct papt_update_state) == 0x18e0, "papt_update_state layout drift"); + +struct alt_ptd_trace_event { + u64 seq; + u64 pa; + u64 caller_elr; + u64 caller_lr; + u64 caller_fp; + u64 types_flags; + u64 retype_params; + u64 ft_idx; + u64 refs_before; + u64 refs_after; + u64 papt_ring_before; + u64 papt_ring_after; + u64 papt_rc; + u64 papt_existing_pte; + u64 papt_new_pte; + u64 papt_va; + u64 caller_sp0; + u64 caller_sp1; + u64 bt_count; + u64 bt_base; + u64 bt[12]; +}; + +struct alt_ptd_trace_state { + u64 ring_idx; + struct alt_ptd_trace_event ring[ALT_PTD_TRACE_RING_SLOTS]; +}; +_Static_assert(sizeof(struct alt_ptd_trace_event) == 0x100, "alt_ptd_trace_event layout drift"); +_Static_assert(sizeof(struct alt_ptd_trace_state) == 0x408, "alt_ptd_trace_state layout drift"); + +struct txm_c_state { + u32 enabled; + u32 violations; + u64 total_calls; + u64 fast_path_calls; + u64 fallthrough_calls; + u64 last_selector; + u64 last_stack_pa; + u64 last_x16; + u64 last_rc; + u64 last_args[8]; + u64 last_words[TXM_STACK_RETURN_WORDS]; + u64 ep_count[TXM_C_MAX_ENDPOINTS]; + u64 secure_chan_pa; + u64 secure_chan_size; + u64 synth_base_va; + u64 synth_base_pa; + u64 synth_size; + u64 synth_next_off; + u64 log_page_va; + u64 log_head_va; + u64 log_sync_va; + u64 txm_rw_data_va; + u64 txm_ro_data_va; + u64 developer_mode_flag_va; + u64 managed_signature_size; + u64 trust_cache_runtime_va; + u64 trust_cache_static_count; + u64 trust_cache_caps0; + u64 trust_cache_caps1; + u64 local_signing_pubkey_va; + u64 kernel_entitlements_va; + u64 ce_ctx_va; + u64 signing_id_va; + u64 image4_exports_va; + u64 nonce_va; + u64 default_address_space_va; + u64 default_signature_va; + u64 default_profile_va; + u64 next_token; + u64 free_count; + u64 free_pages[TXM_C_TRACK_SLOTS]; + u64 loaded_tc_count; + u64 loaded_tc[TXM_C_TRACK_SLOTS]; + u64 secure_chan_dva; + u64 secure_chan_reserved_pa; +}; + +/* Frame types (XNU's per-frame state). */ +#define SPTM_KERNEL_ROOT_TABLE 8 +#define SPTM_PAGE_TABLE 9 +#define XNU_DEFAULT 11 +#define XNU_USER_ROOT_TABLE 18 +#define XNU_SHARED_ROOT_TABLE 19 +#define XNU_PAGE_TABLE 20 +#define XNU_PAGE_TABLE_SHARED 21 +#define XNU_PAGE_TABLE_ROZONE 22 +#define XNU_PAGE_TABLE_COMMPAGE 23 +#define XNU_PAGE_TABLE_ALT 24 +#define XNU_IO 27 +#define XNU_PROTECTED_IO 28 +#define XNU_COPROCESSOR_RO_IO 29 +#define XNU_STAGE2_ROOT_TABLE 33 +#define XNU_STAGE2_PAGE_TABLE 34 +#define XNU_SUBPAGE_USER_ROOT_TABLES 40 +#define TXM_SEP_SECURE_CHANNEL 61 + +#define SPTM_PT_GEOMETRY_16K 0 +#define SPTM_PT_GEOMETRY_4K 1 +#define SPTM_PT_GEOMETRY_16K_KERN 2 +#define ROOT_GEOM_UNKNOWN 0xff +#define TTBR_ASID_SHIFT 48 +#define TTBR_BADDR_MASK 0x0000fffffffffffeUL +#define ROOT_GEOM_SLOTS 64 + +#define PAGE_SIZE_GUEST_EMUL 0x4000UL +#define PAGE_SIZE_4K_EMUL 0x1000UL +#define ARM_TTE_TABLE_MASK 0x0000fffffffff000UL +#define ARM_PTE_PAGE_MASK 0x0000ffffffffc000UL +#define ARM_PTE_PAGE_MASK_4K 0x0000fffffffff000UL +#define ARM_PTE_AP_MASK (3UL << 6) +#define ARM_PTE_ATTRINDX_MASK (7UL << 2) +#define ARM_PTE_SH_MASK (3UL << 8) +#define ARM_PTE_AF (1UL << 10) +#define ARM_PTE_NG (1UL << 11) +#define ARM_PTE_NS (1UL << 5) +#define ARM_PTE_AP_RO (2UL << 6) +#define ARM_PTE_GP (1UL << 50) +#define ARM_PTE_PNX (1UL << 53) +#define ARM_PTE_NX (1UL << 54) +#define ARM_PTE_WIRED (1UL << 58) +#define ARM_PTE_WRITEABLE (1UL << 59) +#define FRAME_PAGE_MASK 0x0000ffffffffc000UL +#define FT_KIND5_REFCNT_OFF 0x04UL +#define FT_TABLE_MAPPING_REFCNT_OFF 0x06UL +#define FT_TABLE_NESTED_REFCNT_OFF 0x08UL +#define FT_DATA_RO_REFCNT_OFF 0x08UL +#define FT_DATA_WX_REFCNT_OFF 0x0cUL +#define DISJOINT_OP_SIZE 24UL +#define SURT_SUBPAGE_SIZE 128UL +#define TTE_CONDEMNED_BIT 0x4UL +#define UINT64_MAX_VALUE 0xffffffffffffffffUL + +/* PT walk indices for XNU's 16K and 4K pmap geometries. */ +static inline u64 va_16k_l1_idx(u64 va) { return (va >> 36) & 0x7ff; } +static inline u64 va_16k_l2_idx(u64 va) { return (va >> 25) & 0x7ff; } +static inline u64 va_16k_l3_idx(u64 va) { return (va >> 14) & 0x7ff; } +static inline u64 va_4k_l0_idx(u64 va) { return (va >> 39) & 0x1ff; } +static inline u64 va_4k_l1_idx(u64 va) { return (va >> 30) & 0x1ff; } +static inline u64 va_4k_l2_idx(u64 va) { return (va >> 21) & 0x1ff; } +static inline u64 va_4k_l3_idx(u64 va) { return (va >> 12) & 0x1ff; } + +static inline u8 ft_get_type(u64 pa); +static inline u8 root_geom_lookup(u64 root_pa); +static inline bool_t root_uses_4k(u64 root_pa, u32 target_level); +static inline bool_t is_pt_type(u8 t); + +/* Translate xnu's PTE template to a plain ARM PTE we can actually install. + * Preserve the architectural leaf fields XNU controls plus the software bits + * XNU later compares in prev-PTE results; force only the valid leaf type. */ +__attribute__((unused)) +static inline u64 xnu_pte_to_arm_geom(u64 xnu_pte, bool_t geom4k) { + if ((xnu_pte & 3) == 0) return 0; + u64 pa = xnu_pte & (geom4k ? ARM_PTE_PAGE_MASK_4K : ARM_PTE_PAGE_MASK); + u64 leaf = xnu_pte & (ARM_PTE_ATTRINDX_MASK | ARM_PTE_SH_MASK | + ARM_PTE_AP_MASK | ARM_PTE_AF | ARM_PTE_NG | + ARM_PTE_NS | ARM_PTE_GP | ARM_PTE_PNX | + ARM_PTE_NX | ARM_PTE_WIRED | + ARM_PTE_WRITEABLE); + /* Preserve Apple-impdef bits (49=SK, 60-62=SPRR idx) so XNU's bookkeeping + * reads see what it wrote. Our MMU ignores the SPRR index bits. */ + u64 apple_bits = xnu_pte & ((1UL << 49) | (1UL << 60) | (1UL << 61) | + (1UL << 62)); + return pa | leaf | apple_bits | 3UL; +} + +/* + * UPDATE_REGION/DISJOINT receive masked attribute templates. XNU may pass only + * the bits selected by options, with valid/type and PA fields clear, such as + * permissions-only templates. Real SPTM still applies those selected fields to + * the existing PTE; do not collapse partial templates to zero just because + * ARM_PTE_TYPE_VALID is absent. + */ +__attribute__((unused)) +static inline u64 xnu_pte_update_template_to_arm_geom(u64 xnu_pte, bool_t geom4k) { + u64 leaf = xnu_pte & (ARM_PTE_ATTRINDX_MASK | ARM_PTE_SH_MASK | + ARM_PTE_AP_MASK | ARM_PTE_AF | ARM_PTE_NG | + ARM_PTE_NS | ARM_PTE_GP | ARM_PTE_PNX | + ARM_PTE_NX | ARM_PTE_WIRED | + ARM_PTE_WRITEABLE); + u64 apple_bits = xnu_pte & ((1UL << 49) | (1UL << 60) | (1UL << 61) | + (1UL << 62)); + + if ((xnu_pte & 3) == 0) + return leaf | apple_bits; + + u64 pa = xnu_pte & (geom4k ? ARM_PTE_PAGE_MASK_4K : ARM_PTE_PAGE_MASK); + return pa | leaf | apple_bits | 3UL; +} + +__attribute__((unused)) +static inline u64 xnu_pte_to_arm(u64 xnu_pte) { + return xnu_pte_to_arm_geom(xnu_pte, 0); +} +static inline u64 xnu_tte_to_arm(u64 xnu_tte) { + if ((xnu_tte & 3) == 0) return 0; + return (xnu_tte & ARM_TTE_TABLE_MASK) | 3UL; +} + +static inline u64 merge_pte_with_mask_geom(u64 existing, u64 new_template, + u64 mask, bool_t geom4k) { + if (mask == SPTM_UPDATE_MASK || mask == 0) + return new_template; + + u64 pa_mask = geom4k ? ARM_PTE_PAGE_MASK_4K : ARM_PTE_PAGE_MASK; + u64 pa_bits = existing & pa_mask; + u64 out = existing; + + if (mask & SPTM_UPDATE_SW_WIRED) + out = (out & ~ARM_PTE_WIRED) | (new_template & ARM_PTE_WIRED); + if (mask & SPTM_UPDATE_PERMS_AND_WAS_WRITABLE) { + u64 clr = ARM_PTE_PNX | ARM_PTE_NX | ARM_PTE_AP_MASK | + ARM_PTE_WRITEABLE; + out = (out & ~clr) | (new_template & clr); + } + if (mask & SPTM_UPDATE_NG) + out = (out & ~ARM_PTE_NG) | (new_template & ARM_PTE_NG); + if (mask & SPTM_UPDATE_AF) + out = (out & ~ARM_PTE_AF) | (new_template & ARM_PTE_AF); + if (mask & SPTM_UPDATE_SH) + out = (out & ~ARM_PTE_SH_MASK) | (new_template & ARM_PTE_SH_MASK); + if (mask & SPTM_UPDATE_MAIR) + out = (out & ~ARM_PTE_ATTRINDX_MASK) | + (new_template & ARM_PTE_ATTRINDX_MASK); + + return (out & ~pa_mask) | pa_bits; +} + +__attribute__((unused)) +static inline u64 merge_pte_with_mask(u64 existing, u64 new_template, u64 mask) { + return merge_pte_with_mask_geom(existing, new_template, mask, 0); +} + +static inline volatile u64 *pt_u64_ptr(u64 pa) { + return (volatile u64 *)pa; +} + +static inline volatile u64 *pt_slot_ptr(u64 table_pa, u64 idx) { + return pt_u64_ptr(table_pa + idx * 8); +} + +static inline u64 pt_read_slot_poc(volatile u64 *slot) { + return *slot; +} + +static inline u64 walk_to_l3_16k(u64 root_pa, u64 va, u64 *out_idx) { + volatile u64 *l1 = pt_u64_ptr(root_pa); + u64 l1e = pt_read_slot_poc(&l1[va_16k_l1_idx(va)]); + if ((l1e & 3) != 3) return 0; + volatile u64 *l2 = pt_u64_ptr(l1e & ARM_TTE_TABLE_MASK); + u64 l2e = pt_read_slot_poc(&l2[va_16k_l2_idx(va)]); + if ((l2e & 3) != 3 || (l2e & TTE_CONDEMNED_BIT)) return 0; + *out_idx = va_16k_l3_idx(va); + return l2e & ARM_TTE_TABLE_MASK; +} + +static inline u64 walk_to_l3_4k(u64 root_pa, u64 va, u64 *out_idx) { + volatile u64 *l0 = pt_u64_ptr(root_pa); + u64 l0e = pt_read_slot_poc(&l0[va_4k_l0_idx(va)]); + if ((l0e & 3) != 3) return 0; + volatile u64 *l1 = pt_u64_ptr(l0e & ARM_TTE_TABLE_MASK); + u64 l1e = pt_read_slot_poc(&l1[va_4k_l1_idx(va)]); + if ((l1e & 3) != 3) return 0; + volatile u64 *l2 = pt_u64_ptr(l1e & ARM_TTE_TABLE_MASK); + u64 l2e = pt_read_slot_poc(&l2[va_4k_l2_idx(va)]); + if ((l2e & 3) != 3 || (l2e & TTE_CONDEMNED_BIT)) return 0; + *out_idx = va_4k_l3_idx(va); + return l2e & ARM_TTE_TABLE_MASK; +} + +/* Walk to the leaf PTE table for either geometry. */ +__attribute__((unused)) +static inline u64 walk_to_l3(u64 root_pa, u64 va, u64 *out_idx) { + return root_uses_4k(root_pa, 3) ? + walk_to_l3_4k(root_pa, va, out_idx) : + walk_to_l3_16k(root_pa, va, out_idx); +} + +/* Walk xnu's pmap geometry to find the parent table at `target_level`. + * 16K roots start at L1; 4K roots can start at L0. Real SPTM gets this from + * root metadata; this emulator records the geometry from sptm_retype params. */ +static inline u64 walk_to_level(u64 root_pa, u64 va, u32 target_level, + u64 *out_idx) { + if (root_uses_4k(root_pa, target_level)) { + if (target_level == 0) { *out_idx = va_4k_l0_idx(va); return root_pa; } + volatile u64 *l0 = pt_u64_ptr(root_pa); + u64 l0e = pt_read_slot_poc(&l0[va_4k_l0_idx(va)]); + if ((l0e & 3) != 3) return 0; + if (target_level == 1) { + *out_idx = va_4k_l1_idx(va); + return l0e & ARM_TTE_TABLE_MASK; + } + volatile u64 *l1 = pt_u64_ptr(l0e & ARM_TTE_TABLE_MASK); + u64 l1e = pt_read_slot_poc(&l1[va_4k_l1_idx(va)]); + if ((l1e & 3) != 3) return 0; + if (target_level == 2) { + *out_idx = va_4k_l2_idx(va); + return l1e & ARM_TTE_TABLE_MASK; + } + volatile u64 *l2 = pt_u64_ptr(l1e & ARM_TTE_TABLE_MASK); + u64 l2e = pt_read_slot_poc(&l2[va_4k_l2_idx(va)]); + if ((l2e & 3) != 3) return 0; + *out_idx = va_4k_l3_idx(va); + return l2e & ARM_TTE_TABLE_MASK; + } + + if (target_level <= 1) { *out_idx = va_16k_l1_idx(va); return root_pa; } + volatile u64 *l1 = pt_u64_ptr(root_pa); + u64 l1e = pt_read_slot_poc(&l1[va_16k_l1_idx(va)]); + if ((l1e & 3) != 3) return 0; + if (target_level == 2) { + *out_idx = va_16k_l2_idx(va); + return l1e & ARM_TTE_TABLE_MASK; + } + volatile u64 *l2 = pt_u64_ptr(l1e & ARM_TTE_TABLE_MASK); + u64 l2e = pt_read_slot_poc(&l2[va_16k_l2_idx(va)]); + if ((l2e & 3) != 3) return 0; + *out_idx = va_16k_l3_idx(va); + return l2e & ARM_TTE_TABLE_MASK; +} + +static inline void barrier(void) { + __asm__ volatile("dsb ish; isb" ::: "memory"); +} + +static inline void tlbi_vmalle1is(void) { + __asm__ volatile( + "dsb ish\n\t" + "tlbi vmalle1is\n\t" + "dsb ish\n\t" + "isb\n\t" + : : : "memory"); +} + +static inline void tlbi_vmalle1os(void) { + __asm__ volatile( + "dsb oshst\n\t" + "sys #0, c8, c1, #0\n\t" // TLBI VMALLE1OS; mnemonic requires tlb-rmi in clang. + "dsb osh\n\t" + "isb\n\t" + : : : "memory"); +} + +static inline void tlbi_aside1os(u64 asid) +{ + __asm__ volatile( + "dsb oshst\n\t" + "sys #0, c8, c1, #2, %0\n\t" // TLBI ASIDE1OS; operand is ASID << 48. + "dsb osh\n\t" + "isb\n\t" + : : "r"(asid) : "memory"); +} + +static inline void tlbi_vmalls12e1is(void) { + __asm__ volatile( + "dsb ish\n\t" + "tlbi vmalls12e1is\n\t" + "dsb ish\n\t" + "isb\n\t" + : : : "memory"); +} + +static inline u64 ttbr1_el12_root_pa(void) { + u64 ttbr; + __asm__ volatile("mrs %0, TTBR1_EL12" : "=r"(ttbr)); + return ttbr & TTBR_BADDR_MASK; +} + +static inline void clean_range(void *ptr, u64 size) { + u64 p = (u64)ptr; + u64 end = p + size; + p &= ~63UL; + while (p < end) { + __asm__ volatile("dc cvau, %0" : : "r"(p) : "memory"); + p += 64; + } + __asm__ volatile("dsb ish" ::: "memory"); +} + +static inline void clean_range_poc(void *ptr, u64 size) { + u64 p = (u64)ptr; + u64 end = p + size; + p &= ~63UL; + while (p < end) { + __asm__ volatile("dc cvac, %0" : : "r"(p) : "memory"); + p += 64; + } + __asm__ volatile("dsb sy" ::: "memory"); +} + +static inline void sptm_coproc_cache_maint_page(u64 pa) { + /* + * M4/A18 SPTM uses guarded implementation-defined ops such as 0x00201401 + * for coprocessor/IOMMU page-cache maintenance. We cannot issue those from + * the emulator context, but the M3 SPTM path uses this older CRn=7 pair for + * the same by-page maintenance, and live probing on A18 Pro shows it does + * not fault from our EL2 context. + */ + register u64 x8 __asm__("x8") = pa & FRAME_PAGE_MASK; + __asm__ volatile( + /* + * LEADING outer-shareable barrier. Real SPTM brackets its per-page + * coprocessor maintenance with an outer-shareable barrier on BOTH sides + * (dsb oshnxs/oshst before, dsb oshnxs/dmb osh after). The caller's + * preceding dc_civac_range (sptm_frame_clean_page) emits no barrier, so + * without this the coprocessor invalidate/refetch below can race AHEAD + * of the CPU's clean reaching the PoC -> the GPU refetches stale page + * contents. The trailing dsb osh alone cannot fix an ordering that has + * already gone wrong before the sys ops execute. + */ + "dsb osh\n\t" + "sys #3, c7, c3, #4, x8\n\t" + "sys #3, c7, c3, #5, x8\n\t" + /* + * The AGX GPU coprocessor is an OUTER-SHAREABLE observer. Real SPTM + * drains its per-page coprocessor maintenance with outer-shareable + * barriers (dsb oshnxs / dmb osh) at every flush site. A `dsb nsh` + * here only guarantees completion for the local PE, so a freshly + * dc_civac'd GPU data page (drained only by this barrier, since + * dc_civac_range emits none) is never published to the GPU coherency + * point -> the GPU reads stale command/ring data, never advances its + * completion stamp, and agx_scheduler spins with faulted=0. + */ + "dsb osh\n\t" + "isb\n\t" + : "+r"(x8) : : "memory"); +} + +static inline void sptm_coproc_cache_maint_range(const void *ptr, u64 size) { + u64 p = (u64)ptr & FRAME_PAGE_MASK; + u64 end = ((u64)ptr + size + PAGE_SIZE_GUEST_EMUL - 1UL) & + FRAME_PAGE_MASK; + + while (p < end) { + sptm_coproc_cache_maint_page(p); + p += PAGE_SIZE_GUEST_EMUL; + } +} + +static inline void sptm_frame_clean_page(u64 pa) { + u64 page = pa & ARM_PTE_PAGE_MASK; + + dc_civac_range((void *)page, PAGE_SIZE_GUEST_EMUL); +} + +static inline void sptm_coproc_cache_maint_data_page(u64 pa) { + u64 page = pa & ARM_PTE_PAGE_MASK; + + sptm_frame_clean_page(page); + sptm_coproc_cache_maint_page(page); +} + +static inline void clean_pt_range_poc(volatile u64 *ptr, u64 size) { + clean_range_poc((void *)ptr, size); + sptm_coproc_cache_maint_range((const void *)ptr, size); + __asm__ volatile("isb" ::: "memory"); +} + +static inline void clean_pt_slot_poc(volatile u64 *slot) { + clean_pt_range_poc(slot, 8); +} + +static inline void zero_16k(u64 pa) { + volatile u64 *q = pt_u64_ptr(pa); + for (u32 i = 0; i < (PAGE_SIZE_GUEST_EMUL / 8); i++) + q[i] = 0; + sptm_coproc_cache_maint_data_page(pa); +} + +static inline void alt_ptd_set_el2_cacheable(u64 pa, bool_t cacheable) { + /* + * Type 24 is firmware kind 5, not an ordinary CPU page-table kind. XNU/PMP + * later read these PTD/ALT frames with non-coherent transactions; if the + * emulator keeps touching them through m1n1's Normal-WB identity alias, AMCC + * can see a stale SLC directory tag and raise UNEXP_RT_HIT_DIR. Mirror the + * /vram fix for m1n1's own alias only: clean the old identity mapping, then + * make future EL2 identity accesses Normal-NC while the frame is type 24. + */ + u64 page = pa & FRAME_PAGE_MASK; + u8 mair = cacheable ? SPTM_EL2_MAIR_NORMAL : SPTM_EL2_MAIR_NORMAL_NC; + + dc_civac_range((void *)page, PAGE_SIZE_GUEST_EMUL); + mmu_add_mapping(page, page, PAGE_SIZE_GUEST_EMUL, mair, SPTM_EL2_PERM_RWX); +} + +/* SEV — wake any guest CPU stuck in WFE waiting on us to complete an op. + * Real SPTM implicitly posts events; we mirror that for every fast-path + * return so xnu's spinlocks don't deadlock waiting for a TLB sync, etc. */ +static inline void sev(void) { + __asm__ volatile("sev" ::: "memory"); +} + +/* Refcount helpers ft_inc_refcount / ft_dec_refcount are defined below the + * g_state extern declaration. */ + +/* ---------------------------------------------------------------------- + * Shared state struct. Python reads counters here for telemetry. + * MUST match proxyclient/tools/sptm/seed.py. + * ---------------------------------------------------------------------- */ +struct sptm_emul_state { + u64 total_calls; /* +0x000 — every HVC the C emulator saw */ + u64 fast_path_calls; /* +0x008 — handled in C without falling through */ + u64 fallthrough_calls; /* +0x010 — fell through to host */ + u64 unknown_calls; /* +0x018 — unrecognized endpoint, fell through */ + u64 last_x16; /* +0x020 — last x16 seen (for debug) */ + u64 last_endpoint; /* +0x028 — last endpoint we saw */ + u64 last_elr; /* +0x030 — last ELR we saw */ + /* Per-endpoint counters: index by endpoint id, 0..63. */ + u64 ep_count[64]; /* +0x038 */ + /* Frame-table state: written at install time by the Python loader. + * RETYPE handler updates frame_table[pa_index(pa)] = new_type so + * xnu's direct frame_table reads (no HVC) stay coherent with our + * emul. Zero frame_table_base = disabled (RETYPE just acks). */ + u64 frame_table_base; /* +0x238 */ + u64 vm_first_phys; /* +0x240 */ + u64 frame_table_size; /* +0x248 — bytes; bounds-check */ + /* AUDIT FIX #1: prev_ptes scratch PA (per-CPU; CPU0 only here) and + * physmap_va base used to compute PAPT VA for sptm_map_page_output_t. + * papt_va = first_papt + (pa - vm_first_phys). xnu reads scratch+0 + * = prev_pte and scratch+8 = ptep PAPT VA. */ + u64 scratch_pa; /* +0x250: base of per-CPU prev_ptes pages */ + u64 first_papt; /* +0x258 */ + /* Circular buffer of LAST 128 calls' (x16, x0, x1, x2, x3, esr, elr, x30) + * for post-hang inspection. ring_idx is the next slot to write. + * Total size 128 * 8 * 8 = 8192 bytes. */ + u64 ring_idx; /* +0x260 */ + u64 ring[128][8]; /* +0x268 — x16, x0, x1, x2, x3, esr, elr, x30 */ + /* SPRR fast-path state. Set by Python at install time. When kc_vmin + * is non-zero, the C handler intercepts EL1t_sync HVCs whose ELR_EL12 + * is in [kc_vmin, kc_vmax) and decodes the trapping instruction by + * reading PA = kc_guest_base + (elr12 - kc_vmin). Avoids a UART round + * trip per pmap_ro_zone_memcpy_internal SPRR write. */ + u64 kc_vmin; /* +0x2268 — kernelcache lowest VA */ + u64 kc_vmax; /* +0x2270 — kernelcache highest VA (exclusive) */ + u64 kc_guest_base; /* +0x2278 — PA where kernelcache is loaded */ + u64 sprr_fast_count; /* +0x2280 — # of SPRR sysreg traps fast-pathed */ + u64 sprr_perm_el1_cache; /* +0x2288 — last MSR'd SPRR_PERM_EL1 value */ + u64 sprr_perm_el0_cache; /* +0x2290 — last MSR'd SPRR_PERM_EL0 value */ + u64 sprr_config_el1_cache;/* +0x2298 — last MSR'd SPRR_CONFIG_EL1 value */ + /* Ring buffer of pmap_ro_zone_memcpy_internal call args. The C + * SPRR fast-path detects when ELR_EL12 is inside the function range + * [pmap_rzm_lo, pmap_rzm_hi) and pushes (zid, va, offset, src, + * size) into this ring. Captured at the SPRR write inside the + * function — by then the args have been moved to x19..x24 by + * the prologue and are stable. Ring size 128 entries × 6 u64s. */ + u64 pmap_rzm_lo; /* +0x22a0 — function start VA */ + u64 pmap_rzm_hi; /* +0x22a8 — function end VA (exclusive) */ + u64 pmap_rzm_idx; /* +0x22b0 — next ring slot to write */ + u64 pmap_rzm_count; /* +0x22b8 — total pushes */ + u64 pmap_rzm_ring[128][6];/* +0x22c0 — (elr, zid, va, offset, src, size) */ + /* Wider-context ring capturing FULL register snapshot at each + * pmap_rzm fast-path. Same length and indexing as pmap_rzm_ring; + * use pmap_rzm_idx to find the matching slot. Captures x0..x31 + * (33 u64 = 264 B per slot, 128 * 264 = 33 KB). */ + u64 pmap_rzm_full[128][33];/* +0x3ac0 — full reg snapshot per slot (128*264B = 33KB) */ + /* DEBUG (2026-05-09): full register snapshot at every SPTM dispatch + * entry — saves ALL 31 GP regs BEFORE any C handler can touch them. + * Used to bisect register-clobber bugs introduced by new C handlers. + * 128 slots * 31 u64 = ~32 KB. */ + u64 sptm_in_idx; /* +0xbec0 — next slot to write */ + u64 sptm_in_regs[128][31];/* +0xbec8 — x0..x30 verbatim from ctx->regs */ + /* Last table/refcount ops. These sit after sptm_in_regs at +0x13ac8. + * Python dumps them on panic to correlate libsptm refcount failures. */ + u64 last_retype[8]; /* pa,current,new,old_type,page_ref,table_ref,idx,flags */ + u64 last_map_table[10]; /* root,va,level,parent,idx,existing,new_tte,arm_tte,child_pa,table_ref */ + u64 last_unmap_table[10]; /* root,va,level,parent,idx,existing,freed_pa,type,table_ref,ft_idx */ + u64 root_geom_pa[ROOT_GEOM_SLOTS]; /* roots/SURT slots with explicit pmap geometry */ + u64 root_geom_attr[ROOT_GEOM_SLOTS]; /* bits[7:0]=SPTM_PT_GEOMETRY_*, bits[31:16]=ASID */ + u64 last_map_page[14]; /* root,va,l3,idx,ptep_pa,ptep_papt,existing,new_pte,arm_pte,target_pa,target_type,page_ref,ptep_table_ref,ptep_ft_idx */ + /* C-side DART object recovery cache. Endpoint 5 used to walk the guest + * frame chain over UART, which is slow enough to wedge boot. Do the same + * verifier-shaped AppleT8110DART object recovery in EL2 and publish the + * decoded object for diagnostics. */ + u64 dart_obj_cache_idx; /* +0x14018 */ + u64 dart_obj_cache_hits; /* +0x14020 */ + u64 dart_obj_cache_misses; /* +0x14028 */ + u64 dart_obj_cache_id[DART_OBJ_CACHE_SLOTS]; /* +0x14030 */ + u64 dart_obj_cache_va[DART_OBJ_CACHE_SLOTS]; /* +0x14230 */ + u64 dart_obj_last[8]; /* +0x14430: table,ep,dart_id,obj,source,gapf_count,inst_count,state */ + u64 dart_obj_last_base_va[16]; /* +0x14470: XNU state MMIO VA per instance */ + u64 dart_obj_last_base_pa[16]; /* +0x144f0: translated MMIO PA per instance */ + struct nvme_c_state nvme; /* +0x14570: C-side NVMe table-6 emulator */ + struct dart_c_state dart; /* C-side T8110 DART table-3/4 emulator */ + /* Appended after the fixed-offset telemetry region so old Python offsets + * remain stable. Real SPTM assigns logical CPU IDs at register_cpu time; + * XNU then uses sptm_prev_ptes + PAGE_SIZE * sptm_cpu_id(phys_id). */ + u64 cpu_count; + u64 cpu_phys_ids[SPTM_MAX_CPUS_EMUL]; + u64 cpu_last_mpidr; + u64 cpu_last_id; + u64 cpu_unknown_mpidr; + struct txm_c_state txm; /* C-side TXM domain=2/table=0 emulator */ + struct sart_c_state sart; /* C-side SART table=5 emulator */ + /* + * Real SPTM's register_cpu endpoint records each CPU's cpu-impl-reg, + * acc-impl-reg, and cpm-impl-reg windows from ADT and creates private SPTM + * mappings for them. Keep the same discovered metadata here. This is + * Mostly bookkeeping; on T8140 the first reserved word can hold a private + * backing page for E-core cpu-impl-reg leaves that fault as locked PIO when + * exposed directly to XNU. + */ + struct sptm_cpu_pio_state cpu_pio; + struct papt_update_state papt_update; + u64 papt_span; + struct uat_c_state uat; /* C-side UAT table=7 emulator */ + struct alt_ptd_trace_state alt_ptd_trace; +}; + +typedef char sptm_emul_state_fits_in_scratch[ + (sizeof(struct sptm_emul_state) <= 0x100000UL) ? 1 : -1]; + +/* The single state instance exported for Python-side boot seeding. Keep this + * 1MiB even though struct sptm_emul_state is smaller: a few emulated SPTM + * devices use the tail as private scratch, matching the old bringup layout. */ +extern u8 g_sptm_state[0x100000]; +#define g_state (*(struct sptm_emul_state *)g_sptm_state) + +static inline bool_t finish_hvc_success(void *ctx, u64 *regs) { + u64 elr2; + __asm__ volatile("mrs %0, elr_el2" : "=r"(elr2)); + u64 *ctx_elr = (u64 *)((unsigned char *)ctx + 0x108); + *ctx_elr = elr2; + regs[0] = SPTM_SUCCESS; + g_state.fast_path_calls++; + sev(); + return 1; +} + +static inline bool_t compat_void_non_xnu_table(u32 table) { + /* + * Temporary parity shim for the removed Python fallback. After TXM, + * DART, SART, NVMe, UAT, and CPUTRACE had first crack, the mixed + * Python/C emulator acknowledged remaining non-XNU dispatch tables with + * x0=0. Keep that behavior only for known auxiliary tables so C-only + * bringup reaches the same wall without hiding real DART/NVMe/UAT + * state-machine failures. + */ + return table == SPTM_TABLE_TXM_BOOTSTRAP || + table == SPTM_TABLE_SK_BOOTSTRAP || + table == SPTM_TABLE_SHART || + table == SPTM_TABLE_HIB; +} + +static inline u64 current_mpidr_phys(void) { + u64 mpidr; + __asm__ volatile("mrs %0, mpidr_el1" : "=r"(mpidr)); + return mpidr & 0xffffffffUL; +} + +static inline u64 cpu_phys_compact(u64 phys) { + return phys & 0x00ffffffUL; +} + +static inline u64 cpu_phys_adt_reg(u64 phys) { + return phys & 0x0000ffffUL; +} + +static inline bool_t cpu_phys_matches(u64 a, u64 b) { + if (a == b) + return 1; + if ((a & 0xffffffffUL) == (b & 0xffffffffUL)) + return 1; + if (cpu_phys_compact(a) == cpu_phys_compact(b)) + return 1; + if (cpu_phys_adt_reg(a) == cpu_phys_adt_reg(b)) + return 1; + return 0; +} + +static inline void sptm_cpu_pio_note_registered(u64 phys, u64 logical_id) { + struct sptm_cpu_pio_state *st = &g_state.cpu_pio; + u64 count = st->seeded_count; + u64 flags = SPTM_CPU_PIO_F_MISSING; + + if (count > SPTM_MAX_CPUS_EMUL) + count = SPTM_MAX_CPUS_EMUL; + + st->last_phys = phys; + st->last_slot = UINT64_MAX_VALUE; + + for (u64 i = 0; i < count; i++) { + struct sptm_cpu_pio_entry *ent = &st->entries[i]; + + if (!cpu_phys_matches(ent->phys_id, phys)) + continue; + + flags = ent->flags | SPTM_CPU_PIO_F_REGISTERED; + if (!ent->registered) + st->registered_count++; + ent->registered++; + ent->logical_id = logical_id; + ent->flags = flags; + st->last_slot = i; + break; + } + + st->last_flags = flags; +} + +#define T8140_CPU_IMPL_LOCKED_REG_OFF 0x9000UL +#define T8140_CPM_DPE_LOCKED_REG_OFF 0x40f8UL + +static inline u64 t8140_cpu_pio_shadow_pa(void) { + return g_state.cpu_pio.reserved[0] & FRAME_PAGE_MASK; +} + +static inline bool_t t8140_cpu_pio_shadow_target(u64 pa, u64 page_mask) { + struct sptm_cpu_pio_state *st = &g_state.cpu_pio; + u64 count = st->seeded_count; + + if (count > SPTM_MAX_CPUS_EMUL) + count = SPTM_MAX_CPUS_EMUL; + + pa &= page_mask; + for (u64 i = 0; i < count; i++) { + struct sptm_cpu_pio_entry *ent = &st->entries[i]; + + if (ent->cpu_base && ent->cpu_size > T8140_CPU_IMPL_LOCKED_REG_OFF) { + u64 locked_page = + (ent->cpu_base + T8140_CPU_IMPL_LOCKED_REG_OFF) & page_mask; + if (pa == locked_page) + return 1; + } + + if (ent->cpm_base && ent->cpm_size > T8140_CPM_DPE_LOCKED_REG_OFF) { + u64 locked_page = + (ent->cpm_base + T8140_CPM_DPE_LOCKED_REG_OFF) & page_mask; + if (pa == locked_page) + return 1; + } + } + + return 0; +} + +static inline u64 t8140_cpu_pio_shadow_pte(u64 arm_pte, bool_t geom4k) { + if ((arm_pte & 3) != 3) + return arm_pte; + + u64 shadow_pa = t8140_cpu_pio_shadow_pa(); + if (!shadow_pa) + return arm_pte; + + u64 page_mask = geom4k ? ARM_PTE_PAGE_MASK_4K : ARM_PTE_PAGE_MASK; + u64 target_pa = arm_pte & page_mask; + if (!t8140_cpu_pio_shadow_target(target_pa, page_mask)) + return arm_pte; + + u64 shadow_target_pa = + shadow_pa + (target_pa & (PAGE_SIZE_GUEST_EMUL - 1UL)); + return (arm_pte & ~page_mask) | (shadow_target_pa & page_mask); +} + +static inline u64 cpu_id_for_phys(u64 phys) { + u64 count = g_state.cpu_count; + if (count > SPTM_MAX_CPUS_EMUL) + count = SPTM_MAX_CPUS_EMUL; + for (u64 i = 0; i < count; i++) { + if (cpu_phys_matches(g_state.cpu_phys_ids[i], phys)) + return i; + } + g_state.cpu_unknown_mpidr = phys; + return 0; +} + +static inline u64 current_sptm_cpu_id(void) { + u64 mpidr = current_mpidr_phys(); + u64 id = cpu_id_for_phys(mpidr); + g_state.cpu_last_mpidr = mpidr; + g_state.cpu_last_id = id; + return id; +} + +static inline volatile u64 *scratch_for_current_cpu(void) { + if (g_state.scratch_pa == 0) + return (volatile u64 *)0; + u64 id = current_sptm_cpu_id(); + if (id >= SPTM_MAX_CPUS_EMUL) + id = 0; + return (volatile u64 *)(g_state.scratch_pa + id * PAGE_SIZE_GUEST_EMUL); +} + +static inline bool_t ft_get_idx(u64 pa, u64 *idx_out) { + pa &= FRAME_PAGE_MASK; + if (g_state.frame_table_base == 0 || pa < g_state.vm_first_phys) return 0; + u64 idx = (pa - g_state.vm_first_phys) >> 14; + if ((idx * 16UL) >= g_state.frame_table_size) return 0; + *idx_out = idx; + return 1; +} + +static inline bool_t papt_va_for_managed_pa(u64 pa, u64 *papt_va_out) { + u64 idx; + + pa &= FRAME_PAGE_MASK; + if (g_state.first_papt == 0) + return 0; + if (!ft_get_idx(pa, &idx)) + return 0; + if (g_state.papt_span && idx * PAGE_SIZE_GUEST_EMUL >= g_state.papt_span) + return 0; + + *papt_va_out = g_state.first_papt + idx * PAGE_SIZE_GUEST_EMUL; + return 1; +} + +static inline void note_papt_nvme_hit_class(u64 cls, u64 rc) { + if (cls >= PAPT_NVME_CLASS_COUNT) + return; + g_state.papt_update.nvme_hits[cls]++; + if (rc == PAPT_UPDATE_RC_WRITE) + g_state.papt_update.nvme_writes[cls]++; +} + +static inline bool_t papt_pa_overlaps(u64 pa, u64 start, u64 size) { + if (start == 0 || start == UINT64_MAX_VALUE || size == 0) + return 0; + + u64 page = pa & FRAME_PAGE_MASK; + u64 first = start & FRAME_PAGE_MASK; + u64 end = (start + size + PAGE_SIZE_GUEST_EMUL - 1UL) & FRAME_PAGE_MASK; + if (end < start) + end = UINT64_MAX_VALUE; + return first <= page && page < end; +} + +static inline void note_papt_nvme_hits(u64 rc, u64 pa) { + u32 queue_count = g_state.nvme.queue_count; + if (queue_count == 0) + queue_count = 1; + if (queue_count > NVME_C_MAX_CID) + queue_count = NVME_C_MAX_CID; + + u64 admin_span = (((u64)queue_count * 0x80UL) + 0xfffUL) & ~0xfffUL; + u64 cq_span = (((u64)queue_count * 0x10UL) + 0xfffUL) & ~0xfffUL; + + if (papt_pa_overlaps(pa, g_state.nvme.admin_asq_kva, admin_span)) + note_papt_nvme_hit_class(PAPT_NVME_SEED_ASQ, rc); + if (papt_pa_overlaps(pa, g_state.nvme.admin_acq_kva, admin_span)) + note_papt_nvme_hit_class(PAPT_NVME_SEED_ACQ, rc); + if (papt_pa_overlaps(pa, g_state.nvme.prp_list_kva, + (u64)queue_count * 0x800UL)) + note_papt_nvme_hit_class(PAPT_NVME_PRP_LIST, rc); + if (papt_pa_overlaps(pa, g_state.nvme.asq_pa_cached, admin_span)) + note_papt_nvme_hit_class(PAPT_NVME_XNU_ASQ, rc); + if (papt_pa_overlaps(pa, g_state.nvme.acq_pa_cached, cq_span)) + note_papt_nvme_hit_class(PAPT_NVME_XNU_ACQ, rc); + if (papt_pa_overlaps(pa, g_state.nvme.iosq_pa_cached, admin_span)) + note_papt_nvme_hit_class(PAPT_NVME_XNU_IOSQ, rc); + if (papt_pa_overlaps(pa, g_state.nvme.iocq_pa_cached, cq_span)) + note_papt_nvme_hit_class(PAPT_NVME_XNU_IOCQ, rc); + + for (u32 cid = 0; cid < queue_count && cid < NVME_C_MAX_CID; cid++) { + if (g_state.nvme.cid_mode[cid] == NVME_CID_AVAILABLE || + g_state.nvme.cid_mode[cid] == NVME_CID_BUSY) + continue; + if (papt_pa_overlaps(pa, g_state.nvme.tcb_prp1[cid], + PAGE_SIZE_GUEST_EMUL) || + papt_pa_overlaps(pa, g_state.nvme.tcb_prp2[cid], + PAGE_SIZE_GUEST_EMUL)) { + note_papt_nvme_hit_class(PAPT_NVME_ACTIVE_PRP, rc); + break; + } + } +} + +static inline u64 sptm_ctx_elr(void *ctx) { + if (!ctx) + return 0; + return *(volatile u64 *)((unsigned char *)ctx + 0x108); +} + +static inline void note_papt_update(u64 rc, u64 pa, u32 options, u64 papt_va, + u64 root_pa, u64 template_pte, + u64 existing_pte, u64 new_pte, + u64 caller_elr, u64 inner_root_pa, + u64 inner_va, u64 inner_template_pte) { + struct papt_update_state *st = &g_state.papt_update; + u64 slot; + + note_papt_nvme_hits(rc, pa); + st->calls++; + switch (rc) { + case PAPT_UPDATE_RC_WRITE: + st->candidates++; + st->writes++; + break; + case PAPT_UPDATE_RC_SKIP_PAPT: + st->skip_skip_papt++; + break; + case PAPT_UPDATE_RC_NO_ATTR: + st->skip_no_attr++; + break; + case PAPT_UPDATE_RC_UNMANAGED: + st->candidates++; + st->skip_unmanaged++; + break; + case PAPT_UPDATE_RC_BAD_TEMPLATE: + st->candidates++; + st->skip_bad_template++; + break; + case PAPT_UPDATE_RC_BAD_ROOT: + st->candidates++; + st->skip_bad_root++; + break; + case PAPT_UPDATE_RC_BAD_WALK: + st->candidates++; + st->skip_bad_walk++; + break; + case PAPT_UPDATE_RC_BAD_EXISTING: + st->candidates++; + st->skip_bad_existing++; + break; + case PAPT_UPDATE_RC_SAME_ATTRS: + st->candidates++; + st->skip_same_attrs++; + break; + default: + break; + } + + slot = st->ring_idx++ & (PAPT_UPDATE_RING_SLOTS - 1); + st->ring[slot].rc = rc; + st->ring[slot].pa = pa; + st->ring[slot].options = options; + st->ring[slot].papt_va = papt_va; + st->ring[slot].root_pa = root_pa; + st->ring[slot].template_pte = template_pte; + st->ring[slot].existing_pte = existing_pte; + st->ring[slot].new_pte = new_pte; + st->ring[slot].caller_elr = caller_elr; + st->ring[slot].inner_root_pa = inner_root_pa; + st->ring[slot].inner_va = inner_va; + st->ring[slot].inner_template_pte = inner_template_pte; +} + +static inline bool_t papt_attrs_are_rt_nc(u64 pte) { + const u64 attr_mask = ARM_PTE_ATTRINDX_MASK | ARM_PTE_SH_MASK; + + return (pte & attr_mask) == (1UL << 2); +} + +static inline bool_t mirror_stage2_for_papt_attrs(u64 pa, u64 old_pte, + u64 new_pte) { + bool_t old_rt = papt_attrs_are_rt_nc(old_pte); + bool_t new_rt = papt_attrs_are_rt_nc(new_pte); + u64 page = pa & ARM_PTE_PAGE_MASK; + u64 attrs; + + if (old_rt == new_rt) + return 0; + + attrs = new_rt ? HV_S2_PTE_NORMAL_NC : HV_S2_PTE_NORMAL_WB; + return hv_map(page, page | attrs, PAGE_SIZE_GUEST_EMUL, 1) == 0; +} + +static inline bool_t update_kernel_papt_attrs(u64 pa, u64 papt_template, + u32 options, u64 caller_elr, + u64 inner_root_pa, u64 inner_va, + u64 inner_template_pte) { + u64 papt_va; + u64 idx; + u64 root_pa; + u64 l3; + volatile u64 *slot; + u64 existing; + u64 template_pte; + u64 new_pte; + const u64 attr_mask = (7UL << 2) | (3UL << 8); + + if (options & SPTM_UPDATE_SKIP_PAPT) { + note_papt_update(PAPT_UPDATE_RC_SKIP_PAPT, pa, options, 0, 0, 0, 0, 0, + caller_elr, inner_root_pa, inner_va, inner_template_pte); + return 0; + } + if ((options & (SPTM_UPDATE_SH | SPTM_UPDATE_MAIR)) == 0) { + note_papt_update(PAPT_UPDATE_RC_NO_ATTR, pa, options, 0, 0, 0, 0, 0, + caller_elr, inner_root_pa, inner_va, inner_template_pte); + return 0; + } + if (!papt_va_for_managed_pa(pa, &papt_va)) { + note_papt_update(PAPT_UPDATE_RC_UNMANAGED, pa, options, 0, 0, 0, 0, 0, + caller_elr, inner_root_pa, inner_va, inner_template_pte); + return 0; + } + + template_pte = xnu_pte_to_arm_geom(papt_template, 0); + if ((template_pte & 3) != 3) { + note_papt_update(PAPT_UPDATE_RC_BAD_TEMPLATE, pa, options, papt_va, 0, + template_pte, 0, 0, caller_elr, inner_root_pa, + inner_va, inner_template_pte); + return 0; + } + if ((template_pte & ARM_PTE_PAGE_MASK) != (pa & ARM_PTE_PAGE_MASK)) { + note_papt_update(PAPT_UPDATE_RC_BAD_TEMPLATE, pa, options, papt_va, 0, + template_pte, 0, 0, caller_elr, inner_root_pa, + inner_va, inner_template_pte); + return 0; + } + + root_pa = ttbr1_el12_root_pa(); + if (root_pa == 0) { + note_papt_update(PAPT_UPDATE_RC_BAD_ROOT, pa, options, papt_va, root_pa, + template_pte, 0, 0, caller_elr, inner_root_pa, + inner_va, inner_template_pte); + return 0; + } + + l3 = walk_to_l3_16k(root_pa, papt_va, &idx); + if (l3 == 0) { + note_papt_update(PAPT_UPDATE_RC_BAD_WALK, pa, options, papt_va, root_pa, + template_pte, 0, 0, caller_elr, inner_root_pa, + inner_va, inner_template_pte); + return 0; + } + + slot = pt_slot_ptr(l3, idx); + existing = pt_read_slot_poc(slot); + if ((existing & 3) != 3) { + note_papt_update(PAPT_UPDATE_RC_BAD_EXISTING, pa, options, papt_va, + root_pa, template_pte, existing, 0, caller_elr, + inner_root_pa, inner_va, inner_template_pte); + return 0; + } + if ((existing & ARM_PTE_PAGE_MASK) != (pa & ARM_PTE_PAGE_MASK)) { + note_papt_update(PAPT_UPDATE_RC_BAD_EXISTING, pa, options, papt_va, + root_pa, template_pte, existing, 0, caller_elr, + inner_root_pa, inner_va, inner_template_pte); + return 0; + } + + new_pte = existing; + if (options & SPTM_UPDATE_SH) + new_pte = (new_pte & ~(3UL << 8)) | (template_pte & (3UL << 8)); + if (options & SPTM_UPDATE_MAIR) + new_pte = (new_pte & ~(7UL << 2)) | (template_pte & (7UL << 2)); + + if ((new_pte & attr_mask) == (existing & attr_mask)) { + note_papt_update(PAPT_UPDATE_RC_SAME_ATTRS, pa, options, papt_va, + root_pa, template_pte, existing, new_pte, caller_elr, + inner_root_pa, inner_va, inner_template_pte); + return 0; + } + + /* + * XNU may be moving a managed DRAM page from cacheable to RT/NC + * semantics. Clean+invalidate the data page while our identity alias is + * still cacheable so the downstream RT mapping cannot hit stale cache/SLC + * directory state. + */ + sptm_coproc_cache_maint_data_page(pa); + mirror_stage2_for_papt_attrs(pa, existing, new_pte); + *slot = new_pte; + clean_pt_slot_poc(slot); + note_papt_update(PAPT_UPDATE_RC_WRITE, pa, options, papt_va, root_pa, + template_pte, existing, new_pte, caller_elr, inner_root_pa, + inner_va, inner_template_pte); + return 1; +} + +static inline bool_t set_kernel_papt_cacheable_for_managed_pa(u64 pa, + bool_t cacheable, + u64 caller_elr) { + u64 papt_va; + u64 root_pa; + u64 idx; + u64 l3; + volatile u64 *slot; + u64 existing; + u64 new_pte; + const u64 attr_mask = ARM_PTE_ATTRINDX_MASK | ARM_PTE_SH_MASK; + u32 options = cacheable ? SPTM_PAPT_OPT_ALT_TO_WB : + SPTM_PAPT_OPT_ALT_TO_NC; + + if (!papt_va_for_managed_pa(pa, &papt_va)) { + note_papt_update(PAPT_UPDATE_RC_UNMANAGED, pa, options, 0, 0, 0, 0, 0, + caller_elr, 0, 0, 0); + return 0; + } + + root_pa = ttbr1_el12_root_pa(); + if (root_pa == 0) { + note_papt_update(PAPT_UPDATE_RC_BAD_ROOT, pa, options, papt_va, root_pa, + 0, 0, 0, caller_elr, 0, 0, 0); + return 0; + } + + l3 = walk_to_l3_16k(root_pa, papt_va, &idx); + if (l3 == 0) { + note_papt_update(PAPT_UPDATE_RC_BAD_WALK, pa, options, papt_va, root_pa, + 0, 0, 0, caller_elr, 0, 0, 0); + return 0; + } + + slot = pt_slot_ptr(l3, idx); + existing = pt_read_slot_poc(slot); + if ((existing & 3) != 3 || + ((existing & ARM_PTE_PAGE_MASK) != (pa & ARM_PTE_PAGE_MASK))) { + note_papt_update(PAPT_UPDATE_RC_BAD_EXISTING, pa, options, papt_va, + root_pa, 0, existing, 0, caller_elr, 0, 0, 0); + return 0; + } + + new_pte = (existing & ~attr_mask) | + (cacheable ? SPTM_PAPT_ATTR_NORMAL_WB : SPTM_PAPT_ATTR_RT_NC); + if ((new_pte & attr_mask) == (existing & attr_mask)) { + note_papt_update(PAPT_UPDATE_RC_SAME_ATTRS, pa, options, papt_va, + root_pa, new_pte, existing, new_pte, caller_elr, + 0, 0, 0); + return 0; + } + + /* + * Real SPTM routes type-24/kind-5 frames through the coprocessor/IOMMU + * coherency path. We cannot issue that GXF/SLC operation here, so prevent + * XNU's PAPT/direct-map alias from creating the cacheable tag in the first + * place while the frame is a coprocessor-visible ALT/PTD page. + */ + sptm_coproc_cache_maint_data_page(pa); + mirror_stage2_for_papt_attrs(pa, existing, new_pte); + *slot = new_pte; + clean_pt_slot_poc(slot); + note_papt_update(PAPT_UPDATE_RC_WRITE, pa, options, papt_va, root_pa, + new_pte, existing, new_pte, caller_elr, 0, 0, 0); + return 1; +} + +static inline u16 ft_read_u16(u64 pa, u64 off) { + u64 idx; + if (!ft_get_idx(pa, &idx)) return 0; + volatile u16 *cnt = (volatile u16 *)(g_state.frame_table_base + idx * 16 + off); + return *cnt; +} + +static inline void ft_clean_entry_idx(u64 idx) { + clean_range_poc((void *)(g_state.frame_table_base + idx * 16), 16); +} + +static inline void ft_write_u16(u64 pa, u64 off, u16 val) { + u64 idx; + if (!ft_get_idx(pa, &idx)) return; + volatile u16 *cnt = (volatile u16 *)(g_state.frame_table_base + idx * 16 + off); + *cnt = val; + ft_clean_entry_idx(idx); +} + +static inline void ft_inc_u16(u64 pa, u64 off) { + u64 idx; + if (!ft_get_idx(pa, &idx)) return; + volatile u16 *cnt = (volatile u16 *)(g_state.frame_table_base + idx * 16 + off); + u16 v = *cnt; + if (v < 0xFFFF) { + *cnt = v + 1; + ft_clean_entry_idx(idx); + } +} + +static inline void ft_dec_u16(u64 pa, u64 off) { + u64 idx; + if (!ft_get_idx(pa, &idx)) return; + volatile u16 *cnt = (volatile u16 *)(g_state.frame_table_base + idx * 16 + off); + u16 v = *cnt; + if (v > 0) { + *cnt = v - 1; + ft_clean_entry_idx(idx); + } +} + +static inline void ft_inc_u32(u64 pa, u64 off) { + u64 idx; + if (!ft_get_idx(pa, &idx)) return; + volatile u32 *cnt = (volatile u32 *)(g_state.frame_table_base + idx * 16 + off); + u32 v = *cnt; + if (v < 0xffffffffU) { + *cnt = v + 1; + ft_clean_entry_idx(idx); + } +} + +static inline void ft_dec_u32(u64 pa, u64 off) { + u64 idx; + if (!ft_get_idx(pa, &idx)) return; + volatile u32 *cnt = (volatile u32 *)(g_state.frame_table_base + idx * 16 + off); + u32 v = *cnt; + if (v > 0) { + *cnt = v - 1; + ft_clean_entry_idx(idx); + } +} + +static inline u32 ft_read_u32(u64 pa, u64 off) { + u64 idx; + if (!ft_get_idx(pa, &idx)) return 0; + volatile u32 *cnt = (volatile u32 *)(g_state.frame_table_base + idx * 16 + off); + return *cnt; +} + +/* Refcount maintenance. + * + * Firmware-visible fields: + * - kind=5 secondary pages: u16 refcount at FTE+0x04. + * - page-table frames: u16 parent/table-link count at FTE+0x06. + * - page-table frames: u16 live child/leaf count at FTE+0x08. + * - kind=3 data pages: u32 ro_refcnt at FTE+0x08, wx_refcnt at FTE+0x0c. + */ +static inline bool_t pte_uses_wx_refcount(u64 pte) { + bool_t writable = (pte & ARM_PTE_WRITEABLE) || + ((pte & ARM_PTE_AP_RO) == 0); + bool_t executable = (pte & (ARM_PTE_PNX | ARM_PTE_NX)) != + (ARM_PTE_PNX | ARM_PTE_NX); + return writable || executable; +} + +static inline void ft_inc_page_refcount_for_pte(u64 pa, u64 pte) { + u8 type = ft_get_type(pa); + if (type == XNU_PAGE_TABLE_ALT) { + ft_inc_u16(pa, FT_KIND5_REFCNT_OFF); + return; + } + if (is_pt_type(type)) + return; + if (pte_uses_wx_refcount(pte)) + ft_inc_u32(pa, FT_DATA_WX_REFCNT_OFF); + else + ft_inc_u32(pa, FT_DATA_RO_REFCNT_OFF); +} + +static inline void ft_dec_page_refcount_for_pte(u64 pa, u64 pte) { + u8 type = ft_get_type(pa); + if (type == XNU_PAGE_TABLE_ALT) { + ft_dec_u16(pa, FT_KIND5_REFCNT_OFF); + return; + } + if (is_pt_type(type)) + return; + if (pte_uses_wx_refcount(pte)) + ft_dec_u32(pa, FT_DATA_WX_REFCNT_OFF); + else + ft_dec_u32(pa, FT_DATA_RO_REFCNT_OFF); +} + +static inline void ft_inc_table_mapping_refcount(u64 pa) { + ft_inc_u16(pa, FT_TABLE_MAPPING_REFCNT_OFF); +} + +static inline void ft_dec_table_mapping_refcount(u64 pa) { + ft_dec_u16(pa, FT_TABLE_MAPPING_REFCNT_OFF); +} + +static inline void ft_inc_table_nested_refcount(u64 pa) { + ft_inc_u16(pa, FT_TABLE_NESTED_REFCNT_OFF); +} + +static inline void ft_dec_table_nested_refcount(u64 pa) { + ft_dec_u16(pa, FT_TABLE_NESTED_REFCNT_OFF); +} + +static inline u32 ft_read_page_refcount(u64 pa) { + if (ft_get_type(pa) == XNU_PAGE_TABLE_ALT) + return ft_read_u16(pa, FT_KIND5_REFCNT_OFF); + return ft_read_u32(pa, FT_DATA_RO_REFCNT_OFF) + + ft_read_u32(pa, FT_DATA_WX_REFCNT_OFF); +} + +static inline u16 ft_read_table_refcount(u64 pa) { + return ft_read_u16(pa, FT_TABLE_NESTED_REFCNT_OFF); +} + +static inline bool_t is_kernel_va(u64 va); +static inline bool_t guest_read64(u64 va, u64 *out); + +static inline void alt_ptd_trace_add_bt(struct alt_ptd_trace_event *ev, + u64 value) { + if (!is_kernel_va(value)) + return; + + for (u64 i = 0; i < ev->bt_count && i < 12; i++) { + if (ev->bt[i] == value) + return; + } + + if (ev->bt_count < 12) + ev->bt[ev->bt_count++] = value; +} + +static inline void alt_ptd_trace_walk_frame(struct alt_ptd_trace_event *ev, + u64 fp) { + for (u32 depth = 0; depth < 8; depth++) { + u64 saved_fp; + u64 saved_lr; + + if (!is_kernel_va(fp)) + break; + if (!guest_read64(fp, &saved_fp) || + !guest_read64(fp + 8, &saved_lr)) + break; + + alt_ptd_trace_add_bt(ev, saved_lr); + + if (saved_fp == fp || saved_fp < fp || saved_fp - fp > 0x200000UL) + break; + fp = saved_fp; + } +} + +static inline void alt_ptd_trace_scan_stack(struct alt_ptd_trace_event *ev, + u64 sp) { + if (!is_kernel_va(sp)) + return; + if (!ev->bt_base) + ev->bt_base = sp; + + for (u32 off = 0; off < 0x180; off += 8) { + u64 value; + + if (!guest_read64(sp + off, &value)) + break; + alt_ptd_trace_add_bt(ev, value); + } +} + +static inline void note_alt_ptd_retype(u64 pa, u64 caller_elr, u64 caller_lr, + u64 caller_fp, u64 caller_sp0, + u64 caller_sp1, u8 current_type, + u8 existing_type, u8 new_type, + u64 retype_params, + u64 page_ref_before, + u64 table_ref_before, + u64 page_ref_after, + u64 table_ref_after, u64 ft_idx, + u64 papt_ring_before, u64 flags) { + struct alt_ptd_trace_state *st = &g_state.alt_ptd_trace; + u64 seq = st->ring_idx++; + struct alt_ptd_trace_event *ev = + &st->ring[seq & (ALT_PTD_TRACE_RING_SLOTS - 1)]; + u64 papt_ring_after = g_state.papt_update.ring_idx; + u64 papt_rc = 0; + u64 papt_existing = 0; + u64 papt_new = 0; + u64 papt_va = 0; + + if (papt_ring_after > papt_ring_before) { + struct papt_update_event *pev = + &g_state.papt_update.ring[(papt_ring_after - 1) & + (PAPT_UPDATE_RING_SLOTS - 1)]; + if ((pev->pa & FRAME_PAGE_MASK) == (pa & FRAME_PAGE_MASK)) { + papt_rc = pev->rc; + papt_existing = pev->existing_pte; + papt_new = pev->new_pte; + papt_va = pev->papt_va; + } + } + + ev->seq = seq; + ev->pa = pa & FRAME_PAGE_MASK; + ev->caller_elr = caller_elr; + ev->caller_lr = caller_lr; + ev->caller_fp = caller_fp; + ev->types_flags = ((u64)current_type) | + ((u64)existing_type << 8) | + ((u64)new_type << 16) | + (flags << 32); + ev->retype_params = retype_params; + ev->ft_idx = ft_idx; + ev->refs_before = (page_ref_before & 0xffffffffUL) | + ((table_ref_before & 0xffffffffUL) << 32); + ev->refs_after = (page_ref_after & 0xffffffffUL) | + ((table_ref_after & 0xffffffffUL) << 32); + ev->papt_ring_before = papt_ring_before; + ev->papt_ring_after = papt_ring_after; + ev->papt_rc = papt_rc; + ev->papt_existing_pte = papt_existing; + ev->papt_new_pte = papt_new; + ev->papt_va = papt_va; + ev->caller_sp0 = caller_sp0; + ev->caller_sp1 = caller_sp1; + ev->bt_count = 0; + ev->bt_base = 0; + for (u32 i = 0; i < 12; i++) + ev->bt[i] = 0; + + alt_ptd_trace_add_bt(ev, caller_lr); + alt_ptd_trace_walk_frame(ev, caller_fp); + alt_ptd_trace_scan_stack(ev, caller_sp0); + alt_ptd_trace_scan_stack(ev, caller_sp1); +} + +static inline u64 ft_idx_or_bad(u64 pa) { + u64 idx; + return ft_get_idx(pa, &idx) ? idx : UINT64_MAX_VALUE; +} + +static inline void ft_inc_refcount(u64 pa) { + ft_inc_page_refcount_for_pte(pa, 0); +} + +static inline void ft_dec_refcount(u64 pa) { + ft_dec_page_refcount_for_pte(pa, 0); +} + +#if 0 +/* Kept disabled to document the old single-refcount model. It was wrong for + * page-table frames because libsptm_get_table_mapping_count only accepts + * frame_type_params kind 1/2, not kind 5. */ +static inline void ft_inc_refcount_old(u64 pa) { + if (g_state.frame_table_base == 0 || pa < g_state.vm_first_phys) return; + u64 idx = (pa - g_state.vm_first_phys) >> 14; + if (idx >= g_state.frame_table_size) return; + volatile u16 *cnt = (volatile u16 *)(g_state.frame_table_base + idx * 16 + 4); + u16 v = *cnt; + if (v < 0xFFFF) *cnt = v + 1; +} + +static inline void ft_dec_refcount_old(u64 pa) { + if (g_state.frame_table_base == 0 || pa < g_state.vm_first_phys) return; + u64 idx = (pa - g_state.vm_first_phys) >> 14; + if (idx >= g_state.frame_table_size) return; + volatile u16 *cnt = (volatile u16 *)(g_state.frame_table_base + idx * 16 + 4); + u16 v = *cnt; + if (v > 0) *cnt = v - 1; +} +#endif + +static inline u8 ft_get_type(u64 pa) { + u64 idx; + if (!ft_get_idx(pa, &idx)) return XNU_DEFAULT; + volatile u8 *ft = (volatile u8 *)g_state.frame_table_base; + return ft[idx * 16 + 2]; +} + +static inline bool_t is_root_type(u8 t) { + return t == SPTM_KERNEL_ROOT_TABLE || + t == XNU_USER_ROOT_TABLE || + t == XNU_SHARED_ROOT_TABLE || + t == XNU_STAGE2_ROOT_TABLE || + t == XNU_SUBPAGE_USER_ROOT_TABLES; +} + +static inline u8 retype_params_attr_idx(u64 params) { + return (u8)(params & 0xff); +} + +static inline u16 retype_params_flags(u64 params) { + return (u16)((params >> 16) & 0xffff); +} + +static inline u16 retype_params_asid(u64 params) { + return (u16)((params >> 32) & 0xffff); +} + +static inline u8 root_geom_lookup(u64 root_pa) { + for (u32 i = 0; i < ROOT_GEOM_SLOTS; i++) { + if (g_state.root_geom_pa[i] == root_pa) + return (u8)(g_state.root_geom_attr[i] & 0xff); + } + + if (ft_get_type(root_pa) == XNU_SUBPAGE_USER_ROOT_TABLES) + return SPTM_PT_GEOMETRY_4K; + return ROOT_GEOM_UNKNOWN; +} + +static inline u16 root_asid_lookup(u64 root_pa) { + for (u32 i = 0; i < ROOT_GEOM_SLOTS; i++) { + if (g_state.root_geom_pa[i] == root_pa) + return (u16)((g_state.root_geom_attr[i] >> 16) & 0xffff); + } + return 0; +} + +static inline void root_geom_set_meta(u64 root_pa, u8 geom, u16 asid) { + u32 empty = ROOT_GEOM_SLOTS; + u64 packed = (u64)geom | ((u64)asid << 16); + + for (u32 i = 0; i < ROOT_GEOM_SLOTS; i++) { + if (g_state.root_geom_pa[i] == root_pa) { + g_state.root_geom_attr[i] = packed; + return; + } + if (empty == ROOT_GEOM_SLOTS && g_state.root_geom_pa[i] == 0) + empty = i; + } + + if (empty == ROOT_GEOM_SLOTS) + empty = (u32)(((root_pa >> 14) ^ (root_pa >> 6)) & (ROOT_GEOM_SLOTS - 1)); + g_state.root_geom_pa[empty] = root_pa; + g_state.root_geom_attr[empty] = packed; +} + +static inline void root_geom_set(u64 root_pa, u8 geom) { + root_geom_set_meta(root_pa, geom, root_asid_lookup(root_pa)); +} + +static inline bool_t root_uses_4k(u64 root_pa, u32 target_level) { + if (target_level == 0) { + /* XNU only calls sptm_map_table(..., level=0, ...) for roots whose + * walk really starts at L0. Record that runtime fact so the next + * level=1/2 operation for this same root descends through the table we + * just installed instead of treating root[0] as an already-present + * 16K L1 entry. */ + root_geom_set(root_pa, SPTM_PT_GEOMETRY_4K); + return 1; + } + + u8 geom = root_geom_lookup(root_pa); + if (geom != ROOT_GEOM_UNKNOWN) + return geom == SPTM_PT_GEOMETRY_4K; + + /* Do not infer 4K from XNU_USER_ROOT_TABLE. Apple Silicon supports mixed + * user geometries, and the normal ARM64 bringup path here uses 16K user + * roots. The earlier frame-type heuristic returned TABLE_NOT_PRESENT for + * a valid 16K user root and sent xnu into pmap_expand()'s WFE loop. */ + return 0; +} + +static inline u32 table_group_count(bool_t geom4k) { + return geom4k ? 4U : 1U; +} + +static inline u64 table_group_base_idx(u64 idx, bool_t geom4k) { + return geom4k ? (idx & ~3UL) : idx; +} + +static inline u64 table_group_child_pa(u64 child_pa, u32 n, bool_t geom4k) { + if (!geom4k) + return child_pa & ARM_TTE_TABLE_MASK; + return (child_pa & FRAME_PAGE_MASK) + ((u64)n * PAGE_SIZE_4K_EMUL); +} + +static inline void ft_set_type(u64 pa, u8 type) { + u64 idx; + if (!ft_get_idx(pa, &idx)) return; + volatile u8 *ft = (volatile u8 *)g_state.frame_table_base; + ft[idx * 16 + 2] = type; + ft_clean_entry_idx(idx); +} + +static inline bool_t is_pt_type(u8 t) { + return t == SPTM_KERNEL_ROOT_TABLE || + t == SPTM_PAGE_TABLE || + t == XNU_USER_ROOT_TABLE || + t == XNU_SHARED_ROOT_TABLE || + t == XNU_PAGE_TABLE || + t == XNU_PAGE_TABLE_SHARED || + t == XNU_PAGE_TABLE_ROZONE || + t == XNU_PAGE_TABLE_COMMPAGE || + t == XNU_STAGE2_ROOT_TABLE || + t == XNU_STAGE2_PAGE_TABLE || + t == XNU_SUBPAGE_USER_ROOT_TABLES; +} + +static inline bool_t is_io_type(u8 t) { + return t == XNU_IO || t == XNU_PROTECTED_IO || t == XNU_COPROCESSOR_RO_IO; +} + +static inline void update_refcounts_for_pte_change_geom(u64 existing, u64 new_pte, + bool_t geom4k) { + bool_t old_valid = ((existing & 3) == 3); + bool_t new_valid = ((new_pte & 3) == 3); + u64 page_mask = geom4k ? ARM_PTE_PAGE_MASK_4K : ARM_PTE_PAGE_MASK; + u64 old_pa = existing & page_mask; + u64 new_pa = new_pte & page_mask; + + if (old_valid && !new_valid) { + ft_dec_page_refcount_for_pte(old_pa, existing); + } else if (!old_valid && new_valid) { + ft_inc_page_refcount_for_pte(new_pa, new_pte); + } else if (old_valid && new_valid && old_pa != new_pa) { + ft_dec_page_refcount_for_pte(old_pa, existing); + ft_inc_page_refcount_for_pte(new_pa, new_pte); + } +} + +static inline void update_refcounts_for_leaf_pte_change(u64 table_pa, + u64 existing, + u64 new_pte, + bool_t geom4k) { + bool_t old_valid = ((existing & 3) == 3); + bool_t new_valid = ((new_pte & 3) == 3); + + update_refcounts_for_pte_change_geom(existing, new_pte, geom4k); + + table_pa &= FRAME_PAGE_MASK; + if (old_valid && !new_valid) + ft_dec_table_nested_refcount(table_pa); + else if (!old_valid && new_valid) + ft_inc_table_nested_refcount(table_pa); +} + +static inline bool_t maybe_map_t8140_pmgr_cpustart_window(u64 root_pa, u64 va, + u64 arm_pte, + bool_t geom4k, + u64 known_l3, + u64 known_idx) { + /* + * T8140 ApplePMGR maps PMGR reg[24] at 0x300730000, but the CPUSTART + * write lands at offset +0x4008 inside that regmap. XNU may install this + * mapping through MAP_PAGE or a region/disjoint update, and it may hand us + * any page inside the 0x14000-byte ADT window. If we leave holes in the + * same real regmap, XNU faults in ApplePMGR::writeReg32 before the + * hypervisor's physical CPUSTART hook can see the access. + * + * This is not generic MMIO expansion: it preserves only this exact ADT + * PMGR regmap window, with the same PTE attributes XNU requested. + */ + const u64 pmgr_base = 0x300730000UL; + const u64 pmgr_size = 0x14000UL; + u64 page_size = geom4k ? PAGE_SIZE_4K_EMUL : PAGE_SIZE_GUEST_EMUL; + u64 page_mask = geom4k ? ARM_PTE_PAGE_MASK_4K : ARM_PTE_PAGE_MASK; + u64 mapped_pa = arm_pte & page_mask; + if ((arm_pte & 3) != 3) + return 0; + if (mapped_pa < pmgr_base || mapped_pa >= (pmgr_base + pmgr_size)) + return 0; + + u64 mapped_off = mapped_pa - pmgr_base; + u64 window_va = va - mapped_off; + bool_t changed = 0; + u64 table_entries = geom4k ? 512UL : 2048UL; + u64 mapped_page_idx = mapped_off / page_size; + bool_t can_fill_direct = + known_l3 != 0 && mapped_off == mapped_page_idx * page_size && + known_idx >= mapped_page_idx; + u64 direct_base_idx = can_fill_direct ? known_idx - mapped_page_idx : 0; + + for (u64 off = 0; off < pmgr_size; off += page_size) { + u64 idx = 0; + u64 l3 = 0; + if (can_fill_direct && direct_base_idx + off / page_size < table_entries) { + l3 = known_l3; + idx = direct_base_idx + off / page_size; + } else { + l3 = walk_to_l3(root_pa, window_va + off, &idx); + if (!l3) + continue; + } + + volatile u64 *slot = pt_slot_ptr(l3, idx); + u64 existing = pt_read_slot_poc(slot); + if ((existing & 3) == 3) + continue; + + u64 extra_pte = (arm_pte & ~page_mask) | ((pmgr_base + off) & page_mask); + *slot = extra_pte; + clean_pt_slot_poc(slot); + update_refcounts_for_leaf_pte_change(l3, existing, extra_pte, geom4k); + changed = 1; + } + + if (changed) + tlbi_vmalle1is(); + return changed; +} + +static inline void write_prev_pte(volatile u64 *prev_out, u32 *prev_count, + u64 existing) { + if (prev_out != (volatile u64 *)0 && *prev_count < 2048) + prev_out[*prev_count] = existing; + *prev_count = *prev_count + 1; +} + +/* Hardcoded offset of regs[] in m1n1's struct exc_info (always 0). */ +#define REGS_OFFSET 0 +#define ESR_OFFSET 0x110 + +static inline bool_t is_dispatch_hvc(u64 esr, u64 x16) { + if ((esr & ESR_ISS_MASK) != SPTM_GENTER_DISPATCH_CALL) return 0; + if (x16 & SPTM_DISPATCH_RESERVED_MASK) return 0; + if (((x16 >> 48) & 0xff) > 4) return 0; + if (((x16 >> 32) & 0xff) > 11) return 0; + return 1; +} + +#define KERNEL_VA_BASE 0xfffffe0000000000UL +#define DART_OBJ_MODE_OFF 0x0b8UL +#define DART_OBJ_STATE_OFF 0x0c50UL +#define DART_OBJ_INST_COUNT_OFF 0x14d8UL +#define DART_OBJ_GAPF_COUNT_OFF 0x17b0UL + +static inline bool_t is_kernel_va(u64 va) { + return (va & KERNEL_VA_BASE) == KERNEL_VA_BASE; +} + +static inline bool_t guest_va_to_pa(u64 va, bool_t write, u64 *pa_out) { + u64 par; + if (write) { + __asm__ volatile( + "at s1e1w, %1\n\t" + "isb\n\t" + "mrs %0, par_el1\n\t" + : "=r"(par) : "r"(va) : "memory"); + } else { + __asm__ volatile( + "at s1e1r, %1\n\t" + "isb\n\t" + "mrs %0, par_el1\n\t" + : "=r"(par) : "r"(va) : "memory"); + } + if (par & 1) + return 0; + *pa_out = (par & 0xfffffffffc000UL) | (va & 0x3fffUL); + return 1; +} + +static inline bool_t guest_read32(u64 va, u32 *out) { + u64 pa; + if (!is_kernel_va(va) || !guest_va_to_pa(va, 0, &pa)) + return 0; + *out = *(volatile u32 *)pa; + return 1; +} + +static inline bool_t guest_read64(u64 va, u64 *out) { + u64 pa; + if (!is_kernel_va(va) || !guest_va_to_pa(va, 0, &pa)) + return 0; + *out = *(volatile u64 *)pa; + return 1; +} + +static inline bool_t guest_write64(u64 va, u64 value) { + u64 pa; + if (!is_kernel_va(va) || !guest_va_to_pa(va, 1, &pa)) + return 0; + *(volatile u64 *)pa = value; + clean_range((void *)pa, 8); + return 1; +} + + +/* Table/module entry points. */ +bool_t dart_handle_table(u32 table, u32 endpoint, u64 *regs); +void dart_cache_xnu_object_from_regs(u32 table, u32 endpoint, u64 *regs); +bool_t nvme_handle_table6(u32 endpoint, u64 *regs); +bool_t sart_handle_table5(u32 endpoint, u64 *regs); +bool_t txm_handle_selector(u32 selector, u64 x16, u64 *regs); +bool_t uat_handle_table7(u32 endpoint, u64 *regs); +bool_t xnu_handle_table0(void *ctx, u32 endpoint, u64 *regs); +bool_t sptm_handler(void *ctx); + +#endif /* SPTM_PRIVATE_H */ diff --git a/src/sptm/sart.c b/src/sptm/sart.c new file mode 100644 index 000000000..be37a18a7 --- /dev/null +++ b/src/sptm/sart.c @@ -0,0 +1,277 @@ +/* SPDX-License-Identifier: MIT */ +#include "private.h" + +int debug_printf(const char *fmt, ...); + +static u32 sart_config_off(u32 idx) { + return idx * 4U; +} + +static u32 sart_paddr_off(u32 idx) { + return 0x40U + idx * 4U; +} + +static u32 sart_size_off(u32 idx) { + return 0x80U + idx * 4U; +} + +static bool_t sart_valid_index(u32 idx) { + return idx < SART_C_MAX_ENTRIES; +} + +static u32 sart_flags_for_perm(u32 perm) { + if (perm > 1) + return 0; + + if (g_state.sart.version == 2 || g_state.sart.version == 3) { + u32 access = perm ? 3U : 2U; + return 0xc0U | access | (access << 2) | (access << 4); + } + + return g_state.sart.flags_allow; +} + +static struct sart_c_entry sart_live_entry(u32 idx) { + struct sart_c_entry ent; + ent.paddr = 0; + ent.size = 0; + ent.flags = 0; + ent.guard = 0; + if (!g_state.sart.base_pa || !sart_valid_index(idx)) + return ent; + + u32 cfg = *(volatile u32 *)(g_state.sart.base_pa + sart_config_off(idx)); + u32 paddr_raw = *(volatile u32 *)(g_state.sart.base_pa + sart_paddr_off(idx)); + u32 size_raw; + + if (g_state.sart.version == 3) { + ent.flags = cfg & 0xffU; + size_raw = *(volatile u32 *)(g_state.sart.base_pa + sart_size_off(idx)); + } else if (g_state.sart.version == 2) { + ent.flags = (cfg >> 24) & 0xffU; + size_raw = cfg & ((1U << 24) - 1U); + } else { + ent.flags = (cfg >> 24) & 0x1fU; + size_raw = cfg & ((1U << 19) - 1U); + } + if (ent.flags && !g_state.sart.exclusive_bounds) + size_raw++; + + ent.paddr = (u64)paddr_raw << g_state.sart.paddr_shift; + ent.size = (u64)size_raw << g_state.sart.size_shift; + return ent; +} + +static bool_t sart_write_entry(u32 idx, u64 paddr, u64 size, u32 flags) { + if (!g_state.sart.base_pa || !sart_valid_index(idx)) + return 0; + if ((paddr & ((1UL << g_state.sart.paddr_shift) - 1UL)) || + (size & ((1UL << g_state.sart.size_shift) - 1UL))) + return 0; + + u32 paddr_raw = (u32)(paddr >> g_state.sart.paddr_shift); + u32 size_raw = (u32)(size >> g_state.sart.size_shift); + if (size_raw > g_state.sart.size_max) + return 0; + if (flags && !g_state.sart.exclusive_bounds) { + if (size_raw == 0) + return 0; + size_raw--; + } + + if (g_state.sart.version == 3) { + *(volatile u32 *)(g_state.sart.base_pa + sart_paddr_off(idx)) = paddr_raw; + *(volatile u32 *)(g_state.sart.base_pa + sart_size_off(idx)) = size_raw; + *(volatile u32 *)(g_state.sart.base_pa + sart_config_off(idx)) = + flags & g_state.sart.flags_allow; + } else { + u32 cfg = ((flags & g_state.sart.flags_allow) << 24) | size_raw; + *(volatile u32 *)(g_state.sart.base_pa + sart_paddr_off(idx)) = paddr_raw; + *(volatile u32 *)(g_state.sart.base_pa + sart_config_off(idx)) = cfg; + } + barrier(); + return 1; +} + +static bool_t sart_power_canary_acquire(u32 guard) { + if (guard != 1 || g_state.sart.power_canary_pa == 0) + return 1; + + if (g_state.sart.power_canary_count == 0) + *(volatile u32 *)(g_state.sart.power_canary_pa + + g_state.sart.power_canary_offset) = 0xabfedeedU; + g_state.sart.power_canary_count++; + if (g_state.sart.power_canary_count == 0) + return 0; + return 1; +} + +static bool_t sart_power_canary_release(u32 guard) { + if (guard != 1 || g_state.sart.power_canary_pa == 0) + return 1; + if (g_state.sart.power_canary_count == 0) + return 0; + if (*(volatile u32 *)(g_state.sart.power_canary_pa + + g_state.sart.power_canary_offset) != 0xabfedeedU) + return 0; + g_state.sart.power_canary_count--; + return 1; +} + +static bool_t sart_validate_range(u64 paddr, u64 size) { + if (size == 0) + return 0; + if (size & ((1UL << g_state.sart.size_shift) - 1UL)) + return 0; + if (paddr & ((1UL << g_state.sart.paddr_shift) - 1UL)) + return 0; + if ((paddr + size) < paddr) + return 0; + if ((size >> g_state.sart.size_shift) > g_state.sart.size_max) + return 0; + return 1; +} + +static int sart_find_exact(u64 paddr, u64 size, struct sart_c_entry *out) { + for (u32 i = 0; i < SART_C_MAX_ENTRIES; i++) { + struct sart_c_entry ent = g_state.sart.shadow[i]; + if (!ent.flags) + ent = sart_live_entry(i); + if (ent.flags && ent.paddr == paddr && ent.size == size) { + if (out) + *out = ent; + return (int)i; + } + } + return -1; +} + +static int sart_find_free(void) { + for (u32 i = 0; i < SART_C_MAX_ENTRIES; i++) { + if (g_state.sart.protected_mask & (1U << i)) + continue; + if (g_state.sart.shadow[i].flags) + continue; + struct sart_c_entry ent = sart_live_entry(i); + if (!ent.flags) + return (int)i; + } + return -1; +} + +static u64 sart_map_region(u64 *regs) { + u64 paddr = regs[0]; + u64 size = regs[1]; + u32 perm = (u32)regs[2]; + u32 guard = (u32)regs[3]; + + if (!sart_validate_range(paddr, size)) + return 1; + if (perm > 1 || guard > 1) + return 1; + + u32 flags = sart_flags_for_perm(perm); + if (flags == 0) + return 1; + + int idx = sart_find_free(); + if (idx < 0) + return 1; + + if (!sart_power_canary_acquire(guard)) + return 1; + if (!sart_write_entry((u32)idx, paddr, size, flags)) { + sart_power_canary_release(guard); + return 1; + } + + g_state.sart.shadow[idx].paddr = paddr; + g_state.sart.shadow[idx].size = size; + g_state.sart.shadow[idx].flags = flags; + g_state.sart.shadow[idx].guard = guard; + if (g_state.nvme.debug_flags & NVME_DBG_PRINT_CALLS) { + u64 ring = (g_state.ring_idx - 1) & 0x7fUL; + debug_printf("C-SART map idx=%d pa=0x%lx size=0x%lx perm=%u guard=%u flags=0x%x pc=0x%lx lr=0x%lx\n", + idx, paddr, size, perm, guard, flags, + g_state.ring[ring][6], g_state.ring[ring][7]); + } + return SPTM_SUCCESS; +} + +static u64 sart_unmap_region(u64 *regs) { + u64 paddr = regs[0]; + u64 size = regs[1]; + struct sart_c_entry ent; + + if (!sart_validate_range(paddr, size)) + return 1; + + int idx = sart_find_exact(paddr, size, &ent); + if (idx < 0) + return 1; + + if (g_state.sart.protected_mask & (1U << (u32)idx)) + return SPTM_SUCCESS; + + u32 guard = g_state.sart.shadow[idx].guard; + + if (!sart_write_entry((u32)idx, 0, 0, 0)) + return 1; + if (!sart_power_canary_release(guard)) + return 1; + g_state.sart.shadow[idx].paddr = 0; + g_state.sart.shadow[idx].size = 0; + g_state.sart.shadow[idx].flags = 0; + g_state.sart.shadow[idx].guard = 0; + if (g_state.nvme.debug_flags & NVME_DBG_PRINT_CALLS) { + u64 ring = (g_state.ring_idx - 1) & 0x7fUL; + debug_printf("C-SART unmap idx=%d pa=0x%lx size=0x%lx guard=%u pc=0x%lx lr=0x%lx\n", + idx, paddr, size, guard, + g_state.ring[ring][6], g_state.ring[ring][7]); + } + return SPTM_SUCCESS; +} + +bool_t sart_handle_table5(u32 endpoint, u64 *regs) { + u64 rc = 1; + + if (!g_state.sart.enabled) + return 0; + if (endpoint >= SART_C_MAX_ENDPOINTS) + return 0; + + g_state.sart.total_calls++; + g_state.sart.ep_count[endpoint]++; + for (u32 i = 0; i < 8; i++) + g_state.sart.last[i] = regs[i]; + + switch (endpoint) { + case 0: + /* sptm_sart_set_state(bool). XNU reaches this through + * sart_ioctl(cmd=0x5ab7), leaving cmd/len in x1/x3. */ + g_state.sart.active = (u32)(regs[0] & 1UL); + rc = SPTM_SUCCESS; + if (g_state.nvme.debug_flags & NVME_DBG_PRINT_CALLS) { + u64 ring = (g_state.ring_idx - 1) & 0x7fUL; + debug_printf("C-SART state active=%u arg=0x%lx pc=0x%lx lr=0x%lx\n", + g_state.sart.active, regs[0], + g_state.ring[ring][6], g_state.ring[ring][7]); + } + break; + case 1: + rc = sart_map_region(regs); + break; + case 2: + rc = sart_unmap_region(regs); + break; + } + + g_state.sart.last_rc = (u32)rc; + if (rc != SPTM_SUCCESS) + g_state.sart.violations++; + regs[0] = rc; + g_state.sart.fast_path_calls++; + g_state.fast_path_calls++; + sev(); + return 1; +} diff --git a/src/sptm/txm.c b/src/sptm/txm.c new file mode 100644 index 000000000..edc0374bf --- /dev/null +++ b/src/sptm/txm.c @@ -0,0 +1,383 @@ +/* SPDX-License-Identifier: MIT */ +#include "private.h" + +#define TXM_SYNTH_RESERVED_TOP 0x200UL + +static inline bool_t txm_synth_pa_for_va(u64 va, u64 *pa_out) { + if (g_state.txm.synth_base_va == 0 || g_state.txm.synth_base_pa == 0) + return 0; + if (va < g_state.txm.synth_base_va) + return 0; + u64 off = va - g_state.txm.synth_base_va; + if (off >= g_state.txm.synth_size) + return 0; + *pa_out = g_state.txm.synth_base_pa + off; + return 1; +} + +static inline void txm_synth_write8(u64 va, u8 value) { + u64 pa; + if (!txm_synth_pa_for_va(va, &pa)) + return; + *(volatile u8 *)pa = value; + clean_range((void *)(pa & ~63UL), 64); +} + +static inline void txm_synth_write16(u64 va, u16 value) { + u64 pa; + if (!txm_synth_pa_for_va(va, &pa)) + return; + *(volatile u16 *)pa = value; + clean_range((void *)(pa & ~63UL), 64); +} + +static inline void txm_synth_write32(u64 va, u32 value) { + u64 pa; + if (!txm_synth_pa_for_va(va, &pa)) + return; + *(volatile u32 *)pa = value; + clean_range((void *)(pa & ~63UL), 64); +} + +static inline void txm_synth_write64(u64 va, u64 value) { + u64 pa; + if (!txm_synth_pa_for_va(va, &pa)) + return; + *(volatile u64 *)pa = value; + clean_range((void *)(pa & ~63UL), 64); +} + +static inline u64 txm_alloc_synth(u64 size, u64 fallback_va) { + if (g_state.txm.synth_base_va == 0 || g_state.txm.synth_size == 0) + return fallback_va; + + size = (size + (TXM_C_SYNTH_ALIGN - 1)) & ~(TXM_C_SYNTH_ALIGN - 1); + u64 off = (g_state.txm.synth_next_off + (TXM_C_SYNTH_ALIGN - 1)) & + ~(TXM_C_SYNTH_ALIGN - 1); + u64 limit = g_state.txm.synth_size; + if (limit > TXM_SYNTH_RESERVED_TOP) + limit -= TXM_SYNTH_RESERVED_TOP; + if (off + size > limit) + return fallback_va ? fallback_va : g_state.txm.synth_base_va; + + g_state.txm.synth_next_off = off + size; + return g_state.txm.synth_base_va + off; +} + +/* Offsets verified against KDK 26.5_25F71 private TXM/libCodeSignature headers. */ +#define TXM_SYNTH_TRUST_CACHE_OFF 0x200UL +#define TXM_SYNTH_TRUST_CACHE_SIZE 0x200UL +#define TXM_SYNTH_TC_OBJECT_OFF 0x000UL +#define TXM_SYNTH_TC_MODULE_OFF 0x080UL +#define TXM_SYNTH_TC_ENTRY_OFF 0x098UL +#define TXM_TRUSTCACHE_TYPE_OFF 0x008UL +#define TXM_TRUSTCACHE_MODULE_SIZE_OFF 0x010UL +#define TXM_TRUSTCACHE_MODULE_OFF 0x018UL +#define TXM_MODULE1_NUM_ENTRIES_OFF 0x014UL +#define TXM_MODULE1_ENTRIES_OFF 0x018UL +#define TXM_ENTRY1_HASH_TYPE_OFF 0x014UL +#define TXM_ENTRY1_FLAGS_OFF 0x015UL +#define TXM_CODE_SIGNATURE_SIZE 0x180UL +#define TXM_CODE_SIGNATURE_TRUST_OFF 0x132UL +#define TXM_ASPACE_MAIN_REGION_OFF 0x040UL +#define TXM_ASPACE_SYNTH_REGION_SIZE 0x080UL +#define TXM_CS_TRUST_STATIC_TC 9U +#define TXM_TC_TYPE_STATIC 0U +#define TXM_TC_TYPE_LTRS 4U +#define TXM_TC_QUERY_LOADABLE 2U +#define TXM_TC_HASH_SHA256_TRUNCATED 3U +#define TXM_TC_CAP_HASH_TYPE (1UL << 0) +#define TXM_TC_CAP_FLAGS (1UL << 1) + +static bool_t txm_fake_trust_cache_token(u64 query_type, u64 *tc_va, u64 *entry_va) { + if (g_state.txm.synth_base_va == 0 || + g_state.txm.synth_size < TXM_SYNTH_TRUST_CACHE_SIZE) + return 0; + + u64 base = g_state.txm.synth_base_va + g_state.txm.synth_size - + TXM_SYNTH_TRUST_CACHE_OFF; + u64 tc = base + TXM_SYNTH_TC_OBJECT_OFF; + u64 module = base + TXM_SYNTH_TC_MODULE_OFF; + u64 entry = base + TXM_SYNTH_TC_ENTRY_OFF; + u8 tc_type = (query_type == TXM_TC_QUERY_LOADABLE) ? TXM_TC_TYPE_LTRS : + TXM_TC_TYPE_STATIC; + + /* TrustCache_t: next=0, type, tombstoned=0, moduleSize, module pointer. */ + txm_synth_write64(tc + 0x00, 0); + txm_synth_write8(tc + TXM_TRUSTCACHE_TYPE_OFF, tc_type); + txm_synth_write8(tc + TXM_TRUSTCACHE_TYPE_OFF + 1, 0); + txm_synth_write64(tc + TXM_TRUSTCACHE_MODULE_SIZE_OFF, 0x18 + 0x16); + txm_synth_write64(tc + TXM_TRUSTCACHE_MODULE_OFF, module); + + /* TrustCacheModule1_t with one synthetic entry. */ + txm_synth_write32(module + 0x00, 1); + txm_synth_write32(module + TXM_MODULE1_NUM_ENTRIES_OFF, 1); + txm_synth_write8(entry + TXM_ENTRY1_HASH_TYPE_OFF, + TXM_TC_HASH_SHA256_TRUNCATED); + txm_synth_write8(entry + TXM_ENTRY1_FLAGS_OFF, 0); + + *tc_va = tc; + *entry_va = entry; + return 1; +} + +static inline void txm_record_call(u32 selector, u64 x16, u64 *regs) { + g_state.txm.total_calls++; + g_state.total_calls++; + g_state.last_x16 = x16; + g_state.last_endpoint = selector; + g_state.txm.last_selector = selector; + g_state.txm.last_stack_pa = regs[0]; + g_state.txm.last_x16 = x16; + for (u32 i = 0; i < 8; i++) + g_state.txm.last_args[i] = regs[i]; + if (selector < TXM_C_MAX_ENDPOINTS) + g_state.txm.ep_count[selector]++; +} + +static bool_t txm_write_stack_return(u64 stack_pa, u64 rc, + const u64 *words, u64 nwords) { + if (stack_pa == 0 || nwords > TXM_STACK_RETURN_WORDS) + return 0; + + u64 shared = stack_pa + TXM_THREAD_STACK_SHARED_OFF; + *(volatile u64 *)(shared + TXM_SHARED_RETURN_CODE_OFF) = rc; + *(volatile u8 *)(shared + TXM_SHARED_RETURN_TYPE_OFF) = 0; /* words */ + *(volatile u64 *)(shared + TXM_SHARED_NUM_WORDS_OFF) = nwords; + for (u64 i = 0; i < TXM_STACK_RETURN_WORDS; i++) { + u64 value = (i < nwords) ? words[i] : 0; + *(volatile u64 *)(shared + TXM_SHARED_WORDS_OFF + i * 8UL) = value; + g_state.txm.last_words[i] = value; + } + + g_state.txm.last_rc = rc; + clean_range((void *)shared, 0x80); + barrier(); + return 1; +} + +static bool_t txm_finish(u32 selector, u64 *regs, u64 rc, + const u64 *words, u64 nwords) { + if (!txm_write_stack_return(regs[0], rc, words, nwords)) { + g_state.txm.violations++; + return 0; + } + regs[0] = rc; + g_state.txm.fast_path_calls++; + g_state.fast_path_calls++; + sev(); + (void)selector; + return 1; +} + +static bool_t txm_success0(u32 selector, u64 *regs) { + u64 words[TXM_STACK_RETURN_WORDS] = {0, 0, 0, 0, 0, 0}; + return txm_finish(selector, regs, TXM_SUCCESS, words, 0); +} + +static bool_t txm_failure(u32 selector, u64 *regs, u64 rc) { + u64 words[TXM_STACK_RETURN_WORDS] = {0, 0, 0, 0, 0, 0}; + return txm_finish(selector, regs, rc, words, 0); +} + +bool_t txm_handle_selector(u32 selector, u64 x16, u64 *regs) { + u64 words[TXM_STACK_RETURN_WORDS] = {0, 0, 0, 0, 0, 0}; + + if (!g_state.txm.enabled) + return 0; + + txm_record_call(selector, x16, regs); + + switch (selector) { + case 0: + /* Selector 0 belongs to the SPTM-facing TXM table, not XNU table 0. */ + return txm_failure(selector, regs, TXM_RETURN_GENERIC); + + case TXM_SEL_GET_LOG_INFO: + words[0] = g_state.txm.log_page_va; + words[1] = g_state.txm.log_head_va; + words[2] = g_state.txm.log_sync_va; + return txm_finish(selector, regs, TXM_SUCCESS, words, 3); + + case TXM_SEL_GET_CODE_SIGNING_INFO: + words[0] = g_state.txm.txm_rw_data_va + 0x690UL; + words[1] = g_state.txm.developer_mode_flag_va; + words[2] = g_state.txm.txm_rw_data_va + 0x918UL; + words[3] = g_state.txm.managed_signature_size; + words[4] = g_state.txm.txm_ro_data_va + 0x190UL; + words[5] = g_state.txm.txm_ro_data_va; + return txm_finish(selector, regs, TXM_SUCCESS, words, 6); + + case TXM_SEL_GET_TRUST_CACHE_INFO: + words[0] = g_state.txm.trust_cache_runtime_va; + words[1] = g_state.txm.trust_cache_static_count; + words[2] = g_state.txm.trust_cache_caps0; + words[3] = g_state.txm.trust_cache_caps1; + return txm_finish(selector, regs, TXM_SUCCESS, words, 4); + + case 4: /* GetBuildVariant */ + words[0] = 0; /* release */ + return txm_finish(selector, regs, TXM_SUCCESS, words, 1); + + case 5: /* EnterLockdownMode */ + case 6: /* EnableRestrictedMode */ + case 8: /* UpdateDeviceState */ + case 9: /* CompleteSecurityBootMode */ + return txm_success0(selector, regs); + + case TXM_SEL_GET_SECURE_CHANNEL: + words[0] = g_state.txm.secure_chan_pa; + words[1] = g_state.txm.secure_chan_size; + return txm_finish(selector, regs, TXM_SUCCESS, words, 2); + + case TXM_SEL_ADD_FREE_LIST_PAGE: + if (regs[1] == 0) + return txm_failure(selector, regs, TXM_RETURN_GENERIC); + if (g_state.txm.free_count < TXM_C_TRACK_SLOTS) + g_state.txm.free_pages[g_state.txm.free_count++] = regs[1]; + return txm_success0(selector, regs); + + case TXM_SEL_GET_FREE_LIST_PAGE: + if (g_state.txm.free_count == 0) + return txm_failure(selector, regs, TXM_RETURN_NOT_FOUND); + words[0] = g_state.txm.free_pages[--g_state.txm.free_count]; + return txm_finish(selector, regs, TXM_SUCCESS, words, 1); + + case TXM_SEL_LOAD_TRUST_CACHE: + if (g_state.txm.loaded_tc_count < TXM_C_TRACK_SLOTS) + g_state.txm.loaded_tc[g_state.txm.loaded_tc_count++] = regs[1]; + /* Return payload VA so xnu can reclaim its temporary copy. */ + words[0] = regs[1]; + return txm_finish(selector, regs, TXM_SUCCESS, words, 1); + + case 13: /* UnloadTrustCache */ + return txm_success0(selector, regs); + + case TXM_SEL_QUERY_TRUST_CACHE: + if (!txm_fake_trust_cache_token(regs[1], &words[0], &words[1])) + return txm_failure(selector, regs, TXM_RETURN_NOT_FOUND); + return txm_finish(selector, regs, TXM_SUCCESS, words, 2); + + case TXM_SEL_QUERY_TC_REM: + words[0] = 0; + return txm_finish(selector, regs, TXM_SUCCESS, words, 1); + + case TXM_SEL_CHECK_TC_UUID: + return txm_failure(selector, regs, TXM_RETURN_NOT_FOUND); + + case TXM_SEL_REGISTER_PROFILE: + words[0] = txm_alloc_synth(0x100, g_state.txm.default_profile_va); + return txm_finish(selector, regs, TXM_SUCCESS, words, 1); + + case 18: /* TrustProvisioningProfile */ + case 20: /* AssociateProvisioningProfile */ + case 21: /* DisassociateProvisioningProfile */ + return txm_success0(selector, regs); + + case TXM_SEL_UNREGISTER_PROFILE: + words[0] = 0; + words[1] = 0; + return txm_finish(selector, regs, TXM_SUCCESS, words, 2); + + case TXM_SEL_REGISTER_SIGNATURE: { + u64 obj = txm_alloc_synth(TXM_CODE_SIGNATURE_SIZE, + g_state.txm.default_signature_va); + txm_synth_write8(obj + TXM_CODE_SIGNATURE_TRUST_OFF, + TXM_CS_TRUST_STATIC_TC); + words[0] = obj; + words[1] = regs[1]; + return txm_finish(selector, regs, TXM_SUCCESS, words, 2); + } + + case TXM_SEL_UNREGISTER_SIGNATURE: + case TXM_SEL_RECONSTITUTE_SIG: + words[0] = 0; + words[1] = 0; + return txm_finish(selector, regs, TXM_SUCCESS, words, 2); + + case 26: /* SetLocalSigningPublicKey */ + case 28: /* AuthorizeLocalSigningCDHash */ + case 29: /* AuthorizeCompilationServiceCDHash */ + case 31: /* DeveloperModeToggle */ + case 33: /* AssociateKernelEntitlements */ + case 36: /* UnregisterAddressSpace */ + case 37: /* SetupNestedAddressSpace */ + case 39: /* AllowJITRegion */ + case 40: /* AssociateJITRegion */ + case 42: /* AssociateDebugRegion */ + return txm_success0(selector, regs); + + case 24: /* ValidateCodeSignature */ + txm_synth_write8(regs[1] + TXM_CODE_SIGNATURE_TRUST_OFF, + TXM_CS_TRUST_STATIC_TC); + return txm_success0(selector, regs); + + case 38: { /* AssociateCodeSignature */ + u64 region = txm_alloc_synth(TXM_ASPACE_SYNTH_REGION_SIZE, regs[2]); + txm_synth_write64(regs[1] + TXM_ASPACE_MAIN_REGION_OFF, region); + return txm_success0(selector, regs); + } + + case TXM_SEL_GET_LOCAL_PUBKEY: + words[0] = g_state.txm.local_signing_pubkey_va; + return txm_finish(selector, regs, TXM_SUCCESS, words, 1); + + case TXM_SEL_MATCH_COMPILATION: + return txm_failure(selector, regs, TXM_RETURN_NOT_FOUND); + + case TXM_SEL_ACQUIRE_SIGNING_ID: + words[0] = g_state.txm.signing_id_va; + return txm_finish(selector, regs, TXM_SUCCESS, words, 1); + + case TXM_SEL_ACCEL_ENTITLEMENTS: + /* Route entitlement evaluation to XNU's non-monitor path. Returning + * NOT_SUPPORTED (what csm_enabled()==false yields) makes + * accelerate_entitlement_queries() (ubc_subr.c:4441) take the + * adjustContextWithoutMonitor branch, which parses the on-disk + * csb_entitlements_blob directly. Previously we returned a NULL + * monitor ce_ctx, which made every per-binary entitlement query + * (incl. com.apple.security.get-movable-control-port) read as absent, + * forcing IMMOVABLE_HARD control ports system-wide and breaking the + * mode-3 cross-task task-port send. See docs/mode3_txm_stub_audit.md. */ + return txm_failure(selector, regs, TXM_RETURN_NOT_SUPPORTED); + + case TXM_SEL_GET_ENTITLEMENTS_CTX: + words[0] = g_state.txm.ce_ctx_va; + return txm_finish(selector, regs, TXM_SUCCESS, words, 1); + + case TXM_SEL_REGISTER_ASPACE: { + u64 obj = txm_alloc_synth(0x100, g_state.txm.default_address_space_va); + /* Offsets verified from KDK 26.5 TXMAddressSpace_t. */ + txm_synth_write64(obj + 0x18, regs[2]); /* addrSpaceFlags */ + txm_synth_write16(obj + 0x24, (u16)regs[1]); /* identifier */ + txm_synth_write16(obj + 0x26, 0); /* type=AddressSpace */ + words[0] = obj; + return txm_finish(selector, regs, TXM_SUCCESS, words, 1); + } + + case 41: /* AllowInvalidCode */ + txm_synth_write8(regs[1] + 0x30, 1); /* allowsInvalidCode */ + return txm_success0(selector, regs); + + case TXM_SEL_RESOLVE_KENT_ASPACE: + return txm_failure(selector, regs, TXM_RETURN_NOT_FOUND); + + case TXM_SEL_IMAGE4_DISPATCH: + return txm_success0(selector, regs); + + case TXM_SEL_IMAGE4_GET_EXPORTS: + case 47: /* Image4SetReleaseType */ + case 48: /* Image4SetBootNonceShadow */ + case 49: /* Image4SetNonce */ + case 50: /* Image4RollNonce */ + case TXM_SEL_IMAGE4_GET_NONCE: + /* The mixed Python/C reference matched the T8140 TXM binary here: + * selectors 46..51 go through the default-fail trampoline and + * return kTXMReturnNotPermitted, not success or synthetic pointers. + */ + return txm_failure(selector, regs, TXM_RETURN_NOT_PERMITTED); + + default: + return txm_failure(selector, regs, TXM_RETURN_NOT_SUPPORTED); + } +} diff --git a/src/sptm/uat.c b/src/sptm/uat.c new file mode 100644 index 000000000..84060a1c6 --- /dev/null +++ b/src/sptm/uat.c @@ -0,0 +1,1323 @@ +/* SPDX-License-Identifier: MIT */ +#include "private.h" + +int debug_printf(const char *fmt, ...); + +#define UAT_TTE_ADDR_MASK 0x000001fffffff000UL +#define UAT_PTE_ADDR_MASK 0x0000fffffffff000UL +#define UAT_TABLE_DESC 3UL +#define UAT_VALID_LEAF_BITS 0x0080000000000400UL +#define UAT_OPTIONS_INVALID_MASK 0xfff0fcf0UL +#define UAT_FW_OWNED_BIT 8UL +#define UAT_DEFAULT_VADDR_SHIFT 0x27UL +#define UAT_MAX_VA_PAGES (1UL << 50) +#define UAT_STATE_ROOT0_OFF 0x08UL +#define UAT_STATE_ROOT1_OFF 0x10UL +#define UAT_STATE_CTX_ID_OFF 0x18UL +#define UAT_STATE_GUARD_OFF 0x1aUL +#define UAT_STATE_WORK0_OFF 0x20UL +#define UAT_STATE_WORK1_OFF 0x28UL +#define UAT_STATE_WORK2_OFF 0x30UL +#define UAT_STATE_WORK3_OFF 0x38UL +#define UAT_STATE_OPTIONS_OFF 0x40UL +#define UAT_STATE_FLUSH_LEVEL_OFF 0x44UL +#define UAT_STATE_MAP_SEGS_OFF 0x48UL +#define UAT_STATE_CTX_FLUSH_OFF 0x248UL +#define UAT_STATE_UNMAP_SEGS_OFF 0x250UL +#define UAT_SAPT_ALIAS_COUNT 12 +#define UAT_SAPT_LAST_PA 13 +#define UAT_SAPT_LAST_OLD 14 +#define UAT_SAPT_LAST_NEW 15 +#define UAT_HANDOFF_MAGIC 0x4b1d000000000002UL +#define UAT_HANDOFF_MAGIC_FW_OFF 0x8UL +#define UAT_HANDOFF_SLOT_BASE 0x20UL +#define UAT_HANDOFF_SLOT_STRIDE 0x18UL +#define UAT_HANDOFF_CUR_CTX_OFF 0x18UL +#define UAT_HANDOFF_READY_OFF 0x638UL +#define UAT_HANDOFF_DEAD_PREFIX 0xdead000000000000UL +#define UAT_HANDOFF_ADDR_MASK 0x0000ffffffffffffUL + +static const u8 uat_attr_table[16] = { + 0, 8, 9, 10, 4, 1, 0xff, 0xff, + 5, 0xff, 2, 0xff, 6, 7, 0xff, 3, +}; + +static inline u8 uat_sapt_field_for_options(u64 options) +{ + switch (options & 3UL) { + case 1: + return 1; + case 3: + return 0; + default: + return 3; + } +} + +static inline bool_t uat_looks_like_pa(u64 pa) +{ + return !is_kernel_va(pa) && pa >= 0x100000000UL && pa < 0x20000000000UL; +} + +static inline bool_t uat_resolve_data_pa(u64 ptr, bool_t write, u64 *pa_out) +{ + if (is_kernel_va(ptr)) + return guest_va_to_pa(ptr, write, pa_out); + if (uat_looks_like_pa(ptr)) { + *pa_out = ptr; + return 1; + } + return 0; +} + +static inline bool_t uat_read64_ptr(u64 ptr, u64 *out) +{ + u64 pa; + if (!uat_resolve_data_pa(ptr, 0, &pa)) + return 0; + *out = *(volatile u64 *)pa; + return 1; +} + +static inline void uat_clean_state_range_poc(u64 pa, u64 size) +{ + /* + * AGX/uPPL observes UAT state objects as a coprocessor-visible protocol, + * not as CPU-local data. The real SPTM runs in the hardware domain that + * makes these stores visible before the guard changes; our EL2 emulator + * must publish them explicitly to PoC plus the coprocessor cache path. + */ + clean_pt_range_poc((volatile u64 *)pa, size); +} + +static inline bool_t uat_sapt_byte_for_pa(u64 pa, u64 *byte_pa, u8 *shift) +{ + u64 base = g_state.uat.sapt_base; + u64 entries = g_state.uat.sapt_entries; + u64 dram_base = g_state.uat.sapt_dram_base; + u64 dram_end = g_state.uat.sapt_dram_end; + u64 page = pa & ~(UAT_C_PAGE_SIZE - 1UL); + u64 offset; + u64 index; + + if (!base || !entries || !dram_base || page < dram_base || page >= dram_end) + return 0; + + offset = page - dram_base; + index = offset >> 16; + if (index >= entries) + return 0; + + *byte_pa = base + index; + *shift = (u8)((offset >> 13) & 6UL); + return 1; +} + +static inline void uat_sapt_set_page(u64 pa, u8 field, bool_t expect_released) +{ + u64 byte_pa; + u8 shift; + u8 mask; + u8 old; + u8 old_field; + u8 new_value; + + if (!uat_sapt_byte_for_pa(pa, &byte_pa, &shift)) + return; + + mask = (u8)(3U << shift); + old = *(volatile u8 *)byte_pa; + old_field = (old >> shift) & 3U; + if (expect_released && old_field != 3U) { + g_state.uat.violations++; + g_state.uat.last[UAT_SAPT_ALIAS_COUNT]++; + g_state.uat.last[UAT_SAPT_LAST_PA] = pa & ~(UAT_C_PAGE_SIZE - 1UL); + g_state.uat.last[UAT_SAPT_LAST_OLD] = old_field; + g_state.uat.last[UAT_SAPT_LAST_NEW] = field & 3U; + } + + new_value = (u8)((old & ~mask) | ((field & 3U) << shift)); + *(volatile u8 *)byte_pa = new_value; + uat_clean_state_range_poc(byte_pa, 1); +} + +static inline void uat_sapt_grant_page(u64 pa, u64 options) +{ + uat_sapt_set_page(pa, uat_sapt_field_for_options(options), 1); +} + +static inline void uat_sapt_release_page(u64 pa) +{ + uat_sapt_set_page(pa, 3, 0); +} + +static u64 uat_publish_fw_unmap_handoff(struct uat_c_root_state *root, + u64 addr, u64 size) +{ + u64 handoff = g_state.uat.gfx_handoff_pa; + u64 slot_pa; + u32 slot_state; + u32 cur_ctx; + u8 ready; + bool_t dead; + + if (root->ctx_id >= 0x41U) + return 0; + + if (!uat_looks_like_pa(handoff) || + g_state.uat.gfx_handoff_size < UAT_HANDOFF_READY_OFF + 1UL) { + g_state.uat.violations++; + return 0; + } + + slot_pa = handoff + UAT_HANDOFF_SLOT_BASE + + (u64)root->ctx_id * UAT_HANDOFF_SLOT_STRIDE; + slot_state = *(volatile u32 *)slot_pa; + if (slot_state != 0) { + g_state.uat.violations++; + root->last[6] = slot_pa; + root->last[7] = slot_state; + return 0; + } + + *(volatile u64 *)(slot_pa + 8) = addr; + *(volatile u64 *)(slot_pa + 0x10) = size; + + cur_ctx = *(volatile u32 *)(handoff + UAT_HANDOFF_CUR_CTX_OFF); + ready = *(volatile u8 *)(handoff + UAT_HANDOFF_READY_OFF); + dead = (ready & 1U) != 0 || + (((root->type & 5U) != 0) && cur_ctx != (u32)root->ctx_id); + + if (dead) { + addr = UAT_HANDOFF_DEAD_PREFIX | (addr & UAT_HANDOFF_ADDR_MASK); + *(volatile u64 *)(slot_pa + 8) = addr; + slot_state = 2; + } else { + slot_state = 1; + } + + /* + * Real SPTM writes addr/size first, publishes state last, then orders the + * handoff with an outer-shareable barrier. In EL2 we also clean to PoC so + * the AGX firmware and UAT hardware see the slot update. + */ + *(volatile u32 *)slot_pa = slot_state; + clean_pt_range_poc((volatile u64 *)slot_pa, UAT_HANDOFF_SLOT_STRIDE); + __asm__ volatile("dsb sy" ::: "memory"); + + root->last[6] = slot_pa; + root->last[7] = ((u64)slot_state << 32) | ready; + return dead ? 0 : 2; +} + +static bool_t uat_handoff_magic_valid(struct uat_c_root_state *root) +{ + u64 handoff = g_state.uat.gfx_handoff_pa; + u64 magic; + + if (root->ctx_id >= 0x41U) + return 1; + + if (!uat_looks_like_pa(handoff) || + g_state.uat.gfx_handoff_size < UAT_HANDOFF_MAGIC_FW_OFF + 8UL) { + g_state.uat.violations++; + root->last[6] = handoff; + root->last[7] = g_state.uat.gfx_handoff_size; + return 0; + } + + magic = *(volatile u64 *)(handoff + UAT_HANDOFF_MAGIC_FW_OFF); + if (magic != UAT_HANDOFF_MAGIC) { + g_state.uat.violations++; + root->last[6] = handoff + UAT_HANDOFF_MAGIC_FW_OFF; + root->last[7] = magic; + return 0; + } + + return 1; +} + +static inline bool_t uat_write64_ptr(u64 ptr, u64 value) +{ + u64 pa; + if (!uat_resolve_data_pa(ptr, 1, &pa)) + return 0; + *(volatile u64 *)pa = value; + uat_clean_state_range_poc(pa, 8); + return 1; +} + +static inline bool_t uat_read16_ptr(u64 ptr, u16 *out) +{ + u64 pa; + if (!uat_resolve_data_pa(ptr, 0, &pa)) + return 0; + *out = *(volatile u16 *)pa; + return 1; +} + +static inline bool_t uat_read8_ptr(u64 ptr, u8 *out) +{ + u64 pa; + if (!uat_resolve_data_pa(ptr, 0, &pa)) + return 0; + *out = *(volatile u8 *)pa; + return 1; +} + +static inline bool_t uat_write16_ptr(u64 ptr, u16 value) +{ + u64 pa; + if (!uat_resolve_data_pa(ptr, 1, &pa)) + return 0; + *(volatile u16 *)pa = value; + uat_clean_state_range_poc(pa, 2); + return 1; +} + +static inline bool_t uat_write8_ptr(u64 ptr, u8 value) +{ + u64 pa; + if (!uat_resolve_data_pa(ptr, 1, &pa)) + return 0; + *(volatile u8 *)pa = value; + uat_clean_state_range_poc(pa, 1); + return 1; +} + +static inline u8 uat_read8_pa_default(u64 pa, u8 def) +{ + if (!uat_looks_like_pa(pa)) + return def; + return *(volatile u8 *)pa; +} + +static inline u64 uat_default_l1_mask(u64 shift) +{ + if (shift <= 36) + return 0; + if (shift >= 47) + return 0x7ff000000000UL; + return (((1UL << (shift - 36)) - 1UL) << 36) & 0x7ff000000000UL; +} + +static inline void uat_set_defaults(void) +{ + struct uat_c_state *st = &g_state.uat; + + if (!st->vaddr_shift) + st->vaddr_shift = UAT_DEFAULT_VADDR_SHIFT; + if (!st->l1_index_mask) + st->l1_index_mask = uat_default_l1_mask(st->vaddr_shift); + if (!st->segment_limit) + st->segment_limit = 0x40; + if (!st->mapping_limit) + st->mapping_limit = 0x100; + if (!st->state_object_size) + st->state_object_size = st->segment_limit * 0x10 + 0x250; + if (!st->gfx_shared_l2_pa) + st->gfx_shared_l2_pa = st->gfx_shared_region_pa; + if (!st->raw_gfx_shared_l2_pa) + st->raw_gfx_shared_l2_pa = st->gfx_shared_l2_pa; + st->enabled = 1; +} + +static inline u64 uat_selector2_state_pa(void) +{ + if (g_state.uat.mode != 1) + return 0; + if (!uat_looks_like_pa(g_state.uat.selector2_pa) || + !uat_looks_like_pa(g_state.uat.selector2_root_pa)) { + g_state.uat.violations++; + return 0; + } + return g_state.uat.selector2_pa; +} + +static inline u64 uat_selector9_papt_va(void) +{ + u64 pa = 0; + + if (!is_kernel_va(g_state.uat.selector9_pa) || + !g_state.uat.raw_gfx_shared_l2_pa || + !guest_va_to_pa(g_state.uat.selector9_pa, 0, &pa) || + (pa & UAT_PTE_ADDR_MASK) != (g_state.uat.raw_gfx_shared_l2_pa & UAT_PTE_ADDR_MASK)) { + g_state.uat.violations++; + return 0; + } + return g_state.uat.selector9_pa; +} + +static inline void uat_refresh_state_object(struct uat_c_root_state *root) +{ + u64 value; + + if (uat_read64_ptr(root->handle + UAT_STATE_ROOT0_OFF, &value)) + root->root0_pa = value; + if (uat_read64_ptr(root->handle + UAT_STATE_ROOT1_OFF, &value)) + root->root1_pa = value; + if (uat_read16_ptr(root->handle + UAT_STATE_CTX_ID_OFF, &root->ctx_id) && + uat_read8_ptr(root->handle, &root->type)) + return; +} + +static inline void uat_record(u32 endpoint, u64 *regs, u64 root_pa, u64 slot_pa, + u64 before, u64 after, u64 rc) +{ + struct uat_c_state *st = &g_state.uat; + u64 idx = st->recent_idx & (UAT_C_RECENT_SLOTS - 1); + struct uat_c_recent_event *ev = &st->recent[idx]; + + ev->seq = st->recent_idx; + ev->endpoint = endpoint; + ev->handle = regs[0]; + for (u32 i = 0; i < 5; i++) + ev->regs[i] = regs[i]; + ev->root_pa = root_pa; + ev->slot_pa = slot_pa; + ev->before = before; + ev->after = after; + ev->rc = rc; + st->recent_idx++; + st->last_rc = rc; +} + +static struct uat_c_root_state *uat_find_state(u64 handle, bool_t alloc) +{ + struct uat_c_root_state *free_slot = (struct uat_c_root_state *)0; + bool_t selector2_shadow; + + if (!handle) + return (struct uat_c_root_state *)0; + + selector2_shadow = handle == g_state.uat.selector2_pa && + uat_looks_like_pa(g_state.uat.selector2_pa) && + uat_looks_like_pa(g_state.uat.selector2_root_pa); + + for (u32 i = 0; i < UAT_C_MAX_STATES; i++) { + struct uat_c_root_state *slot = &g_state.uat.roots[i]; + if (slot->used && slot->handle == handle) { + uat_refresh_state_object(slot); + return slot; + } + if (!slot->used && free_slot == (struct uat_c_root_state *)0) + free_slot = slot; + } + + /* + * Selector 2 returns a firmware-owned UAT state object seeded before any + * table-7 init_state call. Real SPTM can use that object immediately for + * map_table/map/unmap; mirror that by importing it into the shadow root + * table on first use instead of treating map_table as a no-op success. + */ + if ((!alloc && !selector2_shadow) || free_slot == (struct uat_c_root_state *)0) + return (struct uat_c_root_state *)0; + + for (u32 i = 0; i < sizeof(*free_slot); i++) + ((volatile u8 *)free_slot)[i] = 0; + free_slot->used = 1; + free_slot->handle = handle; + free_slot->ctx_id = UAT_C_CTX_NONE; + uat_refresh_state_object(free_slot); + if (selector2_shadow) { + if (!free_slot->root1_pa) + free_slot->root1_pa = g_state.uat.selector2_root_pa; + if (!free_slot->type) + free_slot->type = uat_read8_pa_default(handle, 8); + } + return free_slot; +} + +static inline u64 uat_root_for_va(struct uat_c_root_state *root, u64 va) +{ + u64 shift = g_state.uat.vaddr_shift ? g_state.uat.vaddr_shift : + UAT_DEFAULT_VADDR_SHIFT; + if (((va >> shift) & 1UL) && root->root1_pa) + return root->root1_pa; + return root->root0_pa; +} + +static inline u64 uat_l1_idx(u64 va) +{ + u64 mask = g_state.uat.l1_index_mask; + if (!mask) + mask = uat_default_l1_mask(g_state.uat.vaddr_shift ? + g_state.uat.vaddr_shift : + UAT_DEFAULT_VADDR_SHIFT); + if (mask) + return (va & mask) >> 36; + return (va >> 36) & 0x7ffUL; +} + +static inline u64 uat_l2_idx(u64 va) { return (va >> 25) & 0x7ffUL; } +static inline u64 uat_l3_idx(u64 va) { return (va >> 14) & 0x7ffUL; } + +static bool_t uat_walk_slot(u64 root_pa, u64 va, u32 level, + u64 *slot_pa_out, u64 *before_out) +{ + volatile u64 *l1; + volatile u64 *l2; + u64 l1e, l2e; + u64 slot_pa; + + if (!uat_looks_like_pa(root_pa)) + return 0; + + if (level == 1) { + slot_pa = root_pa + uat_l1_idx(va) * 8; + *slot_pa_out = slot_pa; + *before_out = *(volatile u64 *)slot_pa; + return 1; + } + + l1 = (volatile u64 *)root_pa; + l1e = l1[uat_l1_idx(va)]; + if ((l1e & 3UL) != UAT_TABLE_DESC) + return 0; + + if (level == 2) { + slot_pa = (l1e & UAT_TTE_ADDR_MASK) + uat_l2_idx(va) * 8; + *slot_pa_out = slot_pa; + *before_out = *(volatile u64 *)slot_pa; + return 1; + } + + l2 = (volatile u64 *)(l1e & UAT_TTE_ADDR_MASK); + l2e = l2[uat_l2_idx(va)]; + if ((l2e & 3UL) != UAT_TABLE_DESC) + return 0; + + slot_pa = (l2e & UAT_TTE_ADDR_MASK) + uat_l3_idx(va) * 8; + *slot_pa_out = slot_pa; + *before_out = *(volatile u64 *)slot_pa; + return 1; +} + +static inline void uat_write_slot(u64 slot_pa, u64 value) +{ + *(volatile u64 *)slot_pa = value; + clean_pt_range_poc((volatile u64 *)slot_pa, 8); +} + +static inline void uat_publish_mapped_data_page(struct uat_c_root_state *root, + u64 pa) +{ + u64 page = pa & ARM_PTE_PAGE_MASK; + u64 idx; + + /* + * Real SPTM's UAT map path performs per-mapped-frame side effects in + * addition to installing the leaf PTE. The important correctness effect for + * this emulator is that data the AGX firmware will read is pushed through + * the coprocessor-visible cache path, not just the 8-byte UAT PTE slot. + * + * Some UAT operations carry low/sentinel/non-DRAM values through the same + * segment format. Do not run data-cache maintenance on those addresses: + * EL2 can fault on e.g. PA 0 before XNU even boots. + */ + if (!uat_looks_like_pa(page) || !ft_get_idx(page, &idx)) { + root->last[6] = 0; + root->last[7] = page; + return; + } + + sptm_coproc_cache_maint_data_page(page); + uat_write8_ptr(root->handle + UAT_STATE_FLUSH_LEVEL_OFF, 1); + root->last[6]++; + root->last[7] = page; +} + +static inline void uat_tlbi_ctx_if_bound(struct uat_c_root_state *root) +{ + (void)root; +} + +static inline u64 uat_leaf_pte(struct uat_c_root_state *root, u64 pa, u64 options) +{ + u64 idx = ((options >> 8) & 2UL) | + ((options & 3UL) << 2) | + ((options >> 8) & 1UL); + u64 attr = uat_attr_table[idx & 0xf]; + u64 perm = 3UL; + u64 os_ng = ((root->type & 10U) != 0) ? 0 : 0x800UL; + + if (attr == 0xff) + attr = 0; + if (options & 8UL) + perm = 0xbUL; + if (options & 4UL) + perm = 7UL; + + return os_ng | + ((attr & 3UL) << 53) | + (pa & UAT_PTE_ADDR_MASK) | + perm | + (((attr >> 2) & 3UL) << 6) | + UAT_VALID_LEAF_BITS; +} + +static bool_t uat_read_segment(u64 seglist_pa, u64 index, u64 *first, u64 *pages) +{ + u64 base = seglist_pa + index * 0x10UL; + return uat_read64_ptr(base, first) && + uat_read64_ptr(base + 8, pages); +} + +static bool_t uat_copy_segments_to_state(struct uat_c_root_state *root, + u64 dst_off, u64 seglist_pa, + u64 num_segs) +{ + u64 state_size = g_state.uat.state_object_size ? + g_state.uat.state_object_size : + ((g_state.uat.segment_limit ? g_state.uat.segment_limit : 0x40) * + 0x10UL + 0x250UL); + + if (dst_off + num_segs * 0x10UL > state_size) + return 0; + + for (u64 i = 0; i < num_segs; i++) { + u64 first, pages; + if (!uat_read_segment(seglist_pa, i, &first, &pages) || + !uat_write64_ptr(root->handle + dst_off + i * 0x10UL, first) || + !uat_write64_ptr(root->handle + dst_off + i * 0x10UL + 8, pages)) + return 0; + } + return 1; +} + +static inline void uat_write_state_header(struct uat_c_root_state *root) +{ + u64 handle = root->handle; + + uat_write64_ptr(handle + UAT_STATE_ROOT0_OFF, root->root0_pa); + uat_write64_ptr(handle + UAT_STATE_ROOT1_OFF, root->root1_pa); + uat_write16_ptr(handle + UAT_STATE_CTX_ID_OFF, root->ctx_id); +} + +static inline void uat_release_state_guard(struct uat_c_root_state *root, u8 guard) +{ + uat_write8_ptr(root->handle + UAT_STATE_GUARD_OFF, guard); +} + +static inline void uat_write_idle_state(struct uat_c_root_state *root) +{ + uat_write_state_header(root); + uat_release_state_guard(root, 2); +} + +static void uat_write_map_state(struct uat_c_root_state *root, u8 guard) +{ + u64 handle = root->handle; + + uat_write_state_header(root); + uat_write64_ptr(handle + UAT_STATE_WORK0_OFF, root->current_va); + uat_write64_ptr(handle + UAT_STATE_WORK1_OFF, root->num_segs); + uat_write64_ptr(handle + UAT_STATE_WORK2_OFF, root->current_seg); + uat_write64_ptr(handle + UAT_STATE_WORK3_OFF, root->seg_offset); + uat_write64_ptr(handle + UAT_STATE_OPTIONS_OFF, root->options); + /* + * BUG FIX 2026-07-05: do NOT write CTX_FLUSH_OFF (0x248) here. The map + * seglist copy lives at MAP_SEGS_OFF (0x48); entry 32's first_pa is at + * 0x48 + 32*0x10 = 0x248 == CTX_FLUSH_OFF. For any map with >=33 segments + * this 0-write clobbered the 33rd segment's PA to 0 right after the copy, + * which our old zero-leaf branch then turned into a UAT hole -> GPU FW data + * abort at the corresponding VA (0xfffffc20c0130000). Real SPTM's map-begin + * writes no such field; this write was spurious (CTX_FLUSH_OFF is write-only + * in our emulator, never read). + */ + uat_release_state_guard(root, guard); +} + +static void uat_write_prepare_fw_unmap_state(struct uat_c_root_state *root, + u8 guard, u64 changed) +{ + u64 handle = root->handle; + + uat_write_state_header(root); + uat_write64_ptr(handle + UAT_STATE_WORK0_OFF, root->current_va); + uat_write64_ptr(handle + UAT_STATE_WORK1_OFF, root->current_seg); + uat_write64_ptr(handle + UAT_STATE_WORK2_OFF, root->num_segs); + uat_write64_ptr(handle + UAT_STATE_WORK3_OFF, changed ? 1 : 0); + uat_release_state_guard(root, guard); +} + +static void uat_write_unmap_state(struct uat_c_root_state *root, u8 guard, + u64 op_state) +{ + u64 handle = root->handle; + + uat_write_state_header(root); + uat_write64_ptr(handle + UAT_STATE_WORK0_OFF, op_state); + uat_write64_ptr(handle + UAT_STATE_WORK1_OFF, root->num_segs); + uat_write64_ptr(handle + UAT_STATE_WORK2_OFF, root->current_seg); + uat_write64_ptr(handle + UAT_STATE_WORK3_OFF, root->seg_offset); + uat_write64_ptr(handle + UAT_STATE_OPTIONS_OFF, root->options); + uat_release_state_guard(root, guard); +} + +static u64 uat_map_pending(struct uat_c_root_state *root) +{ + u64 limit = g_state.uat.mapping_limit ? g_state.uat.mapping_limit : 0x100; + u64 mapped = 0; + + while (root->current_seg < root->num_segs) { + u64 first, pages; + if (!uat_read_segment(root->seglist_pa, root->current_seg, &first, &pages)) { + g_state.uat.violations++; + root->state = 0; + return 0; + } + + while (root->seg_offset < pages) { + u64 pa = first + root->seg_offset * UAT_C_PAGE_SIZE; + u64 va = root->current_va; + u64 root_pa = uat_root_for_va(root, va); + u64 slot_pa = 0, before = 0, pte; + + if (!uat_walk_slot(root_pa, va, 3, &slot_pa, &before)) { + g_state.uat.violations++; + root->last[0] = root_pa; + root->last[1] = va; + root->last[2] = pa; + root->last[3] = 3; + root->state = 0; + uat_write_idle_state(root); + return 0; + } + + pte = uat_leaf_pte(root, pa, root->options); + if (before & 1UL) { + g_state.uat.violations++; + root->last[0] = root_pa; + root->last[1] = va; + root->last[2] = pa; + root->last[3] = slot_pa; + root->last[4] = before; + root->last[5] = pte; + root->state = 0; + uat_write_map_state(root, 2); + return 0; + } + if (!uat_looks_like_pa(pa)) { + /* + * XNU can describe holes in GPU queue VA space with a zero + * backing address. Treat those as invalid leaves. Turning PA 0 + * into a valid GPU translation makes AGX read physical low + * memory and trips SAPT at e.g. PA 0x600. + */ + uat_write_slot(slot_pa, 0); + root->last[0] = root_pa; + root->last[1] = va; + root->last[2] = pa; + root->last[3] = slot_pa; + root->last[4] = before; + root->last[5] = 0; + root->last[6]++; + root->last[7] = pa; + + root->current_va += UAT_C_PAGE_SIZE; + root->seg_offset++; + mapped++; + if (mapped >= limit) { + uat_tlbi_ctx_if_bound(root); + root->state = 3; + uat_write_map_state(root, 3); + return 1; + } + continue; + } + uat_write_slot(slot_pa, pte); + uat_sapt_grant_page(pa, root->options); + uat_publish_mapped_data_page(root, pa); + + root->last[0] = root_pa; + root->last[1] = va; + root->last[2] = pa; + root->last[3] = slot_pa; + root->last[4] = before; + root->last[5] = pte; + + root->current_va += UAT_C_PAGE_SIZE; + root->seg_offset++; + mapped++; + if (mapped >= limit) { + uat_tlbi_ctx_if_bound(root); + root->state = 3; + uat_write_map_state(root, 3); + return 1; + } + } + + root->current_seg++; + root->seg_offset = 0; + } + + root->state = 0; + uat_tlbi_ctx_if_bound(root); + uat_write_map_state(root, 2); + return 0; +} + +static inline bool_t uat_fw_unmap_leaf_qualifies(u64 pte) +{ + u32 perm_bits = ((u32)(pte >> 4)) & 0xcU; + u32 attr_hi = (u32)(pte >> 53); + u32 attr_code = perm_bits | (attr_hi & 3U); + + if ((pte & 0x1cUL) != 0) + return 0; + return (perm_bits | (attr_hi & 2U)) == 2U || + (attr_code >= 5U && attr_code < 8U); +} + +static u64 uat_prepare_fw_unmap_pending(struct uat_c_root_state *root) +{ + u64 limit = g_state.uat.mapping_limit ? g_state.uat.mapping_limit : 0x100; + u64 walked = 0; + u64 changed = root->options & 1UL; + + if (!uat_handoff_magic_valid(root)) { + root->state = 0; + root->options = 0; + uat_write_prepare_fw_unmap_state(root, 2, changed); + return 0; + } + + while (root->current_seg < root->num_segs) { + u64 cur = root->current_va + root->current_seg * UAT_C_PAGE_SIZE; + u64 root_pa = uat_root_for_va(root, cur); + u64 slot_pa = 0, before = 0, after = 0; + + if (!uat_walk_slot(root_pa, cur, 3, &slot_pa, &before)) { + g_state.uat.violations++; + root->last[0] = root_pa; + root->last[1] = cur; + root->last[2] = slot_pa; + root->last[3] = before; + root->state = 0; + root->options = 0; + uat_write_prepare_fw_unmap_state(root, 2, changed); + return 0; + } + + after = before; + if ((before & 3UL) == 3UL && uat_fw_unmap_leaf_qualifies(before)) { + after = before | UAT_FW_OWNED_BIT; + uat_write_slot(slot_pa, after); + changed = 1; + } + + root->last[0] = root_pa; + root->last[1] = cur; + root->last[2] = slot_pa; + root->last[3] = before; + root->last[4] = after; + root->last[5] = changed; + + root->current_seg++; + walked++; + if (root->current_seg != root->num_segs && walked >= limit) { + root->state = 4; + root->options = changed; + uat_write_prepare_fw_unmap_state(root, 4, changed); + return 1; + } + } + + root->state = 0; + root->options = 0; + uat_write_prepare_fw_unmap_state(root, 2, changed); + if (!changed) + return 0; + + __asm__ volatile("dsb oshst" ::: "memory"); + tlbi_vmalle1os(); + return uat_publish_fw_unmap_handoff(root, root->current_va, + root->num_segs * UAT_C_PAGE_SIZE); +} + +static u64 uat_unmap_pending(struct uat_c_root_state *root) +{ + u64 limit = g_state.uat.mapping_limit ? g_state.uat.mapping_limit : 0x100; + u64 unmapped = 0; + + while (root->current_seg < root->num_segs) { + u64 va, pages; + if (!uat_read_segment(root->seglist_pa, root->current_seg, &va, &pages)) { + g_state.uat.violations++; + root->state = 0; + uat_write_unmap_state(root, 2, 0); + return 0; + } + + while (root->seg_offset < pages) { + u64 cur = va + root->seg_offset * UAT_C_PAGE_SIZE; + u64 root_pa = uat_root_for_va(root, cur); + u64 slot_pa = 0, before = 0; + + if (!uat_walk_slot(root_pa, cur, 3, &slot_pa, &before) || + !(before & 1UL)) { + g_state.uat.violations++; + root->last[0] = root_pa; + root->last[1] = cur; + root->last[2] = slot_pa; + root->last[3] = before; + root->state = 0; + uat_write_unmap_state(root, 2, 0); + return 0; + } + uat_sapt_release_page(before & UAT_PTE_ADDR_MASK); + uat_write_slot(slot_pa, 0); + + root->last[0] = root_pa; + root->last[1] = cur; + root->last[2] = slot_pa; + root->last[3] = before; + + root->seg_offset++; + unmapped++; + if (unmapped >= limit) { + root->state = 5; + uat_write_unmap_state(root, 5, 1); + tlbi_vmalle1os(); + return 1; + } + } + + root->current_seg++; + root->seg_offset = 0; + } + + root->state = 0; + uat_write_unmap_state(root, 2, 0); + tlbi_vmalle1os(); + return 0; +} + +static u64 uat_unmap_segments(struct uat_c_root_state *root, u64 seglist_pa, + u64 num_segs) +{ + if (num_segs == 0 || + num_segs > (g_state.uat.segment_limit ? g_state.uat.segment_limit : 0x40) || + !uat_copy_segments_to_state(root, UAT_STATE_UNMAP_SEGS_OFF, + seglist_pa, num_segs)) { + g_state.uat.violations++; + return 0; + } + + root->seglist_pa = root->handle + UAT_STATE_UNMAP_SEGS_OFF; + root->num_segs = num_segs; + root->current_seg = 0; + root->seg_offset = 0; + root->options = 0; + root->state = 5; + uat_write_unmap_state(root, 5, 1); + return uat_unmap_pending(root); +} + +static u64 uat_ep_init(u64 *regs) +{ + struct uat_c_root_state *root = uat_find_state(regs[0], 1); + u64 root1 = 0xffffffffUL; + + if (root == (struct uat_c_root_state *)0) { + g_state.uat.violations++; + return 0; + } + + root->type = uat_read8_pa_default(root->handle, 0); + if ((root->type & 5U) == 0) { + if (uat_looks_like_pa(regs[2])) + root->type = 4; + else if (regs[2] == 0xffffffffUL) + root->type = 1; + if (root->type & 5U) + uat_write8_ptr(root->handle, root->type); + } + if ((root->type & 5U) == 0 || + !uat_looks_like_pa(regs[1]) || + (root->root0_pa != 0 && root->root0_pa != 0xffffffffUL) || + (root->type == 1 && regs[2] != 0xffffffffUL) || + (root->type == 4 && !uat_looks_like_pa(regs[2]))) { + g_state.uat.violations++; + return 0; + } + + if (root->type == 4) { + if (root->root1_pa != 0 && root->root1_pa != 0xffffffffUL) { + g_state.uat.violations++; + return 0; + } + root1 = regs[2]; + } + + root->root0_pa = regs[1]; + root->root1_pa = root1; + root->ctx_id = UAT_C_CTX_NONE; + root->state = 0; + uat_write_idle_state(root); + return 0; +} + +static u64 uat_ep_destroy(u64 *regs) +{ + struct uat_c_root_state *root = uat_find_state(regs[0], 0); + if (root == (struct uat_c_root_state *)0) + return 0; + if (root->ctx_id != UAT_C_CTX_NONE) { + g_state.uat.violations++; + return 0; + } + for (u32 i = 0; i < sizeof(*root); i++) + ((volatile u8 *)root)[i] = 0; + tlbi_vmalle1os(); + return 0; +} + +static u64 uat_ep_map_table(u64 *regs, u64 *root_out, u64 *slot_out, + u64 *before_out, u64 *after_out) +{ + struct uat_c_root_state *root = uat_find_state(regs[0], 0); + u64 va = regs[1]; + u32 level = (u32)regs[2]; + u64 table_pa = regs[3]; + u64 root_pa, slot_pa = 0, before = 0, after = 0; + + if (root == (struct uat_c_root_state *)0 || (level != 1 && level != 2) || + !uat_looks_like_pa(table_pa)) { + g_state.uat.violations++; + return 0; + } + + root_pa = uat_root_for_va(root, va); + if (!uat_walk_slot(root_pa, va, level, &slot_pa, &before)) { + g_state.uat.violations++; + return 0; + } + after = (table_pa & UAT_TTE_ADDR_MASK) | UAT_TABLE_DESC; + if (before != 0) { + g_state.uat.violations++; + root->last[0] = root_pa; + root->last[1] = va; + root->last[2] = level; + root->last[3] = slot_pa; + root->last[4] = before; + root->last[5] = after; + *root_out = root_pa; + *slot_out = slot_pa; + *before_out = before; + *after_out = after; + return 0; + } + uat_write_slot(slot_pa, after); + + root->last[0] = root_pa; + root->last[1] = va; + root->last[2] = level; + root->last[3] = slot_pa; + root->last[4] = before; + root->last[5] = after; + *root_out = root_pa; + *slot_out = slot_pa; + *before_out = before; + *after_out = after; + return 0; +} + +static u64 uat_ep_unmap_table(u64 *regs, u64 *root_out, u64 *slot_out, + u64 *before_out) +{ + struct uat_c_root_state *root = uat_find_state(regs[0], 0); + u64 va = regs[1]; + u32 level = (u32)regs[2]; + u64 root_pa, slot_pa = 0, before = 0; + + if (root == (struct uat_c_root_state *)0 || (level != 1 && level != 2)) { + g_state.uat.violations++; + return 0; + } + + root_pa = uat_root_for_va(root, va); + if (!uat_walk_slot(root_pa, va, level, &slot_pa, &before)) { + g_state.uat.violations++; + return 0; + } + if ((before & 3UL) != UAT_TABLE_DESC) { + g_state.uat.violations++; + *root_out = root_pa; + *slot_out = slot_pa; + *before_out = before; + return 0; + } + uat_write_slot(slot_pa, 0); + tlbi_vmalle1os(); + + root->last[0] = root_pa; + root->last[1] = va; + root->last[2] = level; + root->last[3] = slot_pa; + root->last[4] = before; + *root_out = root_pa; + *slot_out = slot_pa; + *before_out = before; + return 0; +} + +static u64 uat_ep_map_begin(u64 *regs) +{ + struct uat_c_root_state *root = uat_find_state(regs[0], 0); + u64 segs_pa = regs[2]; + u64 num_segs = regs[3]; + u64 options = regs[4]; + u64 idx = ((options >> 8) & 2UL) | + ((options & 3UL) << 2) | + ((options >> 8) & 1UL); + + if (root == (struct uat_c_root_state *)0 || + num_segs == 0 || + num_segs > (g_state.uat.segment_limit ? g_state.uat.segment_limit : 0x40) || + (options & UAT_OPTIONS_INVALID_MASK) || + uat_attr_table[idx & 0xf] == 0xff) { + g_state.uat.violations++; + if (root == (struct uat_c_root_state *)0) + return 0; + } + if (!uat_copy_segments_to_state(root, UAT_STATE_MAP_SEGS_OFF, + segs_pa, num_segs)) { + g_state.uat.violations++; + return 0; + } + + root->current_va = regs[1]; + root->seglist_pa = root->handle + UAT_STATE_MAP_SEGS_OFF; + root->num_segs = num_segs; + root->current_seg = 0; + root->seg_offset = 0; + root->options = options; + root->state = 3; + uat_write8_ptr(root->handle + UAT_STATE_FLUSH_LEVEL_OFF, 0); + uat_write_map_state(root, 3); + return uat_map_pending(root); +} + +static u64 uat_ep_prepare_fw_unmap_begin(u64 *regs) +{ + struct uat_c_root_state *root = uat_find_state(regs[0], 0); + u64 va = regs[1]; + u64 pages = regs[2]; + + if (root == (struct uat_c_root_state *)0 || + (va & (UAT_C_PAGE_SIZE - 1)) != 0 || + pages == 0 || pages >= UAT_MAX_VA_PAGES) { + g_state.uat.violations++; + return 0; + } + + if (!uat_handoff_magic_valid(root)) + return 0; + + root->current_va = va; + root->current_seg = 0; + root->num_segs = pages; + root->seg_offset = 0; + root->options = 0; + root->state = 4; + uat_write_prepare_fw_unmap_state(root, 4, 0); + return uat_prepare_fw_unmap_pending(root); +} + +static u64 uat_ep_set_ctx_id(u64 *regs) +{ + struct uat_c_root_state *root = uat_find_state(regs[0], 0); + u64 ctx = regs[1]; + u64 base, root1; + u64 ttbr0, ttbr1; + + if (root == (struct uat_c_root_state *)0 || ctx >= 64 || + root->ctx_id != UAT_C_CTX_NONE || + !g_state.uat.gpu_region_pa) { + g_state.uat.violations++; + return 0; + } + + root1 = root->root1_pa; + base = g_state.uat.gpu_region_pa + ctx * 16UL; + if ((root->type & 0xeU) == 0 || + !uat_looks_like_pa(root->root0_pa) || !uat_looks_like_pa(root1) || + *(volatile u64 *)base != 0 || *(volatile u64 *)(base + 8) != 0) { + g_state.uat.violations++; + return 0; + } + ttbr0 = (root->root0_pa & 0x3ffffffc000UL) | + ((ctx & 0x7fffUL) << 48) | 1UL; + ttbr1 = (root1 & 0x3ffffffc000UL) | + ((ctx & 0xffffUL) << 48) | 1UL; + + uat_write_slot(base, ttbr0); + uat_write_slot(base + 8, ttbr1); + root->ctx_id = (u16)ctx; + uat_write_idle_state(root); + tlbi_vmalle1os(); + return 0; +} + +static u64 uat_ep_remove_ctx_id(u64 *regs) +{ + struct uat_c_root_state *root = uat_find_state(regs[0], 0); + if (root == (struct uat_c_root_state *)0) + return 0; + if (root->ctx_id == UAT_C_CTX_NONE) { + g_state.uat.violations++; + return 0; + } + if (root->ctx_id != UAT_C_CTX_NONE && g_state.uat.gpu_region_pa) { + u64 base = g_state.uat.gpu_region_pa + (u64)root->ctx_id * 16UL; + uat_write_slot(base, *(volatile u64 *)base & ~1UL); + uat_write_slot(base + 8, *(volatile u64 *)(base + 8) & ~1UL); + } + root->ctx_id = UAT_C_CTX_NONE; + uat_write_idle_state(root); + tlbi_vmalle1os(); + return 0; +} + +static u64 uat_ep_get_info(u64 *regs) +{ + u64 selector = regs[0]; + + switch (selector) { + case 0: + return g_state.uat.mode; + case 1: + if (g_state.uat.mode == 0) + return uat_looks_like_pa(g_state.uat.selector2_pa) ? + g_state.uat.selector2_pa : 0; + return 0; + case 2: + return uat_selector2_state_pa(); + case 3: + return g_state.uat.vaddr_shift ? g_state.uat.vaddr_shift : + UAT_DEFAULT_VADDR_SHIFT; + case 4: + return g_state.uat.l1_index_mask ? g_state.uat.l1_index_mask : + uat_default_l1_mask(g_state.uat.vaddr_shift ? + g_state.uat.vaddr_shift : + UAT_DEFAULT_VADDR_SHIFT); + case 5: + return g_state.uat.segment_limit ? g_state.uat.segment_limit : 0x40; + case 6: + return 0x1000000000UL; + case 7: + return 0x7000000000UL; + case 8: + return g_state.uat.state_object_size ? g_state.uat.state_object_size : + ((g_state.uat.segment_limit ? g_state.uat.segment_limit : 0x40) * + 0x10UL + 0x250UL); + case 9: + return uat_selector9_papt_va(); + default: + g_state.uat.violations++; + return 0; + } +} + +bool_t uat_handle_table7(u32 endpoint, u64 *regs) +{ + u64 rc = 0; + u64 root_pa = 0, slot_pa = 0, before = 0, after = 0; + u64 saved_regs[5]; + struct uat_c_root_state *root; + + if (endpoint >= UAT_C_MAX_ENDPOINTS) { + g_state.uat.fallthrough_calls++; + return 0; + } + + uat_set_defaults(); + g_state.uat.total_calls++; + g_state.uat.ep_count[endpoint]++; + for (u32 i = 0; i < 5; i++) { + saved_regs[i] = regs[i]; + g_state.uat.last[i] = regs[i]; + } + g_state.uat.last[5] = endpoint; + + switch (endpoint) { + case UAT_FN_INIT_STATE: + rc = uat_ep_init(regs); + break; + case UAT_FN_DESTROY_STATE: + rc = uat_ep_destroy(regs); + break; + case UAT_FN_MAP_TABLE: + rc = uat_ep_map_table(regs, &root_pa, &slot_pa, &before, &after); + break; + case UAT_FN_UNMAP_TABLE: + rc = uat_ep_unmap_table(regs, &root_pa, &slot_pa, &before); + after = 0; + break; + case UAT_FN_MAP_BEGIN: + rc = uat_ep_map_begin(regs); + break; + case UAT_FN_MAP_CONTINUE: + root = uat_find_state(regs[0], 0); + rc = root ? uat_map_pending(root) : 0; + if (!root) + g_state.uat.violations++; + break; + case UAT_FN_PREP_FW_UNMAP_BEGIN: + rc = uat_ep_prepare_fw_unmap_begin(regs); + root = uat_find_state(saved_regs[0], 0); + if (root) { + root_pa = root->last[0]; + slot_pa = root->last[2]; + before = root->last[3]; + after = root->last[4]; + } + break; + case UAT_FN_PREP_FW_UNMAP_CONTINUE: + root = uat_find_state(regs[0], 0); + if (root && root->state == 4) { + rc = uat_prepare_fw_unmap_pending(root); + root_pa = root->last[0]; + slot_pa = root->last[2]; + before = root->last[3]; + after = root->last[4]; + } else { + g_state.uat.violations++; + } + break; + case UAT_FN_UNMAP_BEGIN: + root = uat_find_state(regs[0], 0); + if (root) + rc = uat_unmap_segments(root, regs[1], regs[2]); + else + g_state.uat.violations++; + break; + case UAT_FN_UNMAP_CONTINUE: + root = uat_find_state(regs[0], 0); + rc = root ? uat_unmap_pending(root) : 0; + if (!root) + g_state.uat.violations++; + break; + case UAT_FN_SET_CTX_ID: + rc = uat_ep_set_ctx_id(regs); + break; + case UAT_FN_REMOVE_CTX_ID: + rc = uat_ep_remove_ctx_id(regs); + break; + case UAT_FN_GET_INFO: + rc = uat_ep_get_info(regs); + break; + default: + g_state.uat.fallthrough_calls++; + return 0; + } + + regs[0] = rc; + g_state.uat.fast_path_calls++; + g_state.fast_path_calls++; + uat_record(endpoint, saved_regs, root_pa, slot_pa, before, after, rc); + sev(); + return 1; +} diff --git a/src/sptm/xnu.c b/src/sptm/xnu.c new file mode 100644 index 000000000..e64291d26 --- /dev/null +++ b/src/sptm/xnu.c @@ -0,0 +1,832 @@ +/* SPDX-License-Identifier: MIT */ +#include "private.h" + +int debug_printf(const char *fmt, ...); + +#define USER_POINTER_BATCH_LIMIT 64U +#define USER_POINTER_OP_SIZE 0x18UL +#define USER_POINTER_VALUE_OFF 0x00UL + +bool_t xnu_handle_table0(void *ctx, u32 endpoint, u64 *regs) +{ + if (endpoint < 64) g_state.ep_count[endpoint]++; + + switch (endpoint) { + case SPTM_FN_MAP_PAGE: { + /* (root_pa, va, new_pte) -> sptm_return_t */ + u64 root_pa = regs[0]; + u64 va = regs[1]; + u64 new_pte = regs[2]; + u64 idx = 0; + bool_t geom4k = root_uses_4k(root_pa, 3); + u64 l3 = walk_to_l3(root_pa, va, &idx); + if (l3 == 0) { + regs[0] = SPTM_TABLE_NOT_PRESENT; + g_state.fast_path_calls++; sev(); + return 1; + } + volatile u64 *slot = pt_slot_ptr(l3, idx); + u64 existing = pt_read_slot_poc(slot); + bool_t existing_valid = ((existing & 3) == 3); + u64 arm_pte = + t8140_cpu_pio_shadow_pte(xnu_pte_to_arm_geom(new_pte, geom4k), + geom4k); + u64 ptep_pa = l3 + idx * 8; + u64 ptep_frame_pa = ptep_pa & FRAME_PAGE_MASK; + u64 page_mask = geom4k ? ARM_PTE_PAGE_MASK_4K : ARM_PTE_PAGE_MASK; + u64 target_pa = new_pte & page_mask; + u64 ptep_papt_va = 0; + if (g_state.first_papt != 0 && ptep_pa >= g_state.vm_first_phys) + ptep_papt_va = g_state.first_papt + (ptep_pa - g_state.vm_first_phys); + g_state.last_map_page[0] = root_pa; + g_state.last_map_page[1] = va; + g_state.last_map_page[2] = l3; + g_state.last_map_page[3] = idx; + g_state.last_map_page[4] = ptep_pa; + g_state.last_map_page[5] = ptep_papt_va; + g_state.last_map_page[6] = existing; + g_state.last_map_page[7] = new_pte; + g_state.last_map_page[8] = arm_pte; + g_state.last_map_page[9] = target_pa; + g_state.last_map_page[10] = ft_get_type(target_pa); + g_state.last_map_page[11] = ft_read_page_refcount(target_pa); + g_state.last_map_page[12] = ft_read_table_refcount(ptep_frame_pa); + g_state.last_map_page[13] = ft_idx_or_bad(ptep_frame_pa); + /* AUDIT FIX #1: write sptm_map_page_output_t. + * Then DC CVAU + DSB to make scratch visible to xnu's read. */ + if (g_state.scratch_pa != 0) { + volatile u64 *out = scratch_for_current_cpu(); + out[0] = existing; + out[1] = ptep_papt_va; + __asm__ volatile( + "dc cvau, %0\n\t" + "dc cvau, %1\n\t" + "dsb ish\n\t" + : : "r"(&out[0]), "r"(&out[1]) : "memory"); + } + /* + * Existing same-PA mappings are upgraded in place. Different-PA + * replacements must go back through XNU's remove/retry path so the + * architecture's break-before-make sequence is preserved. + */ + if (existing_valid && ((existing & page_mask) != (arm_pte & page_mask))) { + regs[0] = SPTM_MAP_PADDR_CONFLICT; + g_state.fast_path_calls++; sev(); + return 1; + } + *slot = arm_pte; + clean_pt_slot_poc(slot); + update_refcounts_for_leaf_pte_change(l3, existing, arm_pte, geom4k); + maybe_map_t8140_pmgr_cpustart_window(root_pa, va, arm_pte, + geom4k, l3, idx); + if (existing_valid) + tlbi_vmalle1is(); + regs[0] = existing_valid ? SPTM_MAP_VALID : SPTM_SUCCESS; + g_state.fast_path_calls++; sev(); + return 1; + } + + case SPTM_FN_MAP_TABLE: { + /* (root_pa, va, target_level, new_tte) -> sptm_return_t */ + u64 root_pa = regs[0]; + u64 va = regs[1]; + u32 target_level = (u32)regs[2]; + u64 new_tte = regs[3]; + u64 idx = 0; + u64 parent = walk_to_level(root_pa, va, target_level, &idx); + if (parent == 0) { + regs[0] = SPTM_TABLE_NOT_PRESENT; + g_state.fast_path_calls++; sev(); + return 1; + } + bool_t geom4k = root_uses_4k(root_pa, target_level); + u64 base_idx = table_group_base_idx(idx, geom4k); + u32 group_count = table_group_count(geom4k); + volatile u64 *slot = pt_slot_ptr(parent, base_idx); + u64 existing = 0; + u64 arm_tte = xnu_tte_to_arm(new_tte); + u64 child_pa = new_tte & ARM_TTE_TABLE_MASK; + bool_t already_present = 0; + for (u32 n = 0; n < group_count; n++) { + u64 cur = pt_read_slot_poc(&slot[n]); + if ((cur & 3) == 3) { + if (!already_present) + existing = cur; + already_present = 1; + } + } + if (already_present) { + g_state.last_map_table[0] = root_pa; + g_state.last_map_table[1] = va; + g_state.last_map_table[2] = target_level; + g_state.last_map_table[3] = parent; + g_state.last_map_table[4] = base_idx; + g_state.last_map_table[5] = existing; + g_state.last_map_table[6] = new_tte; + g_state.last_map_table[7] = arm_tte; + g_state.last_map_table[8] = child_pa & FRAME_PAGE_MASK; + g_state.last_map_table[9] = ft_read_table_refcount(child_pa); + regs[0] = SPTM_TABLE_ALREADY_PRESENT; + g_state.fast_path_calls++; sev(); + return 1; + } + u64 desc_attrs = arm_tte & ~ARM_TTE_TABLE_MASK; + for (u32 n = 0; n < group_count; n++) { + u64 sub_pa = table_group_child_pa(child_pa, n, geom4k); + slot[n] = desc_attrs | (sub_pa & ARM_TTE_TABLE_MASK); + } + clean_pt_range_poc(slot, (u64)group_count * 8); + ft_inc_table_mapping_refcount(child_pa & FRAME_PAGE_MASK); + g_state.last_map_table[0] = root_pa; + g_state.last_map_table[1] = va; + g_state.last_map_table[2] = target_level; + g_state.last_map_table[3] = parent; + g_state.last_map_table[4] = base_idx; + g_state.last_map_table[5] = existing; + g_state.last_map_table[6] = new_tte; + g_state.last_map_table[7] = arm_tte; + g_state.last_map_table[8] = child_pa & FRAME_PAGE_MASK; + g_state.last_map_table[9] = ft_read_table_refcount(child_pa); + regs[0] = SPTM_SUCCESS; + g_state.fast_path_calls++; sev(); + return 1; + } + + case SPTM_FN_UNMAP_TABLE: { + /* (root_pa, va, target_level) -> void. + * Clear the parent TTE and leave the child frame type/counts + * alone. XNU drains leaf mappings first, queries the child table + * count, then performs the retype itself. */ + u64 root_pa = regs[0]; + u64 va = regs[1]; + u32 target_level = (u32)regs[2]; + u64 idx = 0; + u64 parent = walk_to_level(root_pa, va, target_level, &idx); + u64 existing = 0; + u64 freed_pa = 0; + if (parent != 0) { + bool_t geom4k = root_uses_4k(root_pa, target_level); + u64 base_idx = table_group_base_idx(idx, geom4k); + u32 group_count = table_group_count(geom4k); + volatile u64 *slot = pt_slot_ptr(parent, base_idx); + idx = base_idx; + for (u32 n = 0; n < group_count; n++) { + u64 cur = pt_read_slot_poc(&slot[n]); + if (existing == 0 && (cur & 3) == 3) + existing = cur; + slot[n] = 0; + } + clean_pt_range_poc(slot, (u64)group_count * 8); + if ((existing & 3) == 3) { + freed_pa = existing & FRAME_PAGE_MASK; + ft_dec_table_mapping_refcount(freed_pa); + } + } + g_state.last_unmap_table[0] = root_pa; + g_state.last_unmap_table[1] = va; + g_state.last_unmap_table[2] = target_level; + g_state.last_unmap_table[3] = parent; + g_state.last_unmap_table[4] = idx; + g_state.last_unmap_table[5] = existing; + g_state.last_unmap_table[6] = freed_pa; + g_state.last_unmap_table[7] = ft_get_type(freed_pa); + g_state.last_unmap_table[8] = ft_read_table_refcount(freed_pa); + g_state.last_unmap_table[9] = ft_idx_or_bad(freed_pa); + tlbi_vmalle1is(); + regs[0] = SPTM_SUCCESS; + g_state.fast_path_calls++; sev(); + return 1; + } + + case SPTM_FN_RETYPE: { + /* (pa, current_type, new_type, options) -> sptm_return_t. + * Real SPTM validates the old type and zeroes pages that become + * page tables. We keep the functional side effects in C; violations + * remain accepted in this permissive bringup model, but frame_table + * is the single source of truth for XNU/libsptm direct reads. */ + u64 pa = regs[0]; + u8 current_type = (u8)regs[1]; + u8 nt = (u8)regs[2]; + u8 existing_type = ft_get_type(pa); + bool_t was_alt_ptd = existing_type == XNU_PAGE_TABLE_ALT; + bool_t will_alt_ptd = nt == XNU_PAGE_TABLE_ALT; + bool_t alt_papt_attr_write = 0; + u64 page_ref_before = ft_read_page_refcount(pa); + u64 table_ref_before = ft_read_table_refcount(pa); + u64 papt_ring_before = g_state.papt_update.ring_idx; + u8 root_geom = ROOT_GEOM_UNKNOWN; + u16 root_asid = 0; + + if (was_alt_ptd && !will_alt_ptd) { + alt_papt_attr_write = + set_kernel_papt_cacheable_for_managed_pa(pa, 1, + sptm_ctx_elr(ctx)); + alt_ptd_set_el2_cacheable(pa, 1); + } else if (!was_alt_ptd && will_alt_ptd) { + alt_papt_attr_write = + set_kernel_papt_cacheable_for_managed_pa(pa, 0, + sptm_ctx_elr(ctx)); + alt_ptd_set_el2_cacheable(pa, 0); + } + + if (is_pt_type(nt)) { + zero_16k(pa); + ft_write_u16(pa, FT_TABLE_MAPPING_REFCNT_OFF, 0); + ft_write_u16(pa, FT_TABLE_NESTED_REFCNT_OFF, 0); + } + + if (is_pt_type(existing_type) && !is_pt_type(nt)) { + ft_write_u16(pa, FT_TABLE_MAPPING_REFCNT_OFF, 0); + ft_write_u16(pa, FT_TABLE_NESTED_REFCNT_OFF, 0); + } + + ft_set_type(pa, nt); + if (was_alt_ptd || will_alt_ptd) + sptm_coproc_cache_maint_data_page(pa); + if (is_root_type(nt)) { + root_geom = retype_params_attr_idx(regs[3]); + root_asid = retype_params_asid(regs[3]); + root_geom_set_meta(pa, root_geom, root_asid); + } + barrier(); + + g_state.last_retype[0] = pa; + g_state.last_retype[1] = current_type; + g_state.last_retype[2] = nt; + g_state.last_retype[3] = existing_type; + g_state.last_retype[4] = ft_read_page_refcount(pa); + g_state.last_retype[5] = ft_read_table_refcount(pa); + g_state.last_retype[6] = ft_idx_or_bad(pa); + g_state.last_retype[7] = (is_pt_type(existing_type) ? 1UL : 0UL) | + (is_pt_type(nt) ? 2UL : 0UL) | + ((u64)root_geom << 8) | + ((u64)root_asid << 16) | + ((u64)retype_params_flags(regs[3]) << 32); + if (was_alt_ptd || will_alt_ptd) { + u64 flags = (was_alt_ptd ? 1UL : 0UL) | + (will_alt_ptd ? 2UL : 0UL) | + (alt_papt_attr_write ? 4UL : 0UL) | + (is_pt_type(existing_type) ? 8UL : 0UL) | + (is_pt_type(nt) ? 16UL : 0UL); + u64 sp0 = ctx ? *(volatile u64 *)((unsigned char *)ctx + 0x158) : 0; + u64 sp1 = ctx ? *(volatile u64 *)((unsigned char *)ctx + 0x160) : 0; + note_alt_ptd_retype(pa, sptm_ctx_elr(ctx), regs[30], regs[29], + sp0, sp1, current_type, existing_type, nt, + regs[3], + page_ref_before, table_ref_before, + ft_read_page_refcount(pa), + ft_read_table_refcount(pa), + ft_idx_or_bad(pa), papt_ring_before, + flags); + } + + if (alt_papt_attr_write) + tlbi_vmalls12e1is(); + else if (is_pt_type(nt) || is_io_type(nt) || + is_pt_type(existing_type) || is_pt_type(current_type)) + tlbi_vmalle1is(); + + regs[0] = SPTM_SUCCESS; + g_state.fast_path_calls++; sev(); + return 1; + } + + case SPTM_FN_UPDATE_REGION: { + u64 root_pa = regs[0]; + u64 start_va = regs[1]; + u32 num = (u32)(regs[2] & 0xffffffff); + u64 templates_pa = regs[3]; + u64 options = regs[4]; + u64 mask = options & SPTM_UPDATE_MASK; + bool_t geom4k = root_uses_4k(root_pa, 3); + u64 page_size = geom4k ? PAGE_SIZE_4K_EMUL : PAGE_SIZE_GUEST_EMUL; + volatile u64 *prev_out = scratch_for_current_cpu(); + u32 prev_count = 0; + if (num > 2048) num = 2048; + for (u32 i = 0; i < num; i++) { + u64 va = start_va + (u64)i * page_size; + u64 idx; + u64 l3 = walk_to_l3(root_pa, va, &idx); + u64 existing = 0; + if (l3) { + volatile u64 *slot = pt_slot_ptr(l3, idx); + existing = pt_read_slot_poc(slot); + u64 tmpl = ((volatile u64 *)templates_pa)[i]; + u64 arm_template = + xnu_pte_update_template_to_arm_geom(tmpl, geom4k); + u64 arm_pte = + t8140_cpu_pio_shadow_pte( + merge_pte_with_mask_geom(existing, arm_template, + mask, geom4k), + geom4k); + *slot = arm_pte; + clean_pt_slot_poc(slot); + update_refcounts_for_leaf_pte_change(l3, existing, arm_pte, + geom4k); + maybe_map_t8140_pmgr_cpustart_window(root_pa, va, arm_pte, + geom4k, l3, idx); + } + write_prev_pte(prev_out, &prev_count, existing); + } + if (prev_out) clean_range((void *)prev_out, (u64)prev_count * 8); + if (!(options & 0x100)) + tlbi_vmalle1is(); + else + barrier(); + regs[0] = (options & 0x100) ? SPTM_UPDATE_DELAYED_TLBI : SPTM_SUCCESS; + g_state.fast_path_calls++; sev(); + return 1; + } + + case SPTM_FN_UPDATE_DISJOINT: { + u64 ops_pa = regs[1]; + u32 num = (u32)(regs[2] & 0xffffffff); + u64 options = regs[3]; + u64 mask = options & SPTM_UPDATE_MASK; + volatile u64 *prev_out = scratch_for_current_cpu(); + u32 prev_count = 0; + if (num > 2048) num = 2048; + for (u32 i = 0; i < num; i++) { + volatile u64 *op = (volatile u64 *)(ops_pa + (u64)i * DISJOINT_OP_SIZE); + u64 root_pa = op[0]; + u64 va = op[1]; + u64 tmpl = op[2]; + u64 idx; + bool_t geom4k = root_uses_4k(root_pa, 3); + u64 l3 = walk_to_l3(root_pa, va, &idx); + u64 existing = 0; + if (l3) { + volatile u64 *slot = pt_slot_ptr(l3, idx); + existing = pt_read_slot_poc(slot); + u64 arm_template = + xnu_pte_update_template_to_arm_geom(tmpl, geom4k); + u64 arm_pte = + t8140_cpu_pio_shadow_pte( + merge_pte_with_mask_geom(existing, arm_template, + mask, geom4k), + geom4k); + *slot = arm_pte; + clean_pt_slot_poc(slot); + update_refcounts_for_leaf_pte_change(l3, existing, arm_pte, + geom4k); + maybe_map_t8140_pmgr_cpustart_window(root_pa, va, arm_pte, + geom4k, l3, idx); + } + write_prev_pte(prev_out, &prev_count, existing); + } + if (prev_out) clean_range((void *)prev_out, (u64)prev_count * 8); + if (!(options & 0x100)) + tlbi_vmalle1is(); + else + barrier(); + regs[0] = (options & 0x100) ? SPTM_UPDATE_DELAYED_TLBI : SPTM_SUCCESS; + g_state.fast_path_calls++; sev(); + return 1; + } + + case SPTM_FN_UPDATE_DISJOINT_MULTIPAGE: { + u64 ops_pa = regs[0]; + u32 num_entries = (u32)(regs[1] & 0xffffffff); + volatile u64 *prev_out = scratch_for_current_cpu(); + u32 prev_count = 0; + bool_t any_deferred = 0; + bool_t any_papt_attr_write = 0; + u32 i = 0; + if (num_entries > 2048) num_entries = 2048; + while (i < num_entries) { + u64 base = ops_pa + (u64)i * DISJOINT_OP_SIZE; + u64 paddr = *(volatile u64 *)base; + u64 papt_template = *(volatile u64 *)(base + 8); + u32 inner_n = *(volatile u32 *)(base + 16); + u32 opts = *(volatile u32 *)(base + 20); + u64 inner_root_pa = 0; + u64 inner_va = 0; + u64 inner_template = 0; + i++; + if (inner_n != 0 && i < num_entries) { + volatile u64 *first_inner = + (volatile u64 *)(ops_pa + (u64)i * DISJOINT_OP_SIZE); + inner_root_pa = first_inner[0]; + inner_va = first_inner[1]; + inner_template = first_inner[2]; + } + if (opts & SPTM_UPDATE_DEFER_TLBI) + any_deferred = 1; + if (update_kernel_papt_attrs(paddr, papt_template, opts, + sptm_ctx_elr(ctx), inner_root_pa, + inner_va, inner_template)) + any_papt_attr_write = 1; + for (u32 j = 0; j < inner_n && i < num_entries; j++, i++) { + volatile u64 *op = (volatile u64 *)(ops_pa + (u64)i * DISJOINT_OP_SIZE); + u64 root_pa = op[0]; + u64 va = op[1]; + u64 tmpl = op[2]; + u64 idx; + bool_t geom4k = root_uses_4k(root_pa, 3); + u64 l3 = walk_to_l3(root_pa, va, &idx); + u64 existing = 0; + if (l3) { + volatile u64 *slot = pt_slot_ptr(l3, idx); + existing = pt_read_slot_poc(slot); + u64 arm_template = + xnu_pte_update_template_to_arm_geom(tmpl, geom4k); + u64 arm_pte = + t8140_cpu_pio_shadow_pte( + merge_pte_with_mask_geom( + existing, arm_template, + opts & SPTM_UPDATE_MASK, geom4k), + geom4k); + *slot = arm_pte; + clean_pt_slot_poc(slot); + update_refcounts_for_leaf_pte_change(l3, existing, arm_pte, + geom4k); + maybe_map_t8140_pmgr_cpustart_window(root_pa, va, arm_pte, + geom4k, l3, idx); + } + write_prev_pte(prev_out, &prev_count, existing); + } + } + if (prev_out) clean_range((void *)prev_out, (u64)prev_count * 8); + /* + * Deferred TLBI covers the guest-requested VA updates. The PAPT + * mirror is a private kernel alias we update on XNU's behalf, so + * flush it here when its cacheability attributes changed. + */ + if (any_papt_attr_write) + tlbi_vmalls12e1is(); + else if (!any_deferred) + tlbi_vmalle1is(); + else + barrier(); + regs[0] = any_deferred ? SPTM_UPDATE_DELAYED_TLBI : SPTM_SUCCESS; + g_state.fast_path_calls++; sev(); + return 1; + } + + case SPTM_FN_UNMAP_DISJOINT: { + u64 ops_pa = regs[1]; + u32 num = (u32)(regs[2] & 0xffffffff); + volatile u64 *prev_out = scratch_for_current_cpu(); + u32 prev_count = 0; + if (num > 2048) num = 2048; + for (u32 i = 0; i < num; i++) { + volatile u64 *op = (volatile u64 *)(ops_pa + (u64)i * DISJOINT_OP_SIZE); + u64 root_pa = op[0]; + u64 va = op[1]; + u64 tmpl = op[2]; + u64 idx; + bool_t geom4k = root_uses_4k(root_pa, 3); + u64 l3 = walk_to_l3(root_pa, va, &idx); + u64 existing = 0; + if (l3) { + volatile u64 *slot = pt_slot_ptr(l3, idx); + existing = pt_read_slot_poc(slot); + /* + * Disjoint unmap receives an XNU template, but leaving + * template PA/attribute bits in an invalid slot can make + * XNU later interpret the entry as a corrupt compressed + * PTE. Real pmap remove paths fault the slot; keep the + * old PTE only in the prev-PTE scratch output. + */ + (void)tmpl; + *slot = 0; + clean_pt_slot_poc(slot); + update_refcounts_for_leaf_pte_change(l3, existing, 0, geom4k); + } + write_prev_pte(prev_out, &prev_count, existing); + } + if (prev_out) clean_range((void *)prev_out, (u64)prev_count * 8); + tlbi_vmalle1is(); + regs[0] = SPTM_SUCCESS; + g_state.fast_path_calls++; sev(); + return 1; + } + + case SPTM_FN_SWITCH_ROOT: { + u64 root_pa = regs[0]; + if (ft_get_type(root_pa) == SPTM_KERNEL_ROOT_TABLE) { + u64 ttbr = root_pa & TTBR_BADDR_MASK; + __asm__ volatile("msr S3_5_C2_C0_1, %0" : : "r"(ttbr)); + } else { + u64 asid = root_asid_lookup(root_pa); + u64 ttbr = (root_pa & TTBR_BADDR_MASK) | (asid << TTBR_ASID_SHIFT); + __asm__ volatile("msr S3_5_C2_C0_0, %0" : : "r"(ttbr)); + } + tlbi_vmalle1is(); + regs[0] = SPTM_SUCCESS; + g_state.fast_path_calls++; sev(); + return 1; + } + + case SPTM_FN_CONFIGURE_SHAREDREGION: + ft_set_type(regs[0], XNU_SHARED_ROOT_TABLE); + regs[0] = SPTM_SUCCESS; + g_state.fast_path_calls++; sev(); + return 1; + + case SPTM_FN_SET_SHARED_REGION: + regs[0] = SPTM_SUCCESS; + g_state.fast_path_calls++; sev(); + return 1; + + case SPTM_FN_NEST_REGION: { + u64 user_root = regs[0]; + u64 shared_root = regs[1]; + u64 start_va = regs[2]; + u32 page_count = (u32)(regs[3] & 0xffffffff); + bool_t geom4k = root_uses_4k(user_root, 2); + u64 page_size = geom4k ? PAGE_SIZE_4K_EMUL : PAGE_SIZE_GUEST_EMUL; + u64 l2_coverage = (geom4k ? 0x200UL : 0x800UL) * page_size; + u64 end_va = start_va + (u64)page_count * page_size; + u64 va = start_va & ~(l2_coverage - 1); + while (va < end_va) { + u64 sidx = 0, uidx = 0; + u64 sparent = walk_to_level(shared_root, va, 2, &sidx); + u64 uparent = walk_to_level(user_root, va, 2, &uidx); + if (sparent != 0 && uparent != 0 && sidx == uidx) { + volatile u64 *uslot = pt_slot_ptr(uparent, uidx); + u64 tte = pt_read_slot_poc(pt_slot_ptr(sparent, sidx)); + if ((tte & 3) == 3) { + *uslot = tte; + clean_pt_slot_poc(uslot); + } + } + va += l2_coverage; + } + barrier(); + regs[0] = SPTM_SUCCESS; + g_state.fast_path_calls++; sev(); + return 1; + } + + case SPTM_FN_UNNEST_REGION: { + u64 user_root = regs[0]; + u64 start_va = regs[2]; + u32 page_count = (u32)(regs[3] & 0xffffffff); + bool_t geom4k = root_uses_4k(user_root, 2); + u64 page_size = geom4k ? PAGE_SIZE_4K_EMUL : PAGE_SIZE_GUEST_EMUL; + u64 l2_coverage = (geom4k ? 0x200UL : 0x800UL) * page_size; + u64 end_va = start_va + (u64)page_count * page_size; + u64 va = start_va & ~(l2_coverage - 1); + while (va < end_va) { + u64 idx = 0; + u64 parent = walk_to_level(user_root, va, 2, &idx); + if (parent != 0) { + volatile u64 *slot = pt_slot_ptr(parent, idx); + *slot = 0; + clean_pt_slot_poc(slot); + } + va += l2_coverage; + } + tlbi_vmalle1is(); + regs[0] = SPTM_SUCCESS; + g_state.fast_path_calls++; sev(); + return 1; + } + + /* C-only ack-success endpoints. + * + * Historical note: older m1n1 builds also added +4 to ELR for + * handled HVCs, which double-advanced these non-trampoline call + * sites and skipped one real instruction. Core m1n1 now leaves + * handled HVC ELR alone, so resume should use ELR_EL2 directly. */ + case SPTM_FN_FIXUPS_COMPLETE: + case SPTM_FN_CONFIGURE_ROOT: + case SPTM_FN_SLIDE_REGION: + case SPTM_FN_REG_WRITE: + case SPTM_FN_SERIAL_DISABLE: + case SPTM_FN_REGISTER_EXC_RETURN: + case SPTM_FN_DISABLE_KERNEL_MODE_CPA2: + case SPTM_FN_PROGRAM_IRGKEY: + case SPTM_FN_REG_SNAPSHOT: + case SPTM_FN_MAP_SK_DOMAIN: + case SPTM_FN_HIB_BEGIN: + case SPTM_FN_HIB_VERIFY_HASH_NON_WIRED: + case SPTM_FN_HIB_FINALIZE_NON_WIRED: + case SPTM_FN_GUEST_EXIT: + { + return finish_hvc_success(ctx, regs); + } + + case SPTM_FN_GUEST_STAGE1_TLBOP: + /* + * Real SPTM decodes the requested guest S1 operation and issues a + * scoped TLBI while running under the guest's stage-2 context. We + * do not model nested guest contexts yet, but ACK-only is wrong: + * callers use this endpoint specifically to retire stale S1 + * translations. A broadcast S1 invalidate is conservative and + * preserves the architectural effect until the op decoder exists. + */ + tlbi_vmalle1is(); + return finish_hvc_success(ctx, regs); + + case SPTM_FN_GUEST_STAGE2_TLBOP: + /* + * Same conservative treatment for guest stage-2 invalidates. The + * exact firmware path can narrow this to IPA/range later. + */ + tlbi_vmalls12e1is(); + return finish_hvc_success(ctx, regs); + + case SPTM_FN_REGISTER_CPU: { + u64 phys = regs[0]; + u64 count = g_state.cpu_count; + if (count > SPTM_MAX_CPUS_EMUL) + count = SPTM_MAX_CPUS_EMUL; + + u64 slot = SPTM_MAX_CPUS_EMUL; + for (u64 i = 0; i < count; i++) { + if (cpu_phys_matches(g_state.cpu_phys_ids[i], phys)) { + slot = i; + break; + } + } + + if (slot == SPTM_MAX_CPUS_EMUL && count < SPTM_MAX_CPUS_EMUL) { + slot = count; + g_state.cpu_phys_ids[slot] = phys; + g_state.cpu_count = count + 1; + } + + g_state.cpu_last_mpidr = current_mpidr_phys(); + g_state.cpu_last_id = (slot == SPTM_MAX_CPUS_EMUL) ? 0 : slot; + sptm_cpu_pio_note_registered(phys, g_state.cpu_last_id); + return finish_hvc_success(ctx, regs); + } + + case SPTM_FN_SURT_ALLOC: + case SPTM_FN_SURT_FREE: { + u64 surt_frame = regs[0]; + u64 surt_index = regs[1] & 0xff; + u8 attr_idx = (u8)(regs[2] & 0xff); + u64 slot_pa = surt_frame + surt_index * SURT_SUBPAGE_SIZE; + volatile u64 *q = (volatile u64 *)slot_pa; + for (u32 i = 0; i < (SURT_SUBPAGE_SIZE / 8); i++) + q[i] = 0; + clean_range_poc((void *)slot_pa, SURT_SUBPAGE_SIZE); + if (endpoint == SPTM_FN_SURT_ALLOC) + root_geom_set_meta(slot_pa, attr_idx, (u16)(regs[4] & 0xffff)); + regs[0] = SPTM_SUCCESS; + g_state.fast_path_calls++; sev(); + return 1; + } + + case SPTM_FN_REG_READ: + case SPTM_FN_SPTM_SYSCTL: + regs[0] = 0; + g_state.fast_path_calls++; sev(); + return 1; + + case SPTM_FN_SIGN_USER_POINTER: + case SPTM_FN_AUTH_USER_POINTER: + g_state.fast_path_calls++; sev(); + return 1; + + case SPTM_FN_BATCH_SIGN_USER_POINTER: { + u64 ops_pa = regs[0]; + u32 ops_count = (u32)(regs[1] & 0xffffffff); + volatile u64 *out = scratch_for_current_cpu(); + if (out) { + if (ops_count > USER_POINTER_BATCH_LIMIT) + ops_count = USER_POINTER_BATCH_LIMIT; + for (u32 i = 0; i < ops_count; i++) { + u64 op_pa = ops_pa + (u64)i * USER_POINTER_OP_SIZE; + out[i] = *(volatile u64 *)(op_pa + + USER_POINTER_VALUE_OFF); + } + clean_range((void *)out, (u64)ops_count * 8); + } + regs[0] = SPTM_SUCCESS; + g_state.fast_path_calls++; sev(); + return 1; + } + + case SPTM_FN_GUEST_VA_TO_IPA: + regs[0] = UINT64_MAX_VALUE; + g_state.fast_path_calls++; sev(); + return 1; + + case SPTM_FN_IOFILTER_PROTECTED_WRITE: { + /* + * No C model yet. Leave this unhandled so any future caller stops + * at the missing endpoint instead of silently touching guarded + * MMIO or reviving the removed Python fallback policy. + */ + return 0; + } + + case SPTM_FN_CONDEMN_LEAF_TABLE: { + u64 root_pa = regs[0]; + u64 va = regs[1]; + u64 idx = 0; + u64 parent = walk_to_level(root_pa, va, 2, &idx); + if (parent == 0) { + regs[0] = SPTM_TABLE_NOT_PRESENT; + } else { + volatile u64 *slot = pt_slot_ptr(parent, idx); + u64 l2e = pt_read_slot_poc(slot); + if ((l2e & 3) != 3 || (l2e & TTE_CONDEMNED_BIT)) + regs[0] = SPTM_TABLE_NOT_PRESENT; + else { + *slot = l2e | TTE_CONDEMNED_BIT; + clean_pt_slot_poc(slot); + regs[0] = SPTM_SUCCESS; + } + } + g_state.fast_path_calls++; sev(); + return 1; + } + + case SPTM_FN_UNCONDEMN_LEAF_TABLE: { + u64 root_pa = regs[0]; + u64 va = regs[1]; + u64 idx = 0; + u64 parent = walk_to_level(root_pa, va, 2, &idx); + if (parent != 0) { + volatile u64 *slot = pt_slot_ptr(parent, idx); + u64 l2e = pt_read_slot_poc(slot); + *slot = l2e & ~TTE_CONDEMNED_BIT; + clean_pt_slot_poc(slot); + } + regs[0] = SPTM_SUCCESS; + g_state.fast_path_calls++; sev(); + return 1; + } + + case SPTM_FN_SERIAL_PUTC: + /* Avoid per-character UART round trips. Panic diagnostics are + * captured through XNU_PANIC_BEGIN and framebuffer/serial KDP. */ + regs[0] = SPTM_SUCCESS; + g_state.fast_path_calls++; sev(); + return 1; + + case SPTM_FN_LOCKDOWN: + /* Our boot page tables already establish coarse kernel RX/RO/RW. + * Real SPTM would lock CTRR and retype text pages. */ + regs[0] = SPTM_SUCCESS; + g_state.fast_path_calls++; sev(); + return 1; + + /* UNMAP_REGION (ep=7) — port to C, ATTEMPT 3 (subagent fix). + * + * ROOT CAUSE of prior 2 attempts: incorrectly applied the ELR-=4 + * workaround. That workaround is ONLY needed for HVCs at NON- + * trampoline call sites (FIXUPS_COMPLETE etc.). UNMAP_REGION + * goes through the SPTM trampoline at 0xbcee880 — same as + * MAP_PAGE/MAP_TABLE/RETYPE — which has `hvc; retab; bti c`. + * m1n1's auto +4 lands on `bti c` (benign NOP); ELR-=4 made it + * land on `retab` instead, NOT a BTI landing pad → EC=0x1b BTI + * fail → corrupt PC → panic 2-3 calls later. + * + * Plus: real SPTM `f6180 sptm_pte_finalize_unmap` decrements + * the page-FTE refcount on each unmapped target. Mirror that. */ + case 7: { /* SPTM_FN_UNMAP_REGION */ + u64 root_pa = regs[0]; + u64 start_va = regs[1]; + u32 num = (u32)(regs[2] & 0xffffffff); + u64 options = regs[3]; + bool_t geom4k = root_uses_4k(root_pa, 3); + u64 page_size = geom4k ? PAGE_SIZE_4K_EMUL : PAGE_SIZE_GUEST_EMUL; + volatile u64 *prev_out = scratch_for_current_cpu(); + u32 prev_count = 0; + bool_t any_unmapped = 0; + if (num > 2048) num = 2048; + for (u32 i = 0; i < num; i++) { + u64 va = start_va + (u64)i * page_size; + u64 idx; + u64 l3 = walk_to_l3(root_pa, va, &idx); + u64 existing = 0; + if (l3) { + volatile u64 *slot = pt_slot_ptr(l3, idx); + existing = pt_read_slot_poc(slot); + *slot = 0; + clean_pt_slot_poc(slot); + if ((existing & 3) == 3) + any_unmapped = 1; + } + write_prev_pte(prev_out, &prev_count, existing); + update_refcounts_for_leaf_pte_change(l3, existing, 0, geom4k); + } + if (prev_out) clean_range((void *)prev_out, (u64)prev_count * 8); + if (any_unmapped && !(options & 0x100)) + tlbi_vmalle1is(); + else + barrier(); + regs[0] = (any_unmapped && (options & 0x100)) ? + SPTM_UPDATE_DELAYED_TLBI : SPTM_SUCCESS; + g_state.fast_path_calls++; sev(); + return 1; + } + + case SPTM_FN_CPU_ID: + regs[0] = cpu_id_for_phys(regs[0]); + g_state.fast_path_calls++; sev(); + return 1; + + default: + /* Unrecognized — host treats this as a hard diagnostic failure. */ + g_state.unknown_calls++; + g_state.fallthrough_calls++; + return 0; + } +} diff --git a/src/utils.h b/src/utils.h index a7bf86aac..ff3f36bb0 100644 --- a/src/utils.h +++ b/src/utils.h @@ -510,6 +510,7 @@ struct midr_part_features { bool amx; bool actlr_el2; bool counter_redirect; + bool sme_enabled; }; extern bool is_mac;