From 6f20cb831ca9250f09242c719cbb0955afbe4036 Mon Sep 17 00:00:00 2001 From: Yureka Date: Thu, 25 Jun 2026 21:01:43 +0200 Subject: [PATCH 01/33] move Bool, bool_ to constructutils Signed-off-by: Yureka --- proxyclient/m1n1/constructutils.py | 7 ++++++- proxyclient/m1n1/fw/afk/epic.py | 1 + proxyclient/m1n1/fw/common.py | 5 ----- proxyclient/m1n1/fw/dcp/iboot.py | 1 + proxyclient/m1n1/fw/dcp/ipc.py | 2 +- 5 files changed, 9 insertions(+), 7 deletions(-) diff --git a/proxyclient/m1n1/constructutils.py b/proxyclient/m1n1/constructutils.py index 8f7e35781..041328f67 100644 --- a/proxyclient/m1n1/constructutils.py +++ b/proxyclient/m1n1/constructutils.py @@ -890,4 +890,9 @@ def show_struct_trace(log=print): for addr, desc in sorted(list(g_struct_trace)): log(f"{addr:>#18x}: {desc}") -__all__ = ["ConstructClass", "ConstructValueClass", "Dec", "ROPointer", "show_struct_trace", "ZPadding", "Ver"] +def Bool(c): + return ExprAdapter(c, lambda d, ctx: bool(d & 1), lambda d, ctx: int(d)) + +bool_ = Bool(Int8ul) + +__all__ = ["ConstructClass", "ConstructValueClass", "Dec", "ROPointer", "show_struct_trace", "ZPadding", "Ver", "Bool", "bool_"] diff --git a/proxyclient/m1n1/fw/afk/epic.py b/proxyclient/m1n1/fw/afk/epic.py index 090a06cfe..099f1980a 100644 --- a/proxyclient/m1n1/fw/afk/epic.py +++ b/proxyclient/m1n1/fw/afk/epic.py @@ -5,6 +5,7 @@ from construct import * from ..common import * from ...utils import * +from ...constructutils import Bool from ..asc import StandardASC from ..asc.base import * from .rbep import AFKRingBufEndpoint diff --git a/proxyclient/m1n1/fw/common.py b/proxyclient/m1n1/fw/common.py index 479e2dfc8..506c16f07 100644 --- a/proxyclient/m1n1/fw/common.py +++ b/proxyclient/m1n1/fw/common.py @@ -18,9 +18,6 @@ ulong = uint64_t long_ = int64_t -def Bool(c): - return ExprAdapter(c, lambda d, ctx: bool(d & 1), lambda d, ctx: int(d)) - def SizedArray(count, svar, subcon): return Padded(subcon.sizeof() * count, Array(lambda ctx: min(count, ctx.get(svar, ctx._.get(svar))), subcon)) @@ -30,8 +27,6 @@ def SizedBytes(count, svar): def UnkBytes(s): return Default(HexDump(Bytes(s)), b"\x00" * s) -bool_ = Bool(Int8ul) - class OSObject(Construct): TYPE = None diff --git a/proxyclient/m1n1/fw/dcp/iboot.py b/proxyclient/m1n1/fw/dcp/iboot.py index 8124ca93a..111140cba 100644 --- a/proxyclient/m1n1/fw/dcp/iboot.py +++ b/proxyclient/m1n1/fw/dcp/iboot.py @@ -2,6 +2,7 @@ from construct import * from ...utils import * +from ...constructutils import Bool from ..asc import StandardASC from ..afk.epic import * from .dcpav import * diff --git a/proxyclient/m1n1/fw/dcp/ipc.py b/proxyclient/m1n1/fw/dcp/ipc.py index e73331ade..170d22726 100644 --- a/proxyclient/m1n1/fw/dcp/ipc.py +++ b/proxyclient/m1n1/fw/dcp/ipc.py @@ -7,7 +7,7 @@ from ..common import * from m1n1.utils import * from construct import * -from m1n1.constructutils import Ver +from m1n1.constructutils import Ver, Bool, bool_ @dataclass class ByRef: From f3f2ca0e4c823e679031f8a547f6c2fd80d26a3d Mon Sep 17 00:00:00 2001 From: Yureka Date: Thu, 18 Jun 2026 21:28:40 +0200 Subject: [PATCH 02/33] proxy: add P_GET_CPU_FEATURES Signed-off-by: Yureka --- src/proxy.c | 9 +++++++++ src/proxy.h | 1 + 2 files changed, 10 insertions(+) diff --git a/src/proxy.c b/src/proxy.c index 3f020c045..0c1501662 100644 --- a/src/proxy.c +++ b/src/proxy.c @@ -54,6 +54,15 @@ int proxy_process(ProxyRequest *request, ProxyReply *reply) case P_GET_BOOTARGS: reply->retval = boot_args_addr; break; + case P_GET_CPU_FEATURES: + if (request->args[0] == sizeof(struct midr_part_features)) + reply->retval = (u64)cpu_features; + else { + printf("size mismatch: sizeof(struct midr_part_features) = %ld != %ld\n", + sizeof(struct midr_part_features), request->args[0]); + reply->retval = 0; + } + break; case P_GET_BASE: reply->retval = (u64)_base; break; diff --git a/src/proxy.h b/src/proxy.h index 1e8acdf06..ea9992ee5 100644 --- a/src/proxy.h +++ b/src/proxy.h @@ -26,6 +26,7 @@ typedef enum { P_SLEEP, P_EL3_CALL, P_GET_CHIPID, + P_GET_CPU_FEATURES, P_WRITE64 = 0x100, // Generic register functions P_WRITE32, From 3b63a14064615e1107d2f44721e596b61c066121 Mon Sep 17 00:00:00 2001 From: Yureka Date: Thu, 18 Jun 2026 21:28:58 +0200 Subject: [PATCH 03/33] proxyclient: fetch and parse cpu_features Signed-off-by: Yureka --- proxyclient/m1n1/proxy.py | 36 ++++++++++++++++++++++++++++++++++ proxyclient/m1n1/proxyutils.py | 6 ++++++ 2 files changed, 42 insertions(+) diff --git a/proxyclient/m1n1/proxy.py b/proxyclient/m1n1/proxy.py index 16ed89467..ad9faf9a1 100644 --- a/proxyclient/m1n1/proxy.py +++ b/proxyclient/m1n1/proxy.py @@ -5,6 +5,7 @@ from serial.tools.miniterm import Miniterm from .utils import * +from .constructutils import bool_ from .sysreg import * __all__ = ["REGION_RWX_EL0", "REGION_RW_EL0", "REGION_RX_EL1"] @@ -480,6 +481,35 @@ class GUARD(IntFlag): REGION_RW_EL0 = 0xa0000000000 REGION_RX_EL1 = 0xc0000000000 +SleepMode = "SleepMode" / Enum(Int32ul, + SLEEP_NONE = 0, + SLEEP_LEGACY = 1, + SLEEP_GLOBAL = 2, +) + +UncoreVersion = "UncoreVersion" / Enum(Int32ul, + UNCORE_NONE = 0, + UNCORE_V1 = 1, + UNCORE_V2 = 2, +) + +CPUFeatures = Struct( + "sleep_mode" / SleepMode, + "uncore_version" / UncoreVersion, + "disable_dc_mva" / bool_, + "acc_cfg" / bool_, + "apple_sysregs_unlocked" / bool_, + "workaround_cyclone_cache" / bool_, + "nex_powergating" / bool_, + "fast_ipi" / bool_, + "mmu_sprr" / bool_, + "siq_cfg" / bool_, + "amx" / bool_, + "actlr_el2" / bool_, + "counter_redirect" / bool_, + "padding" / Bytes(1), +) + # Uses UartInterface.proxyreq() to send requests to M1N1 and process # responses sent back. class M1N1Proxy(Reloadable): @@ -506,6 +536,7 @@ class M1N1Proxy(Reloadable): P_SLEEP = 0x011 P_EL3_CALL = 0x012 P_GET_CHIPID = 0x013 + P_GET_CPU_FEATURES = 0x014 P_WRITE64 = 0x100 P_WRITE32 = 0x101 @@ -740,6 +771,11 @@ def get_bootargs_rev(self): return (ba_addr, rev) def get_base(self): return self.request(self.P_GET_BASE) + def get_cpu_features(self): + addr = self.request(self.P_GET_CPU_FEATURES, CPUFeatures.sizeof()) + if not addr: + raise ValueError("Size mismatch (Outdated CPUFeatures struct definition?)") + return self.iface.readstruct(addr, CPUFeatures) def set_baud(self, baudrate): self.iface.tty_enable = False def change(): diff --git a/proxyclient/m1n1/proxyutils.py b/proxyclient/m1n1/proxyutils.py index 0824fcf9e..ed95cad80 100644 --- a/proxyclient/m1n1/proxyutils.py +++ b/proxyclient/m1n1/proxyutils.py @@ -44,6 +44,12 @@ def __init__(self, p, heap_size=1024 * 1024 * 1024): self.iface = p.iface self.proxy = p self.base = p.get_base() + + try: + self.cpu_features = p.get_cpu_features() + except ProxyRemoteError: + pass + (self.ba_addr, self.ba_rev) = p.get_bootargs_rev() if self.ba_rev <= 1: From dca7c2a8f09376c844fad4212cbef5a0a5599133 Mon Sep 17 00:00:00 2001 From: Yureka Date: Thu, 18 Jun 2026 22:45:45 +0200 Subject: [PATCH 04/33] proxyclient/hv: add CPUSTART for T8132,T8140,T6034,T6040 Signed-off-by: Yureka --- proxyclient/m1n1/hv/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/proxyclient/m1n1/hv/__init__.py b/proxyclient/m1n1/hv/__init__.py index fd8595698..a2ab2b162 100644 --- a/proxyclient/m1n1/hv/__init__.py +++ b/proxyclient/m1n1/hv/__init__.py @@ -1562,11 +1562,11 @@ def cpustart_wh(base, off, data, width): chip_id = self.u.adt["/chosen"].chip_id if chip_id in (0x8103, 0x6000, 0x6001, 0x6002): cpu_start = 0x54000 + die * 0x20_0000_0000 - elif chip_id in (0x8112, 0x8122, 0x6030): + elif chip_id in (0x8112, 0x8122, 0x8132, 0x8140, 0x6030): cpu_start = 0x34000 + die * 0x20_0000_0000 elif chip_id in (0x6020, 0x6021, 0x6022): cpu_start = 0x28000 + die * 0x20_0000_0000 - elif chip_id in (0x6031,): + elif chip_id in (0x6031, 0x6034, 0x6040): cpu_start = 0x88000 + die * 0x20_0000_0000 else: self.log("CPUSTART unknown for this SoC!") From 37bb4480c3a5511dfeb6cf38f111577c8f029217 Mon Sep 17 00:00:00 2001 From: Yureka Date: Thu, 18 Jun 2026 22:45:47 +0200 Subject: [PATCH 05/33] hv: only write SPRR/GXF/AMX regs if apple_sysregs_unlocked Signed-off-by: Yureka --- src/hv.c | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/src/hv.c b/src/hv.c index 8236ce6cf..5b9cd0a77 100644 --- a/src/hv.c +++ b/src/hv.c @@ -193,7 +193,8 @@ void hv_start(void *entry, u64 regs[4]) static void hv_init_secondary(struct hv_secondary_info_t *info) { - gxf_init(); + if (cpu_features->apple_sysregs_unlocked) + gxf_init(); msr(VBAR_EL1, _hv_vectors_start); @@ -203,18 +204,23 @@ static void hv_init_secondary(struct hv_secondary_info_t *info) msr(VTTBR_EL2, info->vttbr); msr(MDCR_EL2, info->mdcr); msr(MDSCR_EL1, info->mdscr); - msr(SYS_IMP_APL_AMX_CTL_EL2, info->amx_ctl); - msr(SYS_IMP_APL_APVMKEYLO_EL2, info->apvmkeylo); - msr(SYS_IMP_APL_APVMKEYHI_EL2, info->apvmkeyhi); - msr(SYS_IMP_APL_APSTS_EL12, info->apsts); + + if (cpu_features->apple_sysregs_unlocked) { + msr(SYS_IMP_APL_AMX_CTL_EL2, info->amx_ctl); + msr(SYS_IMP_APL_APVMKEYLO_EL2, info->apvmkeylo); + msr(SYS_IMP_APL_APVMKEYHI_EL2, info->apvmkeyhi); + msr(SYS_IMP_APL_APSTS_EL12, info->apsts); + } msr(ACTLR_EL2, info->actlr_el2); if (cpu_features->actlr_el2) msr(SYS_ACTLR_EL12, info->actlr_el1); else msr(SYS_IMP_APL_ACTLR_EL12, info->actlr_el1); msr(CNTHCTL_EL2, info->cnthctl); - msr(SYS_IMP_APL_SPRR_CONFIG_EL1, info->sprr_config); - msr(SYS_IMP_APL_GXF_CONFIG_EL1, info->gxf_config); + if (cpu_features->apple_sysregs_unlocked) { + msr(SYS_IMP_APL_SPRR_CONFIG_EL1, info->sprr_config); + msr(SYS_IMP_APL_GXF_CONFIG_EL1, info->gxf_config); + } if (cpu_features->counter_redirect) { msr(SYS_IMP_APL_AGTCNTRDIR_EL1, info->agt_cnt_rdir_el1); msr(SYS_IMP_APL_AGTCNTRDIR_EL12, info->agt_cnt_rdir_el12); From fb2ed6e869a5a2d1efdb2c4361772abb69c96c69 Mon Sep 17 00:00:00 2001 From: Yureka Date: Thu, 18 Jun 2026 22:45:50 +0200 Subject: [PATCH 06/33] proxyclient/hv: only enable SPRR/GXF/AMX if apple_sysregs_unlocked Signed-off-by: Yureka --- proxyclient/m1n1/hv/__init__.py | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/proxyclient/m1n1/hv/__init__.py b/proxyclient/m1n1/hv/__init__.py index a2ab2b162..2e537a8d7 100644 --- a/proxyclient/m1n1/hv/__init__.py +++ b/proxyclient/m1n1/hv/__init__.py @@ -714,7 +714,10 @@ def handle_msr(self, ctx, iss=None): sys.stdout.flush() if enc in xlate: value = self.p.hv_translate(value, True, False) - self.u.msr(enc2, value, call=self.p.gl2_call) + call=None + if self.u.cpu_features.apple_sysregs_unlocked: + call=self.p.gl2_call + self.u.msr(enc2, value, call=call) self.log(f"Pass: msr {name}, x{iss.Rt} = {value:x} (OK) ({sysreg_name(enc2)})") ctx.elr += 4 @@ -1429,15 +1432,16 @@ def init(self): self.u.msr(MDCR_EL2, mdcr.value) self.u.msr(MDSCR_EL1, MDSCR(MDE=1).value) - # Enable AMX - amx_ctl = AMX_CONFIG(self.u.mrs(AMX_CONFIG_EL1)) - amx_ctl.EN_EL1 = 1 - self.u.msr(AMX_CONFIG_EL1, amx_ctl.value) + if self.u.cpu_features.apple_sysregs_unlocked: + # Enable AMX + amx_ctl = AMX_CONFIG(self.u.mrs(AMX_CONFIG_EL1)) + amx_ctl.EN_EL1 = 1 + self.u.msr(AMX_CONFIG_EL1, amx_ctl.value) - # Set guest AP keys - self.u.msr(VMKEYLO_EL2, 0x4E7672476F6E6147) - self.u.msr(VMKEYHI_EL2, 0x697665596F755570) - self.u.msr(APSTS_EL12, 1) + # Set guest AP keys + self.u.msr(VMKEYLO_EL2, 0x4E7672476F6E6147) + self.u.msr(VMKEYHI_EL2, 0x697665596F755570) + self.u.msr(APSTS_EL12, 1) self.map_vuart() @@ -1968,11 +1972,12 @@ def start(self): print("Shutting down framebuffer...") self.p.fb_shutdown(True) - print("Enabling SPRR...") - self.u.msr(SPRR_CONFIG_EL1, 1) + if self.u.cpu_features.apple_sysregs_unlocked: + print("Enabling SPRR...") + self.u.msr(SPRR_CONFIG_EL1, 1) - print("Enabling GXF...") - self.u.msr(GXF_CONFIG_EL1, 1) + print("Enabling GXF...") + self.u.msr(GXF_CONFIG_EL1, 1) print(f"Jumping to entrypoint at 0x{self.entry:x}") From dd49be16685600e92b59b6e789428a0d92a2468f Mon Sep 17 00:00:00 2001 From: Yureka Date: Fri, 19 Jun 2026 19:47:04 +0200 Subject: [PATCH 07/33] proxyclient/hv: handle RVBAR on !apple_sysregs_unlocked RVBAR (cpu_impl_reg+0) isn't writable on newer chips, so we can't set it to the guest entry, and there is no way the guest can choose a different secondary entry later on. Instead, return the guest entry on reads to make m1n1 happy, and use the initial guest entry as secondary entry. Signed-off-by: Yureka --- proxyclient/m1n1/hv/__init__.py | 35 ++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/proxyclient/m1n1/hv/__init__.py b/proxyclient/m1n1/hv/__init__.py index 2e537a8d7..a24cb547b 100644 --- a/proxyclient/m1n1/hv/__init__.py +++ b/proxyclient/m1n1/hv/__init__.py @@ -1580,6 +1580,19 @@ def cpustart_wh(base, off, data, width): self.map_hook(pmgr0_start + cpu_start, 0x20, write=cpustart_wh) self.add_tracer(zone, "CPU_START", TraceMode.RESERVED) + if not self.u.cpu_features.apple_sysregs_unlocked: + + def rvbar_rh(base, off, width): + ret = self.entry & ~0xfff | 1 + self.log(f"RVBAR R {base:x}+{off:x}:{width} -> 0x{ret:x}") + return ret + + for cpu in self.adt["cpus"]: + addr, _ = cpu.cpu_impl_reg + zone = irange(addr, 4) + self.map_hook(addr, 4, read=rvbar_rh) + self.add_tracer(zone, "RVBAR", TraceMode.RESERVED) + def start_secondary(self, die, cluster, cpu): self.log(f"Starting guest secondary {die}:{cluster}:{cpu}") @@ -1590,7 +1603,10 @@ def start_secondary(self, die, cluster, cpu): self.log("CPU not found!") return - entry = self.p.read64(node.cpu_impl_reg[0]) & 0xfffffffffff + if self.u.cpu_features.apple_sysregs_unlocked: + entry = self.p.read64(node.cpu_impl_reg[0]) & 0xfffffffffff + else: + entry = self.entry & ~0xfff index = node.cpu_id self.log(f" CPU #{index}: RVBAR = {entry:#x}") @@ -1783,14 +1799,15 @@ def remove_oslog(node): elif self.tba.revision == 3: self.iface.writemem(guest_base + self.bootargs_off, BootArgs_r3.build(self.tba)) - print("Setting secondary CPU RVBARs...") - rvbar = self.entry & ~0xfff - for cpu in self.adt["cpus"]: - if cpu.state == "running": - continue - addr, size = cpu.cpu_impl_reg - print(f" {cpu.name}: [0x{addr:x}] = 0x{rvbar:x}") - self.p.write64(addr, rvbar) + if self.u.cpu_features.apple_sysregs_unlocked: + print("Setting secondary CPU RVBARs...") + rvbar = self.entry & ~0xfff + for cpu in self.adt["cpus"]: + if cpu.state == "running": + continue + addr, size = cpu.cpu_impl_reg + print(f" {cpu.name}: [0x{addr:x}] = 0x{rvbar:x}") + self.p.write64(addr, rvbar) def _load_macho_symbols(self): self.symbol_dict = self.macho.symbols From 42b932b4935b5ccb751f218cc02caa55f566a05b Mon Sep 17 00:00:00 2001 From: Yureka Date: Fri, 19 Jun 2026 19:47:09 +0200 Subject: [PATCH 08/33] proxyclient/hv: use pmgr_dev_get_addr helper This fixes use with the new pmgr on A18 Pro / M4 Pro. Signed-off-by: Yureka --- proxyclient/m1n1/hv/__init__.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/proxyclient/m1n1/hv/__init__.py b/proxyclient/m1n1/hv/__init__.py index a24cb547b..baf5bf79a 100644 --- a/proxyclient/m1n1/hv/__init__.py +++ b/proxyclient/m1n1/hv/__init__.py @@ -1491,13 +1491,11 @@ def rh(base, off, width): pmgr_hooks = [] def hook_pmgr_dev(dev): - ps = pmgr.ps_regs[dev.psreg] - if dev.psidx or dev.psreg: - addr = pmgr.get_reg(ps.reg)[0] + ps.offset + dev.psidx * 8 - pmgr_hooks.append(addr) - for idx in self.adt.pmgr_dev_get_parents(dev): - if idx in dev_by_id: - hook_pmgr_dev(dev_by_id[idx]) + addr = self.adt.pmgr_dev_get_addr(dev) + pmgr_hooks.append(addr) + for idx in self.adt.pmgr_dev_get_parents(dev): + if idx in dev_by_id: + hook_pmgr_dev(dev_by_id[idx]) for name in hook_devs: dev = dev_by_name[name] From df7656cf17d04ec19de8acb8a78d57cc9b48fda1 Mon Sep 17 00:00:00 2001 From: Cody Ho Date: Wed, 8 Jul 2026 16:42:12 -0700 Subject: [PATCH 09/33] chainload: skip RVBAR write if locked Writing a locked RVBAR causes an SError, so skip the write. Signed-off-by: Cody Ho --- proxyclient/tools/chainload.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/proxyclient/tools/chainload.py b/proxyclient/tools/chainload.py index 414b548cf..d01d706d1 100755 --- a/proxyclient/tools/chainload.py +++ b/proxyclient/tools/chainload.py @@ -99,7 +99,12 @@ def remove_oslog(node): for cpu in u.adt["cpus"]: if cpu.state == "running": continue + addr, size = cpu.cpu_impl_reg + if p.read64(addr) & 1: + print(f" {cpu.name}: [0x{addr:x}] RVBAR locked, skipping") + continue + print(f" {cpu.name}: [0x{addr:x}] = 0x{rvbar:x}") p.write64(addr, rvbar) From 7a19e541372c7df2b69083465f24eea46f9cbbd1 Mon Sep 17 00:00:00 2001 From: Cody Ho Date: Wed, 8 Jul 2026 19:28:58 -0700 Subject: [PATCH 10/33] dapf: search all instances for config T8140 also needs DAPF programmed for more DARTs. Other SoCs gate on the ADT, so this will not break existing chips. Signed-off-by: Cody Ho --- proxyclient/m1n1/proxy.py | 4 ++-- src/dapf.c | 27 ++++++++++++++++++++------- src/proxy.c | 2 +- 3 files changed, 23 insertions(+), 10 deletions(-) diff --git a/proxyclient/m1n1/proxy.py b/proxyclient/m1n1/proxy.py index ad9faf9a1..b75f5268b 100644 --- a/proxyclient/m1n1/proxy.py +++ b/proxyclient/m1n1/proxy.py @@ -1189,8 +1189,8 @@ def display_is_external(self): def dapf_init_all(self): return self.request(self.P_DAPF_INIT_ALL) - def dapf_init(self, path): - return self.request(self.P_DAPF_INIT, path) + def dapf_init(self, path, index=1): + return self.request(self.P_DAPF_INIT, path, index) def cpufreq_init(self): return self.request(self.P_CPUFREQ_INIT) diff --git a/src/dapf.c b/src/dapf.c index 3b4b773dc..a39edba9b 100644 --- a/src/dapf.c +++ b/src/dapf.c @@ -21,7 +21,7 @@ struct dapf_t8020_config { static int dapf_init_t8020(const char *path, u64 base, int node) { - u32 length; + u32 length = 0; const char *prop = "filter-data-instance-0"; const struct dapf_t8020_config *config = adt_getprop(adt, node, prop, &length); @@ -101,12 +101,22 @@ static int dapf_init_t8110b(u64 base, struct dapf_t8110b_config *config, u32 len static int dapf_init_t8110(const char *path, u64 base, int node) { - u32 length; - const char *prop = "dapf-instance-0"; - const void *config = adt_getprop(adt, node, prop, &length); + u32 length = 0; + char prop[32]; + const void *config = NULL; + + // The dapf config is not always under instance 0; find the first present + // dapf-instance-N. + for (int i = 0; i < 8; i++) { + snprintf(prop, sizeof(prop), "dapf-instance-%d", i); + length = 0; + config = adt_getprop(adt, node, prop, &length); + if (config && length) + break; + } if (!config || !length) { - printf("dapf: Error getting ADT node %s property %s.\n", path, prop); + printf("dapf: Error getting ADT node %s dapf-instance property.\n", path); return -1; } @@ -171,8 +181,11 @@ struct entry { }; struct entry dapf_entries[] = { - {"/arm-io/dart-aop", 1}, {"/arm-io/dart-mtp", 1}, {"/arm-io/dart-pmp", 1}, - {"/arm-io/dart-isp", 5}, {"/arm-io/dart-isp0", 5}, {NULL, -1}, + {"/arm-io/dart-aop", 1}, {"/arm-io/dart-mtp", 1}, + {"/arm-io/dart-pmp", 1}, {"/arm-io/dart-dcp", 1}, + {"/arm-io/dart-dcpext0", 1}, {"/arm-io/dart-isp", 5}, + {"/arm-io/dart-isp0", 5}, {"/arm-io/dart-ane", 3}, + {"/arm-io/dart-ave", 3}, {NULL, -1}, }; int dapf_init_all(void) diff --git a/src/proxy.c b/src/proxy.c index 0c1501662..a30e60e44 100644 --- a/src/proxy.c +++ b/src/proxy.c @@ -614,7 +614,7 @@ int proxy_process(ProxyRequest *request, ProxyReply *reply) reply->retval = dapf_init_all(); break; case P_DAPF_INIT: - reply->retval = dapf_init((const char *)request->args[0], 1); + reply->retval = dapf_init((const char *)request->args[0], request->args[1]); break; case P_CPUFREQ_INIT: From 20cb4ff035fcd9aea21b4396ec11ac65cb4489be Mon Sep 17 00:00:00 2001 From: Cody Ho Date: Fri, 10 Jul 2026 00:48:38 -0700 Subject: [PATCH 11/33] mcc: fall back to ADT protected carveouts Some platforms do not expose usable MCC TZ window registers to m1n1, but iBoot still publishes the protected memory ranges under /chosen/carveout-memory-map. Factor the unmap/bookkeeping logic into a helper and, if the MCC TZ register walk yields no regions, unmap region-id-4 and region-id-2 from the ADT carveout map instead. Existing SoCs keep the register-derived path; the ADT fallback only runs when no MCC TZ carveouts were recorded. Signed-off-by: Cody Ho --- src/mcc.c | 98 +++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 69 insertions(+), 29 deletions(-) diff --git a/src/mcc.c b/src/mcc.c index 375388399..15d072f6b 100644 --- a/src/mcc.c +++ b/src/mcc.c @@ -176,6 +176,48 @@ int mcc_enable_cache(void) return ret; } +static void mcc_unmap_carveout(const char *name, u64 start, u64 size) +{ + if (!start || !size) + return; + + if (mcc_carveout_count >= ARRAY_SIZE(mcc_carveouts) - 1) { + printf("MMU: carveout list full, cannot record %s at 0x%lx..0x%lx\n", name, start, + start + size); + return; + } + + printf("MMU: Unmapping %s carveout at 0x%lx..0x%lx\n", name, start, start + size); + mmu_rm_mapping(start, size); + mmu_rm_mapping(start | REGION_RWX_EL0, size); + mmu_rm_mapping(start | REGION_RW_EL0, size); + mmu_rm_mapping(start | REGION_RX_EL1, size); + mcc_carveouts[mcc_carveout_count].base = start; + mcc_carveouts[mcc_carveout_count].size = size; + mcc_carveout_count++; +} + +static void mcc_unmap_adt_carveout(int node, const char *name) +{ + u64 range[2]; + + if (ADT_GETPROP_ARRAY(adt, node, name, range) != sizeof(range)) + return; + + mcc_unmap_carveout(name, range[0], range[1]); +} + +static void mcc_unmap_adt_tz_carveouts(void) +{ + int node = adt_path_offset(adt, "/chosen/carveout-memory-map"); + + if (node < 0) + return; + + mcc_unmap_adt_carveout(node, "region-id-4"); + mcc_unmap_adt_carveout(node, "region-id-2"); +} + int mcc_unmap_carveouts(void) { if (!mcc_initialized) @@ -184,39 +226,37 @@ int mcc_unmap_carveouts(void) mcc_carveout_count = 0; memset(mcc_carveouts, 0, sizeof mcc_carveouts); - // All MCCs and planes should have identical configs - // Note: For unhandled machines, the TZ regions can be found (on m1, m2, m3) by looking at - // region-id-2 and region-id-4 on a booted macos, in the /chosen/carveout-memory-map DT node. - // This can be used along with dumping the mcc reg space to find the correct start/end/enable - // above. - for (u32 i = 0; i < mcc_regs[0].tz->count; i++) { - uint64_t off = mcc_regs[0].tz->stride * i; - uint64_t start = plane_read32(0, 0, mcc_regs[0].tz->start + off); - uint64_t end = plane_read32(0, 0, mcc_regs[0].tz->end + off); - bool enabled = plane_read32(0, 0, mcc_regs[0].tz->enable + off); - - if (enabled) { - if (!start || start == end) { - printf("MMU: TZ%d region has bad bounds 0x%lx..0x%lx (iBoot bug?)\n", i, start, - end); - continue; + if (mcc_regs[0].tz) { + // All MCCs and planes should have identical configs + // Note: For unhandled machines, the TZ regions can be found (on m1, m2, m3) by looking at + // region-id-2 and region-id-4 on a booted macos, in the /chosen/carveout-memory-map DT node. + // This can be used along with dumping the mcc reg space to find the correct start/end/enable + // above. + for (u32 i = 0; i < mcc_regs[0].tz->count; i++) { + uint64_t off = mcc_regs[0].tz->stride * i; + uint64_t start = plane_read32(0, 0, mcc_regs[0].tz->start + off); + uint64_t end = plane_read32(0, 0, mcc_regs[0].tz->end + off); + bool enabled = plane_read32(0, 0, mcc_regs[0].tz->enable + off); + + if (enabled) { + if (!start || start == end) { + printf("MMU: TZ%d region has bad bounds 0x%lx..0x%lx (iBoot bug?)\n", i, + start, end); + continue; + } + + start = start << 12; + end = (end + 1) << 12; + start |= ram_base; + end |= ram_base; + mcc_unmap_carveout("TZ", start, end - start); } - - start = start << 12; - end = (end + 1) << 12; - start |= ram_base; - end |= ram_base; - printf("MMU: Unmapping TZ%d region at 0x%lx..0x%lx\n", i, start, end); - mmu_rm_mapping(start, end - start); - mmu_rm_mapping(start | REGION_RWX_EL0, end - start); - mmu_rm_mapping(start | REGION_RW_EL0, end - start); - mmu_rm_mapping(start | REGION_RX_EL1, end - start); - mcc_carveouts[mcc_carveout_count].base = start; - mcc_carveouts[mcc_carveout_count].size = end - start; - mcc_carveout_count++; } } + if (!mcc_carveout_count) + mcc_unmap_adt_tz_carveouts(); + return 0; } From bf7d4efa7fa5df76a87fcdb65c5ea2e1950c1e3f Mon Sep 17 00:00:00 2001 From: Cody Ho Date: Fri, 10 Jul 2026 00:49:17 -0700 Subject: [PATCH 12/33] mcc: add T8140 AMCC initialization Add the T8140 AMCC register discovery path. T8140 publishes the AMCC register index and topology through ADT properties and /chosen/lock-regs/amcc rather than matching the older MCC register layouts. The existing cache enable path controls MCC/system-level-cache registers, not CPU caches. Mark older supported layouts as having that cache-control interface, and let T8140 skip it because this boot path does not have usable PLANE_CACHE_ENABLE/STATUS programming for the guarded AMCC state. T8140 also leaves tz unset so protected carveouts are taken from the ADT fallback added by the previous commit. Signed-off-by: Cody Ho --- src/mcc.c | 73 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/src/mcc.c b/src/mcc.c index 15d072f6b..9d75f79bd 100644 --- a/src/mcc.c +++ b/src/mcc.c @@ -29,6 +29,8 @@ static bool mcc_initialized = false; #define T6031_DCS_OFFSET 0x400000 #define T6031_DCS_STRIDE 0x200000 +#define T8140_PLANE_STRIDE 0x40000 + #define PLANE_TZ_MAX_REGS 4 struct tz_regs { @@ -124,6 +126,7 @@ struct mcc_regs { u32 cache_status_mask; u32 cache_status_val; u32 cache_disable; + bool has_cache_ctrl; struct tz_regs *tz; }; @@ -154,6 +157,11 @@ int mcc_enable_cache(void) if (!mcc_initialized) return -1; + if (!mcc_regs[0].has_cache_ctrl) { + printf("MCC: cache enable skipped (unsupported)\n"); + return 0; + } + /* The 6030 memory controller supports setting a waymask, but the desktop chips do not appear to use it */ for (int mcc = 0; mcc < mcc_count; mcc++) { @@ -296,6 +304,7 @@ int mcc_init_t8103(int node, int *path, bool t8112) mcc_regs[0].cache_status_mask = T8103_CACHE_STATUS_MASK; mcc_regs[0].cache_status_val = T8103_CACHE_STATUS_VAL; mcc_regs[0].cache_disable = t8112 ? T8112_CACHE_DISABLE : 0; + mcc_regs[0].has_cache_ctrl = true; mcc_regs[0].tz = &t8103_tz_regs; printf("MCC: Initialized T8103 MCC (%d channels)\n", val); @@ -346,6 +355,7 @@ int mcc_init_t6000(int node, int *path, bool t602x) mcc_regs[i].cache_status_mask = T6000_CACHE_STATUS_MASK; mcc_regs[i].cache_status_val = T6000_CACHE_STATUS_VAL; mcc_regs[i].cache_disable = 0; + mcc_regs[i].has_cache_ctrl = true; mcc_regs[i].tz = t602x ? &t602x_tz_regs : &t8103_tz_regs; } @@ -412,6 +422,7 @@ int mcc_init_t6031(int node, int *path) mcc_regs[i].cache_status_mask = T6031_CACHE_STATUS_MASK; mcc_regs[i].cache_status_val = T6031_CACHE_STATUS_VAL; mcc_regs[i].cache_disable = 0; + mcc_regs[i].has_cache_ctrl = true; mcc_regs[i].tz = &t603x_tz_regs; } @@ -424,6 +435,66 @@ int mcc_init_t6031(int node, int *path) return 0; } +int mcc_init_t8140(int node, int *path) +{ + u32 amcc_count = 1; + u32 amcc_reg_idx = 0; + u32 dcs_count = 0; + u32 plane_count = 0; + u32 plane_stride = T8140_PLANE_STRIDE; + int lock_node; + + printf("MCC: Initializing T8140 AMCCs...\n"); + + if (ADT_GETPROP(adt, node, "amcc-reg-idx", &amcc_reg_idx) < 0) { + printf("MCC: Failed to get amcc-reg-idx property!\n"); + return -1; + } + + ADT_GETPROP(adt, node, "amcc-count", &amcc_count); + ADT_GETPROP(adt, node, "dcs-count-per-amcc", &dcs_count); + + lock_node = adt_path_offset(adt, "/chosen/lock-regs/amcc"); + if (lock_node >= 0) { + ADT_GETPROP(adt, lock_node, "plane-count", &plane_count); + ADT_GETPROP(adt, lock_node, "plane-stride", &plane_stride); + } + if (!plane_count) { + u32 plane_mask = 0; + + if (ADT_GETPROP(adt, node, "amcc-plane-enable-mask", &plane_mask) > 0) + plane_count = __builtin_popcount(plane_mask); + } + if (!plane_count) + plane_count = 4; + + mcc_count = amcc_count; + if (mcc_count > MAX_MCC_INSTANCES) { + printf("MCC: Too many instances, increase MAX_MCC_INSTANCES!\n"); + mcc_count = MAX_MCC_INSTANCES; + } + + for (int i = 0; i < mcc_count; i++) { + if (adt_get_reg(adt, path, "reg", amcc_reg_idx + i, &mcc_regs[i].plane_base, NULL)) { + printf("MCC: Failed to get AMCC reg index %d!\n", amcc_reg_idx + i); + return -1; + } + + mcc_regs[i].plane_stride = plane_stride; + mcc_regs[i].plane_count = plane_count; + mcc_regs[i].dcs_count = dcs_count; + mcc_regs[i].has_cache_ctrl = false; + mcc_regs[i].tz = NULL; + } + + printf("MCC: Initialized T8140 AMCCs (%d instances, %d planes, %d channels)\n", mcc_count, + mcc_regs[0].plane_count, mcc_regs[0].dcs_count); + + mcc_initialized = true; + + return 0; +} + int mcc_init(void) { int path[8]; @@ -444,6 +515,8 @@ int mcc_init(void) return mcc_init_t6000(node, path, true); } else if (adt_is_compatible(adt, node, "mcc,t6031")) { return mcc_init_t6031(node, path); + } else if (adt_is_compatible(adt, node, "mcc,t8140")) { + return mcc_init_t8140(node, path); } else { printf("MCC: Unsupported version:%s\n", adt_get_property(adt, node, "compatible")->value); return -1; From d919a38d59e8f5e1dd167abef6fb66b0dbe60336 Mon Sep 17 00:00:00 2001 From: Cody Ho Date: Thu, 9 Jul 2026 00:32:38 -0700 Subject: [PATCH 13/33] memory: map iBoot handoff and framebuffer Normal-NC for AMCC coherency m1n1 maps RAM Normal-WB, which allocates AMCC SLC directory tags. Two regions are read non-coherently after handoff and then hit those stale tags, raising AMCC UNEXP_RT_HIT_DIR / RO_RGN_ACCESS_VIO: - The iBoot handoff carveout (/chosen/iboot-handoff, or carveout region-id-1): keep m1n1's aliases uncached over that sub-range so no new WB state is added for the iBoot-RO-locked page. - The boot framebuffer (/vram): XNU's non-coherent framebuffer writes hit stale SLC tags -> PEH panic. Remap it Normal-NC (which also cleans the lines), avoiding the panic without disabling the SLC. Signed-off-by: Cody Ho --- src/memory.c | 127 +++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 123 insertions(+), 4 deletions(-) diff --git a/src/memory.c b/src/memory.c index e6e1abaec..f25ba52c6 100644 --- a/src/memory.c +++ b/src/memory.c @@ -429,11 +429,103 @@ void mmu_map_framebuffer(u64 addr, size_t size) mmu_add_mapping(addr, addr, size, MAIR_IDX_NORMAL_NC, PERM_RW_EL0); } +int display_get_vram(u64 *paddr, u64 *size); + +static int mmu_get_iboot_handoff_range(u64 *start, u64 *size) +{ + u64 range[2]; + int chosen = adt_path_offset(adt, "/chosen"); + + if (chosen >= 0 && + ADT_GETPROP_ARRAY(adt, chosen, "iboot-handoff", range) == sizeof(range)) + goto found; + + int carveouts = adt_path_offset(adt, "/chosen/carveout-memory-map"); + if (carveouts >= 0 && + ADT_GETPROP_ARRAY(adt, carveouts, "region-id-1", range) == sizeof(range)) + goto found; + + return -1; + +found: + if (!range[0] || !range[1]) + return -1; + + *start = ALIGN_DOWN(range[0], PAGE_SIZE); + *size = ALIGN_UP(range[0] + range[1], PAGE_SIZE) - *start; + return 0; +} + +static void mmu_add_ram_mapping_handoff_nc(u64 from, u64 to, size_t size, + u8 attribute_index, u64 perms, + u64 handoff_start, + u64 handoff_size) +{ + u64 end = to + size; + u64 handoff_end = handoff_start + handoff_size; + + if (!handoff_start || !handoff_size || handoff_end <= to || + handoff_start >= end) { + mmu_add_mapping(from, to, size, attribute_index, perms); + return; + } + + u64 cur_to = to; + if (handoff_start > cur_to) { + u64 before_size = handoff_start - cur_to; + mmu_add_mapping(from, cur_to, before_size, attribute_index, perms); + from += before_size; + cur_to += before_size; + } + + u64 nc_end = min(handoff_end, end); + u64 nc_size = nc_end - cur_to; + mmu_add_mapping(from, cur_to, nc_size, MAIR_IDX_NORMAL_NC, perms); + from += nc_size; + cur_to += nc_size; + + if (cur_to < end) + mmu_add_mapping(from, cur_to, end - cur_to, attribute_index, perms); +} + +static void mmu_map_nc_aliases(u64 addr, size_t size, const char *name) +{ + u64 start = ALIGN_DOWN(addr, PAGE_SIZE); + u64 end = ALIGN_UP(addr + size, PAGE_SIZE); + size_t map_size = end - start; + + if (!addr || !size) + return; + + printf("MMU: Adding Normal-NC mappings at 0x%lx..0x%lx for %s\n", + start, end, name); + mmu_add_mapping(start, start, map_size, MAIR_IDX_NORMAL_NC, PERM_RWX); + mmu_add_mapping(start | REGION_RWX_EL0, start, map_size, + MAIR_IDX_NORMAL_NC, PERM_RWX_EL0); + mmu_add_mapping(start | REGION_RW_EL0, start, map_size, + MAIR_IDX_NORMAL_NC, PERM_RW_EL0); + mmu_add_mapping(start | REGION_RX_EL1, start, map_size, + MAIR_IDX_NORMAL_NC, PERM_RX_EL0); +} + +static void mmu_map_iboot_handoff_nc(void) +{ + u64 start, size; + + if (mmu_get_iboot_handoff_range(&start, &size) < 0) + return; + + mmu_map_nc_aliases(start, size, "/chosen/iboot-handoff"); + +} + static void mmu_add_default_mappings(void) { ram_base = ALIGN_DOWN(cur_boot_args.phys_base, BIT(32)); uint64_t ram_size = cur_boot_args.mem_size + cur_boot_args.phys_base - ram_base; ram_size = ALIGN_DOWN(ram_size, PAGE_SIZE); + u64 handoff_start = 0, handoff_size = 0; + mmu_get_iboot_handoff_range(&handoff_start, &handoff_size); printf("MMU: RAM base: 0x%lx\n", ram_base); printf("MMU: Top of normal RAM: 0x%lx\n", ram_base + ram_size); @@ -445,11 +537,25 @@ static void mmu_add_default_mappings(void) * With SPRR enabled, this becomes RW. * This range includes all real RAM, including carveouts */ - mmu_add_mapping(ram_base, ram_base, mem_size_actual, MAIR_IDX_NORMAL, PERM_RWX); + mmu_add_ram_mapping_handoff_nc(ram_base, ram_base, mem_size_actual, + MAIR_IDX_NORMAL, PERM_RWX, + handoff_start, handoff_size); /* Unmap carveout regions */ mcc_unmap_carveouts(); + /* + * Re-map the boot framebuffer (/vram) Normal-NC. The Normal-WB identity map + * lets m1n1/iBoot cacheable accesses allocate AMCC SLC directory tags + * for /vram; after handoff XNU's non-coherent framebuffer writes hit those + * stale tags and the AMCC raises UNEXP_RT_HIT_DIR -> PEH panic. + */ + { + u64 fb_pa = 0, fb_size = 0; + if (display_get_vram(&fb_pa, &fb_size) == 0 && fb_pa && fb_size) + mmu_map_framebuffer(fb_pa, fb_size); + } + /* * Remap m1n1 executable code as RX. */ @@ -466,19 +572,32 @@ static void mmu_add_default_mappings(void) * read/writable/exec by EL0 (but not executable by EL1) * With SPRR enabled, this becomes RX_EL0. */ - mmu_add_mapping(ram_base | REGION_RWX_EL0, ram_base, ram_size, MAIR_IDX_NORMAL, PERM_RWX_EL0); + mmu_add_ram_mapping_handoff_nc(ram_base | REGION_RWX_EL0, ram_base, + ram_size, MAIR_IDX_NORMAL, PERM_RWX_EL0, + handoff_start, handoff_size); /* * Create mapping for RAM from 0x98_0000_0000, * read/writable by EL0 (but not executable by EL1) * With SPRR enabled, this becomes RW_EL0. */ - mmu_add_mapping(ram_base | REGION_RW_EL0, ram_base, ram_size, MAIR_IDX_NORMAL, PERM_RW_EL0); + mmu_add_ram_mapping_handoff_nc(ram_base | REGION_RW_EL0, ram_base, + ram_size, MAIR_IDX_NORMAL, PERM_RW_EL0, + handoff_start, handoff_size); /* * Create mapping for RAM from 0xa8_0000_0000, * read/executable by EL1 * This allows executing from dynamic regions in EL1 */ - mmu_add_mapping(ram_base | REGION_RX_EL1, ram_base, ram_size, MAIR_IDX_NORMAL, PERM_RX_EL0); + mmu_add_ram_mapping_handoff_nc(ram_base | REGION_RX_EL1, ram_base, + ram_size, MAIR_IDX_NORMAL, PERM_RX_EL0, + handoff_start, handoff_size); + + /* + * The iBoot handoff page is a protected low carveout on T8140 + * (/chosen/iboot-handoff, region-id-1). Keep m1n1's aliases uncached so + * we do not add any new WB state for this page. + */ + mmu_map_iboot_handoff_nc(); /* * Create four separate full mappings of MMIO space, with different access types From de6e294c6823f2693cd51783f9104fc642923c79 Mon Sep 17 00:00:00 2001 From: Cody Ho Date: Thu, 9 Jul 2026 01:22:20 -0700 Subject: [PATCH 14/33] xnutools: add XNU msgbuf dump helper Add find_msgbuf_va() and dump_msgbuf() to recover the XNU kernel message buffer when the serial/panic path is unavailable: Signed-off-by: Cody Ho --- proxyclient/m1n1/xnutools.py | 184 +++++++++++++++++++++++++++++++++++ 1 file changed, 184 insertions(+) diff --git a/proxyclient/m1n1/xnutools.py b/proxyclient/m1n1/xnutools.py index f7fb59fa8..4c04027b1 100644 --- a/proxyclient/m1n1/xnutools.py +++ b/proxyclient/m1n1/xnutools.py @@ -1,7 +1,12 @@ # SPDX-License-Identifier: MIT +import pathlib import re +import struct +import time from construct import * +from .macho import MachOLoadCmdType + __all__ = [] DebuggerState = Struct( @@ -113,5 +118,184 @@ def format_arg(match): format_arg, string) print(string + "\n", end="") +_msgbuf_va_cache = {} +_msgbuf_dump_seq = 0 +MAX_MSGBUF_DUMP_SIZE = 8 * 1024 * 1024 + + +def _macho_bytes(macho): + pos = macho.io.tell() + try: + macho.io.seek(macho.off) + return macho.io.read(macho.size) + finally: + macho.io.seek(pos) + + +def _macho_fileoff_to_va(macho, fileoff): + for image in [macho] + list(getattr(macho, "subfiles", {}).values()): + for cmd in image.get_cmds(MachOLoadCmdType.SEGMENT_64): + start = image.off + cmd.args.fileoff + end = start + cmd.args.filesize + if start <= fileoff < end: + return cmd.args.vmaddr + (fileoff - start) + return None + + +def find_msgbuf_va(macho): + cached = _msgbuf_va_cache.get(id(macho), False) + if cached is not False: + return cached + + data = _macho_bytes(macho) + magic = struct.pack("> 56) == 0xff + + +def _strip_pac(va): + va = int(va) + if (va & 0xffffff00_00000000) == 0xfffffe00_00000000: + return va + return (va & 0x000000ff_ffffffff) | 0xfffffe00_00000000 + + +def _decode_msgbuf(raw, msg_bufx, msg_bufr): + size = len(raw) + if size == 0: + return b"" + + def printable_score(blob): + if not blob: + return 0.0 + keep = sum(1 for b in blob if b in (9, 10, 13) or 0x20 <= b < 0x7f) + return keep / max(1, len(blob)) + + ordered = raw[:msg_bufx] if 0 <= msg_bufx <= size else raw + if 0 <= msg_bufx < size and any(raw[msg_bufx:]): + ring = raw[msg_bufx:] + raw[:msg_bufx] + if printable_score(ring.replace(b"\x00", b"")) >= 0.70: + ordered = ring + if 0 <= msg_bufr < size and 0 <= msg_bufx < size and msg_bufr != msg_bufx: + unread = raw[msg_bufr:msg_bufx] if msg_bufr < msg_bufx else raw[msg_bufr:] + raw[:msg_bufx] + if printable_score(unread.replace(b"\x00", b"")) >= 0.70: + ordered = unread + + return ordered.replace(b"\x00", b"") + + +def dump_msgbuf(u, macho, dump_dir="logs", tail_lines=80): + global _msgbuf_dump_seq + + msgbuf_va = find_msgbuf_va(macho) + if msgbuf_va is None: + return None + + hdr = _read_kernel_va(u, msgbuf_va, 24) + if hdr is None or len(hdr) < 24: + print(f"[host] XNU-MSGBUF: header read failed at 0x{msgbuf_va:x}", + flush=True) + return None + + magic, msg_size, msg_bufx, msg_bufr, msg_bufc = struct.unpack(" MAX_MSGBUF_DUMP_SIZE: + print(f"[host] XNU-MSGBUF: invalid header magic=0x{magic:x} " + f"size={msg_size} bufx={msg_bufx} bufr={msg_bufr} " + f"bufc=0x{msg_bufc:x}", flush=True) + return None + + if not _is_kernel_va(msg_bufc): + msg_bufc = _strip_pac(msg_bufc) + if not _is_kernel_va(msg_bufc): + print(f"[host] XNU-MSGBUF: non-kernel msg_bufc=0x{msg_bufc:x}", + flush=True) + return None + + raw = _read_kernel_va(u, msg_bufc, msg_size) + if raw is None: + print(f"[host] XNU-MSGBUF: buffer read failed at 0x{msg_bufc:x} " + f"size=0x{msg_size:x}", flush=True) + return None + + text_bytes = _decode_msgbuf(raw, msg_bufx, msg_bufr) + text = text_bytes.decode("utf-8", errors="replace") + if text and not text.endswith("\n"): + text += "\n" + + dump_dir = pathlib.Path(dump_dir) + dump_dir.mkdir(parents=True, exist_ok=True) + stamp = time.strftime("%Y%m%d_%H%M%S") + _msgbuf_dump_seq += 1 + stem = f"xnu_msgbuf_{stamp}_{_msgbuf_dump_seq:02d}" + raw_path = dump_dir / f"{stem}.raw" + txt_path = dump_dir / f"{stem}.txt" + raw_path.write_bytes(raw) + txt_path.write_text(text, encoding="utf-8", errors="replace") + + print(f"[host] XNU-MSGBUF: dumped {len(text_bytes)} text bytes " + f"(raw {msg_size} bytes) msgbuf=0x{msgbuf_va:x} " + f"bufc=0x{msg_bufc:x} bufx={msg_bufx} bufr={msg_bufr} " + f"path={txt_path}", flush=True) + tail = [line for line in text.splitlines() if line.strip()] + if tail and tail_lines: + print(f"[host] XNU-MSGBUF tail ({min(tail_lines, len(tail))}/{len(tail)} lines):", + flush=True) + for line in tail[-tail_lines:]: + print(f"[xnu-log] {line}", flush=True) + return txt_path + + __all__.extend(k for k, v in globals().items() if (callable(v) or isinstance(v, type)) and v.__module__ == __name__) From c9a9f966b9447d760540348f7e0474e13f0f9876 Mon Sep 17 00:00:00 2001 From: Cody Ho Date: Thu, 9 Jul 2026 18:20:56 -0700 Subject: [PATCH 15/33] hv: skip ATC PMGR hooks for on debugusb Signed-off-by: Cody Ho --- proxyclient/m1n1/hv/__init__.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/proxyclient/m1n1/hv/__init__.py b/proxyclient/m1n1/hv/__init__.py index baf5bf79a..bbfe37bca 100644 --- a/proxyclient/m1n1/hv/__init__.py +++ b/proxyclient/m1n1/hv/__init__.py @@ -1479,10 +1479,12 @@ def rh(base, off, width): self.log(f"PMGR R {base:x}+{off:x}:{width} = 0x{data:x} -> 0x{ret:x}") return ret - atc = f"ATC{self.iodev - IODEV.USB0}_USB" - atc_aon = f"ATC{self.iodev - IODEV.USB0}_USB_AON" - - hook_devs = ["UART0", atc, atc_aon] + # When the proxy is on the debugusb link there is no ATC PHY to protect + # and the index would go negative (ie, ATC-3_USB) + hook_devs = ["UART0"] + if self.iodev >= IODEV.USB0: + idx = self.iodev - IODEV.USB0 + hook_devs += [f"ATC{idx}_USB", f"ATC{idx}_USB_AON"] pmgr = self.adt["/arm-io/pmgr"] dev_by_name = {dev.name: dev for dev in pmgr.devices} From eb76a7e2b52813c2deb37bb161321003bf8d24af Mon Sep 17 00:00:00 2001 From: Cody Ho Date: Thu, 9 Jul 2026 23:20:17 -0700 Subject: [PATCH 16/33] hv: apply stolen-time offset before guest entry CNTVOFF_EL2 is per physical CPU. Apply the current stolen-time offset before entering the boot CPU and secondary CPUs, as well as when returning from an exception, so newly entered guest CPUs do not run with a stale virtual counter offset until their first trap. Signed-off-by: Cody Ho --- src/hv.c | 2 ++ src/hv.h | 1 + src/hv_exc.c | 8 +++++++- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/hv.c b/src/hv.c index 9d147e6cb..5519e2b57 100644 --- a/src/hv.c +++ b/src/hv.c @@ -156,6 +156,7 @@ void hv_start(void *entry, u64 regs[4]) hv_pinned_cpu = -1; hv_want_cpu = -1; hv_cpus_in_guest = BIT(smp_id()); + hv_apply_time_stealing_offset(); hv_enter_guest(regs[0], regs[1], regs[2], regs[3], entry); @@ -237,6 +238,7 @@ static void hv_init_secondary(struct hv_secondary_info_t *info) static void hv_enter_secondary(void *entry, u64 regs[4]) { + hv_apply_time_stealing_offset(); hv_enter_guest(regs[0], regs[1], regs[2], regs[3], entry); spin_lock(&bhl); diff --git a/src/hv.h b/src/hv.h index 783fad86c..a1e9ce15a 100644 --- a/src/hv.h +++ b/src/hv.h @@ -78,6 +78,7 @@ void virtio_put_buffer(u64 base, int qu, u32 id, u32 len); /* Exceptions */ void hv_exc_proxy(struct exc_info *ctx, uartproxy_boot_reason_t reason, u32 type, void *extra); void hv_set_time_stealing(bool enabled, bool reset); +void hv_apply_time_stealing_offset(void); void hv_add_time(s64 time); /* WDT */ diff --git a/src/hv_exc.c b/src/hv_exc.c index e56c7f5f5..d4dd86d29 100644 --- a/src/hv_exc.c +++ b/src/hv_exc.c @@ -144,6 +144,12 @@ void hv_set_time_stealing(bool enabled, bool reset) time_stealing = enabled; if (reset) stolen_time = 0; + hv_apply_time_stealing_offset(); +} + +void hv_apply_time_stealing_offset(void) +{ + msr(CNTVOFF_EL2, stolen_time); } void hv_add_time(s64 time) @@ -420,7 +426,7 @@ static void hv_exc_exit(struct exc_info *ctx) hv_update_fiq(); /* reenable PMU counters */ reg_set(SYS_IMP_APL_PMCR0, PERCPU(exc_entry_pmcr0_cnt)); - msr(CNTVOFF_EL2, stolen_time); + hv_apply_time_stealing_offset(); spin_unlock(&bhl); hv_maybe_exit(); __atomic_or_fetch(&hv_cpus_in_guest, BIT(smp_id()), __ATOMIC_ACQUIRE); From 918a6274db9e02c93595b26575b832c871640527 Mon Sep 17 00:00:00 2001 From: Cody Ho Date: Thu, 9 Jul 2026 04:25:05 -0700 Subject: [PATCH 17/33] hv: initialize guest secondary EL1 state On SPTM-enabled SoCs XNU boots with the mmu already enabled and expect seeded state (previous SoCs would enable the mmu and create the state themselves). Leaving stale m1n1 EL2 state causes the secondaries to fault; program them with the running guest EL1 state instead. Signed-off-by: Cody Ho --- src/hv.c | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/src/hv.c b/src/hv.c index 5519e2b57..eacbbf0b4 100644 --- a/src/hv.c +++ b/src/hv.c @@ -49,10 +49,53 @@ struct hv_secondary_info_t { uint64_t gxf_config; uint64_t agt_cnt_rdir_el1; uint64_t agt_cnt_rdir_el12; + uint64_t guest_ttbr0; + uint64_t guest_ttbr1; + uint64_t guest_tcr; + uint64_t guest_mair; + uint64_t guest_amair; + uint64_t guest_contextidr; + uint64_t guest_cpacr; + uint64_t guest_vbar; + uint64_t guest_sctlr; }; static struct hv_secondary_info_t hv_secondary_info; +static void hv_capture_guest_el12_state(struct hv_secondary_info_t *info) +{ + info->guest_ttbr0 = mrs(TTBR0_EL12); + info->guest_ttbr1 = mrs(TTBR1_EL12); + info->guest_tcr = mrs(TCR_EL12); + info->guest_mair = mrs(MAIR_EL12); + info->guest_amair = mrs(AMAIR_EL12); + info->guest_contextidr = mrs(CONTEXTIDR_EL12); + info->guest_cpacr = mrs(CPACR_EL12); + info->guest_vbar = mrs(VBAR_EL12); + info->guest_sctlr = mrs(SCTLR_EL12); +} + +static void hv_restore_guest_el12_state(const struct hv_secondary_info_t *info) +{ + /* + * On SPTM-enabled SoCs XNU boots with the mmu already enabled and expect + * seeded state (previous SoCs would enable the mmu and create the state + * themselves). Leaving stale m1n1 EL2 state causes the secondaries to + * fault; program them with the running guest EL1 state instead. + */ + msr(TTBR0_EL12, info->guest_ttbr0); + msr(TTBR1_EL12, info->guest_ttbr1); + msr(TCR_EL12, info->guest_tcr); + msr(MAIR_EL12, info->guest_mair); + msr(AMAIR_EL12, info->guest_amair); + msr(CONTEXTIDR_EL12, info->guest_contextidr); + msr(CPACR_EL12, info->guest_cpacr); + msr(VBAR_EL12, info->guest_vbar); + sysop("isb"); + msr(SCTLR_EL12, info->guest_sctlr); + sysop("isb"); +} + void hv_init(void) { pcie_shutdown(); @@ -151,6 +194,8 @@ void hv_start(void *entry, u64 regs[4]) hv_secondary_info.agt_cnt_rdir_el1 = mrs(SYS_IMP_APL_AGTCNTRDIR_EL1); hv_secondary_info.agt_cnt_rdir_el12 = mrs(SYS_IMP_APL_AGTCNTRDIR_EL12); } + if (!cpu_features->apple_sysregs_unlocked) + hv_capture_guest_el12_state(&hv_secondary_info); hv_arm_tick(false); hv_pinned_cpu = -1; @@ -227,6 +272,9 @@ static void hv_init_secondary(struct hv_secondary_info_t *info) msr(SYS_IMP_APL_AGTCNTRDIR_EL12, info->agt_cnt_rdir_el12); } + if (!cpu_features->apple_sysregs_unlocked) + hv_restore_guest_el12_state(info); + if (cpu_features->apple_sysregs_unlocked) reg_mask(SYS_IMP_APL_CYC_OVRD, CYC_OVRD_WFI_MODE_MASK, CYC_OVRD_WFI_MODE(0)); @@ -256,6 +304,8 @@ void hv_start_secondary(int cpu, void *entry, u64 regs[4]) printf("HV: Initializing secondary %d\n", cpu); iodev_console_flush(); + if (!cpu_features->apple_sysregs_unlocked) + hv_capture_guest_el12_state(&hv_secondary_info); mmu_init_secondary(cpu); iodev_console_flush(); smp_call4(cpu, hv_init_secondary, (u64)&hv_secondary_info, 0, 0, 0); From a51787aad4884ae10ba8ca215e8de9e10025be5f Mon Sep 17 00:00:00 2001 From: Cody Ho Date: Thu, 9 Jul 2026 18:13:13 -0700 Subject: [PATCH 18/33] hv: support custom guest start arguments Allow callers to override the primary and secondary guest entry ABI while preserving the existing defaults: primary guests still receive the bootargs pointer, and secondaries still use the discovered RVBAR/entry path unless explicit override state is set. Signed-off-by: Cody Ho --- proxyclient/m1n1/hv/__init__.py | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/proxyclient/m1n1/hv/__init__.py b/proxyclient/m1n1/hv/__init__.py index bbfe37bca..d662f40b4 100644 --- a/proxyclient/m1n1/hv/__init__.py +++ b/proxyclient/m1n1/hv/__init__.py @@ -109,6 +109,10 @@ def __init__(self, iface, proxy, utils): self.hook_exceptions = False self.started_cpus = {} self.started = False + self.start_entry = None + self.start_args = None + self.secondary_start_entry = None + self.secondary_start_args = None self.ctx = None self.hvcall_handlers = {} self.switching_context = False @@ -1603,7 +1607,12 @@ def start_secondary(self, die, cluster, cpu): self.log("CPU not found!") return - if self.u.cpu_features.apple_sysregs_unlocked: + secondary_entry = self.secondary_start_entry + secondary_args = self.secondary_start_args + + if secondary_entry is not None: + entry = secondary_entry + elif self.u.cpu_features.apple_sysregs_unlocked: entry = self.p.read64(node.cpu_impl_reg[0]) & 0xfffffffffff else: entry = self.entry & ~0xfff @@ -1612,7 +1621,10 @@ def start_secondary(self, die, cluster, cpu): self.sysreg[index] = {} self.started_cpus[index] = (die, cluster, cpu) - self.p.hv_start_secondary(index, entry) + if secondary_args is not None: + self.p.hv_start_secondary(index, entry, *secondary_args) + else: + self.p.hv_start_secondary(index, entry) def setup_adt(self): self.adt["product"].product_name += " on m1n1 hypervisor" @@ -1996,7 +2008,12 @@ def start(self): print("Enabling GXF...") self.u.msr(GXF_CONFIG_EL1, 1) - print(f"Jumping to entrypoint at 0x{self.entry:x}") + entry = self.start_entry if self.start_entry is not None else self.entry + args = self.start_args + if args is None: + args = (self.guest_base + self.bootargs_off,) + + print(f"Jumping to entrypoint at 0x{entry:x}") self.iface.dev.timeout = None self.default_sigint = signal.signal(signal.SIGINT, self._handle_sigint) @@ -2013,6 +2030,6 @@ def start(self): break self.started_cpus[cpu_node.cpu_id] = (getattr(cpu_node, "die_id", 0), cpu_node.cluster_id, cpu_node.cpu_id) self.sysreg[cpu_node.cpu_id] = {} - self.p.hv_start(self.entry, self.guest_base + self.bootargs_off) + self.p.hv_start(entry, *args) from .. import trace From 0390abe90a34fef6a3d8408dca18637d9817f0a5 Mon Sep 17 00:00:00 2001 From: Cody Ho Date: Thu, 9 Jul 2026 00:04:46 -0700 Subject: [PATCH 19/33] HACK: hv: T8140 CPU/ACC/CPM stage-2 write guard Passing through MMIO acceses to these pages results in EL2 SErrors that we have not figured out how to avoid. This file selectively guards MMIO to prevent the SErrors: reads pass through, writes are selectively forwarded or blackholed by writer-PC range and page index. The exact policy (which writes to forward vs drop) was arrived at over nearly two weeks of trial-and-error. We are not happy with this file, as it is a very ugly hack, however we cannot figure out how to get rid of it. Signed-off-by: Cody Ho --- Makefile | 2 +- src/hv.c | 4 + src/hv.h | 3 + src/hv_t8140.c | 108 ++ src/sptm/README.md | 7 + src/sptm/dart.c | 2348 +++++++++++++++++++++++++++++++++++++++++ src/sptm/emul.c | 326 ++++++ src/sptm/emul.h | 7 + src/sptm/nvme.c | 1252 ++++++++++++++++++++++ src/sptm/private.h | 2472 ++++++++++++++++++++++++++++++++++++++++++++ src/sptm/sart.c | 277 +++++ src/sptm/txm.c | 383 +++++++ src/sptm/uat.c | 1323 ++++++++++++++++++++++++ src/sptm/xnu.c | 832 +++++++++++++++ 14 files changed, 9343 insertions(+), 1 deletion(-) create mode 100644 src/hv_t8140.c create mode 100644 src/sptm/README.md create mode 100644 src/sptm/dart.c create mode 100644 src/sptm/emul.c create mode 100644 src/sptm/emul.h create mode 100644 src/sptm/nvme.c create mode 100644 src/sptm/private.h create mode 100644 src/sptm/sart.c create mode 100644 src/sptm/txm.c create mode 100644 src/sptm/uat.c create mode 100644 src/sptm/xnu.c diff --git a/Makefile b/Makefile index 148aaccc1..73fcb5eb2 100644 --- a/Makefile +++ b/Makefile @@ -141,7 +141,7 @@ OBJECTS := \ firmware.o \ gxf.o gxf_asm.o \ heapblock.o \ - hv.o hv_vm.o hv_exc.o hv_vuart.o hv_wdt.o hv_asm.o hv_aic.o hv_virtio.o \ + hv.o hv_vm.o hv_exc.o hv_vuart.o hv_wdt.o hv_asm.o hv_aic.o hv_t8140.o hv_virtio.o \ i2c.o \ iodev.o \ iova.o \ diff --git a/src/hv.c b/src/hv.c index eacbbf0b4..04b47f92f 100644 --- a/src/hv.c +++ b/src/hv.c @@ -167,6 +167,10 @@ void hv_start(void *entry, u64 regs[4]) hv_started_cpus[boot_cpu_idx] = true; + // Install the T8140 CPU/ACC/CPM write guard before the guest runs (no-op on + // other SoCs). Must be after the guest's device mappings are established. + hv_t8140_map_accumulators(); + msr(VBAR_EL1, _hv_vectors_start); if (gxf_enabled()) diff --git a/src/hv.h b/src/hv.h index a1e9ce15a..291ec8a3d 100644 --- a/src/hv.h +++ b/src/hv.h @@ -58,6 +58,9 @@ int hv_unmap(u64 from, u64 size); int hv_map_hw(u64 from, u64 to, u64 size); int hv_map_sw(u64 from, u64 to, u64 size); int hv_map_hook(u64 from, hv_hook_t *hook, u64 size); + +/* T8140 CPU/ACC/CPM write guard (no-op on other SoCs); see hv_t8140.c. */ +int hv_t8140_map_accumulators(void); u64 hv_translate(u64 addr, bool s1only, bool w, u64 *par_out); u64 hv_pt_walk(u64 addr); bool hv_handle_dabort(struct exc_info *ctx); diff --git a/src/hv_t8140.c b/src/hv_t8140.c new file mode 100644 index 000000000..d6f0c8092 --- /dev/null +++ b/src/hv_t8140.c @@ -0,0 +1,108 @@ +/* SPDX-License-Identifier: MIT */ +/* + * This T8140 CPU/ACC/CPM hook is a horrible hack that we don't know how to + * get rid of. We spent almost two weeks experimenting and ultimately + * converged to this file. + * + * Known failed edits: + * + * - Deleting this hook or forwarding all writes causes PEH/DPE bringup + * failures. + * - Dropping all writes produced SPMI errors and hard-rebooted the machine, + * and did not fix SPMI. + * + */ + +#include "hv.h" +#include "cpu_regs.h" +#include "memory.h" +#include "smp.h" +#include "soc.h" +#include "string.h" +#include "utils.h" + +#define T8140_ACC_PAGE_SIZE 0x4000UL +#define T8140_ACC_GUARD_FIRST_PAGE 2 + +static const u64 t8140_acc_pages[] = { + 0x210074000UL, 0x211074000UL, 0x210e44000UL, 0x211e44000UL, + 0x210e48000UL, 0x211e48000UL, 0x210058000UL, 0x210158000UL, + 0x210258000UL, 0x210358000UL, 0x211058000UL, 0x211158000UL, +}; + +static void t8140_flush_stage2_hooks(void) +{ + sysop("dsb ishst"); + sysop("tlbi vmalls12e1is"); + sysop("dsb ish"); + sysop("isb"); +} + +static int t8140_acc_page_index(u64 ipa) +{ + u64 page = ipa & ~(T8140_ACC_PAGE_SIZE - 1); + + for (u32 i = 0; i < ARRAY_SIZE(t8140_acc_pages); i++) + if (page == t8140_acc_pages[i]) + return i; + + return -1; +} + +#define T8140_ACC_PC_DPE_LOOP_LO 0xfffffe00097b7b00UL +#define T8140_ACC_PC_DPE_LOOP_HI 0xfffffe00097b8d00UL + +static bool t8140_acc_hook(struct exc_info *ctx, u64 ipa, u64 *val, bool write, int width) +{ + if (width < 0 || width > 6) + return false; + + int idx = t8140_acc_page_index(ipa); + if (idx < 0) + return false; + + u64 bytes = 1UL << width; + u64 off = ipa & (T8140_ACC_PAGE_SIZE - 1); + if (off + bytes > T8140_ACC_PAGE_SIZE) + return false; + + if (write) { + u64 pc = ctx->elr; + if (pc >= T8140_ACC_PC_DPE_LOOP_LO && pc < T8140_ACC_PC_DPE_LOOP_HI) { + return hv_pa_rw(ctx, ipa, val, true, width); + } + if (idx >= 6 && idx <= 9 && (off == 0x1000 || off == 0x1008)) { + return hv_pa_rw(ctx, ipa, val, true, width); + } + /* + * For E-core CPUs (idx 6..9 -> pages 0x21x058000), forward so the + * silicon power state matches XNU's view; otherwise cpu0 re-enters + * peh.c with a stale SError that never clears. P-core CPUs (idx + * 10..11) are HV-managed; drop those. + */ + if (idx >= 6 && idx <= 9) { + return hv_pa_rw(ctx, ipa, val, true, width); + } + return true; + } + + return hv_pa_rw(ctx, ipa, val, false, width); +} + +int hv_t8140_map_accumulators(void) +{ + if (chip_id != T8140) + return 0; + + int failures = 0; + for (u32 i = T8140_ACC_GUARD_FIRST_PAGE; i < ARRAY_SIZE(t8140_acc_pages); i++) + failures += hv_map_hook(t8140_acc_pages[i], t8140_acc_hook, + T8140_ACC_PAGE_SIZE) != 0; + + u32 requested = ARRAY_SIZE(t8140_acc_pages) - T8140_ACC_GUARD_FIRST_PAGE; + u32 installed = requested - failures; + printf("HV: T8140 CPU/ACC/CPM write guard for %lu/%lu pages\n", + (u64)installed, (u64)requested); + t8140_flush_stage2_hooks(); + return failures ? -failures : 0; +} diff --git a/src/sptm/README.md b/src/sptm/README.md new file mode 100644 index 000000000..eb3228073 --- /dev/null +++ b/src/sptm/README.md @@ -0,0 +1,7 @@ +# SPTM Emulator + +This directory is tainted bringup code. Keep reverse-engineered SPTM endpoint +behavior here while the generic hypervisor and loader plumbing stay separate. + +The emulator is linked into the streamed stage2 image when `SPTM_EMUL=1`; the +stage1 boot object does not include it. diff --git a/src/sptm/dart.c b/src/sptm/dart.c new file mode 100644 index 000000000..38f9b57f2 --- /dev/null +++ b/src/sptm/dart.c @@ -0,0 +1,2348 @@ +/* SPDX-License-Identifier: MIT */ +#include "private.h" + +int debug_printf(const char *fmt, ...); + +static inline bool_t dart_validate_xnu_object(u64 obj, u32 *gapf_count_out, + u32 *inst_count_out, + u64 *state_out) { + u32 mode, gapf_count, inst_count; + u64 state; + + if (!is_kernel_va(obj)) + return 0; + if (!guest_read32(obj + DART_OBJ_MODE_OFF, &mode) || mode != 2) + return 0; + if (!guest_read32(obj + DART_OBJ_GAPF_COUNT_OFF, &gapf_count)) + return 0; + if (gapf_count == 0 || gapf_count > 128) + return 0; + if (!guest_read32(obj + DART_OBJ_INST_COUNT_OFF, &inst_count)) + return 0; + if (inst_count == 0 || inst_count > 16) + return 0; + if (!guest_read64(obj + DART_OBJ_STATE_OFF, &state) || !is_kernel_va(state)) + return 0; + + *gapf_count_out = gapf_count; + *inst_count_out = inst_count; + *state_out = state; + return 1; +} + +static inline void dart_record_xnu_object(u32 table, u32 endpoint, u64 dart_id, + u64 obj, u64 source, + u32 gapf_count, u32 inst_count, + u64 state) { + u32 slot = DART_OBJ_CACHE_SLOTS; + + for (u32 i = 0; i < DART_OBJ_CACHE_SLOTS; i++) { + if (g_state.dart_obj_cache_va[i] != 0 && + g_state.dart_obj_cache_id[i] == dart_id) { + slot = i; + break; + } + } + if (slot == DART_OBJ_CACHE_SLOTS) { + slot = (u32)(g_state.dart_obj_cache_idx & (DART_OBJ_CACHE_SLOTS - 1)); + g_state.dart_obj_cache_idx++; + } + + g_state.dart_obj_cache_id[slot] = dart_id; + g_state.dart_obj_cache_va[slot] = obj; + g_state.dart_obj_last[0] = table; + g_state.dart_obj_last[1] = endpoint; + g_state.dart_obj_last[2] = dart_id; + g_state.dart_obj_last[3] = obj; + g_state.dart_obj_last[4] = source; + g_state.dart_obj_last[5] = gapf_count; + g_state.dart_obj_last[6] = inst_count; + g_state.dart_obj_last[7] = state; + for (u32 i = 0; i < 16; i++) { + g_state.dart_obj_last_base_va[i] = 0; + g_state.dart_obj_last_base_pa[i] = 0; + } + for (u32 i = 0; i < inst_count && i < 16; i++) { + u64 mmio_va, mmio_pa; + if (!guest_read64(state + 0x20UL + (u64)i * 0x70UL, &mmio_va)) + continue; + g_state.dart_obj_last_base_va[i] = mmio_va; + if (is_kernel_va(mmio_va) && guest_va_to_pa(mmio_va, 1, &mmio_pa)) { + g_state.dart_obj_last_base_pa[i] = mmio_pa; + } else if (mmio_va >= 0x100000000UL && + (mmio_va & (DART_C_PAGE_SIZE - 1)) == 0) { + g_state.dart_obj_last_base_pa[i] = mmio_va; + } + } + g_state.dart_obj_cache_hits++; +} + +static inline bool_t dart_try_candidate(u32 table, u32 endpoint, u64 dart_id, + u64 candidate, u64 source) { + u32 gapf_count, inst_count; + u64 state; + + if (!dart_validate_xnu_object(candidate, &gapf_count, &inst_count, &state)) + return 0; + dart_record_xnu_object(table, endpoint, dart_id, candidate, source, + gapf_count, inst_count, state); + return 1; +} + +void dart_cache_xnu_object_from_regs(u32 table, u32 endpoint, + u64 *regs) { + u64 dart_id = regs[0]; + + if (table != SPTM_TABLE_T8110_DART_XNU && + table != SPTM_TABLE_T8110_DART_SK && + table != SPTM_TABLE_GEN3_DART_XNU && + table != SPTM_TABLE_GEN3_DART_SK) + return; + + /* Endpoint 6 (init) also needs the recovered AppleT8110DART object so + * Python can import locked live TTBR roots without doing UART-backed + * object/frame probing. Map/unmap endpoints still stay out of this path. */ + if (endpoint != 4 && endpoint != 5 && endpoint != 6) + return; + + if (dart_try_candidate(table, endpoint, dart_id, regs[19], 0x19)) + return; + + u64 fp = regs[29]; + for (u32 depth = 0; depth < 16; depth++) { + u64 saved_fp, saved_lr, candidate; + + if (!is_kernel_va(fp)) + break; + if (!guest_read64(fp, &saved_fp) || !guest_read64(fp + 8, &saved_lr)) + break; + + if (guest_read64(fp - 8, &candidate) && + dart_try_candidate(table, endpoint, dart_id, candidate, + 0x100000UL | ((u64)depth << 8) | 19)) + return; + if (guest_read64(fp - 16, &candidate) && + dart_try_candidate(table, endpoint, dart_id, candidate, + 0x100000UL | ((u64)depth << 8) | 20)) + return; + if (guest_read64(fp - 24, &candidate) && + dart_try_candidate(table, endpoint, dart_id, candidate, + 0x100000UL | ((u64)depth << 8) | 21)) + return; + if (guest_read64(fp - 32, &candidate) && + dart_try_candidate(table, endpoint, dart_id, candidate, + 0x100000UL | ((u64)depth << 8) | 22)) + return; + + if (saved_fp == fp) + break; + fp = saved_fp; + } + + g_state.dart_obj_cache_misses++; + g_state.dart_obj_last[0] = table; + g_state.dart_obj_last[1] = endpoint; + g_state.dart_obj_last[2] = dart_id; + g_state.dart_obj_last[3] = 0; + g_state.dart_obj_last[4] = 0; + g_state.dart_obj_last[5] = 0; + g_state.dart_obj_last[6] = 0; + g_state.dart_obj_last[7] = 0; +} + +static inline bool_t dart_c_looks_like_mmio(u64 pa) { + return !is_kernel_va(pa) && + pa >= 0x100000000UL && + (pa & (DART_C_PAGE_SIZE - 1)) == 0; +} + +static inline u32 dart_c_read32(u64 addr) { + return *(volatile u32 *)addr; +} + +static inline void dart_c_write32(u64 addr, u32 value) { + *(volatile u32 *)addr = value; +} + +static inline u64 dart_c_pte_from_pa(u64 paddr) { + return (((paddr >> 14) & 0x0fffffffUL) << 10) | 1UL; +} + +static inline u64 dart_c_pa_from_pte(u64 pte) { + return ((pte >> 10) & 0x0fffffffUL) << 14; +} + +static inline u32 dart_c_ttbr_from_pa(u64 paddr) { + return (u32)((((paddr >> 14) << 2) & 0xfffffffcU) | DART_T8110_TTBR_VALID); +} + +static inline u64 dart_c_pa_from_ttbr(u32 ttbr) { + return (u64)((ttbr & 0xfffffffcU) >> 2) << 14; +} + +static inline u32 dart_c_idx(u64 dva, u32 level) { + u64 page = dva >> 14; + if (level == 0) return (u32)((page >> 33) & 0x7ff); + if (level == 1) return (u32)((page >> 22) & 0x7ff); + if (level == 2) return (u32)((page >> 11) & 0x7ff); + return (u32)(page & 0x7ff); +} + +static inline u64 dart_c_effective_dva(struct dart_c_instance *inst, u32 sid, + struct dart_c_sid_state *st, u64 dva) { + (void)inst; + (void)sid; + (void)st; + + /* + * The firmware validates the full DVA, but a non-FOUR_LEVELS stream starts + * its TTBR root at the DVA[25:35] table. DVA[36:46] is not part of the page + * table walk for that mode; root_level encodes that below. + */ + return dva; +} + +static inline u8 dart_c_root_level_from_tcr(u32 tcr) { + return (tcr & DART_T8110_TCR_FOUR_LEVEL) ? 1 : 2; +} + +static inline u32 dart_c_tcr_from_root_level(u8 root_level) { + u32 tcr = DART_T8110_TCR_TRANSLATE_EN; + if (root_level <= 1) + tcr |= DART_T8110_TCR_FOUR_LEVEL; + return tcr; +} + +static inline void dart_c_clean64(u64 addr) { + clean_pt_range_poc((volatile u64 *)addr, 8); +} + +static inline u64 dart_c_leaf_pte(u64 paddr, u64 dva, u64 bytes, u64 bits) { + u64 start = (dva & (DART_C_PAGE_SIZE - 1)) >> 2; + u64 end = ((dva & (DART_C_PAGE_SIZE - 1)) + bytes - 1) >> 2; + if (end > 0xfff) + end = 0xfff; + return dart_c_pte_from_pa(paddr) | + (bits & 0xfUL) | + (start << 52) | + (end << 40); +} + +static inline u64 dart_c_perm_bits(u64 flags) { + /* + * 26.6b2 SPTM no longer uses the older 4-bit reversal here. The valid bit + * is supplied by dart_c_pte_from_pa(); only prot bits 0/1 contribute. + */ + return ((flags << 1) & 0x4UL) | ((flags & 1UL) << 3); +} + +static inline u64 dart_c_leaf_perm_bits(struct dart_c_instance *inst, + u64 paddr, u64 flags) { + (void)paddr; + + /* + * 26.6 SPTM records relaxed-rw-protections and then consults an internal + * frame-type policy table before changing PTE permission bits. Keep the + * flag in state, but leave PTE permissions unchanged until that policy is + * recovered instead of guessing a write-protect set. + */ + (void)inst->relaxed_rw_protections; + return dart_c_perm_bits(flags); +} + +static inline bool_t dart_c_map_needs_tlbi(struct dart_c_instance *inst, + u64 old_pte, u64 new_pte) { + if (old_pte == new_pte) + return 0; + if (!inst->avoid_tlbi_in_map) + return 1; + return (old_pte & 1UL) != 0; +} + +static inline u64 dart_c_validate_dva_range(u64 dva, u64 size) { + if (!size) + return SPTM_SUCCESS; + if (size > 0x2000000UL) + return DART_C_STATUS_VIOLATION; + if (dva >> 42) + return DART_C_STATUS_VIOLATION; + if ((size - 1UL) > (((1UL << 42) - 1UL) - dva)) + return DART_C_STATUS_VIOLATION; + + u64 pages = ((dva & (DART_C_PAGE_SIZE - 1UL)) + size + + DART_C_PAGE_SIZE - 1UL) / DART_C_PAGE_SIZE; + if (pages > 0x800UL) + return DART_C_STATUS_VIOLATION; + return SPTM_SUCCESS; +} + +static inline struct dart_c_instance *dart_c_get(u64 dart_id) { + u32 empty = DART_C_MAX_DARTS; + + for (u32 i = 0; i < DART_C_MAX_DARTS; i++) { + if (g_state.dart.inst[i].used && g_state.dart.inst[i].dart_id == dart_id) + return &g_state.dart.inst[i]; + if (!g_state.dart.inst[i].used && empty == DART_C_MAX_DARTS) + empty = i; + } + + if (empty == DART_C_MAX_DARTS) + empty = (u32)(dart_id & (DART_C_MAX_DARTS - 1)); + + struct dart_c_instance *inst = &g_state.dart.inst[empty]; + for (u32 i = 0; i < sizeof(*inst); i++) + ((volatile u8 *)inst)[i] = 0; + inst->dart_id = dart_id; + inst->used = 1; + return inst; +} + +static inline struct dart_c_sid_state *dart_c_sid(struct dart_c_instance *inst, + u64 sid) { + if (sid >= DART_C_MAX_SIDS) + return (struct dart_c_sid_state *)0; + struct dart_c_sid_state *st = &inst->sid[sid]; + st->valid = 1; + return st; +} + +static inline struct dart_c_sid_state * +dart_c_sid_lookup(struct dart_c_instance *inst, u64 sid) { + if (sid >= DART_C_MAX_SIDS) + return (struct dart_c_sid_state *)0; + struct dart_c_sid_state *st = &inst->sid[sid]; + return st->valid ? st : (struct dart_c_sid_state *)0; +} + +static inline bool_t dart_c_sid_exclave(struct dart_c_sid_state *st) { + return st && st->exclave; +} + +static inline bool_t dart_c_sid_translates(struct dart_c_sid_state *st) { + return st && ((st->tcr & DART_T8110_TCR_TRANSLATE_EN) != 0); +} + +static inline u32 dart_c_num_streams(u64 base) { + u32 num = dart_c_read32(base + DART_T8110_PARAMS4) & 0x1ffU; + if (num == 0 || num > DART_C_MAX_STREAMS) + num = DART_C_MAX_STREAMS; + return num; +} + +static inline u32 dart_c_stream_words(u32 num) { + return (num + 31U) >> 5; +} + +static inline u64 dart_c_first_mmio_base(struct dart_c_instance *inst) { + for (u32 i = 0; i < inst->base_count && i < DART_C_MAX_BASES; i++) { + u64 base = inst->base_pa[i]; + if (dart_c_looks_like_mmio(base)) + return base; + } + return 0; +} + +static inline u64 dart_c_pack_hw_sid_regs(struct dart_c_instance *inst, + u32 sid) { + u64 base = dart_c_first_mmio_base(inst); + if (!base || sid >= DART_C_MAX_SIDS) + return 0; + return ((u64)dart_c_read32(base + DART_T8110_TCR + sid * 4) << 32) | + dart_c_read32(base + DART_T8110_TTBR + sid * 4); +} + +static inline u64 dart_c_pack_hw_stream_error(struct dart_c_instance *inst, + u32 sid) { + u64 base = dart_c_first_mmio_base(inst); + if (!base) + return 0; + u32 stream_word = 0; + u32 num = dart_c_num_streams(base); + if (sid < num) { + stream_word = dart_c_read32(base + DART_T8110_ENABLE_STREAMS + + ((sid >> 5) * 4)); + } + return ((u64)dart_c_read32(base + DART_T8110_ERROR) << 32) | stream_word; +} + +static inline bool_t dart_c_base_needs_new_tlb_cmd(u64 base) { + return (dart_c_read32(base + DART_T8110_PARAMS3) & 0xffffU) >= 0x0202U; +} + +static inline u64 dart_c_tlb_raw_command(u64 base, u32 command) { + dart_c_write32(base + DART_T8110_TLB_CMD, command); + for (u32 i = 0; i < 100; i++) { + u32 status = dart_c_read32(base + DART_T8110_TLB_CMD); + if (!(status & DART_T8110_TLB_CMD_BUSY)) + return ((u64)status << 32); + } + return DART_C_ERR_TIMEOUT | ((u64)dart_c_read32(base + DART_T8110_TLB_CMD) << 32); +} + +static inline u64 dart_c_tlb_command(u64 base, u32 op, u32 sid) { + u32 command = ((op & 7U) << 8) | (sid & 0xffU); + + if (op == DART_T8110_TLB_OP_FLUSH_SID && + dart_c_base_needs_new_tlb_cmd(base)) + command |= DART_T8110_TLB_CMD_NEW_DART; + + return dart_c_tlb_raw_command(base, command); +} + +static inline u64 dart_c_flush_sid_base(u64 base, u32 sid) { + return dart_c_tlb_command(base, DART_T8110_TLB_OP_FLUSH_SID, sid); +} + +static inline u64 dart_c_flush_sid_range_base(u64 base, u32 sid, + u64 start_dva, u64 end_dva) { + u32 start_page = (u32)((start_dva >> 14) & 0x0fffffffU); + u32 end_page = (u32)((end_dva >> 14) & 0x0fffffffU); + u32 command = DART_T8110_TLB_CMD_VA_RANGE | + DART_T8110_TLB_CMD_STT_FLUSH | + ((DART_T8110_TLB_OP_FLUSH_SID & 7U) << 8) | + (sid & 0xffU); + + if (dart_c_base_needs_new_tlb_cmd(base)) + command |= DART_T8110_TLB_CMD_NEW_DART; + + dart_c_write32(base + DART_T8110_TLB_START_DVA_PAGE, start_page << 2); + __asm__ volatile("dsb sy" ::: "memory"); + dart_c_write32(base + DART_T8110_TLB_END_DVA_PAGE, end_page << 2); + __asm__ volatile("dsb sy" ::: "memory"); + return dart_c_tlb_raw_command(base, command); +} + +static inline void dart_c_clear_errors_base(u64 base) { + u32 num = dart_c_num_streams(base); + u32 words = dart_c_stream_words(num); + u32 error = dart_c_read32(base + DART_T8110_ERROR); + dart_c_write32(base + DART_T8110_ERROR, error); + for (u32 i = 0; i < words; i++) + dart_c_write32(base + DART_T8110_ERROR_STREAMS + i * 4, 0xffffffffU); + dart_c_write32(base + DART_T8110_ERROR_MASK, 0xffffffffU); +} + +static inline u32 dart_c_import_live_base(struct dart_c_instance *inst, u64 base, + bool_t *preserve_out) { + u32 num = dart_c_num_streams(base); + if (num > DART_C_MAX_SIDS) + num = DART_C_MAX_SIDS; + + u32 protect = dart_c_read32(base + DART_T8110_PROTECT); + u32 protect_lock = dart_c_read32(base + DART_T8110_PROTECT_LOCK); + bool_t preserve = ((protect | protect_lock) & DART_T8110_PROTECT_TTBR_TCR) ? 1 : 0; + u32 enable_words[(DART_C_MAX_STREAMS + 31U) >> 5]; + u32 words = dart_c_stream_words(num); + if (words > ((DART_C_MAX_STREAMS + 31U) >> 5)) + words = ((DART_C_MAX_STREAMS + 31U) >> 5); + + for (u32 i = 0; i < words; i++) + enable_words[i] = dart_c_read32(base + DART_T8110_ENABLE_STREAMS + i * 4); + + for (u32 sid = 0; sid < num; sid++) { + u32 tcr = dart_c_read32(base + DART_T8110_TCR + sid * 4); + u32 ttbr = dart_c_read32(base + DART_T8110_TTBR + sid * 4); + if (ttbr & DART_T8110_TTBR_VALID) + preserve = 1; + if (tcr & DART_T8110_TCR_REMAP_EN) + preserve = 1; + if ((tcr & DART_T8110_TCR_TRANSLATE_EN) && ttbr) + preserve = 1; + } + + *preserve_out = preserve; + if (!preserve) + return 0; + + inst->locked = 1; + u32 imported = 0; + for (u32 sid = 0; sid < num; sid++) { + u32 tcr = dart_c_read32(base + DART_T8110_TCR + sid * 4); + u32 ttbr = dart_c_read32(base + DART_T8110_TTBR + sid * 4); + u32 word = (sid >> 5) < words ? enable_words[sid >> 5] : 0; + u8 stream_enabled = (word & (1U << (sid & 31))) ? 1 : 0; + if (!tcr && !ttbr && !stream_enabled) + continue; + if ((tcr & DART_T8110_TCR_TRANSLATE_EN) && + !(ttbr & DART_T8110_TTBR_VALID) && !ttbr) { + /* + * Several T8110-class DART register files contain non-reset-looking + * stale values in unused SID slots at raw boot. Treating TE with a + * non-valid TTBR as live state later replays a translated stream + * with no root, which is not a coherent DART configuration. + */ + if (!stream_enabled) + continue; + } + + struct dart_c_sid_state *st = dart_c_sid_lookup(inst, sid); + if (dart_c_sid_exclave(st)) { + if (stream_enabled) + st->stream_enabled = 1; + imported++; + continue; + } + if (st && (st->tcr & (DART_T8110_TCR_REMAP_EN | + DART_T8110_TCR_BYPASS_DART | + DART_T8110_TCR_BYPASS_DAPF))) { + imported++; + continue; + } + if (st && st->root_valid && !(ttbr & DART_T8110_TTBR_VALID)) { + if (stream_enabled) + st->stream_enabled = 1; + imported++; + continue; + } + st = dart_c_sid(inst, sid); + if (!st) + continue; + st->tcr = tcr; + st->ttbr = ttbr; + /* + * SPTM keeps ADT initial streams as state and replays them during + * finalize. A pre-finalize ENABLE_STREAMS read may still be clear, so + * live import may set this bit but must not erase seeded state. + */ + if (stream_enabled) + st->stream_enabled = 1; + st->translation_enabled = (tcr & DART_T8110_TCR_TRANSLATE_EN) ? 1 : 0; + st->root_level = dart_c_root_level_from_tcr(tcr); + if (ttbr & DART_T8110_TTBR_VALID) { + st->root_valid = 1; + st->root_pt_pa = dart_c_pa_from_ttbr(ttbr); + st->raw_replay = 0; + } else { + st->root_valid = 0; + st->root_pt_pa = 0; + st->raw_replay = + ((tcr & DART_T8110_TCR_TRANSLATE_EN) && ttbr) ? 1 : 0; + } + imported++; + } + + inst->last[10] = imported; + inst->last[11] = num; + inst->last[12] = protect; + inst->last[13] = protect_lock; + g_state.dart.last[10] = imported; + g_state.dart.last[11] = num; + g_state.dart.last[12] = protect; + g_state.dart.last[13] = protect_lock; + return imported; +} + +static inline bool_t dart_c_has_seeded_live_sid(struct dart_c_instance *inst) { + for (u32 sid = 0; sid < DART_C_MAX_SIDS; sid++) { + struct dart_c_sid_state *st = &inst->sid[sid]; + if (!st->valid) + continue; + if (st->ttbr || st->root_valid || st->stream_enabled) + return 1; + if (st->tcr && !(st->tcr & DART_T8110_TCR_REMAP_EN)) + return 1; + } + return 0; +} + +static inline void dart_c_for_each_base_flush_sid(struct dart_c_instance *inst, + u32 sid) { + for (u32 i = 0; i < inst->base_count && i < DART_C_MAX_BASES; i++) { + if (dart_c_looks_like_mmio(inst->base_pa[i])) + dart_c_flush_sid_base(inst->base_pa[i], sid); + } +} + +static inline void dart_c_for_each_base_flush_sid_range(struct dart_c_instance *inst, + u32 sid, + u64 start_dva, + u64 end_dva) { + for (u32 i = 0; i < inst->base_count && i < DART_C_MAX_BASES; i++) { + if (dart_c_looks_like_mmio(inst->base_pa[i])) + dart_c_flush_sid_range_base(inst->base_pa[i], sid, start_dva, + end_dva); + } +} + +static inline void dart_c_for_each_base_flush_map_change(struct dart_c_instance *inst, + u32 sid, + u64 start_dva, + u64 end_dva) { + if (inst->flush_by_dva) + dart_c_for_each_base_flush_sid_range(inst, sid, start_dva, end_dva); + else + dart_c_for_each_base_flush_sid(inst, sid); +} + +static inline void dart_c_flush_known_sids_base(struct dart_c_instance *inst, + u64 base) { + u32 touched[DART_C_MAX_SIDS / 32]; + + if (!dart_c_looks_like_mmio(base)) + return; + + for (u32 i = 0; i < DART_C_MAX_SIDS / 32; i++) + touched[i] = 0; + + u32 num = dart_c_num_streams(base); + if (num > DART_C_MAX_SIDS) + num = DART_C_MAX_SIDS; + u32 words = dart_c_stream_words(num); + if (words > DART_C_MAX_SIDS / 32) + words = DART_C_MAX_SIDS / 32; + + for (u32 word = 0; word < words; word++) { + u32 enabled = dart_c_read32(base + DART_T8110_ENABLE_STREAMS + + (u64)word * 4UL); + touched[word] |= enabled; + } + + for (u32 sid = 0; sid < DART_C_MAX_SIDS; sid++) { + struct dart_c_sid_state *st = &inst->sid[sid]; + if (!st->valid && !st->root_valid && !st->stream_enabled && + !st->ttbr) + continue; + touched[sid >> 5] |= 1U << (sid & 31); + } + + for (u32 sid = 0; sid < num; sid++) + if (touched[sid >> 5] & (1U << (sid & 31))) + dart_c_flush_sid_base(base, sid); +} + +static inline void dart_c_add_base(struct dart_c_instance *inst, u64 base) { + if (!dart_c_looks_like_mmio(base)) + return; + for (u32 i = 0; i < inst->base_count && i < DART_C_MAX_BASES; i++) { + if (inst->base_pa[i] == base) + return; + } + if (inst->base_count < DART_C_MAX_BASES) { + inst->base_pa[inst->base_count++] = base; + /* + * Real SPTM maps through an already-recovered DART object and flushes + * after leaf updates. If our object recovery learns the MMIO base after + * a software PTE update, catch the hardware up once the base is known. + */ + dart_c_flush_known_sids_base(inst, base); + } +} + +static inline u64 dart_c_range_sid(struct dart_c_identity_range *range) { + return range->sid & DART_C_RANGE_SID_MASK; +} + +static inline u64 dart_c_range_flags(struct dart_c_identity_range *range) { + return range->sid & ~DART_C_RANGE_SID_MASK; +} + +static inline bool_t dart_c_range_matches_inst(struct dart_c_instance *inst, + struct dart_c_identity_range *range) { + for (u32 bi = 0; bi < inst->base_count && bi < DART_C_MAX_BASES; bi++) { + if (inst->base_pa[bi] == range->base_pa) + return 1; + } + return 0; +} + +static inline bool_t dart_c_restore_boot_premap_page(struct dart_c_instance *inst, + u32 sid, u64 dva, + u64 slot) { + u64 page = dva & ~(DART_C_PAGE_SIZE - 1UL); + + for (u32 ri = 0; ri < g_state.dart.identity_count && + ri < DART_C_MAX_IDENTITY_RANGES; ri++) { + struct dart_c_identity_range *range = &g_state.dart.identity[ri]; + u64 flags = dart_c_range_flags(range); + u64 start, end; + + if (!(flags & DART_C_RANGE_FLAG_BOOT_PREMAP) || + (flags & DART_C_RANGE_FLAG_REMAP) || + !dart_c_range_matches_inst(inst, range) || + dart_c_range_sid(range) != sid || + range->end <= range->start) + continue; + + start = range->start & ~(DART_C_PAGE_SIZE - 1UL); + end = (range->end + DART_C_PAGE_SIZE - 1UL) & + ~(DART_C_PAGE_SIZE - 1UL); + if (page < start || page >= end) + continue; + + *(volatile u64 *)slot = + dart_c_leaf_pte(page, page, DART_C_PAGE_SIZE, 0); + dart_c_clean64(slot); + return 1; + } + + return 0; +} + +static inline void dart_c_program_sid_tcr_ttbr(struct dart_c_instance *inst, + u32 sid, u32 tcr, u32 ttbr) { + u32 writes = 0; + for (u32 i = 0; i < inst->base_count && i < DART_C_MAX_BASES; i++) { + u64 base = inst->base_pa[i]; + if (!dart_c_looks_like_mmio(base)) + continue; + u64 tcr_reg = base + DART_T8110_TCR + sid * 4; + u64 ttbr_reg = base + DART_T8110_TTBR + sid * 4; + if (dart_c_read32(tcr_reg) != tcr) { + dart_c_write32(tcr_reg, tcr); + writes++; + } + if (dart_c_read32(ttbr_reg) != ttbr) { + dart_c_write32(ttbr_reg, ttbr); + writes++; + } + } + if (writes) { + __asm__ volatile("dsb sy" ::: "memory"); + dart_c_for_each_base_flush_sid(inst, sid); + } +} + +static inline void dart_c_configure_sid_remap(struct dart_c_instance *inst, + u32 sid, u32 target) { + struct dart_c_sid_state *st = dart_c_sid_lookup(inst, sid); + if (!st) + return; + + st->tcr = DART_T8110_TCR_REMAP_EN | DART_T8110_TCR_REMAP_TARGET(target); + st->ttbr = 0; + st->root_pt_pa = 0; + st->root_valid = 0; + st->root_level = 2; + st->translation_enabled = 0; + st->raw_replay = 0; + dart_c_program_sid_tcr_ttbr(inst, sid, st->tcr, 0); +} + +static inline u64 dart_c_walk_leaf_slot(struct dart_c_instance *inst, u32 sid, + struct dart_c_sid_state *st, u64 dva, + u64 *slot_out); +static inline u64 dart_c_ensure_leaf_table(struct dart_c_instance *inst, u32 sid, + struct dart_c_sid_state *st, u64 dva, + u64 *table_out, u64 *allocs_out, + u64 *imports_out); + +static inline bool_t dart_c_import_live_sid(struct dart_c_instance *inst, u32 sid) { + if (sid >= DART_C_MAX_SIDS) + return 0; + + for (u32 i = 0; i < inst->base_count && i < DART_C_MAX_BASES; i++) { + u64 base = inst->base_pa[i]; + if (!dart_c_looks_like_mmio(base)) + continue; + + u32 tcr = dart_c_read32(base + DART_T8110_TCR + sid * 4); + u32 ttbr = dart_c_read32(base + DART_T8110_TTBR + sid * 4); + if (!(ttbr & DART_T8110_TTBR_VALID) && + !(ttbr && (tcr & DART_T8110_TCR_TRANSLATE_EN))) + continue; + + struct dart_c_sid_state *st = dart_c_sid(inst, sid); + if (!st) + return 0; + + u32 enable_word = dart_c_read32(base + DART_T8110_ENABLE_STREAMS + + ((sid >> 5) * 4)); + struct dart_c_sid_state *existing = dart_c_sid_lookup(inst, sid); + if (dart_c_sid_exclave(existing)) { + if (enable_word & (1U << (sid & 31))) + existing->stream_enabled = 1; + return 1; + } + + st->tcr = tcr; + st->ttbr = ttbr; + st->valid = 1; + st->root_level = dart_c_root_level_from_tcr(tcr); + if (enable_word & (1U << (sid & 31))) + st->stream_enabled = 1; + st->translation_enabled = (tcr & DART_T8110_TCR_TRANSLATE_EN) ? 1 : 0; + if (ttbr & DART_T8110_TTBR_VALID) { + st->root_valid = 1; + st->root_pt_pa = dart_c_pa_from_ttbr(ttbr); + st->raw_replay = 0; + } else { + st->root_valid = 0; + st->root_pt_pa = 0; + st->raw_replay = + ((tcr & DART_T8110_TCR_TRANSLATE_EN) && ttbr) ? 1 : 0; + } + inst->locked = 1; + return 1; + } + + return 0; +} + +static inline bool_t dart_c_pt_region_page_referenced(u64 start, u64 end, + u64 pa) { + start &= ~(DART_C_PAGE_SIZE - 1UL); + end = (end + DART_C_PAGE_SIZE - 1UL) & ~(DART_C_PAGE_SIZE - 1UL); + for (u64 table = start; table < end; table += DART_C_PAGE_SIZE) { + volatile u64 *entries = (volatile u64 *)table; + for (u32 i = 0; i < DART_C_TT_ENTRIES; i++) { + u64 pte = entries[i]; + if ((pte & 1UL) && dart_c_pa_from_pte(pte) == pa) + return 1; + } + } + return 0; +} + +static inline u64 dart_c_alloc_pt_region_table(struct dart_c_instance *inst, + u32 sid, + struct dart_c_sid_state *st) { + if (!inst || !st) + return 0; + + for (u32 ri = 0; ri < g_state.dart.identity_count && + ri < DART_C_MAX_IDENTITY_RANGES; ri++) { + struct dart_c_identity_range *range = &g_state.dart.identity[ri]; + u64 flags = dart_c_range_flags(range); + if (!(flags & DART_C_RANGE_FLAG_PT_REGION) || + !dart_c_range_matches_inst(inst, range) || + dart_c_range_sid(range) != sid || + range->end <= range->start) + continue; + + u64 scan_start = st->root_pt_pa; + if (scan_start < range->start || scan_start >= range->end) + scan_start = range->start; + scan_start &= ~(DART_C_PAGE_SIZE - 1UL); + + u64 cur = (range->start + DART_C_PAGE_SIZE - 1UL) & + ~(DART_C_PAGE_SIZE - 1UL); + u64 end = range->end & ~(DART_C_PAGE_SIZE - 1UL); + while (cur < end) { + if (cur == st->root_pt_pa || + dart_c_pt_region_page_referenced(scan_start, range->end, cur)) { + cur += DART_C_PAGE_SIZE; + continue; + } + + volatile u64 *entries = (volatile u64 *)cur; + for (u32 i = 0; i < DART_C_TT_ENTRIES; i++) + entries[i] = 0; + clean_range_poc((void *)cur, DART_C_PAGE_SIZE); + sptm_coproc_cache_maint_range((const void *)cur, DART_C_PAGE_SIZE); + g_state.dart.pt_pool_allocs++; + return cur; + } + } + + g_state.dart.pt_pool_failures++; + return 0; +} + +static inline bool_t dart_c_pa_in_pt_region(struct dart_c_instance *inst, + u32 sid, u64 pa) { + pa &= ~(DART_C_PAGE_SIZE - 1UL); + for (u32 ri = 0; ri < g_state.dart.identity_count && + ri < DART_C_MAX_IDENTITY_RANGES; ri++) { + struct dart_c_identity_range *range = &g_state.dart.identity[ri]; + u64 flags = dart_c_range_flags(range); + if (!(flags & DART_C_RANGE_FLAG_PT_REGION) || + !dart_c_range_matches_inst(inst, range) || + dart_c_range_sid(range) != sid) + continue; + if (range->start <= pa && pa < range->end) + return 1; + } + return 0; +} + +static inline u64 dart_c_shadow_child_table_if_needed( + struct dart_c_instance *inst, u32 sid, struct dart_c_sid_state *st, + u64 pt_pa) { + if (!inst || !st || !st->root_valid || !st->root_pt_pa) + return pt_pa; + + if (dart_c_pa_in_pt_region(inst, sid, pt_pa)) + return pt_pa; + + u64 shadow = dart_c_alloc_pt_region_table(inst, sid, st); + if (!shadow) + return pt_pa; + + g_state.dart.last[8] = inst->dart_id; + g_state.dart.last[9] = sid; + g_state.dart.last[10] = pt_pa; + g_state.dart.last[11] = shadow; + g_state.dart.last[12] = 0x53484457UL; /* "SHDW" */ + return shadow; +} + +static inline u32 dart_c_map_identity_range_page(struct dart_c_instance *inst, + u32 sid, u64 dva, u64 flags) { + struct dart_c_sid_state *st = dart_c_sid(inst, sid); + u64 table, slot; + (void)flags; + + if (!st || !st->root_valid || !st->root_pt_pa) + return 0; + + dva &= ~(DART_C_PAGE_SIZE - 1); + + if (dart_c_ensure_leaf_table(inst, sid, st, dva, &table, 0, 0)) + return 0; + + slot = table + (u64)dart_c_idx(dva, 3) * 8UL; + *(volatile u64 *)slot = dart_c_leaf_pte(dva, dva, DART_C_PAGE_SIZE, 0); + dart_c_clean64(slot); + + inst->last[8] = 0x44415046UL; /* "DAPF" */ + inst->last[9] = sid; + inst->last[10] = dva; + inst->last[11] = slot; + inst->last[12] = 0; + inst->last[13] = 0; + g_state.dart.last[8] = inst->dart_id; + g_state.dart.last[9] = sid; + g_state.dart.last[10] = dva; + g_state.dart.last[11] = slot; + return 1; +} + +static inline u32 dart_c_apply_identity_ranges(struct dart_c_instance *inst) { + u32 pages = 0; + + for (u32 ri = 0; ri < g_state.dart.identity_count && + ri < DART_C_MAX_IDENTITY_RANGES; ri++) { + struct dart_c_identity_range *range = &g_state.dart.identity[ri]; + u64 flags = dart_c_range_flags(range); + u64 sid = dart_c_range_sid(range); + u64 flush_start = 0; + u64 flush_end = 0; + if (!(flags & DART_C_RANGE_FLAG_BOOT_PREMAP) || + (flags & DART_C_RANGE_FLAG_REMAP) || + !dart_c_range_matches_inst(inst, range) || + sid >= DART_C_MAX_SIDS || + range->end <= range->start) + continue; + + u64 start = range->start & ~(DART_C_PAGE_SIZE - 1); + u64 end = (range->end + DART_C_PAGE_SIZE - 1) & + ~(DART_C_PAGE_SIZE - 1); + for (u64 dva = start; dva < end && pages < 4096; dva += DART_C_PAGE_SIZE) { + if (dart_c_map_identity_range_page(inst, (u32)sid, dva, flags)) { + if (!flush_end) + flush_start = dva; + flush_end = dva + DART_C_PAGE_SIZE; + pages++; + } + } + + if (flush_end > flush_start) { + /* + * These leaves are written by the emulator into persistent + * bootloader-owned DART roots. Real t8110dart_map flushes hardware + * by DVA range immediately after leaf updates; do the same for + * bootstrap leaves so stale negative display translations cannot + * survive until scanout. + */ + dart_c_for_each_base_flush_map_change(inst, (u32)sid, + flush_start, + flush_end - 1UL); + } + } + + if (pages) { + inst->last[14] = pages; + g_state.dart.last[12] = pages; + g_state.dart.last[13] = g_state.dart.identity_count; + } + return pages; +} + +static inline u32 dart_c_apply_bootstrap_remaps(struct dart_c_instance *inst) { + u32 count = 0; + + for (u32 ri = 0; ri < g_state.dart.identity_count && + ri < DART_C_MAX_IDENTITY_RANGES; ri++) { + struct dart_c_identity_range *range = &g_state.dart.identity[ri]; + if (!(dart_c_range_flags(range) & DART_C_RANGE_FLAG_REMAP)) + continue; + if (!dart_c_range_matches_inst(inst, range)) + continue; + + u32 sid = (u32)dart_c_range_sid(range); + u32 target = (u32)(range->start & DART_C_RANGE_SID_MASK); + if (sid >= DART_C_MAX_SIDS || target >= DART_C_MAX_SIDS) + continue; + dart_c_configure_sid_remap(inst, sid, target); + count++; + } + + if (count) { + inst->last[13] = 0x524d5444UL; /* "DTMR" */ + inst->last[14] = count; + g_state.dart.last[13] = inst->dart_id; + g_state.dart.last[14] = count; + } + return count; +} + +static inline bool_t dart_c_import_object_details(struct dart_c_instance *inst, + u64 obj) { + u32 gapf_count, inst_count; + u64 state; + + if (!dart_validate_xnu_object(obj, &gapf_count, &inst_count, &state)) + return 0; + + inst->obj_va = obj; + inst->state_va = state; + for (u32 i = 0; i < inst_count && i < 16; i++) { + u64 mmio_va, mmio_pa; + if (!guest_read64(state + 0x20UL + (u64)i * 0x70UL, &mmio_va)) + continue; + if (is_kernel_va(mmio_va) && guest_va_to_pa(mmio_va, 1, &mmio_pa)) + dart_c_add_base(inst, mmio_pa); + else if (dart_c_looks_like_mmio(mmio_va)) + dart_c_add_base(inst, mmio_va); + } + return 1; +} + +static inline void dart_c_import_cached_bases(struct dart_c_instance *inst) { + if (g_state.dart_obj_last[2] == inst->dart_id && + g_state.dart_obj_last[3] != 0) { + inst->obj_va = g_state.dart_obj_last[3]; + inst->state_va = g_state.dart_obj_last[7]; + for (u32 i = 0; i < 16; i++) + dart_c_add_base(inst, g_state.dart_obj_last_base_pa[i]); + return; + } + + /* + * Endpoint stack layouts differ enough that a later recovery attempt can + * miss even though an earlier endpoint already validated this DART object. + * Real SPTM keeps a per-DART state object; mirror that by falling back to + * the per-ID cache instead of letting a transient miss erase live state. + */ + for (u32 i = 0; i < DART_OBJ_CACHE_SLOTS; i++) { + if (g_state.dart_obj_cache_id[i] == inst->dart_id && + g_state.dart_obj_cache_va[i] != 0 && + dart_c_import_object_details(inst, g_state.dart_obj_cache_va[i])) + return; + } +} + +static inline u64 dart_c_walk_to_table(struct dart_c_instance *inst, u32 sid, + struct dart_c_sid_state *st, u64 dva, + u32 table_level, u64 *table_out, + u32 *idx_out) { + if (!st || !st->root_valid || !st->root_pt_pa) + return DART_C_ERR_NO_TTBR; + + dva = dart_c_effective_dva(inst, sid, st, dva); + + if (table_level < st->root_level || table_level > 3) + return DART_C_ERR_NO_TABLE; + if (table_level == st->root_level) { + *table_out = st->root_pt_pa; + *idx_out = dart_c_idx(dva, table_level); + return 0; + } + + u64 table = st->root_pt_pa; + for (u32 level = st->root_level; level < table_level; level++) { + u32 idx = dart_c_idx(dva, level); + u64 pte = *(volatile u64 *)(table + (u64)idx * 8UL); + if (!(pte & 1UL)) + return DART_C_ERR_NO_TABLE; + table = dart_c_pa_from_pte(pte); + } + *table_out = table; + *idx_out = dart_c_idx(dva, table_level); + return 0; +} + +static inline u64 dart_c_walk_leaf_slot(struct dart_c_instance *inst, u32 sid, + struct dart_c_sid_state *st, u64 dva, + u64 *slot_out) { + u64 table; + u32 idx; + u64 rc = dart_c_walk_to_table(inst, sid, st, dva, 3, &table, &idx); + if (rc) + return rc; + *slot_out = table + (u64)idx * 8UL; + return 0; +} + +static inline u64 dart_c_ensure_leaf_table(struct dart_c_instance *inst, u32 sid, + struct dart_c_sid_state *st, u64 dva, + u64 *table_out, u64 *allocs_out, + u64 *imports_out) { + if (!st || !st->root_valid || !st->root_pt_pa) + return DART_C_ERR_NO_TTBR; + + dva = dart_c_effective_dva(inst, sid, st, dva); + + u64 table = st->root_pt_pa; + u64 allocs = 0; + u64 imports = 0; + + for (u32 level = st->root_level; level < 3; level++) { + u32 idx = dart_c_idx(dva, level); + u64 slot = table + (u64)idx * 8UL; + u64 pte = *(volatile u64 *)slot; + if (pte & 1UL) { + table = dart_c_pa_from_pte(pte); + imports++; + continue; + } + + u64 child = dart_c_alloc_pt_region_table(inst, sid, st); + if (!child) + return DART_C_ERR_NO_TABLE; + *(volatile u64 *)slot = dart_c_pte_from_pa(child); + dart_c_clean64(slot); + table = child; + allocs++; + } + + *table_out = table; + if (allocs_out) + *allocs_out = allocs; + if (imports_out) + *imports_out = imports; + return 0; +} + +static inline void dart_c_snapshot_leaf(struct dart_c_instance *inst, u64 sid, + u64 dva, u64 *slot_out, + u64 *pte_out) { + *slot_out = 0; + *pte_out = 0; + struct dart_c_sid_state *st = dart_c_sid_lookup(inst, sid); + if (!st) + return; + u64 slot; + if (dart_c_walk_leaf_slot(inst, (u32)sid, st, dva, &slot)) + return; + *slot_out = slot; + *pte_out = *(volatile u64 *)slot; +} + +static inline void dart_c_record_recent(struct dart_c_instance *inst, + u32 table, u32 endpoint, u64 *regs, + u64 rc, u64 sid, u64 before_slot, + u64 before_pte, u64 after_slot, + u64 after_pte) { + u64 seq = ++g_state.dart.recent_idx; + struct dart_c_recent_event *ev = + &g_state.dart.recent[(seq - 1) & (DART_C_RECENT_SLOTS - 1)]; + + ev->seq = seq; + ev->table = table; + ev->endpoint = endpoint; + ev->dart_id = inst ? inst->dart_id : 0; + ev->sid = sid; + for (u32 i = 0; i < 6; i++) + ev->regs[i] = regs[i]; + ev->before_slot = before_slot; + ev->before_pte = before_pte; + ev->after_slot = after_slot; + ev->after_pte = after_pte; + ev->rc = rc; +} + +#define DART_C_SID0_TRACE_DVA 0x300704048UL +#define DART_C_SID0_REASON_DIRECT (1UL << 0) +#define DART_C_SID0_REASON_GLOBAL (1UL << 2) +#define DART_C_SID0_REASON_FLUSH (1UL << 3) + +static inline u64 dart_c_sid0_pack_tcr_ttbr(struct dart_c_sid_state *st) { + if (!st) + return 0; + return ((u64)st->ttbr << 32) | st->tcr; +} + +static inline u64 dart_c_sid0_pack_root_flags(struct dart_c_sid_state *st) { + if (!st) + return 0; + return (st->root_pt_pa & 0x00ffffffffffffffUL) | + ((u64)(st->valid & 1U) << 56) | + ((u64)(st->root_valid & 1U) << 57) | + ((u64)(st->root_level & 3U) << 58) | + ((u64)(st->stream_enabled & 1U) << 60) | + ((u64)(dart_c_sid_translates(st) & 1U) << 61) | + ((u64)(st->raw_replay & 1U) << 62) | + ((u64)(st->initial_stream_enabled & 1U) << 63); +} + +static inline void dart_c_snapshot_sid0(struct dart_c_instance *inst, u64 dva, + u64 *tcr_ttbr_out, + u64 *root_flags_out, + u64 *slot_out, u64 *pte_out) { + struct dart_c_sid_state *st = dart_c_sid_lookup(inst, 0); + *tcr_ttbr_out = dart_c_sid0_pack_tcr_ttbr(st); + *root_flags_out = dart_c_sid0_pack_root_flags(st); + dart_c_snapshot_leaf(inst, 0, dva, slot_out, pte_out); +} + +static inline bool_t dart_c_sid0_trace_reason(struct dart_c_instance *inst, + bool_t xnu_table, + bool_t sk_table, + u32 endpoint, u64 *regs, + u64 *reason_out) { + u64 reason = 0; + (void)inst; + + if (sk_table) { + if (endpoint == 0 && regs[1] == 0) + reason = DART_C_SID0_REASON_DIRECT; + if (endpoint == 1 || endpoint == 2 || endpoint == 3) + reason = DART_C_SID0_REASON_GLOBAL; + } else if (xnu_table) { + switch (endpoint) { + case 0: + case 1: + case 8: + case 9: + case 13: + if (regs[1] == 0) + reason = DART_C_SID0_REASON_DIRECT; + break; + case 2: + case 3: + if (regs[1] == 0) + reason = DART_C_SID0_REASON_DIRECT; + break; + case 4: + case 5: + case 6: + reason = DART_C_SID0_REASON_GLOBAL; + break; + case 7: + if (regs[1] == 0) + reason = DART_C_SID0_REASON_FLUSH; + break; + case 11: + case 16: + reason = DART_C_SID0_REASON_FLUSH; + break; + default: + break; + } + } + + *reason_out = reason; + return reason != 0; +} + +static inline void dart_c_record_sid0_trace(struct dart_c_instance *inst, + u32 table, u32 endpoint, u64 *regs, + u64 rc, u64 reason, + u64 before_tcr_ttbr, + u64 before_root_flags, + u64 before_pte, + u64 after_tcr_ttbr, + u64 after_root_flags, + u64 after_pte) { + u64 seq = ++g_state.dart.sid0_trace_idx; + struct dart_c_sid0_event *ev = + &g_state.dart.sid0_trace[(seq - 1) & (DART_C_SID0_TRACE_SLOTS - 1)]; + + ev->seq = seq; + ev->meta = ((u64)(table & 0xffffU)) | + ((u64)(endpoint & 0xffffU) << 16) | + ((u64)((inst ? inst->dart_id : 0) & 0xffffU) << 32) | + ((u64)(reason & 0xffffU) << 48); + for (u32 i = 0; i < 4; i++) + ev->regs[i] = regs[i]; + ev->before_tcr_ttbr = before_tcr_ttbr; + ev->before_root_flags = before_root_flags; + ev->before_pte = before_pte; + ev->after_tcr_ttbr = after_tcr_ttbr; + ev->after_root_flags = after_root_flags; + ev->after_pte = after_pte; + ev->rc = rc; +} + +static inline u64 dart_c_ep0_install_table(struct dart_c_instance *inst, u64 *regs) { + u64 sid = regs[1]; + u64 dva = regs[2]; + u32 level = (u32)regs[3]; + u64 pt_pa = regs[4]; + struct dart_c_sid_state *st = dart_c_sid(inst, sid); + if (!st || (pt_pa & (DART_C_PAGE_SIZE - 1)) || level > 2) + return DART_C_STATUS_VIOLATION; + if (dart_c_sid_exclave(st)) + return DART_C_STATUS_VIOLATION; + + /* + * Firmware map_table uses level 0 as the root install for FOUR_LEVELS + * streams and level 1 as the root install for normal streams. Once a + * level-0 root is installed, later level-1 and level-2 requests install + * child tables below that root; they must not replace it. + */ + bool_t root_install = (!st->root_valid || level < st->root_level); + u32 root_install_level = (level == 0) ? 1U : 2U; + + g_state.dart.last[8] = sid; + g_state.dart.last[9] = level; + g_state.dart.last[10] = pt_pa; + g_state.dart.last[11] = 0; + g_state.dart.last[12] = 0; + g_state.dart.last[13] = st->root_level; + g_state.dart.last[14] = st->root_valid; + g_state.dart.last[15] = st->root_pt_pa; + + if (root_install && level > 1) + return DART_C_STATUS_VIOLATION; + + if (root_install && st->root_valid) + dart_c_import_live_sid(inst, (u32)sid); + + if (root_install) { + st->root_valid = 1; + st->root_level = (u8)root_install_level; + st->root_pt_pa = pt_pa; + st->ttbr = dart_c_ttbr_from_pa(pt_pa); + st->tcr = dart_c_tcr_from_root_level(st->root_level); + st->raw_replay = 0; + st->translation_enabled = 1; + dart_c_program_sid_tcr_ttbr(inst, (u32)sid, st->tcr, st->ttbr); + return SPTM_SUCCESS; + } + + if (level < st->root_level || level > 2) + return DART_C_STATUS_VIOLATION; + + u64 parent; + u32 idx; + u64 rc = dart_c_walk_to_table(inst, (u32)sid, st, dva, level, + &parent, &idx); + if (rc) + return SPTM_SUCCESS; /* Adapter glue: missing imported parent state is non-fatal. */ + u64 live_pt_pa = dart_c_shadow_child_table_if_needed( + inst, (u32)sid, st, pt_pa); + *(volatile u64 *)(parent + (u64)idx * 8UL) = dart_c_pte_from_pa(live_pt_pa); + dart_c_clean64(parent + (u64)idx * 8UL); + g_state.dart.last[11] = parent; + g_state.dart.last[12] = idx; + g_state.dart.last[13] = st->root_level; + g_state.dart.last[14] = st->root_valid; + g_state.dart.last[15] = st->root_pt_pa; + dart_c_for_each_base_flush_sid(inst, (u32)sid); + return SPTM_SUCCESS; +} + +static inline u64 dart_c_ep1_unmap_table(struct dart_c_instance *inst, u64 *regs) { + u64 sid = regs[1]; + u64 dva = regs[2]; + u32 level = (u32)regs[3]; + struct dart_c_sid_state *st = dart_c_sid(inst, sid); + if (!st) + return DART_C_STATUS_VIOLATION; + if (dart_c_sid_exclave(st)) + return DART_C_STATUS_VIOLATION; + if (!st->root_valid) + return SPTM_SUCCESS; + + if ((st->root_level == 1 && level == 0) || + (st->root_level == 2 && level == 1)) { + st->root_valid = 0; + st->root_pt_pa = 0; + st->ttbr = 0; + st->tcr &= ~DART_T8110_TCR_TRANSLATE_EN; + st->translation_enabled = 0; + st->raw_replay = 0; + for (u32 i = 0; i < inst->base_count && i < DART_C_MAX_BASES; i++) { + u64 base = inst->base_pa[i]; + if (dart_c_looks_like_mmio(base)) + dart_c_write32(base + DART_T8110_TTBR + sid * 4, 0); + } + dart_c_for_each_base_flush_sid(inst, (u32)sid); + return SPTM_SUCCESS; + } + + if (level < st->root_level || level > 2) + return DART_C_STATUS_VIOLATION; + + u64 parent; + u32 idx; + u64 rc = dart_c_walk_to_table(inst, (u32)sid, st, dva, level, + &parent, &idx); + if (!rc) { + *(volatile u64 *)(parent + (u64)idx * 8UL) = 0; + dart_c_clean64(parent + (u64)idx * 8UL); + } + dart_c_for_each_base_flush_sid(inst, (u32)sid); + return SPTM_SUCCESS; +} + +static inline bool_t dart_c_is_txm_secure_channel_dva(u64 dva) { + if (!g_state.txm.secure_chan_dva) + return 0; + return (dva & ~(DART_C_PAGE_SIZE - 1UL)) == + (g_state.txm.secure_chan_dva & ~(DART_C_PAGE_SIZE - 1UL)); +} + +static inline bool_t dart_c_is_txm_secure_channel_pa(u64 pa) { + if (!pa) + return 0; + return ft_get_type(pa & ~(DART_C_PAGE_SIZE - 1UL)) == + TXM_SEP_SECURE_CHANNEL; +} + +static inline bool_t dart_c_is_txm_secure_channel_dart(struct dart_c_instance *inst) { + /* + * txm-secure-channel-base is a /arm-io/dart-sep property. DCP and other + * DARTs use the same high DVA numbering and can legitimately have ordinary + * mappings at the same numeric DVA, so keep this reservation scoped to the + * SEP DART that owns the property. + */ + return inst && inst->dart_id == 3; +} + +static inline bool_t dart_c_validate_txm_secure_channel_map(struct dart_c_instance *inst, + u64 dva, u64 pa) { + if (!g_state.txm.secure_chan_dva) + return 1; + if (!dart_c_is_txm_secure_channel_dart(inst)) + return 1; + + /* + * Real sptm_t8110dart_map treats /arm-io/dart-sep's + * txm-secure-channel-base/size as a reserved DVA window. With that DT + * state present, the reserve window may only map frame type 0x3d, and + * frame type 0x3d may only be mapped inside that window. + */ + return dart_c_is_txm_secure_channel_dva(dva) == + dart_c_is_txm_secure_channel_pa(pa); +} + +static inline bool_t dart_c_leaf_slot_is_txm_secure_channel(u64 slot) { + u64 pte = *(volatile u64 *)slot; + if ((pte & 1UL) == 0) + return 0; + return ft_get_type(dart_c_pa_from_pte(pte)) == TXM_SEP_SECURE_CHANNEL; +} + +static inline u64 dart_c_ep2_map_from_pa_list(struct dart_c_instance *inst, + u64 *regs) { + u64 sid = regs[1]; + u64 dva = regs[2]; + u64 scratch_pa = regs[3]; + u64 size = regs[4]; + u64 flags = regs[5]; + struct dart_c_sid_state *st = dart_c_sid(inst, sid); + u64 first_old_pa = 0; + u64 first_new_pa = 0; + bool_t flush_sid = 0; + if (dart_c_sid_exclave(st)) + return DART_C_STATUS_VIOLATION; + if (!st || !size || (scratch_pa & 7UL)) + return SPTM_SUCCESS; + u64 range_rc = dart_c_validate_dva_range(dva, size); + if (range_rc) + return range_rc; + + u64 pages = ((dva & (DART_C_PAGE_SIZE - 1)) + size + DART_C_PAGE_SIZE - 1) / + DART_C_PAGE_SIZE; + + u64 cur = dva; + u64 remaining = size; + for (u64 i = 0; i < pages; i++) { + u64 table, slot; + u64 page_bytes = DART_C_PAGE_SIZE - (cur & (DART_C_PAGE_SIZE - 1)); + if (page_bytes > remaining) + page_bytes = remaining; + if (dart_c_ensure_leaf_table(inst, (u32)sid, st, cur, &table, 0, 0)) + break; + slot = table + (u64)dart_c_idx(cur, 3) * 8UL; + u64 scratch_slot = scratch_pa + i * 8UL; + u64 old_pte = *(volatile u64 *)slot; + u64 old_pa = (old_pte & 1UL) ? dart_c_pa_from_pte(old_pte) : 0; + u64 new_pa = *(volatile u64 *)scratch_slot; + if (i == 0) { + first_old_pa = old_pa; + first_new_pa = new_pa; + } + if (!dart_c_validate_txm_secure_channel_map(inst, cur, new_pa)) { + inst->last[8] = 0x54584d56UL; /* "TXMV" */ + inst->last[9] = sid; + inst->last[10] = cur; + inst->last[11] = new_pa; + inst->last[12] = ft_get_type(new_pa); + inst->last[13] = old_pa; + g_state.dart.last[8] = 0x54584d56UL; + g_state.dart.last[9] = sid; + g_state.dart.last[10] = cur; + g_state.dart.last[11] = new_pa; + g_state.dart.last[12] = ft_get_type(new_pa); + g_state.dart.last[13] = old_pa; + return DART_C_STATUS_VIOLATION; + } + /* + * Real t8110dart_map copies the caller's PA-list page into SPTM-owned + * scratch, consumes the new PAs from that copy, and keeps old PAs only + * for internal release bookkeeping. Do not write old mappings back into + * XNU's supplied PA-list page. + */ + u64 new_pte = new_pa ? + dart_c_leaf_pte(new_pa, cur, page_bytes, + dart_c_leaf_perm_bits(inst, new_pa, flags)) : 0; + *(volatile u64 *)slot = new_pte; + dart_c_clean64(slot); + if (dart_c_map_needs_tlbi(inst, old_pte, new_pte)) + flush_sid = 1; + remaining -= page_bytes; + cur += page_bytes; + } + if (flush_sid) { + u64 flush_start = dva & ~(DART_C_PAGE_SIZE - 1UL); + u64 flush_end = (dva + size + DART_C_PAGE_SIZE - 1UL) & + ~(DART_C_PAGE_SIZE - 1UL); + if (flush_end > flush_start) + dart_c_for_each_base_flush_map_change(inst, (u32)sid, + flush_start, + flush_end - 1UL); + } + inst->last[8] = 0x4d41504cUL; /* "MAPL" */ + inst->last[9] = sid; + inst->last[10] = dva; + inst->last[11] = size; + inst->last[12] = pages; + inst->last[13] = first_old_pa; + inst->last[14] = first_new_pa; + inst->last[15] = flags; + g_state.dart.last[8] = 0x4d41504cUL; + g_state.dart.last[9] = first_old_pa; + g_state.dart.last[10] = first_new_pa; + g_state.dart.last[11] = pages; + g_state.dart.last[12] = flags; + return SPTM_SUCCESS; +} + +static inline u64 dart_c_ep3_unmap_range(struct dart_c_instance *inst, u64 *regs, + bool_t flush) { + u64 sid = regs[1]; + u64 start = regs[2]; + u64 size_or_end = regs[3]; + u64 end = flush ? size_or_end : start + size_or_end; + struct dart_c_sid_state *st = dart_c_sid(inst, sid); + u64 unmapped = 0; + if (dart_c_sid_exclave(st)) + return DART_C_STATUS_VIOLATION; + if (!st || end < start) + return SPTM_SUCCESS; + if (!flush) { + u64 range_rc = dart_c_validate_dva_range(start, size_or_end); + if (range_rc) + return range_rc; + } + start &= ~(DART_C_PAGE_SIZE - 1); + end = (end + DART_C_PAGE_SIZE - 1) & ~(DART_C_PAGE_SIZE - 1); + for (u64 dva = start; dva < end; dva += DART_C_PAGE_SIZE) { + u64 slot; + if (!dart_c_walk_leaf_slot(inst, (u32)sid, st, dva, &slot)) { + if (dart_c_leaf_slot_is_txm_secure_channel(slot)) + return DART_C_STATUS_VIOLATION; + if (!dart_c_restore_boot_premap_page(inst, (u32)sid, dva, slot)) { + *(volatile u64 *)slot = 0; + dart_c_clean64(slot); + } + unmapped++; + } + } + if (end > start) + dart_c_for_each_base_flush_map_change(inst, (u32)sid, start, + end - 1UL); + inst->last[8] = 0x554e4d50UL; /* "UNMP" */ + inst->last[9] = sid; + inst->last[10] = start; + inst->last[11] = end - start; + inst->last[12] = unmapped; + g_state.dart.last[8] = 0x554e4d50UL; + g_state.dart.last[9] = sid; + g_state.dart.last[10] = start; + g_state.dart.last[11] = end; + g_state.dart.last[12] = unmapped; + return SPTM_SUCCESS; +} + +static inline u64 dart_c_ep12_translate(struct dart_c_instance *inst, u64 *regs) { + u64 sid = regs[1]; + u64 dva = regs[2]; + struct dart_c_sid_state *st = dart_c_sid_lookup(inst, sid); + if (!st && sid < DART_C_MAX_SIDS && dart_c_import_live_sid(inst, (u32)sid)) + st = dart_c_sid_lookup(inst, sid); + u64 slot; + if (!st || dart_c_walk_leaf_slot(inst, (u32)sid, st, dva, &slot)) { + regs[1] = 0; + return SPTM_SUCCESS; + } + u64 pte = *(volatile u64 *)slot; + regs[1] = (pte & 1UL) ? (dart_c_pa_from_pte(pte) | (dva & (DART_C_PAGE_SIZE - 1))) : 0; + return SPTM_SUCCESS; +} + +#define DART_DAPF_SLICE_STRIDE 0x40UL +#define DART_DAPF_CTRL 0x00UL +#define DART_DAPF_START_LO 0x08UL +#define DART_DAPF_START_HI 0x0cUL +#define DART_DAPF_END_LO 0x10UL +#define DART_DAPF_END_HI 0x14UL +#define DART_DAPF_WORD_BASE 0x20UL + +static inline u32 dart_c_dapf_count(struct dart_c_instance *inst) { + u32 count = inst->dapf.count; + if (count > DART_C_MAX_DAPF_SLICES) + count = DART_C_MAX_DAPF_SLICES; + return count; +} + +static inline u32 dart_c_dapf_word_count(struct dart_c_instance *inst) { + u32 sid_count = inst->dapf.sid_count; + if (sid_count == 0 || sid_count > DART_C_MAX_STREAMS) + sid_count = DART_C_MAX_STREAMS; + u32 words = (sid_count + 31U) >> 5; + if (words == 0) + words = 1; + if (words > 8) + words = 8; + return words; +} + +static inline u32 dart_c_program_dapf_power_up(struct dart_c_instance *inst) { + u64 base = inst->dapf.base_pa; + u32 count = dart_c_dapf_count(inst); + u32 words = dart_c_dapf_word_count(inst); + u32 writes = 0; + + if (!base || !count || !dart_c_looks_like_mmio(base)) + return 0; + + for (u32 i = 0; i < count; i++) { + struct dart_c_dapf_slice *desc = &inst->dapf.slice[i]; + u64 off = base + (u64)i * DART_DAPF_SLICE_STRIDE; + dart_c_write32(off + DART_DAPF_START_LO, (u32)desc->start); + dart_c_write32(off + DART_DAPF_START_HI, (u32)(desc->start >> 32)); + dart_c_write32(off + DART_DAPF_END_LO, (u32)desc->end); + dart_c_write32(off + DART_DAPF_END_HI, (u32)(desc->end >> 32)); + writes += 4; + + for (u32 word = 0; word < words; word++) { + dart_c_write32(off + DART_DAPF_WORD_BASE + (u64)word * 4UL, + desc->words[word]); + writes++; + } + + dart_c_write32(off + DART_DAPF_CTRL, desc->ctrl); + writes++; + } + + __asm__ volatile("dsb sy" ::: "memory"); + return writes; +} + +#define DART_PIOGW_PROT_BASE 0x1120UL +#define DART_PIOGW_PROT_STRIDE 0x20UL +#define DART_PIOGW_PROT_CTRL 0x00UL +#define DART_PIOGW_PROT_START_LO 0x04UL +#define DART_PIOGW_PROT_START_HI 0x08UL +#define DART_PIOGW_PROT_END_LO 0x0cUL +#define DART_PIOGW_PROT_END_HI 0x10UL + +static inline u32 dart_c_piogw_desc_count(struct dart_c_instance *inst) { + u32 count = inst->piogw_desc_count; + if (count > DART_C_MAX_PIOGW_DESCS) + count = DART_C_MAX_PIOGW_DESCS; + return count; +} + +static inline u32 dart_c_piogw_count(struct dart_c_instance *inst) { + u32 count = inst->piogw_count; + if (count > DART_C_MAX_PIOGW_RANGES) + count = DART_C_MAX_PIOGW_RANGES; + return count; +} + +static inline u32 dart_c_program_piogw_power_up(struct dart_c_instance *inst) { + u32 desc_count = dart_c_piogw_desc_count(inst); + u32 piogw_count = dart_c_piogw_count(inst); + u32 writes = 0; + + if (!piogw_count || !desc_count) + return 0; + + for (u32 i = 0; i < piogw_count; i++) { + u64 base = inst->piogw_base_pa[i]; + if (!dart_c_looks_like_mmio(base)) + continue; + if (dart_c_read32(base + 0x10UL) != 0 || + dart_c_read32(base + 0x0cUL) != 0) + continue; + + dart_c_write32(base + DART_PIOGW_PROT_BASE + + DART_PIOGW_PROT_START_LO, 0); + dart_c_write32(base + DART_PIOGW_PROT_BASE + + DART_PIOGW_PROT_START_HI, 0); + dart_c_write32(base + DART_PIOGW_PROT_BASE + + DART_PIOGW_PROT_END_LO, 0xffffffffU); + dart_c_write32(base + DART_PIOGW_PROT_BASE + + DART_PIOGW_PROT_END_HI, 0xffffffffU); + dart_c_write32(base + DART_PIOGW_PROT_BASE + + DART_PIOGW_PROT_CTRL, 3); + writes += 5; + + for (u32 j = 0; j < desc_count; j++) { + struct dart_c_piogw_desc *desc = &inst->piogw_desc[j]; + u32 slice = desc->slice; + u64 addr = desc->addr; + if (slice < 1 || slice > 9 || addr == 0) + continue; + u64 off = DART_PIOGW_PROT_BASE + (u64)slice * DART_PIOGW_PROT_STRIDE; + dart_c_write32(base + off + DART_PIOGW_PROT_START_LO, (u32)addr); + dart_c_write32(base + off + DART_PIOGW_PROT_START_HI, + (u32)(addr >> 32)); + dart_c_write32(base + off + DART_PIOGW_PROT_END_LO, (u32)(addr + 3)); + dart_c_write32(base + off + DART_PIOGW_PROT_END_HI, + (u32)((addr + 3) >> 32)); + dart_c_write32(base + off + DART_PIOGW_PROT_CTRL, 6); + writes += 5; + } + + dart_c_write32(base + 4UL, dart_c_read32(base + 4UL) | 1U); + writes++; + } + + return writes; +} + +static inline u32 dart_c_program_piogw_power_down(struct dart_c_instance *inst) { + u32 desc_count = dart_c_piogw_desc_count(inst); + u32 piogw_count = dart_c_piogw_count(inst); + u32 writes = 0; + + if (!piogw_count || !desc_count) + return 0; + + for (u32 i = 0; i < piogw_count; i++) { + u64 base = inst->piogw_base_pa[i]; + if (!dart_c_looks_like_mmio(base)) + continue; + if (dart_c_read32(base + 0x10UL) != 0 || + dart_c_read32(base + 0x0cUL) != 0) + continue; + + for (u32 j = 0; j < desc_count; j++) { + struct dart_c_piogw_desc *desc = &inst->piogw_desc[j]; + u32 slice = desc->slice; + if (slice < 1 || slice > 9) + continue; + u64 off = DART_PIOGW_PROT_BASE + (u64)slice * DART_PIOGW_PROT_STRIDE; + dart_c_write32(base + off + DART_PIOGW_PROT_CTRL, 0); + writes++; + } + } + + return writes; +} + +static inline u32 dart_c_snapshot_live_streams(struct dart_c_instance *inst) { + u32 seen = 0; + + for (u32 sid = 0; sid < DART_C_MAX_SIDS; sid++) { + struct dart_c_sid_state *st = &inst->sid[sid]; + if (st->valid) + st->stream_enabled = 0; + } + + for (u32 i = 0; i < inst->base_count && i < DART_C_MAX_BASES; i++) { + u64 base = inst->base_pa[i]; + if (!dart_c_looks_like_mmio(base)) + continue; + u32 num = dart_c_num_streams(base); + if (num > DART_C_MAX_SIDS) + num = DART_C_MAX_SIDS; + u32 words = dart_c_stream_words(num); + for (u32 word = 0; word < words; word++) { + u32 bits = dart_c_read32(base + DART_T8110_ENABLE_STREAMS + word * 4); + while (bits) { + u32 bit = __builtin_ctz(bits); + u32 sid = word * 32 + bit; + bits &= bits - 1; + if (sid >= num) + continue; + struct dart_c_sid_state *st = dart_c_sid(inst, sid); + if (st) + st->stream_enabled = 1; + seen++; + } + } + } + + return seen; +} + +static inline u32 dart_c_disable_translated_streams(struct dart_c_instance *inst) { + u32 writes = 0; + + for (u32 sid = 0; sid < DART_C_MAX_SIDS; sid++) { + struct dart_c_sid_state *st = dart_c_sid_lookup(inst, sid); + if (dart_c_sid_exclave(st) || !dart_c_sid_translates(st)) + continue; + for (u32 i = 0; i < inst->base_count && i < DART_C_MAX_BASES; i++) { + u64 base = inst->base_pa[i]; + if (!dart_c_looks_like_mmio(base)) + continue; + dart_c_write32(base + DART_T8110_DISABLE_STREAMS + + ((sid >> 5) * 4), 1U << (sid & 31)); + writes++; + } + } + + return writes; +} + +static inline u32 dart_c_finalize_replay(struct dart_c_instance *inst) { + u32 writes = 0; + u32 touched[DART_C_MAX_SIDS / 32]; + + for (u32 i = 0; i < DART_C_MAX_SIDS / 32; i++) + touched[i] = 0; + + for (u32 i = 0; i < inst->base_count && i < DART_C_MAX_BASES; i++) { + u64 base = inst->base_pa[i]; + if (!dart_c_looks_like_mmio(base)) + continue; + + for (u32 sid = 0; sid < DART_C_MAX_SIDS; sid++) { + struct dart_c_sid_state *st = &inst->sid[sid]; + if (!st->valid) + continue; + bool_t replay_regs = + !dart_c_sid_exclave(st) && + (st->root_valid || st->raw_replay || + (st->tcr & (DART_T8110_TCR_REMAP_EN | + DART_T8110_TCR_BYPASS_DART | + DART_T8110_TCR_BYPASS_DAPF))); + if (replay_regs) { + u64 tcr_reg = base + DART_T8110_TCR + sid * 4; + u64 ttbr_reg = base + DART_T8110_TTBR + sid * 4; + touched[sid >> 5] |= 1U << (sid & 31); + if (dart_c_read32(tcr_reg) != st->tcr) { + dart_c_write32(tcr_reg, st->tcr); + writes++; + } + if (dart_c_read32(ttbr_reg) != st->ttbr) { + dart_c_write32(ttbr_reg, st->ttbr); + writes++; + } + } + } + + u32 num = dart_c_num_streams(base); + if (num > DART_C_MAX_SIDS) + num = DART_C_MAX_SIDS; + u32 words = dart_c_stream_words(num); + for (u32 word = 0; word < words; word++) { + u32 bits = 0; + for (u32 bit = 0; bit < 32; bit++) { + u32 sid = word * 32 + bit; + if (sid >= num) + break; + struct dart_c_sid_state *st = &inst->sid[sid]; + if (st->valid && + (st->stream_enabled || st->initial_stream_enabled)) { + bits |= 1U << bit; + touched[sid >> 5] |= 1U << (sid & 31); + } + } + dart_c_write32(base + DART_T8110_ENABLE_STREAMS + word * 4, bits); + writes++; + } + } + + /* + * Persistent display/DCP roots can contain valid leaves before any map call + * touches them in this boot. Once finalize replays that root or enables a + * stream, force the hardware translation cache to forget older negative + * entries before firmware starts reading through the DART. + */ + __asm__ volatile("dsb sy" ::: "memory"); + for (u32 sid = 0; sid < DART_C_MAX_SIDS; sid++) + if (touched[sid >> 5] & (1U << (sid & 31))) + dart_c_for_each_base_flush_sid(inst, sid); + + return writes; +} + +static inline u32 dart_c_count_raw_replay_state(struct dart_c_instance *inst) { + u32 count = 0; + + for (u32 sid = 0; sid < DART_C_MAX_SIDS; sid++) { + struct dart_c_sid_state *st = &inst->sid[sid]; + if (st->valid && st->raw_replay) + count++; + } + + return count; +} + +static inline u32 dart_c_xnu_clock_protection(struct dart_c_instance *inst, + bool_t enable) { + u64 obj = inst->obj_va; + u32 gapf_count, inst_count; + u64 state; + + if (!dart_validate_xnu_object(obj, &gapf_count, &inst_count, &state)) { + dart_c_import_cached_bases(inst); + obj = inst->obj_va; + } + if (!dart_validate_xnu_object(obj, &gapf_count, &inst_count, &state)) + return 0; + inst->state_va = state; + + u32 writes = 0; + for (u32 gapf_i = 0; gapf_i < gapf_count; gapf_i++) { + u64 base_va; + if (!guest_read64(obj + 0x17d0UL + (u64)gapf_i * 0x18UL, &base_va)) + continue; + if (!is_kernel_va(base_va)) + continue; + + for (u32 inst_i = 0; inst_i < inst_count; inst_i++) { + u32 slice_idx; + u64 entry = obj + 0x14f8UL + (u64)inst_i * 0x70UL + + (u64)gapf_i * 4UL; + if (!guest_read32(entry, &slice_idx) || slice_idx > 0x3ffU) + continue; + + u64 reg_va = base_va + 0x100UL + (u64)slice_idx * 0x40UL; + u64 reg_pa; + if (!guest_va_to_pa(reg_va, 1, ®_pa)) + continue; + + u32 old = dart_c_read32(reg_pa); + u32 new_value = enable ? (old | 0x10U) : (old & ~0x10U); + dart_c_write32(reg_pa, new_value); + writes++; + } + } + + inst->clock_protection = enable ? (writes != 0) : 0; + inst->last[8] = enable; + inst->last[9] = writes; + g_state.dart.last[8] = enable; + g_state.dart.last[9] = writes; + return writes; +} + +bool_t dart_handle_table(u32 table, u32 endpoint, u64 *regs) { + bool_t t8110_xnu = table == SPTM_TABLE_T8110_DART_XNU; + bool_t t8110_sk = table == SPTM_TABLE_T8110_DART_SK; + bool_t gen3_xnu = table == SPTM_TABLE_GEN3_DART_XNU; + bool_t gen3_sk = table == SPTM_TABLE_GEN3_DART_SK; + bool_t xnu_table = t8110_xnu || gen3_xnu; + bool_t sk_table = t8110_sk || gen3_sk; + + if (!xnu_table && !sk_table) + return 0; + + struct dart_c_instance *inst = dart_c_get(regs[0]); + if (!inst) + return 0; + + g_state.dart.enabled = 1; + g_state.dart.total_calls++; + inst->total_calls++; + if (endpoint < DART_C_MAX_ENDPOINTS) { + inst->ep_count[endpoint]++; + } + g_state.dart.last[0] = table; + g_state.dart.last[1] = endpoint; + g_state.dart.last[2] = regs[0]; + g_state.dart.last[3] = regs[1]; + g_state.dart.last[4] = regs[2]; + g_state.dart.last[5] = regs[3]; + for (u32 i = 0; i < 16; i++) + inst->last[i] = (i < 8) ? regs[i] : 0; + + u64 rc = SPTM_SUCCESS; + bool_t record_dart_lifetime = 0; + u64 record_sid = 0; + u64 record_dva = 0; + u64 before_slot = 0; + u64 before_pte = 0; + u64 after_slot = 0; + u64 after_pte = 0; + bool_t record_sid0_trace = 0; + u64 sid0_reason = 0; + u64 sid0_before_tcr_ttbr = 0; + u64 sid0_before_root_flags = 0; + u64 sid0_before_slot = 0; + u64 sid0_before_pte = 0; + u64 sid0_after_tcr_ttbr = 0; + u64 sid0_after_root_flags = 0; + u64 sid0_after_slot = 0; + u64 sid0_after_pte = 0; + + if (xnu_table && (endpoint == 2 || endpoint == 3)) { + record_dart_lifetime = 1; + record_sid = regs[1]; + record_dva = regs[2]; + if (record_dart_lifetime) + dart_c_snapshot_leaf(inst, record_sid, record_dva, + &before_slot, &before_pte); + } else if (xnu_table && + (endpoint == 0 || endpoint == 1 || + endpoint == 4 || endpoint == 5 || endpoint == 6 || + endpoint == 7 || endpoint == 8 || endpoint == 9 || + endpoint == 10 || endpoint == 11 || endpoint == 16)) { + record_dart_lifetime = 1; + record_sid = (endpoint == 7 || endpoint == 8 || endpoint == 14) ? + regs[1] : 0; + record_dva = regs[2]; + if (endpoint == 0 || endpoint == 1) { + dart_c_snapshot_leaf(inst, record_sid, record_dva, + &before_slot, &before_pte); + } else { + before_slot = dart_c_pack_hw_sid_regs(inst, (u32)record_sid); + before_pte = dart_c_pack_hw_stream_error(inst, (u32)record_sid); + } + } + + record_sid0_trace = dart_c_sid0_trace_reason(inst, xnu_table, sk_table, + endpoint, regs, &sid0_reason); + if (record_sid0_trace) { + dart_c_snapshot_sid0(inst, DART_C_SID0_TRACE_DVA, + &sid0_before_tcr_ttbr, + &sid0_before_root_flags, + &sid0_before_slot, + &sid0_before_pte); + } + + if (sk_table) { + switch (endpoint) { + case 0: + rc = dart_c_ep3_unmap_range(inst, regs, 1); + break; + case 1: + inst->locked = 0; + rc = SPTM_SUCCESS; + break; + case 2: + inst->clock_protection++; + rc = SPTM_SUCCESS; + break; + case 3: + regs[1] = inst->clock_protection; + rc = SPTM_SUCCESS; + break; + default: + rc = SPTM_SUCCESS; + break; + } + } else { + if (endpoint == 4 || endpoint == 5 || endpoint == 6) { + dart_cache_xnu_object_from_regs(table, endpoint, regs); + dart_c_import_cached_bases(inst); + } + if (endpoint == 6 && dart_c_looks_like_mmio(regs[1])) + dart_c_add_base(inst, regs[1]); + + switch (endpoint) { + case 0: + rc = dart_c_ep0_install_table(inst, regs); + if (rc == SPTM_SUCCESS) + dart_c_apply_identity_ranges(inst); + break; + case 1: + rc = dart_c_ep1_unmap_table(inst, regs); + break; + case 2: + rc = dart_c_ep2_map_from_pa_list(inst, regs); + break; + case 3: + rc = dart_c_ep3_unmap_range(inst, regs, 0); + break; + case 4: + if (t8110_xnu || gen3_xnu) { + u32 stream_seen = dart_c_snapshot_live_streams(inst); + u32 stream_disables = dart_c_disable_translated_streams(inst); + u32 piogw_writes = dart_c_program_piogw_power_down(inst); + inst->init_state = 1; + dart_c_xnu_clock_protection(inst, 0); + inst->last[13] = stream_seen; + inst->last[14] = stream_disables; + inst->last[15] = piogw_writes; + g_state.dart.last[13] = stream_seen; + g_state.dart.last[14] = stream_disables; + g_state.dart.last[15] = piogw_writes; + } else { + inst->last[8] = 0x494f4354UL; /* "IOCT" */ + inst->last[9] = endpoint; + } + rc = SPTM_SUCCESS; + break; + case 5: { + u32 piogw_writes = dart_c_program_piogw_power_up(inst); + u32 dapf_writes = dart_c_program_dapf_power_up(inst); + u32 raw_replay_sids = dart_c_count_raw_replay_state(inst); + inst->init_state = 2; + inst->locked = 1; + dart_c_apply_bootstrap_remaps(inst); + dart_c_apply_identity_ranges(inst); + u32 replay_writes = dart_c_finalize_replay(inst); + dart_c_xnu_clock_protection(inst, 1); + inst->last[10] = raw_replay_sids; + inst->last[11] = replay_writes; + inst->last[14] = dapf_writes; + inst->last[15] = piogw_writes; + g_state.dart.last[10] = raw_replay_sids; + g_state.dart.last[11] = replay_writes; + g_state.dart.last[14] = dapf_writes; + g_state.dart.last[15] = piogw_writes; + rc = SPTM_SUCCESS; + break; + } + case 6: { + inst->init_state = 1; + u32 preserved = 0; + u32 reset = 0; + u32 imported = 0; + for (u32 i = 0; i < inst->base_count && i < DART_C_MAX_BASES; i++) { + u64 base = inst->base_pa[i]; + if (!dart_c_looks_like_mmio(base)) + continue; + bool_t preserve = 0; + imported += dart_c_import_live_base(inst, base, &preserve); + if (!preserve && dart_c_has_seeded_live_sid(inst)) + preserve = 1; + if (preserve) { + /* + * Bootloader-owned display/AOP/DCP DARTs can have + * protected TTBR/TCR registers. On those DARTs, resetting + * TCR/TTBR is ineffective, but enabling every stream is + * very effective and can expose stale DMA as a real XNU + * DART panic. Preserve the live root and only clear stale + * interrupt state. + */ + dart_c_clear_errors_base(base); + preserved++; + continue; + } + dart_c_clear_errors_base(base); + reset++; + } + inst->last[10] = imported; + inst->last[11] = preserved; + inst->last[12] = reset; + g_state.dart.last[10] = imported; + g_state.dart.last[11] = preserved; + g_state.dart.last[12] = reset; + dart_c_apply_bootstrap_remaps(inst); + dart_c_apply_identity_ranges(inst); + rc = SPTM_SUCCESS; + break; + } + case 7: { + u32 sid = (u32)regs[1]; + struct dart_c_sid_state *st = dart_c_sid_lookup(inst, sid); + if (!st || dart_c_sid_exclave(st)) { + rc = DART_C_STATUS_VIOLATION; + break; + } + if (dart_c_sid_translates(st)) { + st->stream_enabled = 0; + for (u32 i = 0; i < inst->base_count && i < DART_C_MAX_BASES; i++) { + u64 base = inst->base_pa[i]; + if (!dart_c_looks_like_mmio(base)) + continue; + dart_c_write32(base + DART_T8110_DISABLE_STREAMS + + ((sid >> 5) * 4), 1U << (sid & 31)); + } + } + rc = SPTM_SUCCESS; + break; + } + case 8: { + u32 sid = (u32)regs[1]; + struct dart_c_sid_state *st = dart_c_sid_lookup(inst, sid); + if (!st || dart_c_sid_exclave(st)) { + rc = DART_C_STATUS_VIOLATION; + break; + } + if (dart_c_sid_translates(st)) { + st->stream_enabled = 1; + for (u32 i = 0; i < inst->base_count && i < DART_C_MAX_BASES; i++) { + u64 base = inst->base_pa[i]; + if (!dart_c_looks_like_mmio(base)) + continue; + dart_c_write32(base + DART_T8110_ENABLE_STREAMS + + ((sid >> 5) * 4), 1U << (sid & 31)); + } + __asm__ volatile("dsb sy" ::: "memory"); + dart_c_for_each_base_flush_sid(inst, sid); + } + rc = SPTM_SUCCESS; + break; + } + case 9: { + u32 instance = (u32)regs[1]; + u32 mask = (u32)regs[2] & 0x7fffU; + if (instance >= inst->base_count || + instance >= DART_C_MAX_BASES || + !dart_c_looks_like_mmio(inst->base_pa[instance]) || + !mask || (mask & (mask - 1U))) { + rc = DART_C_STATUS_VIOLATION; + break; + } + dart_c_write32(inst->base_pa[instance] + DART_T8110_ERROR, mask); + inst->last[8] = 0x45525243UL; /* "ERRC" */ + inst->last[9] = instance; + inst->last[10] = mask; + rc = SPTM_SUCCESS; + break; + } + case 10: { + u32 instance = (u32)regs[1]; + u32 mask = (u32)regs[2] & 0x777U; + if (instance >= inst->base_count || + instance >= DART_C_MAX_BASES || + !dart_c_looks_like_mmio(inst->base_pa[instance])) { + rc = DART_C_STATUS_VIOLATION; + break; + } + if (mask) + dart_c_write32(inst->base_pa[instance] + + DART_T8110_ERROR_MASK, mask); + inst->last[8] = 0x4552524dUL; /* "ERRM" */ + inst->last[9] = instance; + inst->last[10] = mask; + rc = SPTM_SUCCESS; + break; + } + case 11: + for (u32 i = 0; i < inst->base_count && i < DART_C_MAX_BASES; i++) { + u64 base = inst->base_pa[i]; + if (!dart_c_looks_like_mmio(base)) + continue; + dart_c_clear_errors_base(base); + } + rc = SPTM_SUCCESS; + break; + case 12: { + u32 instance = (u32)regs[1]; + if (instance >= inst->base_count || + instance >= DART_C_MAX_BASES || + !dart_c_looks_like_mmio(inst->base_pa[instance])) { + rc = DART_C_STATUS_VIOLATION; + break; + } + u32 command = (((u32)regs[3] & 0xfU) << 4) | + (((u32)regs[2] & 0x3fffU) << 8) | + ((u32)regs[4] & 7U); + u64 base = inst->base_pa[instance]; + dart_c_write32(base + DART_T8110_TLB_CMD, command); + for (u32 poll = 0; poll < 100; poll++) { + if (!(dart_c_read32(base + DART_T8110_TLB_CMD) & + DART_T8110_TLB_CMD_BUSY)) + break; + } + regs[1] = dart_c_read32(base + 0x88UL); + regs[2] = dart_c_read32(base + 0x90UL); + inst->last[8] = 0x544c4243UL; /* "TLBC" */ + inst->last[9] = instance; + inst->last[10] = command; + break; + } + case 13: + inst->last[8] = 0x41504653UL; /* "APFS" */ + inst->last[9] = regs[1]; + inst->last[10] = regs[2]; + rc = SPTM_SUCCESS; + break; + case 14: { + struct dart_c_sid_state *st = dart_c_sid_lookup(inst, regs[1]); + regs[1] = st ? st->status : 0; + rc = SPTM_SUCCESS; + break; + } + case 15: { + inst->last[8] = 0x494f4354UL; /* "IOCT" */ + inst->last[9] = endpoint; + inst->last[10] = regs[1]; + inst->last[11] = regs[2]; + rc = SPTM_SUCCESS; + break; + } + case 16: + for (u32 i = 0; i < inst->base_count && i < DART_C_MAX_BASES; i++) + if (dart_c_looks_like_mmio(inst->base_pa[i])) + dart_c_clear_errors_base(inst->base_pa[i]); + rc = SPTM_SUCCESS; + break; + case 17: + rc = DART_C_STATUS_VIOLATION; + break; + case 18: + for (u32 i = 0; i < inst->base_count && i < DART_C_MAX_BASES; i++) + if (dart_c_looks_like_mmio(inst->base_pa[i])) + dart_c_clear_errors_base(inst->base_pa[i]); + inst->last[8] = 0x45525252UL; /* "ERRR" */ + inst->last[9] = endpoint; + rc = SPTM_SUCCESS; + break; + default: + rc = DART_C_STATUS_VIOLATION; + break; + } + } + + if (record_dart_lifetime) { + if (endpoint == 0 || endpoint == 1 || endpoint == 2 || endpoint == 3) { + dart_c_snapshot_leaf(inst, record_sid, record_dva, + &after_slot, &after_pte); + } else { + after_slot = dart_c_pack_hw_sid_regs(inst, (u32)record_sid); + after_pte = dart_c_pack_hw_stream_error(inst, (u32)record_sid); + } + dart_c_record_recent(inst, table, endpoint, regs, rc, record_sid, + before_slot, before_pte, after_slot, after_pte); + } + if (record_sid0_trace) { + dart_c_snapshot_sid0(inst, DART_C_SID0_TRACE_DVA, + &sid0_after_tcr_ttbr, + &sid0_after_root_flags, + &sid0_after_slot, + &sid0_after_pte); + bool_t changed = + sid0_before_tcr_ttbr != sid0_after_tcr_ttbr || + sid0_before_root_flags != sid0_after_root_flags || + sid0_before_slot != sid0_after_slot || + sid0_before_pte != sid0_after_pte; + if (changed || !(sid0_reason & DART_C_SID0_REASON_FLUSH)) { + dart_c_record_sid0_trace(inst, table, endpoint, regs, rc, sid0_reason, + sid0_before_tcr_ttbr, + sid0_before_root_flags, + sid0_before_pte, + sid0_after_tcr_ttbr, + sid0_after_root_flags, + sid0_after_pte); + } + } + + if (rc == DART_C_STATUS_VIOLATION) { + g_state.dart.violations++; + inst->violations++; + } else { + g_state.dart.fast_path_calls++; + inst->fast_path_calls++; + } + g_state.dart.last[6] = rc; + g_state.dart.last[7] = inst->base_count; + regs[0] = rc; + sev(); + return 1; +} diff --git a/src/sptm/emul.c b/src/sptm/emul.c new file mode 100644 index 000000000..04a3d18a1 --- /dev/null +++ b/src/sptm/emul.c @@ -0,0 +1,326 @@ +/* SPDX-License-Identifier: MIT */ +/* + * DO NOT MERGE: tainted SPTM endpoint emulator imported from the bringup tree. + * + * This is intentionally isolated under src/sptm while we separate the clean + * hypervisor plumbing from the reverse-engineered SPTM behavior. The old tree + * built this as a hot-loaded C object; this branch links it directly into the + * streamed stage2 hypervisor so endpoint state has one C source of truth. + */ + +#include "emul.h" +#include "private.h" + +#include "../memory.h" +#include "../utils.h" +#include "../xnuboot.h" + +int debug_printf(const char *fmt, ...); + +/* The single state instance exported for Python-side boot seeding. Keep this + * 1MiB even though struct sptm_emul_state is smaller: a few emulated SPTM + * devices use the tail as private scratch, matching the old bringup layout. */ +u8 g_sptm_state[0x100000] __attribute__((aligned(0x4000))); + +bool_t sptm_handler(void *ctx) { + /* ctx points at struct exc_info. regs[] is at offset 0. */ + u64 *regs = (u64 *)((unsigned char *)ctx + REGS_OFFSET); + u64 esr = *(u64 *)((unsigned char *)ctx + ESR_OFFSET); + u64 *ctx_spsr_ptr = (u64 *)((unsigned char *)ctx + 0x100); + + /* XNU's kernel-mode invariant is DAIF.D == 1. Most SPTM calls are + * direct in-kernel HVCs, so returning with the trapped SPSR unchanged can + * propagate a transient D-clear state until ml_set_interrupts_enabled() + * panics with "debug exceptions enabled in kernel mode". Enforce only D + * here; do not touch A/I/F, since the timer/FIQ path depends on preserving + * XNU's interrupt mask state exactly. */ + *ctx_spsr_ptr |= 0x200; + + /* DEBUG: snapshot ALL guest GP regs at trap entry — BEFORE any C + * code can touch them. Lets us bisect register-clobber bugs + * introduced by new C handlers. Cheap (31 u64 stores). */ + { + u64 idx = g_state.sptm_in_idx & 0x7f; + for (int r = 0; r < 31; r++) { + g_state.sptm_in_regs[idx][r] = regs[r]; + } + g_state.sptm_in_idx = (g_state.sptm_in_idx + 1) & 0x7f; + } + + /* --------------------------------------------------------------- + * SPRR sysreg fast-path. EL1's vbar slot 0 has been patched to + * HVC #0, so when xnu (e.g. _pmap_ro_zone_memcpy_internal in + * libT8140) writes SPRR_PERM_EL1 from EL1, the UNDEF lands here. + * SPRR is GL2-gated and m1n1 EL2 cannot complete the write — we + * soft-cache it. Without this fast path each trap pays a UART + * round-trip (~hundreds of ms × thousands of RO-zone init writes). + * + * Detect by inspecting the trapping instruction at ELR_EL12. If + * it's any MSR/MRS to S3_6_C15_C1_x (op1=6, CRn=15, CRm=1): + * - MSR: cache write value (no actual sysreg touch) + * - MRS: return cached/seeded value into Rt + * Then advance ELR_EL12 by 4 by setting ctx->elr to elr12+4 and + * let m1n1 ERET past xnu's vector handler back to user code at + * elr12+4. + * --------------------------------------------------------------- */ + if (g_state.kc_vmin != 0) { + u64 esr12, elr12, spsr12, elr2; + /* ESR_EL12 = S3_5_C5_C2_0; ELR_EL12 = S3_5_C4_C0_1; SPSR_EL12 = S3_5_C4_C0_0 */ + __asm__ volatile("mrs %0, S3_5_C5_C2_0" : "=r"(esr12)); + __asm__ volatile("mrs %0, S3_5_C4_C0_1" : "=r"(elr12)); + __asm__ volatile("mrs %0, S3_5_C4_C0_0" : "=r"(spsr12)); + __asm__ volatile("mrs %0, elr_el2" : "=r"(elr2)); + u32 ec12 = (u32)((esr12 >> 26) & 0x3f); + /* CRITICAL: distinguish vector-induced HVC (real SPRR sysreg trap) + * from in-kernel HVC dispatch (e.g. SPTM trampoline calls + * `hvc #0` directly with x16=selector). For the latter, ELR_EL12 + * and ESR_EL12 are STALE from the prior EL1 vector entry — they + * retain the LAST real SPRR trap's state. If we don't filter, + * we mis-fast-path SPTM HVCs as if they were SPRR sysreg traps, + * ERET-ing to a random PC inside pmap_ro_zone. ELR_EL2 is always + * fresh: for vector-induced traps it points at the patched HVC + * inside vbar_el1; for in-kernel HVCs it points at the dispatch + * site in xnu code (outside vbar_el1). + * + * vbar_el1 in the current 26.5 setup = 0xfffffe000b388000 (vector table is + * 0x800 bytes). Only treat as SPRR if ELR_EL2 falls in this range. */ + bool_t in_vbar = (elr2 >= 0xfffffe000b388000UL && + elr2 < 0xfffffe000b388800UL); + if (ec12 == 0 && in_vbar && + elr12 >= g_state.kc_vmin && elr12 < g_state.kc_vmax) { + u64 insn_pa = g_state.kc_guest_base + (elr12 - g_state.kc_vmin); + /* DEFENSIVE GUARD (2026-05-09): ensure insn_pa is a low PA, not a + * high VA. Symptom: m1n1 takes EL2 sync exception with FAR in + * 0xfffffe000c282xxx range when reading insn at "insn_pa" — that's + * a guest VA, not a PA, so EL2 stage-1 doesn't have it mapped. + * Cause: elr12 can be stale/bogus from a previous EL1 vector entry + * (the in_vbar+ec12==0 filter isn't perfect across all paths), so + * the computed insn_pa can land outside our PA range. Guard: + * - insn_pa must NOT have any high-VA bits set (PAs are < 4 TB) + * - insn_pa must be >= kc_guest_base (no arith underflow) + * - insn_pa must be < kc_guest_base + kc_size (kept in bounds) + * On failure: return unhandled so the host can stop with full + * diagnostics instead of crashing EL2. */ + u64 kc_size_va = g_state.kc_vmax - g_state.kc_vmin; + bool_t insn_pa_ok = (insn_pa >= g_state.kc_guest_base) && + ((insn_pa - g_state.kc_guest_base) < kc_size_va) && + ((insn_pa & 0xfffffe0000000000UL) == 0); + if (!insn_pa_ok) { + g_state.fallthrough_calls++; + return 0; /* unsafe insn_pa — hard-fail in host diagnostics */ + } + u32 insn = *(volatile u32 *)insn_pa; + /* MSR S3_6_C15_C1_x, xN → 0xd51ef1 ?? */ + /* MRS xN, S3_6_C15_C1_x → 0xd53ef1 ?? */ + u32 base = insn & 0xffffff00u; + if (base == 0xd51ef100u || base == 0xd53ef100u) { + u32 is_read = (insn >> 21) & 1; + u32 op2 = (insn >> 5) & 7; + u32 Rt = insn & 0x1f; + if (is_read) { + u64 val; + if (op2 == 6) val = g_state.sprr_perm_el1_cache; + else if (op2 == 5) val = g_state.sprr_perm_el0_cache; + else if (op2 == 0) val = g_state.sprr_config_el1_cache; + else val = 0; + if (Rt < 31) regs[Rt] = val; + } else { + u64 val = (Rt < 31) ? regs[Rt] : 0; + if (op2 == 6) g_state.sprr_perm_el1_cache = val; + else if (op2 == 5) g_state.sprr_perm_el0_cache = val; + else if (op2 == 0) g_state.sprr_config_el1_cache = val; + } + /* If we're inside pmap_ro_zone_memcpy_internal, capture + * its args into the ring buffer. By the time the first + * SPRR MSR fires (~+0x90 from function start), the + * prologue has stashed args into x19..x24 — they're + * stable for the rest of the call. Records: (elr, x22, + * x24, x23, x20, x19) = (pc, zid, va, offset, src, size). */ + if (g_state.pmap_rzm_lo != 0 && + elr12 >= g_state.pmap_rzm_lo && + elr12 < g_state.pmap_rzm_hi) { + u64 idx = g_state.pmap_rzm_idx & 0x7f; + g_state.pmap_rzm_ring[idx][0] = elr12; + g_state.pmap_rzm_ring[idx][1] = regs[22]; /* zid */ + g_state.pmap_rzm_ring[idx][2] = regs[24]; /* va */ + g_state.pmap_rzm_ring[idx][3] = regs[23]; /* offset */ + g_state.pmap_rzm_ring[idx][4] = regs[20]; /* new_data */ + g_state.pmap_rzm_ring[idx][5] = regs[19]; /* new_data_size */ + /* Also capture full register snapshot in parallel ring. + * Helps tell whether it's args that are wrong or the whole + * machine state that's corrupted. */ + for (int r = 0; r < 31; r++) { + g_state.pmap_rzm_full[idx][r] = regs[r]; + } + g_state.pmap_rzm_full[idx][31] = spsr12; + g_state.pmap_rzm_full[idx][32] = elr12; + g_state.pmap_rzm_idx = (g_state.pmap_rzm_idx + 1) & 0x7f; + g_state.pmap_rzm_count++; + } + + /* Skip xnu's vector handler entirely. ctx->elr currently + * points at the HVC in vbar_el1; redirect ERET straight to + * elr12+4, the original post-fault instruction boundary. + * + * CRITICAL: also restore ctx->spsr from SPSR_EL12. Without + * this, m1n1 ERETs with SPSR_EL2 which is EL1h (the vector + * handler's mode after entry), NOT the EL1t/EL1h state the + * function was actually running in. Result: SPSel switches + * mid-function, ldp/retab use the wrong stack — guaranteed + * PAC failure on signed-return functions. We mask DAIF for + * good measure (matches the Python EL1 vector handler). */ + u64 *ctx_elr = (u64 *)((unsigned char *)ctx + 0x108); + *ctx_elr = elr12 + 4; + /* Only mask DAIF.D (bit 9) to avoid xnu's + * "debug exceptions enabled in kernel mode" panic in + * ml_set_interrupts_enabled_with_debug. Preserve + * DAIF.I so vIRQ injection can actually fire. */ + *ctx_spsr_ptr = spsr12 | 0x200; + g_state.sprr_fast_count++; + g_state.fast_path_calls++; + sev(); + return 1; + } + } + } + + u64 x16 = regs[16]; + if (!is_dispatch_hvc(esr, x16)) { + u64 elr; + __asm__ volatile("mrs %0, elr_el2" : "=r"(elr)); + if ((esr & ESR_ISS_MASK) == SPTM_GENTER_DISPATCH_CALL && + (elr == XNU_265_GENTER_NODISPATCH_RET_PC || + elr == XNU_266_GENTER_NODISPATCH_RET_PC)) { + regs[0] = SPTM_SUCCESS; + g_state.fast_path_calls++; + sev(); + return 1; + } + g_state.fallthrough_calls++; + return 0; + } + u32 domain = (u32)((x16 >> 48) & 0xff); + u32 table = (u32)((x16 >> 32) & 0xff); + u32 endpoint = (u32)(x16 & 0xffffffff); + + if (domain == TXM_DOMAIN && table == TXM_TABLE_XNU) { + if (txm_handle_selector(endpoint, x16, regs)) + return 1; + g_state.fallthrough_calls++; + return 0; + } + + /* Only handle domain=0 (SPTM/XNU) here. Other domains have different + * endpoint numbering and must never be decoded as XNU_BOOTSTRAP calls. */ + if (domain != 0) { + g_state.fallthrough_calls++; + return 0; + } + + g_state.total_calls++; + g_state.last_x16 = x16; + g_state.last_endpoint = endpoint; + /* Record into ring buffer (last 128 calls). Also read ESR_EL2 + ELR_EL2 + * so we can distinguish SPTM HVC #0 from vector-trap HVC #0xFEEx and + * see which xnu code issued the HVC. */ + { + u64 idx = g_state.ring_idx & 0x7f; + u64 esr, elr; + __asm__ volatile("mrs %0, esr_el2" : "=r"(esr)); + __asm__ volatile("mrs %0, elr_el2" : "=r"(elr)); + g_state.ring[idx][0] = x16; + g_state.ring[idx][1] = regs[0]; + g_state.ring[idx][2] = regs[1]; + g_state.ring[idx][3] = regs[2]; + g_state.ring[idx][4] = regs[3]; + g_state.ring[idx][5] = esr; + g_state.ring[idx][6] = elr; + g_state.ring[idx][7] = regs[30]; /* x30 = LR = caller of trampoline */ + g_state.ring_idx = (g_state.ring_idx + 1) & 0x7f; + } + + /* Do not force periodic Python fall-throughs. UART round trips are too + * expensive for boot progress; visibility comes from the shared rings and + * explicit panic/watchdog dumps. */ + + if (table == SPTM_TABLE_NVME) { + if (nvme_handle_table6(endpoint, regs)) + return 1; + g_state.fallthrough_calls++; + return 0; + } + + if (table == SPTM_TABLE_T8110_DART_XNU || + table == SPTM_TABLE_T8110_DART_SK || + table == SPTM_TABLE_GEN3_DART_XNU || + table == SPTM_TABLE_GEN3_DART_SK) { + if (dart_handle_table(table, endpoint, regs)) + return 1; + g_state.fallthrough_calls++; + return 0; + } + + if (table == SPTM_TABLE_SART) { + if (sart_handle_table5(endpoint, regs)) + return 1; + g_state.fallthrough_calls++; + return 0; + } + + if (table == SPTM_TABLE_UAT) { + if (uat_handle_table7(endpoint, regs)) + return 1; + g_state.fallthrough_calls++; + return 0; + } + + if (table == SPTM_TABLE_CPUTRACE) { + /* CPUTRACE is a diagnostic/performance facility, not a boot-critical + * memory-management endpoint. The firmware exposes 13 known selectors + * on this table; acknowledge them here instead of forcing a UART + * round-trip to the removed Python fallback. regs[0]=0 is both the + * normal success code and the conservative "unsupported/no carveout" + * answer for query-shaped helpers. */ + if (endpoint <= 12) { + return finish_hvc_success(ctx, regs); + } + g_state.fallthrough_calls++; + return 0; + } + + if (compat_void_non_xnu_table(table)) { + return finish_hvc_success(ctx, regs); + } + + /* Fast-path the XNU_BOOTSTRAP table. Other non-XNU tables still fall + * through to host, except NVMe table 6 above which is intentionally in C + * to avoid Python/C split state on the ANS boot path. */ + if (table != SPTM_TABLE_XNU_BOOTSTRAP) { + dart_cache_xnu_object_from_regs(table, endpoint, regs); + g_state.fallthrough_calls++; + return 0; /* not handled */ + } + + + return xnu_handle_table0(ctx, endpoint, regs); +} + +extern void *hv_hvc_handler; + +void sptm_emul_init(void) +{ + hv_hvc_handler = (void *)sptm_handler; + + /* + * Map the RW_EL0 RAM alias Normal-NC. The emulator places the coprocessor / + * IOMMU page tables it manages in this alias, and they must be uncached so + * m1n1 does not allocate AMCC SLC directory tags on pages that AMCC/PMP + * agents later read non-coherently (otherwise: UNEXP_RT_HIT_DIR -> panic). + * This lives here, not in memory.c, so upstream memory.c stays generic and + * carries no SPTM layout knowledge. + */ + u64 ram_base = ALIGN_DOWN(cur_boot_args.phys_base, BIT(32)); + u64 ram_size = ALIGN_DOWN(cur_boot_args.mem_size + cur_boot_args.phys_base - ram_base, + get_page_size()); + mmu_add_mapping(ram_base | REGION_RW_EL0, ram_base, ram_size, MAIR_IDX_NORMAL_NC, PERM_RW_EL0); +} diff --git a/src/sptm/emul.h b/src/sptm/emul.h new file mode 100644 index 000000000..8882b87ab --- /dev/null +++ b/src/sptm/emul.h @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef SPTM_EMUL_H +#define SPTM_EMUL_H + +void sptm_emul_init(void); + +#endif diff --git a/src/sptm/nvme.c b/src/sptm/nvme.c new file mode 100644 index 000000000..1d9d8403a --- /dev/null +++ b/src/sptm/nvme.c @@ -0,0 +1,1252 @@ +/* SPDX-License-Identifier: MIT */ +#include "private.h" +#include "../memory.h" + +int debug_printf(const char *fmt, ...); +extern u64 hv_translate(u64 addr, _Bool s1only, _Bool w, u64 *par_out); + +#define NVME_NORMAL_BAR_FROM_NVMMU(nvmmu) ((nvmmu) - 0x28000UL) +#define NVME_NORMAL_BAR_UNKNOWN_CTRL 0x24008U +#define NVME_NORMAL_BAR_LINEAR_SQ_CTRL 0x24908U +#define NVME_NORMAL_BAR_MODESEL 0x1304U + +static inline void nvme_record_last(u32 endpoint, u64 rc, u64 *regs) { + g_state.nvme.last[0] = endpoint; + g_state.nvme.last[1] = rc; + g_state.nvme.last[2] = regs[0]; + g_state.nvme.last[3] = regs[1]; + g_state.nvme.last[4] = regs[2]; + g_state.nvme.last[5] = regs[3]; + g_state.nvme.last[6] = regs[4]; + g_state.nvme.last[7] = regs[5]; + g_state.nvme.last_rc = (u32)rc; +} + +static inline u64 nvme_violation(u32 endpoint, u64 *regs, u32 reason) { + g_state.nvme.violations++; + g_state.nvme.last[8] = reason; + nvme_record_last(endpoint, NVME_STATUS_VIOLATION, regs); + return NVME_STATUS_VIOLATION; +} + +static inline bool_t nvme_mmio_base_valid(u64 base) { + return base != 0 && base != ~0UL; +} + +static inline void nvme_mmio_write32(u64 base, u32 off, u32 val) { + *(volatile u32 *)(base + off) = val; +} + +static inline u32 nvme_mmio_read32(u64 base, u32 off) { + return *(volatile u32 *)(base + off); +} + +static inline u64 nvme_mmio_read64(u64 base, u32 off) { + u64 lo = nvme_mmio_read32(base, off); + u64 hi = nvme_mmio_read32(base, off + 4); + return lo | (hi << 32); +} + +static inline void nvme_bar_write32(u32 off, u32 val) { + nvme_mmio_write32(g_state.nvme.bar_pa, off, val); +} + +static inline void nvme_bar_write64(u32 off, u64 val) { + nvme_bar_write32(off, (u32)(val & 0xffffffffUL)); + nvme_bar_write32(off + 4, (u32)(val >> 32)); +} + +static inline void nvme_nvmmu_write32(u32 off, u32 val) { + nvme_mmio_write32(g_state.nvme.nvmmu_pa, off, val); +} + +static inline void nvme_nvmmu_write64(u32 off, u64 val) { + nvme_nvmmu_write32(off, (u32)(val & 0xffffffffUL)); + nvme_nvmmu_write32(off + 4, (u32)(val >> 32)); +} + +static inline u64 nvme_cntpct(void) { + u64 val; + __asm__ volatile("mrs %0, CNTPCT_EL0" : "=r"(val)); + return val; +} + +static inline u64 nvme_cntfrq(void) { + u64 val; + __asm__ volatile("mrs %0, CNTFRQ_EL0" : "=r"(val)); + return val; +} + +static inline void nvme_wfe(void) { + __asm__ volatile("wfe" ::: "memory"); +} + +static inline void nvme_dsb_sy(void) { + sysop("dsb sy"); +} + +static void nvme_diag_sart_live_entry(u32 idx, u64 *paddr, u64 *size, + u32 *flags) { + u32 cfg; + u32 paddr_raw; + u32 size_raw; + + *paddr = 0; + *size = 0; + *flags = 0; + + if (!g_state.sart.base_pa || idx >= SART_C_MAX_ENTRIES) + return; + + cfg = *(volatile u32 *)(g_state.sart.base_pa + idx * 4U); + paddr_raw = *(volatile u32 *)(g_state.sart.base_pa + 0x40U + idx * 4U); + + if (g_state.sart.version == 3) { + *flags = cfg & 0xffU; + size_raw = *(volatile u32 *)(g_state.sart.base_pa + 0x80U + idx * 4U); + } else if (g_state.sart.version == 2) { + *flags = (cfg >> 24) & 0xffU; + size_raw = cfg & ((1U << 24) - 1U); + } else { + *flags = (cfg >> 24) & 0x1fU; + size_raw = cfg & ((1U << 19) - 1U); + } + + if (*flags && !g_state.sart.exclusive_bounds) + size_raw++; + + *paddr = (u64)paddr_raw << g_state.sart.paddr_shift; + *size = (u64)size_raw << g_state.sart.size_shift; +} + +static void nvme_diag_sart_readback_once(void) { + static u32 printed; + + if (printed) + return; + printed = 1; + + if (!(g_state.nvme.debug_flags & NVME_DBG_PRINT_CALLS)) + return; + + debug_printf("C-NVME sart enabled=%u active=%u base=0x%lx protected=0x%x total=%lu last_rc=0x%x\n", + g_state.sart.enabled, g_state.sart.active, + g_state.sart.base_pa, g_state.sart.protected_mask, + g_state.sart.total_calls, g_state.sart.last_rc); + debug_printf("C-NVME ranges trusted=0x%lx..0x%lx secondary=0x%lx..0x%lx asq=0x%lx acq=0x%lx admin_kva=0x%lx/0x%lx\n", + g_state.nvme.trusted_io_base, + g_state.nvme.trusted_io_base + g_state.nvme.trusted_io_size, + g_state.nvme.secondary_io_base, + g_state.nvme.secondary_io_base + g_state.nvme.secondary_io_size, + g_state.nvme.asq_pa_cached, g_state.nvme.acq_pa_cached, + g_state.nvme.admin_asq_kva, g_state.nvme.admin_acq_kva); + + for (u32 i = 0; i < SART_C_MAX_ENTRIES; i++) { + u64 live_pa, live_size; + u32 live_flags; + struct sart_c_entry shadow = g_state.sart.shadow[i]; + + nvme_diag_sart_live_entry(i, &live_pa, &live_size, &live_flags); + if (!live_flags && !shadow.flags) + continue; + + debug_printf("C-NVME sart[%u] live=0x%lx+0x%lx f=0x%x shadow=0x%lx+0x%lx f=0x%x g=%u prot=%u\n", + i, live_pa, live_size, live_flags, + shadow.paddr, shadow.size, shadow.flags, shadow.guard, + (g_state.sart.protected_mask & (1U << i)) ? 1U : 0U); + } +} + +static bool_t nvme_guest_read64(u64 va, u64 *out) { + u64 pa; + + if ((va >> 48) != 0xffff || (va & 7)) + return 0; + + pa = hv_translate(va, 0, 0, 0); + if (!pa) + return 0; + + *out = *(volatile u64 *)pa; + return 1; +} + +static inline u16 nvme_load_le16(u8 *p) { + return (u16)p[0] | ((u16)p[1] << 8); +} + +static inline u32 nvme_load_le32(u8 *p) { + return (u32)p[0] | ((u32)p[1] << 8) | ((u32)p[2] << 16) | + ((u32)p[3] << 24); +} + +static void nvme_print_backtrace(u64 fp) { + debug_printf(" fp=0x%lx bt=", fp); + + for (u32 depth = 0; depth < 8; depth++) { + u64 saved_fp; + u64 saved_lr; + + if (!nvme_guest_read64(fp, &saved_fp) || + !nvme_guest_read64(fp + 8, &saved_lr)) + break; + + debug_printf("%s0x%lx", depth ? "," : "", saved_lr); + + if (saved_fp == fp || saved_fp < fp) + break; + fp = saved_fp; + } +} + +static void nvme_diag_ep5_readback_once(void) { + static u32 printed; + + if (printed) + return; + printed = 1; + + if (!nvme_mmio_base_valid(g_state.nvme.bar_pa) || + !nvme_mmio_base_valid(g_state.nvme.nvmmu_pa)) { + debug_printf("C-NVME ep5rb missing bar=0x%lx nvmmu=0x%lx\n", + g_state.nvme.bar_pa, g_state.nvme.nvmmu_pa); + return; + } + + nvme_dsb_sy(); + u32 aqa = nvme_mmio_read32(g_state.nvme.bar_pa, NVME_BAR_AQA); + u64 asq = nvme_mmio_read64(g_state.nvme.bar_pa, NVME_BAR_ASQ); + u64 acq = nvme_mmio_read64(g_state.nvme.bar_pa, NVME_BAR_ACQ); + u32 ioqa = nvme_mmio_read32(g_state.nvme.bar_pa, NVME_BAR_IOQA); + u32 cg = nvme_mmio_read32(g_state.nvme.nvmmu_pa, NVME_BAR_COASTGUARD); + u64 cgasq = nvme_mmio_read64(g_state.nvme.nvmmu_pa, NVME_BAR_CG_ASQ); + u64 cgacq = nvme_mmio_read64(g_state.nvme.nvmmu_pa, NVME_BAR_CG_ACQ); + u64 normal_bar = NVME_NORMAL_BAR_FROM_NVMMU(g_state.nvme.nvmmu_pa); + u32 unknown_ctrl = nvme_mmio_read32(normal_bar, + NVME_NORMAL_BAR_UNKNOWN_CTRL); + u32 linear_sq_ctrl = nvme_mmio_read32(normal_bar, + NVME_NORMAL_BAR_LINEAR_SQ_CTRL); + u32 modesel = nvme_mmio_read32(normal_bar, NVME_NORMAL_BAR_MODESEL); + + g_state.nvme.last[9] = 0xe5; + g_state.nvme.last[10] = ((u64)aqa << 32) | ioqa; + g_state.nvme.last[11] = asq; + g_state.nvme.last[12] = acq; + g_state.nvme.last[13] = ((u64)cg << 32) | unknown_ctrl; + g_state.nvme.last[14] = cgasq; + g_state.nvme.last[15] = cgacq; + + if (!(g_state.nvme.debug_flags & + (NVME_DBG_PRINT_CALLS | NVME_DBG_PRINT_MINIMAL))) + return; + + if (g_state.nvme.debug_flags & NVME_DBG_PRINT_MINIMAL) { + debug_printf("C-NVME ep5min allowed=0x%x aqa=0x%x asq=0x%lx acq=0x%lx ioqa=0x%x csts=0x%x\n", + g_state.nvme.allowed_functions, aqa, asq, acq, ioqa, + nvme_mmio_read32(normal_bar, 0x1c)); + } + + if (g_state.nvme.debug_flags & NVME_DBG_PRINT_CALLS) { + debug_printf("C-NVME ep5rb bar=0x%lx nvmmu=0x%lx allowed=0x%x\n", + g_state.nvme.bar_pa, g_state.nvme.nvmmu_pa, + g_state.nvme.allowed_functions); + debug_printf("C-NVME ep5rb aqa=0x%x asq=0x%lx acq=0x%lx ioqa=0x%x\n", + aqa, asq, acq, ioqa); + debug_printf("C-NVME ep5rb cg=0x%x cgasq=0x%lx cgacq=0x%lx cached aqa=0x%x ioqa=0x%x\n", + cg, cgasq, cgacq, g_state.nvme.aqa_cached, + g_state.nvme.ioqa_cached); + debug_printf("C-NVME ep5rb normal unknown=0x%x linear=0x%x modesel=0x%x\n", + unknown_ctrl, linear_sq_ctrl, modesel); + nvme_diag_sart_readback_once(); + } +} + +static void nvme_diag_ep1_readback(u32 qid, u32 cid, u32 count, u16 nlb, + u16 dir_bits, u8 mode, u64 first, + u64 second_or_list) { + u64 tcb_base = 0; + u16 word0 = 0; + u16 word1 = 0; + u16 word2 = 0; + u16 word3 = 0; + + if (qid < 2 && g_state.nvme.tcb_kva[qid] != 0) { + tcb_base = g_state.nvme.tcb_kva[qid] + (u64)cid * 0x80UL; + word0 = *(volatile u16 *)(tcb_base + 0); + word1 = *(volatile u16 *)(tcb_base + 2); + word2 = *(volatile u16 *)(tcb_base + 4); + word3 = *(volatile u16 *)(tcb_base + 6); + } + + g_state.nvme.last[9] = 0xe1; + g_state.nvme.last[10] = ((u64)qid << 48) | ((u64)cid << 32) | + ((u64)count << 16) | nlb; + g_state.nvme.last[11] = ((u64)dir_bits << 32) | mode; + g_state.nvme.last[12] = first; + g_state.nvme.last[13] = second_or_list; + g_state.nvme.last[14] = tcb_base; + g_state.nvme.last[15] = ((u64)word0 << 48) | ((u64)word1 << 32) | + ((u64)word2 << 16) | word3; + + if (!(g_state.nvme.debug_flags & NVME_DBG_PRINT_CALLS)) + return; + + debug_printf("C-NVME ep1rb qid=%u cid=%u count=%u nlb=0x%x dir=0x%x mode=%u\n", + qid, cid, count, nlb, dir_bits, mode); + debug_printf("C-NVME ep1rb first=0x%lx second_or_list=0x%lx tcb=0x%lx words=%x,%x,%x,%x\n", + first, second_or_list, tcb_base, word0, word1, word2, word3); +} + +static void nvme_diag_cmd_map(u32 qid, u32 cid, u32 count, u16 nlb, + u16 dir_bits, u8 mode, u64 first, + u64 second_or_list, u64 template_pa, + u64 seglist_pa, u8 *tmpl) { + if (!(g_state.nvme.debug_flags & NVME_DBG_CMD_TRACE)) + return; + + debug_printf("C-NVME cmd map qid=%u cid=%u count=%u nlb=0x%x dir=0x%x mode=%u tpl=0x%lx seg=0x%lx prp1=0x%lx prp2=0x%lx\n", + qid, cid, count, nlb, dir_bits, mode, template_pa, seglist_pa, + first, second_or_list); + debug_printf("C-NVME cmd map words=%x,%x,%x,%x,%x,%x,%x,%x cdw10=%x cdw11=%x cdw12=%x cdw13=%x\n", + nvme_load_le16(&tmpl[0x00]), nvme_load_le16(&tmpl[0x02]), + nvme_load_le16(&tmpl[0x04]), nvme_load_le16(&tmpl[0x06]), + nvme_load_le16(&tmpl[0x08]), nvme_load_le16(&tmpl[0x0a]), + nvme_load_le16(&tmpl[0x0c]), nvme_load_le16(&tmpl[0x0e]), + nvme_load_le32(&tmpl[0x28]), nvme_load_le32(&tmpl[0x2c]), + nvme_load_le32(&tmpl[0x30]), nvme_load_le32(&tmpl[0x34])); +} + +static void nvme_diag_ep2_prp_readback(u32 qid, u32 cid, u32 count, + u16 dir_bits, u8 mode, u64 first) { + static u32 printed; + u64 q0 = 0; + u64 q1 = 0; + u64 q2 = 0; + u64 q3 = 0; + u64 q178 = 0; + u64 q180 = 0; + u64 q188 = 0; + u8 b180 = 0; + + if (!(g_state.nvme.debug_flags & NVME_DBG_PRINT_CALLS) || + printed >= 8 || count == 0 || first == 0) + return; + printed++; + + q0 = *(volatile u64 *)(first + 0x00); + q1 = *(volatile u64 *)(first + 0x08); + q2 = *(volatile u64 *)(first + 0x10); + q3 = *(volatile u64 *)(first + 0x18); + if (g_state.nvme.debug_flags & NVME_DBG_PRP_WINDOW) { + q178 = *(volatile u64 *)(first + 0x178); + q180 = *(volatile u64 *)(first + 0x180); + q188 = *(volatile u64 *)(first + 0x188); + b180 = *(volatile u8 *)(first + 0x180); + } + + g_state.nvme.last[9] = 0xe2; + g_state.nvme.last[10] = ((u64)qid << 48) | ((u64)cid << 32) | + ((u64)count << 16) | dir_bits; + g_state.nvme.last[11] = ((u64)mode << 56) | first; + g_state.nvme.last[12] = q0; + g_state.nvme.last[13] = q1; + g_state.nvme.last[14] = q2; + g_state.nvme.last[15] = q3; + + debug_printf("C-NVME ep2rb qid=%u cid=%u count=%u dir=0x%x mode=%u prp1=0x%lx q=%lx,%lx,%lx,%lx\n", + qid, cid, count, dir_bits, mode, first, q0, q1, q2, q3); + if (g_state.nvme.debug_flags & NVME_DBG_PRP_WINDOW) { + debug_printf("C-NVME ep2rb180 qid=%u cid=%u prp1=0x%lx b180=0x%x q178=%lx q180=%lx q188=%lx\n", + qid, cid, first, b180, q178, q180, q188); + } +} + +static void nvme_diag_cmd_unmap(u32 qid, u32 cid, u32 retry, u32 count, + u16 dir_bits, u8 mode, u64 first, + u64 second_or_list) { + if (!(g_state.nvme.debug_flags & NVME_DBG_CMD_TRACE)) + return; + + debug_printf("C-NVME cmd unmap qid=%u cid=%u retry=%u count=%u dir=0x%x mode=%u prp1=0x%lx prp2=0x%lx\n", + qid, cid, retry, count, dir_bits, mode, first, + second_or_list); +} + +static inline bool_t nvme_range_contains(u64 base, u64 size, u64 pa, u64 len) { + if (base == 0 || size == 0) + return 0; + if (len == 0) + len = 1; + if (pa < base) + return 0; + if ((pa - base) > size) + return 0; + if (len > (size - (pa - base))) + return 0; + return 1; +} + +static inline bool_t nvme_in_trusted_io(u64 pa, u64 len) { + if (nvme_range_contains(g_state.nvme.trusted_io_base, + g_state.nvme.trusted_io_size, pa, len)) + return 1; + if (nvme_range_contains(g_state.nvme.secondary_io_base, + g_state.nvme.secondary_io_size, pa, len)) + return 1; + if (g_state.nvme.trusted_io_base == 0 && g_state.nvme.trusted_io_size == 0 && + g_state.nvme.secondary_io_base == 0 && g_state.nvme.secondary_io_size == 0) + return 1; + return 0; +} + +static inline bool_t nvme_validate_page_pa(u64 pa, u64 page_size) { + if (pa == 0) + return 0; + if (pa & (page_size - 1)) + return 0; + if (!nvme_in_trusted_io(pa, page_size)) + return 0; + return 1; +} + +static inline bool_t nvme_validate_ans_sha_page_pa(u64 pa) { + if (!nvme_validate_page_pa(pa, NVME_PRP_PAGE_SIZE)) + return 0; + if (pa & 0x3000UL) + return 0; + return 1; +} + +static inline bool_t nvme_validate_buffer(u64 pa, u64 len) { + if (pa == 0) + return 0; + return nvme_in_trusted_io(pa, len ? len : 1); +} + +static inline bool_t nvme_in_main_ram(u64 pa, u64 len) { + return nvme_range_contains(g_state.nvme.secondary_io_base, + g_state.nvme.secondary_io_size, + pa, len ? len : 1); +} + +static inline bool_t nvme_validate_guest_copy_buffer(u64 pa, u64 len) { + if (pa == 0 || len == 0) + return 0; + if (!nvme_in_main_ram(pa, len)) + return 0; + if (((pa & (NVME_PAGE_SIZE - 1)) + len) > NVME_PAGE_SIZE) + return 0; + return 1; +} + +static inline bool_t nvme_validate_prp(u64 pa, bool_t allow_offset) { + if (!nvme_validate_buffer(pa, 1)) + return 0; + if (!allow_offset && (pa & (NVME_PRP_PAGE_SIZE - 1))) + return 0; + return 1; +} + +static inline bool_t nvme_func_enter(u32 idx) { + if (idx >= 16) + return 0; + if (g_state.nvme.func_state[idx] == 1) + return 0; + g_state.nvme.func_state[idx] = 1; + return 1; +} + +static inline void nvme_func_exit(u32 idx) { + if (idx < 16) + g_state.nvme.func_state[idx] = 0; +} + +static inline bool_t nvme_cid_enter(u32 cid) { + if (cid >= g_state.nvme.max_cid || cid >= NVME_C_MAX_CID) + return 0; + if (g_state.nvme.cid_mode[cid] != NVME_CID_AVAILABLE) + return 0; + g_state.nvme.cid_mode[cid] = NVME_CID_BUSY; + return 1; +} + +static inline void nvme_cid_clear(u32 cid) { + if (cid < g_state.nvme.max_cid && cid < NVME_C_MAX_CID) + g_state.nvme.cid_mode[cid] = NVME_CID_AVAILABLE; +} + +static void nvme_diag_queue_summary(u32 event, u32 qid, u32 cid, u64 rc) { + static u64 last_map1; + static u64 last_unmap1; + u32 zero = 0; + u32 busy = 0; + u32 active0 = 0; + u32 active1 = 0; + u32 retry0 = 0; + u32 retry1 = 0; + u32 other = 0; + u32 max_cid = g_state.nvme.max_cid; + bool_t force; + + if (!(g_state.nvme.debug_flags & NVME_DBG_SUMMARY)) + return; + + if (max_cid > NVME_C_MAX_CID) + max_cid = NVME_C_MAX_CID; + + force = (event >= 3U); + if (!force) { + if (event == 1U && qid == 1) { + u64 n = g_state.nvme.ep_count[10]; + if ((n & 31UL) != 0 || n == last_map1) + return; + last_map1 = n; + } else if (event == 2U && qid == 1) { + u64 n = g_state.nvme.ep_count[12]; + if ((n & 31UL) != 0 || n == last_unmap1) + return; + last_unmap1 = n; + } else { + return; + } + } + + for (u32 i = 0; i < max_cid; i++) { + switch (g_state.nvme.cid_mode[i]) { + case NVME_CID_AVAILABLE: zero++; break; + case NVME_CID_BUSY: busy++; break; + case 0x03: active0++; break; + case 0x13: active1++; break; + case 0x04: retry0++; break; + case 0x14: retry1++; break; + default: other++; break; + } + } + + debug_printf("C-NVME qsum ev=%u qid=%u cid=%u rc=0x%lx " + "map0=%lu map1=%lu unmap0=%lu unmap1=%lu " + "retry=%lu mapfail=%lu unmapfail=%lu " + "zero=%u busy=%u act0=%u act1=%u retry0=%u retry1=%u other=%u " + "tl=%u viol=%u\n", + event, qid, cid, rc, + g_state.nvme.ep_count[9], g_state.nvme.ep_count[10], + g_state.nvme.ep_count[11], g_state.nvme.ep_count[12], + g_state.nvme.ep_count[13], g_state.nvme.ep_count[14], + g_state.nvme.ep_count[15], + zero, busy, active0, active1, retry0, retry1, other, + g_state.nvme.tl_timeouts, g_state.nvme.violations); +} + +static inline u32 nvme_ft_read_u32(u64 pa, u64 off) { + u64 idx; + if (!ft_get_idx(pa, &idx)) + return 0; + volatile u32 *cnt = + (volatile u32 *)(g_state.frame_table_base + idx * 16 + off); + return *cnt; +} + +static void nvme_diag_iommu_ref(const char *op, u64 pa, u8 mode, + u32 before, u32 after) { + static u32 printed; + u64 idx = 0; + + if (!(g_state.nvme.debug_flags & NVME_DBG_IOMMU)) + return; + if (printed >= 96) + return; + printed++; + + bool_t indexed = ft_get_idx(pa, &idx); + debug_printf("C-NVME iommu %s pa=0x%lx mode=%u indexed=%u idx=0x%lx " + "type=0x%x page_ref=%u table_ref=%u ro=%u rw=%u " + "before=%u after=%u\n", + op, pa, mode, indexed, idx, ft_get_type(pa), + ft_read_u16(pa, 4), ft_read_u16(pa, 8), + nvme_ft_read_u32(pa, 8), nvme_ft_read_u32(pa, 12), + before, after); +} + +static inline bool_t nvme_iommu_acquire(u64 pa, u8 mode) { + u64 off = (mode == NVME_IOMMU_RO) ? 8 : 12; + u32 before = nvme_ft_read_u32(pa, off); + if (mode == NVME_IOMMU_RO) + ft_inc_u32(pa, 8); + else + ft_inc_u32(pa, 12); + nvme_diag_iommu_ref("acq", pa, mode, before, nvme_ft_read_u32(pa, off)); + return 1; +} + +static inline void nvme_iommu_release_mode(u64 pa, u8 mode) { + if (pa == 0) + return; + u64 off = (mode == NVME_IOMMU_RO) ? 8 : 12; + u32 before = nvme_ft_read_u32(pa, off); + if (mode == NVME_IOMMU_RO) + ft_dec_u32(pa, 8); + else + ft_dec_u32(pa, 12); + nvme_diag_iommu_ref("rel", pa, mode, before, nvme_ft_read_u32(pa, off)); +} + +static inline void nvme_store_le64(u8 *p, u64 v) { + for (u32 i = 0; i < 8; i++) + p[i] = (u8)(v >> (i * 8)); +} + +static bool_t nvme_tl_wa_wait(u32 qid, u32 cid, u16 dir_bits, bool_t retry) { + if (!(g_state.nvme.quirks & NVME_QUIRK_TL_WA) || !(dir_bits & 0x300U)) + return 1; + if (!nvme_mmio_base_valid(g_state.nvme.tl_status_pa) || + !nvme_mmio_base_valid(g_state.nvme.tl_mask_pa) || + !nvme_mmio_base_valid(g_state.nvme.tl_ctrl_pa)) + return 0; + if (g_state.nvme.tl_num_sl > 16) + return 0; + + if (!retry) { + bool_t h2f = (dir_bits & 0x100U) != 0; + u32 mask = h2f ? 0x800000U : 0x80U; + u32 ctrl = h2f ? 0x400000U : 0x40U; + + nvme_mmio_write32(g_state.nvme.tl_mask_pa, 0x4, mask); + nvme_mmio_write32(g_state.nvme.tl_mask_pa, 0xc, mask); + nvme_mmio_write32(g_state.nvme.tl_ctrl_pa, 0x4, ctrl); + g_state.nvme.tl_slot_done[cid] = 0; + } + + nvme_dsb_sy(); + u64 start = nvme_cntpct(); + u64 freq = nvme_cntfrq(); + u64 timeout = (freq & ~0x1fUL) / 4000; + u64 nap_threshold = (freq & ~0x1fUL) / 100000; + u64 nap_period = (freq & ~0x7fUL) / 10000000; + if (timeout == 0) + timeout = 1; + if (nap_threshold == 0) + nap_threshold = 1; + if (nap_period == 0) + nap_period = 1; + + for (;;) { + u16 done = g_state.nvme.tl_slot_done[cid]; + u32 complete = 0; + + for (u32 slot = 0; slot < g_state.nvme.tl_num_sl; slot++) { + u16 bit = (u16)(1U << slot); + + if (done & bit) { + complete++; + continue; + } + + u32 status = nvme_mmio_read32(g_state.nvme.tl_status_pa, + slot * 0x10000U); + u32 pending; + if (dir_bits & 0x100U) + pending = (~status) & 0xffU; + else + pending = (~status) & 0x7f0000U; + if (pending == 0) { + done |= bit; + complete++; + } + } + g_state.nvme.tl_slot_done[cid] = done; + + if (complete == g_state.nvme.tl_num_sl) { + nvme_mmio_write32(g_state.nvme.tl_mask_pa, 0x4, 0x800080U); + nvme_mmio_write32(g_state.nvme.tl_mask_pa, 0xc, 0x800080U); + nvme_mmio_write32(g_state.nvme.tl_ctrl_pa, 0x4, 0x400040U); + g_state.nvme.tl_slot_done[cid] = 0; + return 1; + } + + nvme_dsb_sy(); + u64 elapsed = nvme_cntpct() - start; + if (elapsed >= timeout) + break; + if (elapsed > nap_threshold) { + nvme_dsb_sy(); + u64 nap = nvme_cntpct(); + while ((nvme_cntpct() - nap) < nap_period) { + nvme_dsb_sy(); + nvme_wfe(); + } + } + } + + g_state.nvme.tl_timeouts++; + g_state.nvme.cid_mode[cid] = (u8)((qid << 4) | 0x4U); + return 0; +} + +static u64 nvme_ep3_validate(u64 *regs) { + u32 queue_entries = (u32)regs[0]; + u32 proto = (u32)regs[1]; + u32 expected_proto = g_state.nvme.linear_sq ? 2U : 1U; + + if (!(g_state.nvme.allowed_functions & NVME_AF_QUEUE_VALIDATE)) + return nvme_violation(NVME_FN_VALIDATE_QENTRIES, regs, 1); + if (queue_entries != g_state.nvme.queue_count || proto != expected_proto) + return nvme_violation(NVME_FN_VALIDATE_QENTRIES, regs, 3); + return SPTM_SUCCESS; +} + +static u64 nvme_ep0_enable_coastguard(u64 *regs) { + if (!(g_state.nvme.allowed_functions & NVME_AF_ENABLE_COASTGUARD)) + return nvme_violation(NVME_FN_ENABLE_COASTGUARD, regs, 1); + if (g_state.nvme.admin_asq_kva == 0 || g_state.nvme.admin_acq_kva == 0) + return nvme_violation(NVME_FN_ENABLE_COASTGUARD, regs, 2); + if (!nvme_mmio_base_valid(g_state.nvme.nvmmu_pa)) + return nvme_violation(NVME_FN_ENABLE_COASTGUARD, regs, 3); + + nvme_nvmmu_write64(NVME_BAR_CG_ASQ, g_state.nvme.admin_asq_kva); + nvme_nvmmu_write64(NVME_BAR_CG_ACQ, g_state.nvme.admin_acq_kva); + nvme_nvmmu_write32(NVME_BAR_COASTGUARD, NVME_COASTGUARD_VALUE); + nvme_dsb_sy(); + + if (g_state.nvme.packed_writes_present) + g_state.nvme.allowed_functions |= NVME_AF_BAR_ADMIN_Q; + else + g_state.nvme.allowed_functions |= (NVME_AF_TCB_REGISTER | + NVME_AF_TCB_INVALIDATE); + return SPTM_SUCCESS; +} + +static u64 nvme_ep4_admin_queue(u64 *regs) { + u64 asq_pa = regs[0]; + u32 asq_size = (u32)regs[1]; + u64 acq_pa = regs[2]; + u32 acq_size = (u32)regs[3]; + u32 aqa; + u64 rc = NVME_STATUS_VIOLATION; + bool_t new_asq = 0; + bool_t new_acq = 0; + + if (!(g_state.nvme.allowed_functions & NVME_AF_BAR_ADMIN_Q)) + return nvme_violation(NVME_FN_BAR_ADMIN_QUEUE_REGS, regs, 1); + if (!nvme_func_enter(4)) + return nvme_violation(NVME_FN_BAR_ADMIN_QUEUE_REGS, regs, 2); + if (!nvme_mmio_base_valid(g_state.nvme.bar_pa)) + goto out; + + if (asq_size > NVME_QSIZE_MAX || acq_size > NVME_QSIZE_MAX) + goto out; + aqa = (asq_size & 0xfffU) | ((acq_size & 0xfffU) << 16); + if (g_state.nvme.aqa_cached != ~0U && + g_state.nvme.aqa_cached != aqa) + goto out; + if (g_state.nvme.asq_pa_cached != ~0UL && + g_state.nvme.asq_pa_cached != asq_pa) + goto out; + if (g_state.nvme.acq_pa_cached != ~0UL && + g_state.nvme.acq_pa_cached != acq_pa) + goto out; + if (!nvme_validate_page_pa(asq_pa, NVME_PRP_PAGE_SIZE) || + !nvme_validate_page_pa(acq_pa, NVME_PRP_PAGE_SIZE)) + goto out; + new_asq = g_state.nvme.asq_pa_cached == ~0UL; + new_acq = g_state.nvme.acq_pa_cached == ~0UL; + if (new_asq && !nvme_iommu_acquire(asq_pa, NVME_IOMMU_RW)) + goto out; + if (new_acq && !nvme_iommu_acquire(acq_pa, NVME_IOMMU_RW)) { + if (new_asq) + nvme_iommu_release_mode(asq_pa, NVME_IOMMU_RW); + goto out; + } + nvme_bar_write32(NVME_BAR_AQA, aqa); + nvme_bar_write64(NVME_BAR_ASQ, asq_pa); + nvme_bar_write64(NVME_BAR_ACQ, acq_pa); + barrier(); + + g_state.nvme.aqa_cached = aqa; + g_state.nvme.asq_pa_cached = asq_pa; + g_state.nvme.acq_pa_cached = acq_pa; + g_state.nvme.allowed_functions |= (NVME_AF_TCB_REGISTER | + NVME_AF_TCB_INVALIDATE | + NVME_AF_BAR_IOQA); + rc = SPTM_SUCCESS; +out: + nvme_func_exit(4); + if (rc != SPTM_SUCCESS) + return nvme_violation(NVME_FN_BAR_ADMIN_QUEUE_REGS, regs, 3); + return rc; +} + +static u64 nvme_ep5_ioqa(u64 *regs) { + u32 iosq_size = (u32)regs[0]; + u32 iocq_size = (u32)regs[1]; + u32 ioqa; + u64 rc = NVME_STATUS_VIOLATION; + + if (!(g_state.nvme.allowed_functions & NVME_AF_BAR_IOQA)) + return nvme_violation(NVME_FN_BAR_IOQA_REG, regs, 1); + if (!nvme_func_enter(5)) + return nvme_violation(NVME_FN_BAR_IOQA_REG, regs, 2); + if (!nvme_mmio_base_valid(g_state.nvme.bar_pa)) + goto out; + + if (iosq_size > g_state.nvme.queue_count || + iocq_size > g_state.nvme.queue_count) + goto out; + ioqa = (iosq_size & 0xffffU) | ((iocq_size & 0xffffU) << 16); + if (g_state.nvme.ioqa_cached != ~0U && g_state.nvme.ioqa_cached != ioqa) + goto out; + nvme_bar_write32(NVME_BAR_IOQA, ioqa); + barrier(); + g_state.nvme.ioqa_cached = ioqa; + if (g_state.nvme.packed_writes_present) + g_state.nvme.allowed_functions |= NVME_AF_BAR_IOCQ; + rc = SPTM_SUCCESS; +out: + nvme_func_exit(5); + if (rc != SPTM_SUCCESS) + return nvme_violation(NVME_FN_BAR_IOQA_REG, regs, 3); + return rc; +} + +static u64 nvme_ep6_iosq(u64 *regs) { + u64 iosq_pa = regs[0]; + u64 rc = NVME_STATUS_VIOLATION; + + if (!(g_state.nvme.allowed_functions & NVME_AF_BAR_IOSQ)) + return nvme_violation(NVME_FN_BAR_IOSQ_REG, regs, 1); + if (!nvme_func_enter(6)) + return nvme_violation(NVME_FN_BAR_IOSQ_REG, regs, 2); + if (!nvme_mmio_base_valid(g_state.nvme.bar_pa)) + goto out; + if (!nvme_validate_page_pa(iosq_pa, NVME_PRP_PAGE_SIZE)) + goto out; + if (g_state.nvme.iosq_pa_cached != ~0UL && + g_state.nvme.iosq_pa_cached != iosq_pa) + goto out; + if (g_state.nvme.iosq_pa_cached == ~0UL && + !nvme_iommu_acquire(iosq_pa, NVME_IOMMU_RW)) + goto out; + nvme_bar_write64(NVME_BAR_IOSQ_BASE, iosq_pa); + barrier(); + g_state.nvme.iosq_pa_cached = iosq_pa; + rc = SPTM_SUCCESS; +out: + nvme_func_exit(6); + if (rc != SPTM_SUCCESS) + return nvme_violation(NVME_FN_BAR_IOSQ_REG, regs, 3); + return rc; +} + +static u64 nvme_ep7_iocq(u64 *regs) { + u64 iocq_pa = regs[0]; + u64 rc = NVME_STATUS_VIOLATION; + + if (!(g_state.nvme.allowed_functions & NVME_AF_BAR_IOCQ)) + return nvme_violation(NVME_FN_BAR_IOCQ_REG, regs, 1); + if (!nvme_func_enter(7)) + return nvme_violation(NVME_FN_BAR_IOCQ_REG, regs, 2); + if (!nvme_mmio_base_valid(g_state.nvme.bar_pa)) + goto out; + if (!nvme_validate_page_pa(iocq_pa, NVME_PRP_PAGE_SIZE)) + goto out; + if (g_state.nvme.iocq_pa_cached != ~0UL && + g_state.nvme.iocq_pa_cached != iocq_pa) + goto out; + if (g_state.nvme.iocq_pa_cached == ~0UL && + !nvme_iommu_acquire(iocq_pa, NVME_IOMMU_RW)) + goto out; + nvme_bar_write64(NVME_BAR_IOCQ_BASE, iocq_pa); + barrier(); + g_state.nvme.iocq_pa_cached = iocq_pa; + if (g_state.nvme.packed_writes_present) + g_state.nvme.allowed_functions |= NVME_AF_BAR_IOSQ; + rc = SPTM_SUCCESS; +out: + nvme_func_exit(7); + if (rc != SPTM_SUCCESS) + return nvme_violation(NVME_FN_BAR_IOCQ_REG, regs, 3); + return rc; +} + +static u64 nvme_ep8_ans_sha(u64 *regs) { + u64 ans_sha_pa = regs[0]; + u64 ans_sha_size = regs[1]; + u32 pwc = (u32)regs[2]; + u32 pwc_mask = pwc & 3U; + u64 rc = NVME_STATUS_VIOLATION; + + if (!(g_state.nvme.allowed_functions & NVME_AF_ANS_SHA)) + return nvme_violation(NVME_FN_ANS_SHA_REG, regs, 1); + if (!g_state.nvme.ans_sha_present) + return nvme_violation(NVME_FN_ANS_SHA_REG, regs, 2); + if (!nvme_func_enter(8)) + return nvme_violation(NVME_FN_ANS_SHA_REG, regs, 3); + if (ans_sha_size != (u64)g_state.nvme.queue_count * NVME_PAGE_SIZE) + goto out; + if (g_state.nvme.ans_sha_pa_cached != ~0UL && + (g_state.nvme.ans_sha_pa_cached != ans_sha_pa || + g_state.nvme.ans_sha_pwc_cached != pwc_mask)) + goto out; + if (g_state.nvme.ans_sha_pwc_cached != ~0U && + g_state.nvme.ans_sha_pwc_cached != pwc_mask) + goto out; + for (u64 off = 0; off < ans_sha_size; off += NVME_PAGE_SIZE) { + if (!nvme_validate_ans_sha_page_pa(ans_sha_pa + off)) + goto out; + } + if (g_state.nvme.ans_sha_pa_cached == ~0UL) { + for (u64 off = 0; off < ans_sha_size; off += NVME_PAGE_SIZE) { + if (!nvme_iommu_acquire(ans_sha_pa + off, NVME_IOMMU_RW)) + goto out; + } + } + if (g_state.nvme.ans_sha_kva != 0) { + if (g_state.nvme.ans_sha_pwc_cached == ~0U) + g_state.nvme.ans_sha_pwc_cached = pwc_mask; + *(volatile u32 *)(g_state.nvme.ans_sha_kva + 0x0) = + (u32)(ans_sha_pa >> 14); + *(volatile u32 *)(g_state.nvme.ans_sha_kva + 0x4) = + g_state.nvme.ans_sha_pwc_cached; + } else { + goto out; + } + barrier(); + g_state.nvme.ans_sha_pa_cached = ans_sha_pa; + rc = SPTM_SUCCESS; +out: + nvme_func_exit(8); + if (rc != SPTM_SUCCESS) + return nvme_violation(NVME_FN_ANS_SHA_REG, regs, 4); + return rc; +} + +static u64 nvme_ep1_map_pages(u64 *regs) { + u32 qid = (u32)regs[0]; + u32 cid = (u32)regs[1]; + u64 template_pa = regs[2]; + u64 seglist_pa = regs[3]; + u32 count = (u32)regs[4]; + u8 tmpl[0x80]; + u16 nlb; + u16 dir_bits; + u8 mode; + u64 first = 0, second = 0, second_or_list = 0; + u32 acquired_count = 0; + bool_t cid_entered = 0; + + if (!(g_state.nvme.allowed_functions & NVME_AF_TCB_REGISTER)) + return nvme_violation(NVME_FN_MAP_PAGES, regs, 1); + if (qid > 1 || cid >= g_state.nvme.queue_count || + cid >= g_state.nvme.max_cid || cid >= NVME_C_MAX_CID || + count > 0x101U) + return nvme_violation(NVME_FN_MAP_PAGES, regs, 2); + if (!nvme_cid_enter(cid)) + return nvme_violation(NVME_FN_MAP_PAGES, regs, 3); + cid_entered = 1; + + if (!nvme_validate_guest_copy_buffer(template_pa, 0x80)) + goto fail; + for (u32 i = 0; i < 0x80; i++) + tmpl[i] = *(volatile u8 *)(template_pa + i); + + nlb = nvme_load_le16(&tmpl[4]); + dir_bits = nvme_load_le16(&tmpl[0]) & 0x1f00U; + if ((g_state.nvme.quirks & NVME_QUIRK_TL_WA) && + (dir_bits & 0x300U) == 0x300U) + goto fail; + mode = (dir_bits & 0x100U) ? NVME_IOMMU_RW : NVME_IOMMU_RO; + g_state.nvme.tcb_dir[cid] = dir_bits; + g_state.nvme.tcb_nlb[cid] = nlb; + for (u32 i = 0x18; i < 0x28; i++) + tmpl[i] = 0; + + if (count == 0) { + if (nlb != 0) + goto fail; + } else { + bool_t first_aligned; + u32 expected_nlb; + + if (!nvme_validate_guest_copy_buffer(seglist_pa, (u64)count * 8UL)) + goto fail; + first = *(volatile u64 *)seglist_pa; + first_aligned = ((first & (NVME_PRP_PAGE_SIZE - 1)) == 0); + if (!first_aligned && count == 1) + goto fail; + expected_nlb = count - (first_aligned ? 1U : 2U); + if (expected_nlb != nlb) + goto fail; + if (!nvme_validate_prp(first, 1) || + !nvme_iommu_acquire(first, mode)) + goto fail; + acquired_count = 1; + nvme_store_le64(&tmpl[0x18], first); + + if (count >= 2) { + second = *(volatile u64 *)(seglist_pa + 8); + if (!nvme_validate_prp(second, 0) || + !nvme_iommu_acquire(second, mode)) + goto fail; + acquired_count = 2; + if (count == 2) { + second_or_list = second; + nvme_store_le64(&tmpl[0x20], second); + } else { + u64 prp_list = g_state.nvme.prp_list_kva + (u64)cid * 0x800UL; + if (g_state.nvme.prp_list_kva == 0) + goto fail; + second_or_list = prp_list; + nvme_store_le64(&tmpl[0x20], prp_list); + for (u32 i = 1; i < count; i++) { + u64 pa = *(volatile u64 *)(seglist_pa + (u64)i * 8UL); + if (!nvme_validate_prp(pa, 0)) + goto fail; + if (i > 1) { + if (!nvme_iommu_acquire(pa, mode)) + goto fail; + acquired_count++; + } + *(volatile u64 *)(prp_list + (u64)(i - 1) * 8UL) = pa; + } + } + } + } + + tmpl[0] = 0; + tmpl[1] = 0; + + if (g_state.nvme.tcb_kva[qid] != 0) { + u64 tcb_base = g_state.nvme.tcb_kva[qid] + (u64)cid * 0x80UL; + if (*(volatile u16 *)tcb_base != 0 || + *(volatile u16 *)(tcb_base + 4) != 0) + goto fail; + for (u32 i = 0; i < 0x80; i++) + *(volatile u8 *)(tcb_base + i) = tmpl[i]; + *(volatile u16 *)(tcb_base + 2) = (u16)cid; + if (g_state.nvme.linear_sq) { + barrier(); + *(volatile u16 *)tcb_base = dir_bits; + } + } + + if ((g_state.nvme.quirks & NVME_QUIRK_PRP_FLUSH_WA) && count > 0x10U) { + u64 prp_list = g_state.nvme.prp_list_kva + (u64)cid * 0x800UL; + u32 flush_len = 0x1000U; + + if (count < 0x100U) + flush_len = ((count * 8U) + 0x7fU) & 0x1f80U; + sysop("dsb sy"); + if (flush_len != 0) + dc_civac_range((void *)prp_list, flush_len); + } + + g_state.nvme.tcb_count[cid] = count; + g_state.nvme.tcb_mode[cid] = mode; + g_state.nvme.tcb_qid[cid] = (u8)qid; + g_state.nvme.tcb_prp1[cid] = first; + g_state.nvme.tcb_prp2[cid] = second_or_list; + g_state.nvme.cid_mode[cid] = + (u8)((qid << 4) | (g_state.nvme.linear_sq ? 0x3U : 0x2U)); + barrier(); + if (qid < 2) + g_state.nvme.ep_count[9 + qid]++; + nvme_diag_queue_summary(1, qid, cid, SPTM_SUCCESS); + nvme_diag_cmd_map(qid, cid, count, nlb, dir_bits, mode, first, + second_or_list, template_pa, seglist_pa, tmpl); + nvme_diag_ep1_readback(qid, cid, count, nlb, dir_bits, mode, first, + second_or_list); + return SPTM_SUCCESS; + +fail: + if (cid_entered) + nvme_cid_clear(cid); + if (acquired_count >= 1) + nvme_iommu_release_mode(first, mode); + if (acquired_count >= 2) + nvme_iommu_release_mode(second, mode); + for (u32 i = 2; i < acquired_count; i++) { + u64 pa = *(volatile u64 *)(seglist_pa + (u64)i * 8UL); + nvme_iommu_release_mode(pa, mode); + } + g_state.nvme.ep_count[14]++; + nvme_diag_queue_summary(3, qid, cid, NVME_STATUS_VIOLATION); + return nvme_violation(NVME_FN_MAP_PAGES, regs, 4); +} + +static u64 nvme_ep2_violation(u64 *regs, u32 qid, u32 cid, u32 reason) { + g_state.nvme.ep_count[15]++; + nvme_diag_queue_summary(4, qid, cid, NVME_STATUS_VIOLATION); + return nvme_violation(NVME_FN_UNMAP_PAGES, regs, reason); +} + +static u64 nvme_ep2_unmap_pages(u64 *regs) { + u32 qid = (u32)regs[0]; + u32 cid = (u32)regs[1]; + u32 retry = (u32)regs[2] & 1U; + u8 cur; + u8 expected; + u32 count; + u16 dir_bits; + + if (!(g_state.nvme.allowed_functions & NVME_AF_TCB_INVALIDATE)) + return nvme_ep2_violation(regs, qid, cid, 1); + if (qid > 1 || cid >= g_state.nvme.queue_count || + cid >= g_state.nvme.max_cid || cid >= NVME_C_MAX_CID) + return nvme_ep2_violation(regs, qid, cid, 2); + if (retry) + g_state.nvme.ep_count[13]++; + cur = g_state.nvme.cid_mode[cid]; + expected = (u8)((qid << 4) | (retry ? 0x4U : 0x3U)); + if (cur != expected) + return nvme_ep2_violation(regs, qid, cid, 3); + g_state.nvme.cid_mode[cid] = NVME_CID_BUSY; + + count = g_state.nvme.tcb_count[cid]; + dir_bits = g_state.nvme.tcb_dir[cid]; + + if (!retry) { + if (g_state.nvme.tcb_kva[qid] != 0) { + u64 tcb_base = g_state.nvme.tcb_kva[qid] + (u64)cid * 0x80UL; + *(volatile u16 *)tcb_base = 0; + } + if ((g_state.nvme.quirks & NVME_QUIRK_VDMA_WA) && + (!nvme_mmio_base_valid(g_state.nvme.vdma_status_pa) || + (*(volatile u32 *)(g_state.nvme.vdma_status_pa + + (u64)cid * 0x20UL) & 0x300U) != 0)) + return nvme_ep2_violation(regs, qid, cid, 5); + if (!nvme_mmio_base_valid(g_state.nvme.nvmmu_pa)) + return nvme_ep2_violation(regs, qid, cid, 6); + barrier(); + nvme_nvmmu_write32(NVME_BAR_TCB_INVALIDATE, cid); + u32 stat = nvme_mmio_read32(g_state.nvme.nvmmu_pa, + ((cid >> 2) & 0x3fffU) * 4U); + u32 shift = (cid & 3U) | ((cid & 3U) << 2); + if (((stat >> shift) & 0xfU) != 0) + return nvme_ep2_violation(regs, qid, cid, 7); + } + + if (!nvme_tl_wa_wait(qid, cid, dir_bits, retry)) { + g_state.nvme.ep_count[15]++; + nvme_diag_queue_summary(5, qid, cid, 0); + return 0; + } + + nvme_diag_cmd_unmap(qid, cid, retry, count, dir_bits, + g_state.nvme.tcb_mode[cid], + g_state.nvme.tcb_prp1[cid], + g_state.nvme.tcb_prp2[cid]); + nvme_diag_ep2_prp_readback(qid, cid, count, dir_bits, + g_state.nvme.tcb_mode[cid], + g_state.nvme.tcb_prp1[cid]); + + if (count >= 1) + nvme_iommu_release_mode(g_state.nvme.tcb_prp1[cid], + g_state.nvme.tcb_mode[cid]); + if (count == 2) { + nvme_iommu_release_mode(g_state.nvme.tcb_prp2[cid], + g_state.nvme.tcb_mode[cid]); + } else if (count > 2) { + u64 prp_list = g_state.nvme.prp_list_kva ? + g_state.nvme.prp_list_kva + (u64)cid * 0x800UL : + g_state.nvme.tcb_prp2[cid]; + for (u32 i = 0; i < count - 1; i++) { + u64 pa = *(volatile u64 *)(prp_list + (u64)i * 8UL); + nvme_iommu_release_mode(pa, g_state.nvme.tcb_mode[cid]); + } + for (u32 i = 0; i < count - 1; i++) + *(volatile u64 *)(prp_list + (u64)i * 8UL) = 0; + } + + if (g_state.nvme.tcb_kva[qid] != 0) { + u64 tcb_base = g_state.nvme.tcb_kva[qid] + (u64)cid * 0x80UL; + for (u32 i = 0; i < 0x80; i++) + *(volatile u8 *)(tcb_base + i) = 0; + } + if ((g_state.nvme.quirks & NVME_QUIRK_PRP_FLUSH_WA) && count > 0xffU) { + u64 prp_list = g_state.nvme.tcb_prp2[cid]; + + sysop("dsb sy"); + if (prp_list != 0) + dc_civac_range((void *)(prp_list + 0x800UL), 0x800); + } + + g_state.nvme.tcb_count[cid] = 0; + g_state.nvme.tcb_prp1[cid] = 0; + g_state.nvme.tcb_prp2[cid] = 0; + g_state.nvme.tcb_dir[cid] = 0; + nvme_cid_clear(cid); + barrier(); + if (qid < 2) + g_state.nvme.ep_count[11 + qid]++; + nvme_diag_queue_summary(2, qid, cid, 1); + /* sptm_nvme_unmap_pages is the odd table-6 endpoint whose RE'd return is + * boolean-like: 1 means success, 0 is the retry-path failure value. Keep + * this as 1 even though most SPTM endpoints use SPTM_SUCCESS == 0. */ + return 1; +} + +bool_t nvme_handle_table6(u32 endpoint, u64 *regs) { + u64 rc = NVME_STATUS_VIOLATION; + u64 orig0 = regs[0]; + u64 orig1 = regs[1]; + u64 orig2 = regs[2]; + u64 orig3 = regs[3]; + + if (!g_state.nvme.enabled) + return 0; + + g_state.nvme.total_calls++; + if (endpoint < 16) + g_state.nvme.ep_count[endpoint]++; + + switch (endpoint) { + case NVME_FN_ENABLE_COASTGUARD: rc = nvme_ep0_enable_coastguard(regs); break; + case NVME_FN_MAP_PAGES: rc = nvme_ep1_map_pages(regs); break; + case NVME_FN_UNMAP_PAGES: rc = nvme_ep2_unmap_pages(regs); break; + case NVME_FN_VALIDATE_QENTRIES: rc = nvme_ep3_validate(regs); break; + case NVME_FN_BAR_ADMIN_QUEUE_REGS: rc = nvme_ep4_admin_queue(regs); break; + case NVME_FN_BAR_IOQA_REG: rc = nvme_ep5_ioqa(regs); break; + case NVME_FN_BAR_IOSQ_REG: rc = nvme_ep6_iosq(regs); break; + case NVME_FN_BAR_IOCQ_REG: rc = nvme_ep7_iocq(regs); break; + case NVME_FN_ANS_SHA_REG: rc = nvme_ep8_ans_sha(regs); break; + default: + rc = nvme_violation(endpoint, regs, 0xff); + break; + } + + nvme_record_last(endpoint, rc, regs); + if (endpoint == NVME_FN_BAR_IOQA_REG && rc == SPTM_SUCCESS) + nvme_diag_ep5_readback_once(); + if (g_state.nvme.debug_flags & NVME_DBG_PRINT_MINIMAL) { + if (endpoint == NVME_FN_ENABLE_COASTGUARD || + endpoint == NVME_FN_MAP_PAGES || + endpoint == NVME_FN_UNMAP_PAGES || + endpoint == NVME_FN_VALIDATE_QENTRIES || + endpoint == NVME_FN_BAR_ADMIN_QUEUE_REGS || + endpoint == NVME_FN_BAR_IOQA_REG || + endpoint == NVME_FN_BAR_IOSQ_REG || + endpoint == NVME_FN_BAR_IOCQ_REG) { + debug_printf("C-NVME min ep=%u rc=0x%lx allowed=0x%x args=%lx,%lx,%lx,%lx\n", + endpoint, rc, g_state.nvme.allowed_functions, + orig0, orig1, orig2, orig3); + } + } + if ((g_state.nvme.debug_flags & NVME_DBG_PRINT_CALLS) && + (g_state.nvme.total_calls <= 128 || + endpoint == NVME_FN_BAR_IOSQ_REG || + endpoint == NVME_FN_BAR_IOCQ_REG || + (rc != SPTM_SUCCESS && + !(endpoint == NVME_FN_UNMAP_PAGES && rc == 1)))) { + u64 elr; + __asm__ volatile("mrs %0, elr_el2" : "=r"(elr)); + debug_printf("C-NVME ep=%u rc=0x%lx allowed=0x%x pc=0x%lx lr=0x%lx args=%lx,%lx,%lx,%lx", + endpoint, rc, g_state.nvme.allowed_functions, + elr, regs[30], orig0, orig1, orig2, orig3); + nvme_print_backtrace(regs[29]); + debug_printf("\n"); + } + regs[0] = rc; + g_state.nvme.fast_path_calls++; + g_state.fast_path_calls++; + sev(); + return 1; +} diff --git a/src/sptm/private.h b/src/sptm/private.h new file mode 100644 index 000000000..3185745ca --- /dev/null +++ b/src/sptm/private.h @@ -0,0 +1,2472 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef SPTM_PRIVATE_H +#define SPTM_PRIVATE_H + +/* + * DO NOT MERGE: tainted SPTM endpoint emulator imported from the bringup tree. + * + * This is intentionally isolated under src/sptm while we separate the clean + * hypervisor plumbing from the reverse-engineered SPTM behavior. The old tree + * built this as a hot-loaded C object; this branch links it directly into the + * streamed stage2 hypervisor so endpoint state has one C source of truth. + */ + +typedef unsigned long u64; +typedef unsigned int u32; +typedef unsigned short u16; +typedef unsigned char u8; +typedef int bool_t; /* avoid in freestanding */ + +void dc_civac_range(void *addr, u64 length); +void mmu_add_mapping(u64 from, u64 to, u64 size, u8 attribute_index, u64 perms); +int hv_map(u64 from, u64 to, u64 size, u64 incr); +u64 hv_pt_walk(u64 addr); + +/* SPTM constants — match the proxyclient/tools/sptm Python helpers. */ +#define SPTM_SUCCESS 0 +#define SPTM_MAP_VALID 1 +#define SPTM_UPDATE_DELAYED_TLBI 5 +#define SPTM_MAP_PADDR_CONFLICT 6 +#define SPTM_TABLE_NOT_PRESENT 7 +#define SPTM_TABLE_ALREADY_PRESENT 8 + +#define SPTM_UPDATE_SW_WIRED (1U << 0) +#define SPTM_UPDATE_PERMS_AND_WAS_WRITABLE (1U << 1) +#define SPTM_UPDATE_NG (1U << 2) +#define SPTM_UPDATE_AF (1U << 3) +#define SPTM_UPDATE_SH (1U << 4) +#define SPTM_UPDATE_MAIR (1U << 5) +#define SPTM_UPDATE_MASK 0x3fU +#define SPTM_UPDATE_DEFER_TLBI (1U << 8) +#define SPTM_UPDATE_SKIP_PAPT (1U << 9) + +#define HV_S2_PTE_ACCESS (1UL << 10) +#define HV_S2_PTE_SH_NS (3UL << 8) +#define HV_S2_PTE_S2AP_RW (3UL << 6) +#define HV_S2_MEMATTR_WB (0xfUL << 2) +/* With HCR_EL2.FWB set, stage-2 MemAttr 5 is Normal Non-cacheable. */ +#define HV_S2_MEMATTR_NORMAL_NC (0x5UL << 2) +/* + * GPU-shared coherency type: Outer-WB, Inner-NC (matches XNU's MAIR 0xf4 "Shared" + * and the AGX UAT AttrIndex-2). Inner-NC makes CPU writes skip L1/L2 and reach the + * SLC (Point of Coherency) the GPU reads, so the GPU sees ongoing CPU writes with + * no per-write flush. Under FWB=1: bits[3:2]=outer,[1:0]=inner,{01=NC,11=WB} -> 0b1101. + */ +#define HV_S2_MEMATTR_SHARED (0xdUL << 2) +#define HV_S2_PTE_VALID 1UL +#define HV_S2_PTE_NORMAL_WB (HV_S2_PTE_ACCESS | HV_S2_PTE_SH_NS | \ + HV_S2_PTE_S2AP_RW | HV_S2_MEMATTR_WB | \ + HV_S2_PTE_VALID) +#define HV_S2_PTE_NORMAL_NC (HV_S2_PTE_ACCESS | HV_S2_PTE_SH_NS | \ + HV_S2_PTE_S2AP_RW | \ + HV_S2_MEMATTR_NORMAL_NC | \ + HV_S2_PTE_VALID) +#define HV_S2_PTE_SHARED (HV_S2_PTE_ACCESS | HV_S2_PTE_SH_NS | \ + HV_S2_PTE_S2AP_RW | \ + HV_S2_MEMATTR_SHARED | \ + HV_S2_PTE_VALID) + +#define SPTM_EL2_MAIR_NORMAL 0U +#define SPTM_EL2_MAIR_NORMAL_NC 1U +#define SPTM_EL2_PERM_RWX 0UL +#define SPTM_PAPT_ATTR_RT_NC (1UL << 2) /* AttrIdx=1, SH=0 */ +#define SPTM_PAPT_ATTR_NORMAL_WB (2UL << 8) /* AttrIdx=0, inner-shareable */ +#define SPTM_PAPT_OPT_ALT_TO_NC 0x80000000U +#define SPTM_PAPT_OPT_ALT_TO_WB 0x40000000U + +#define PAPT_UPDATE_RING_SLOTS 64 +#define PAPT_UPDATE_RC_WRITE 1 +#define PAPT_UPDATE_RC_SKIP_PAPT 2 +#define PAPT_UPDATE_RC_NO_ATTR 3 +#define PAPT_UPDATE_RC_UNMANAGED 4 +#define PAPT_UPDATE_RC_BAD_TEMPLATE 5 +#define PAPT_UPDATE_RC_BAD_ROOT 6 +#define PAPT_UPDATE_RC_BAD_WALK 7 +#define PAPT_UPDATE_RC_BAD_EXISTING 8 +#define PAPT_UPDATE_RC_SAME_ATTRS 9 +#define ALT_PTD_TRACE_RING_SLOTS 4 +#define PAPT_NVME_CLASS_COUNT 8 +#define PAPT_NVME_SEED_ASQ 0 +#define PAPT_NVME_SEED_ACQ 1 +#define PAPT_NVME_PRP_LIST 2 +#define PAPT_NVME_XNU_ASQ 3 +#define PAPT_NVME_XNU_ACQ 4 +#define PAPT_NVME_XNU_IOSQ 5 +#define PAPT_NVME_XNU_IOCQ 6 +#define PAPT_NVME_ACTIVE_PRP 7 + +#define SPTM_TABLE_XNU_BOOTSTRAP 0 +#define SPTM_TABLE_TXM_BOOTSTRAP 1 +#define SPTM_TABLE_SK_BOOTSTRAP 2 +#define SPTM_TABLE_T8110_DART_XNU 3 +#define SPTM_TABLE_T8110_DART_SK 4 +#define SPTM_TABLE_SART 5 +#define SPTM_TABLE_NVME 6 +#define SPTM_TABLE_UAT 7 +#define SPTM_TABLE_SHART 8 +/* Public SPTM tables call 9 "reserved", but the t8140 SPTM firmware uses it + * for CPU trace/perf-trace helpers. Boot does not need a real trace buffer: + * report unsupported/no-carveout/success and keep moving. */ +#define SPTM_TABLE_CPUTRACE 9 +#define SPTM_TABLE_HIB 10 +#define SPTM_TABLE_GEN3_DART_XNU 11 +#define SPTM_TABLE_GEN3_DART_SK 12 +#define SPTM_TABLE_T6000_DART_XNU 13 +#define SPTM_TABLE_INVALID 14 +#define DART_OBJ_CACHE_SLOTS 64 +#define NVME_C_MAX_CID 0x101 +#define SART_C_MAX_ENTRIES 16 +#define SART_C_MAX_ENDPOINTS 3 + +#define SPTM_GENTER_DISPATCH_CALL 0 +#define SPTM_DISPATCH_RESERVED_MASK 0xff00ff0000000000UL +#define ESR_ISS_MASK 0x1ffffffUL + +/* t8140 top-level non-dispatch GENTER return sites. Mechanical + * genter->HVC patching erases the original immediate, so GENTER #1..#4 + * reach us as HVC #0 with arbitrary/poison x16. Only the observed returning + * stub is fast-pathed here; other invalid HVC #0 sites still fall through + * loudly instead of hiding a corrupted dispatch call. */ +#define XNU_265_GENTER_NODISPATCH_RET_PC 0xfffffe000bcf4d9cUL +#define XNU_266_GENTER_NODISPATCH_RET_PC 0xfffffe000bcfac08UL + +#define SPTM_FN_LOCKDOWN 0 +#define SPTM_FN_RETYPE 1 +#define SPTM_FN_MAP_PAGE 2 +#define SPTM_FN_MAP_TABLE 3 +#define SPTM_FN_UNMAP_TABLE 4 +#define SPTM_FN_UPDATE_REGION 5 +#define SPTM_FN_UPDATE_DISJOINT 6 +#define SPTM_FN_UNMAP_REGION 7 +#define SPTM_FN_UNMAP_DISJOINT 8 +#define SPTM_FN_CONFIGURE_SHAREDREGION 9 +#define SPTM_FN_NEST_REGION 10 +#define SPTM_FN_UNNEST_REGION 11 +#define SPTM_FN_CONFIGURE_ROOT 12 +#define SPTM_FN_SWITCH_ROOT 13 +#define SPTM_FN_REGISTER_CPU 14 +#define SPTM_FN_FIXUPS_COMPLETE 15 +#define SPTM_FN_SIGN_USER_POINTER 16 +#define SPTM_FN_AUTH_USER_POINTER 17 +#define SPTM_FN_REGISTER_EXC_RETURN 18 +#define SPTM_FN_CPU_ID 19 +#define SPTM_FN_SLIDE_REGION 20 +#define SPTM_FN_UPDATE_DISJOINT_MULTIPAGE 21 +#define SPTM_FN_REG_READ 22 +#define SPTM_FN_REG_WRITE 23 +#define SPTM_FN_GUEST_VA_TO_IPA 24 +#define SPTM_FN_GUEST_STAGE1_TLBOP 25 +#define SPTM_FN_GUEST_STAGE2_TLBOP 26 +#define SPTM_FN_GUEST_DISPATCH 27 +#define SPTM_FN_GUEST_EXIT 28 +#define SPTM_FN_MAP_SK_DOMAIN 29 +#define SPTM_FN_HIB_BEGIN 30 +#define SPTM_FN_HIB_VERIFY_HASH_NON_WIRED 31 +#define SPTM_FN_HIB_FINALIZE_NON_WIRED 32 +#define SPTM_FN_IOFILTER_PROTECTED_WRITE 33 +#define SPTM_FN_SPTM_SYSCTL 37 +#define SPTM_FN_DISABLE_KERNEL_MODE_CPA2 38 +#define SPTM_FN_SET_SHARED_REGION 39 +#define SPTM_FN_BATCH_SIGN_USER_POINTER 40 +#define SPTM_FN_SURT_ALLOC 41 +#define SPTM_FN_SURT_FREE 42 +#define SPTM_FN_CONDEMN_LEAF_TABLE 43 +#define SPTM_FN_UNCONDEMN_LEAF_TABLE 44 +#define SPTM_FN_SERIAL_PUTC 45 +#define SPTM_FN_SERIAL_DISABLE 46 +#define SPTM_FN_PROGRAM_IRGKEY 48 +#define SPTM_FN_REG_SNAPSHOT 49 + +#define SPTM_MAX_CPUS_EMUL 16 + +#define SPTM_CPU_PIO_F_SEEDED (1ULL << 0) +#define SPTM_CPU_PIO_F_REGISTERED (1ULL << 1) +#define SPTM_CPU_PIO_F_MISSING (1ULL << 2) + +/* NVMe table 6 endpoint IDs and state-machine constants. Endpoint semantics + * live here; Python only seeds ADT-derived configuration and SPTM-owned + * buffers before guest start. Endpoint 1 receives a TCB template PA and a + * segment-list PA; it does not receive direct PRP1/PRP2 arguments. */ +#define NVME_FN_ENABLE_COASTGUARD 0 +#define NVME_FN_MAP_PAGES 1 +#define NVME_FN_UNMAP_PAGES 2 +#define NVME_FN_VALIDATE_QENTRIES 3 +#define NVME_FN_BAR_ADMIN_QUEUE_REGS 4 +#define NVME_FN_BAR_IOQA_REG 5 +#define NVME_FN_BAR_IOSQ_REG 6 +#define NVME_FN_BAR_IOCQ_REG 7 +#define NVME_FN_ANS_SHA_REG 8 + +#define NVME_AF_ENABLE_COASTGUARD (1u << 0) +#define NVME_AF_TCB_REGISTER (1u << 1) +#define NVME_AF_TCB_INVALIDATE (1u << 2) +#define NVME_AF_QUEUE_VALIDATE (1u << 3) +#define NVME_AF_BAR_ADMIN_Q (1u << 4) +#define NVME_AF_BAR_IOQA (1u << 5) +#define NVME_AF_BAR_IOSQ (1u << 6) +#define NVME_AF_BAR_IOCQ (1u << 7) +#define NVME_AF_ANS_SHA (1u << 8) + +#define NVME_STATUS_VIOLATION 0x100UL +#define NVME_PAGE_SIZE 0x4000UL +#define NVME_PRP_PAGE_SIZE 0x1000UL +#define NVME_QSIZE_MAX 0xffeU +#define NVME_COASTGUARD_VALUE 0x3fU + +#define NVME_BAR_AQA 0x24U +#define NVME_BAR_ASQ 0x28U +#define NVME_BAR_ACQ 0x30U +#define NVME_BAR_COASTGUARD 0x100U +#define NVME_BAR_CG_ASQ 0x108U +#define NVME_BAR_CG_ACQ 0x110U +#define NVME_BAR_IOSQ_BASE 0x1200U +#define NVME_BAR_IOCQ_BASE 0x1208U +#define NVME_BAR_IOQA 0x1210U +#define NVME_BAR_TCB_INVALIDATE 0x118U + +#define NVME_CID_AVAILABLE 0 +#define NVME_CID_BUSY 1 +#define NVME_IOMMU_RO 1 +#define NVME_IOMMU_RW 2 + +#define NVME_QUIRK_PRP_FLUSH_WA (1u << 0) +#define NVME_QUIRK_TL_WA (1u << 1) +#define NVME_QUIRK_VDMA_WA (1u << 2) + +#define NVME_DBG_PRINT_CALLS (1u << 0) +#define NVME_DBG_PRP_WINDOW (1u << 1) +#define NVME_DBG_PRINT_MINIMAL (1u << 2) +#define NVME_DBG_SUMMARY (1u << 3) +#define NVME_DBG_IOMMU (1u << 4) +#define NVME_DBG_CMD_TRACE (1u << 5) + +/* + * T8110 DART C emulator. + * + * The register constants and TLB/error sequencing below intentionally follow + * the public Asahi Linux apple-dart.c T8110 model: + * archive/references/asahi_linux/apple-dart.c + * + * This is SPTM endpoint emulation, not Linux's ownership driver. Endpoint 6 + * preserves bootloader-owned state instead of doing apple_dart_hw_reset()'s + * destructive TCR/TTBR clear. Later endpoints install roots, stream enables, + * and TLB operations explicitly, matching the SPTM ABI observed in the real + * t8140 firmware. + */ +#define DART_C_MAX_DARTS 32 +#define DART_C_MAX_BASES 16 +#define DART_C_MAX_SIDS 256 +#define DART_C_MAX_IDENTITY_RANGES 1024 +#define DART_C_MAX_ENDPOINTS 19 +#define DART_C_RECENT_SLOTS 1024 +#define DART_C_SID0_TRACE_SLOTS 4096 +#define DART_C_MAX_DAPF_SLICES 32 +#define DART_C_MAX_PIOGW_RANGES 2 +#define DART_C_MAX_PIOGW_DESCS 2 + +#define UAT_C_MAX_ENDPOINTS 13 +#define UAT_C_MAX_STATES 32 +#define UAT_C_RECENT_SLOTS 32 +#define UAT_C_PAGE_SIZE 0x4000UL +#define UAT_C_CTX_NONE 0xffffU + +#define UAT_FN_INIT_STATE 0 +#define UAT_FN_DESTROY_STATE 1 +#define UAT_FN_MAP_TABLE 2 +#define UAT_FN_UNMAP_TABLE 3 +#define UAT_FN_MAP_BEGIN 4 +#define UAT_FN_MAP_CONTINUE 5 +#define UAT_FN_PREP_FW_UNMAP_BEGIN 6 +#define UAT_FN_PREP_FW_UNMAP_CONTINUE 7 +#define UAT_FN_UNMAP_BEGIN 8 +#define UAT_FN_UNMAP_CONTINUE 9 +#define UAT_FN_SET_CTX_ID 10 +#define UAT_FN_REMOVE_CTX_ID 11 +#define UAT_FN_GET_INFO 12 + +#define DART_C_STATUS_VIOLATION 0x100UL +#define DART_C_PAGE_SIZE 0x4000UL +#define DART_C_TT_ENTRIES 2048 +#define DART_C_MAX_STREAMS 256 +#define DART_C_PTE_NO_CACHE (1UL << 1) + +#define DART_C_RANGE_SID_MASK 0xffffUL +#define DART_C_RANGE_FLAG_REMAP (1UL << 63) +#define DART_C_RANGE_FLAG_BOOT_PREMAP (1UL << 60) +#define DART_C_RANGE_FLAG_VM_WINDOW (1UL << 59) +#define DART_C_RANGE_FLAG_PT_REGION (1UL << 58) +#define DART_C_RANGE_FLAG_BYPASS (1UL << 57) +#define DART_C_RANGE_FLAG_APF_BYPASS (1UL << 56) +#define DART_C_RANGE_FLAG_REAL_TIME (1UL << 55) + +/* TXM peer domain (domain=2/table=0) constants. XNU 26.5 passes the TXM + * selector in x16's endpoint field, x0 is the physical TXM thread-stack page, + * and TXM writes TXMSharedContextData_t at PAGE_SIZE - 1024 inside that page. */ +#define TXM_DOMAIN 2 +#define TXM_TABLE_XNU 0 +#define TXM_C_MAX_ENDPOINTS 64 +#define TXM_C_TRACK_SLOTS 32 +#define TXM_C_SYNTH_ALIGN 0x40UL +#define TXM_THREAD_STACK_SHARED_OFF 0x3c00UL +#define TXM_SHARED_RETURN_CODE_OFF 0x08UL +#define TXM_SHARED_RETURN_TYPE_OFF 0x10UL +#define TXM_SHARED_NUM_WORDS_OFF 0x18UL +#define TXM_SHARED_WORDS_OFF 0x20UL +#define TXM_STACK_RETURN_WORDS 6 + +#define TXM_SUCCESS 0UL +#define TXM_RETURN_GENERIC 1UL +#define TXM_RETURN_NOT_FOUND 8UL +#define TXM_RETURN_NOT_PERMITTED 38UL +#define TXM_RETURN_NOT_SUPPORTED 41UL + +#define TXM_SEL_GET_LOG_INFO 1U +#define TXM_SEL_GET_CODE_SIGNING_INFO 2U +#define TXM_SEL_GET_TRUST_CACHE_INFO 3U +#define TXM_SEL_GET_SECURE_CHANNEL 7U +#define TXM_SEL_ADD_FREE_LIST_PAGE 10U +#define TXM_SEL_GET_FREE_LIST_PAGE 11U +#define TXM_SEL_LOAD_TRUST_CACHE 12U +#define TXM_SEL_QUERY_TRUST_CACHE 14U +#define TXM_SEL_QUERY_TC_REM 15U +#define TXM_SEL_CHECK_TC_UUID 16U +#define TXM_SEL_REGISTER_PROFILE 17U +#define TXM_SEL_UNREGISTER_PROFILE 19U +#define TXM_SEL_REGISTER_SIGNATURE 22U +#define TXM_SEL_UNREGISTER_SIGNATURE 23U +#define TXM_SEL_RECONSTITUTE_SIG 25U +#define TXM_SEL_GET_LOCAL_PUBKEY 27U +#define TXM_SEL_MATCH_COMPILATION 30U +#define TXM_SEL_ACQUIRE_SIGNING_ID 32U +#define TXM_SEL_ACCEL_ENTITLEMENTS 34U +#define TXM_SEL_REGISTER_ASPACE 35U +#define TXM_SEL_GET_ENTITLEMENTS_CTX 43U +#define TXM_SEL_RESOLVE_KENT_ASPACE 44U +#define TXM_SEL_IMAGE4_DISPATCH 45U +#define TXM_SEL_IMAGE4_GET_EXPORTS 46U +#define TXM_SEL_IMAGE4_GET_NONCE 51U + +#define DART_T8110_PARAMS3 0x008U +#define DART_T8110_PARAMS4 0x00cU +#define DART_T8110_TLB_CMD 0x080U +#define DART_T8110_TLB_CMD_BUSY (1U << 31) +#define DART_T8110_TLB_CMD_NEW_DART (1U << 15) +#define DART_T8110_TLB_CMD_VA_RANGE (1U << 14) +#define DART_T8110_TLB_CMD_STT_FLUSH (1U << 13) +#define DART_T8110_TLB_CMD_NO_STC_FLUSH (1U << 12) +#define DART_T8110_TLB_OP_FLUSH_ALL 0U +#define DART_T8110_TLB_OP_FLUSH_SID 1U +#define DART_T8110_TLB_START_DVA_PAGE 0x098U +#define DART_T8110_TLB_END_DVA_PAGE 0x0a0U +#define DART_T8110_ERROR 0x100U +#define DART_T8110_ERROR_MASK 0x104U +#define DART_T8110_ERROR_STREAMS 0x1c0U +#define DART_T8110_PROTECT 0x200U +#define DART_T8110_PROTECT_TTBR_TCR (1U << 0) +#define DART_T8110_PROTECT_LOCK 0x208U +#define DART_T8110_ENABLE_STREAMS 0xc00U +#define DART_T8110_DISABLE_STREAMS 0xc20U +#define DART_T8110_TCR 0x1000U +#define DART_T8110_TCR_TRANSLATE_EN (1U << 0) +#define DART_T8110_TCR_BYPASS_DART (1U << 1) +#define DART_T8110_TCR_BYPASS_DAPF (1U << 2) +#define DART_T8110_TCR_FOUR_LEVEL (1U << 3) +#define DART_T8110_TCR_REMAP_EN (1U << 7) +#define DART_T8110_TCR_REMAP_TARGET(x) (((x) & 0xffU) << 8) +#define DART_T8110_TCR_DISABLED (DART_T8110_TCR_BYPASS_DART | \ + DART_T8110_TCR_BYPASS_DAPF) +#define DART_T8110_TTBR 0x1400U +#define DART_T8110_TTBR_VALID (1U << 0) + +#define DART_C_HOT_OK 0U +#define DART_C_ERR_LOCKED (1U << 0) +#define DART_C_ERR_TIMEOUT (1U << 1) +#define DART_C_ERR_ALIGN (1U << 2) +#define DART_C_ERR_NO_TTBR (1U << 3) +#define DART_C_ERR_NO_TABLE (1U << 4) + +struct dart_c_sid_state { + u32 tcr; + u32 ttbr; + u32 protection; + u32 status; + u64 root_pt_pa; + u8 valid; + u8 root_valid; + u8 root_level; + u8 stream_enabled; + u8 translation_enabled; + u8 exclave; + u8 raw_replay; + u8 initial_stream_enabled; +}; + +struct dart_c_piogw_desc { + u32 slice; + u32 pad; + u64 addr; +}; + +struct dart_c_dapf_slice { + u64 start; + u64 end; + u32 ctrl; + u32 words[8]; +}; + +struct dart_c_dapf_state { + u64 base_pa; + u32 count; + u32 sid_count; + struct dart_c_dapf_slice slice[DART_C_MAX_DAPF_SLICES]; + u8 reserved[0x800 - 0x10 - DART_C_MAX_DAPF_SLICES * 0x38]; +}; + +struct dart_c_instance { + u64 dart_id; + u64 obj_va; + u64 state_va; + u64 base_pa[DART_C_MAX_BASES]; + u32 base_count; + u32 init_state; + u32 clock_protection; + u32 locked; + u32 used; + u8 avoid_tlbi_in_map; + u8 relaxed_rw_protections; + u8 flush_by_dva; + u8 pad0[1]; + u64 total_calls; + u64 fast_path_calls; + u64 violations; + u64 last[16]; + u64 ep_count[DART_C_MAX_ENDPOINTS]; + struct dart_c_dapf_state dapf; + struct dart_c_sid_state sid[DART_C_MAX_SIDS]; + u64 piogw_base_pa[DART_C_MAX_PIOGW_RANGES]; + u32 piogw_count; + u32 piogw_desc_count; + struct dart_c_piogw_desc piogw_desc[DART_C_MAX_PIOGW_DESCS]; +}; + +struct dart_c_recent_event { + u64 seq; + u64 table; + u64 endpoint; + u64 dart_id; + u64 sid; + u64 regs[6]; + u64 before_slot; + u64 before_pte; + u64 after_slot; + u64 after_pte; + u64 rc; +}; + +struct dart_c_sid0_event { + u64 seq; + u64 meta; /* table, endpoint, dart_id, reason: 16 bits each */ + u64 regs[4]; + u64 before_tcr_ttbr; + u64 before_root_flags; + u64 before_pte; + u64 after_tcr_ttbr; + u64 after_root_flags; + u64 after_pte; + u64 rc; +}; + +struct dart_c_identity_range { + u64 base_pa; + u64 sid; + u64 start; + u64 end; +}; + +struct dart_c_state { + u32 enabled; + u32 pad0; + u32 violations; + u32 pad1; + u64 total_calls; + u64 fast_path_calls; + u64 fallthrough_calls; + u64 last[16]; + struct dart_c_instance inst[DART_C_MAX_DARTS]; + u64 pt_pool_base; + u64 pt_pool_end; + u64 pt_pool_next; + u64 pt_pool_allocs; + u64 pt_pool_failures; + u64 identity_count; + struct dart_c_identity_range identity[DART_C_MAX_IDENTITY_RANGES]; + u64 recent_idx; + struct dart_c_recent_event recent[DART_C_RECENT_SLOTS]; + u64 sid0_trace_idx; + struct dart_c_sid0_event sid0_trace[DART_C_SID0_TRACE_SLOTS]; +}; + +struct nvme_c_state { + u32 enabled; + u32 queue_count; + u32 allowed_functions; + u8 secure_reg_layout; + u8 packed_writes_present; + u8 linear_sq; + u8 ans_sha_present; + u8 debug_flags; + u8 quirks; + u16 max_cid; + u32 violations; + u64 total_calls; + u64 fast_path_calls; + u64 fallthrough_calls; + u64 bar_pa; + u64 trusted_io_base; + u64 trusted_io_size; + u64 secondary_io_base; + u64 secondary_io_size; + u64 admin_asq_kva; + u64 admin_acq_kva; + u64 prp_list_kva; + u64 ans_sha_kva; + u64 tcb_kva[2]; + u32 aqa_cached; + u32 ioqa_cached; + u32 ans_sha_pwc_cached; + u32 last_rc; + u64 asq_pa_cached; + u64 acq_pa_cached; + u64 iosq_pa_cached; + u64 iocq_pa_cached; + u64 ans_sha_pa_cached; + u64 last[16]; + u64 ep_count[16]; + u8 cid_mode[NVME_C_MAX_CID]; + u8 func_state[16]; + u16 tcb_dir[NVME_C_MAX_CID]; + u16 tcb_nlb[NVME_C_MAX_CID]; + u32 tcb_count[NVME_C_MAX_CID]; + u8 tcb_mode[NVME_C_MAX_CID]; + u8 tcb_qid[NVME_C_MAX_CID]; + u16 pad0[NVME_C_MAX_CID]; + u64 tcb_prp1[NVME_C_MAX_CID]; + u64 tcb_prp2[NVME_C_MAX_CID]; + u64 nvmmu_pa; + u64 tl_status_pa; + u64 tl_mask_pa; + u64 tl_ctrl_pa; + u32 tl_num_sl; + u32 tl_timeouts; + u16 tl_slot_done[NVME_C_MAX_CID]; + u64 vdma_status_pa; +}; +_Static_assert(sizeof(struct nvme_c_state) == 0x2128, "nvme_c_state layout drift"); + +struct sart_c_entry { + u64 paddr; + u64 size; + u32 flags; + u32 guard; +}; + +struct sart_c_state { + u32 enabled; + u32 active; + u32 version; + u32 flags_allow; + u32 size_shift; + u32 paddr_shift; + u32 size_max; + u32 protected_mask; + u32 violations; + u32 last_rc; + u64 total_calls; + u64 fast_path_calls; + u64 fallthrough_calls; + u64 base_pa; + u64 power_canary_pa; + u32 power_canary_offset; + u32 power_canary_count; + u32 exclusive_bounds; + u32 reserved; + u64 last[8]; + u64 ep_count[SART_C_MAX_ENDPOINTS]; + struct sart_c_entry shadow[SART_C_MAX_ENTRIES]; +}; +_Static_assert(sizeof(struct sart_c_entry) == 0x18, "sart_c_entry layout drift"); +_Static_assert(sizeof(struct sart_c_state) == 0x238, "sart_c_state layout drift"); + +struct uat_c_root_state { + u64 handle; + u64 root0_pa; + u64 root1_pa; + u64 current_va; + u64 seglist_pa; + u64 num_segs; + u64 current_seg; + u64 seg_offset; + u64 options; + u64 last[8]; + u16 ctx_id; + u8 used; + u8 state; + u8 type; + u8 pad[19]; +}; + +struct uat_c_recent_event { + u64 seq; + u64 endpoint; + u64 handle; + u64 regs[5]; + u64 root_pa; + u64 slot_pa; + u64 before; + u64 after; + u64 rc; +}; + +struct uat_c_state { + u32 enabled; + u32 violations; + u64 total_calls; + u64 fast_path_calls; + u64 fallthrough_calls; + u64 last_rc; + u64 mode; + u64 vaddr_shift; + u64 l1_index_mask; + u64 segment_limit; + u64 mapping_limit; + u64 state_object_size; + u64 gpu_region_pa; + u64 gpu_region_size; + u64 gfx_shared_region_pa; + u64 gfx_shared_region_size; + u64 gfx_shared_l2_pa; + u64 gfx_shared_l2_size; + u64 gfx_handoff_pa; + u64 gfx_handoff_size; + u64 last[16]; + u64 ep_count[UAT_C_MAX_ENDPOINTS]; + struct uat_c_root_state roots[UAT_C_MAX_STATES]; + u64 recent_idx; + struct uat_c_recent_event recent[UAT_C_RECENT_SLOTS]; + u64 selector2_pa; + u64 selector9_pa; + u64 raw_gfx_shared_l2_pa; + u64 selector2_root_pa; + u64 sapt_base; + u64 sapt_entries; + u64 sapt_dram_base; + u64 sapt_dram_end; + u64 sapt_policy; +}; +_Static_assert(sizeof(struct uat_c_root_state) == 0xa0, "uat_c_root_state layout drift"); +_Static_assert(sizeof(struct uat_c_recent_event) == 0x68, "uat_c_recent_event layout drift"); +_Static_assert(sizeof(struct uat_c_state) == 0x22d0, "uat_c_state layout drift"); + +struct sptm_cpu_pio_entry { + u64 phys_id; + u64 cpu_base; + u64 cpu_size; + u64 acc_base; + u64 acc_size; + u64 cpm_base; + u64 cpm_size; + u64 cpu_pages; + u64 acc_pages; + u64 cpm_pages; + u64 registered; + u64 logical_id; + u64 flags; + u64 reserved[3]; +}; + +struct sptm_cpu_pio_state { + u64 seeded_count; + u64 registered_count; + u64 last_phys; + u64 last_slot; + u64 last_flags; + u64 reserved[3]; + struct sptm_cpu_pio_entry entries[SPTM_MAX_CPUS_EMUL]; +}; + +struct papt_update_event { + u64 rc; + u64 pa; + u64 options; + u64 papt_va; + u64 root_pa; + u64 template_pte; + u64 existing_pte; + u64 new_pte; + u64 caller_elr; + u64 inner_root_pa; + u64 inner_va; + u64 inner_template_pte; +}; + +struct papt_update_state { + u64 calls; + u64 candidates; + u64 writes; + u64 skip_no_attr; + u64 skip_skip_papt; + u64 skip_unmanaged; + u64 skip_bad_template; + u64 skip_bad_root; + u64 skip_bad_walk; + u64 skip_bad_existing; + u64 skip_same_attrs; + u64 ring_idx; + struct papt_update_event ring[PAPT_UPDATE_RING_SLOTS]; + u64 nvme_hits[PAPT_NVME_CLASS_COUNT]; + u64 nvme_writes[PAPT_NVME_CLASS_COUNT]; +}; +_Static_assert(sizeof(struct papt_update_event) == 0x60, "papt_update_event layout drift"); +_Static_assert(sizeof(struct papt_update_state) == 0x18e0, "papt_update_state layout drift"); + +struct alt_ptd_trace_event { + u64 seq; + u64 pa; + u64 caller_elr; + u64 caller_lr; + u64 caller_fp; + u64 types_flags; + u64 retype_params; + u64 ft_idx; + u64 refs_before; + u64 refs_after; + u64 papt_ring_before; + u64 papt_ring_after; + u64 papt_rc; + u64 papt_existing_pte; + u64 papt_new_pte; + u64 papt_va; + u64 caller_sp0; + u64 caller_sp1; + u64 bt_count; + u64 bt_base; + u64 bt[12]; +}; + +struct alt_ptd_trace_state { + u64 ring_idx; + struct alt_ptd_trace_event ring[ALT_PTD_TRACE_RING_SLOTS]; +}; +_Static_assert(sizeof(struct alt_ptd_trace_event) == 0x100, "alt_ptd_trace_event layout drift"); +_Static_assert(sizeof(struct alt_ptd_trace_state) == 0x408, "alt_ptd_trace_state layout drift"); + +struct txm_c_state { + u32 enabled; + u32 violations; + u64 total_calls; + u64 fast_path_calls; + u64 fallthrough_calls; + u64 last_selector; + u64 last_stack_pa; + u64 last_x16; + u64 last_rc; + u64 last_args[8]; + u64 last_words[TXM_STACK_RETURN_WORDS]; + u64 ep_count[TXM_C_MAX_ENDPOINTS]; + u64 secure_chan_pa; + u64 secure_chan_size; + u64 synth_base_va; + u64 synth_base_pa; + u64 synth_size; + u64 synth_next_off; + u64 log_page_va; + u64 log_head_va; + u64 log_sync_va; + u64 txm_rw_data_va; + u64 txm_ro_data_va; + u64 developer_mode_flag_va; + u64 managed_signature_size; + u64 trust_cache_runtime_va; + u64 trust_cache_static_count; + u64 trust_cache_caps0; + u64 trust_cache_caps1; + u64 local_signing_pubkey_va; + u64 kernel_entitlements_va; + u64 ce_ctx_va; + u64 signing_id_va; + u64 image4_exports_va; + u64 nonce_va; + u64 default_address_space_va; + u64 default_signature_va; + u64 default_profile_va; + u64 next_token; + u64 free_count; + u64 free_pages[TXM_C_TRACK_SLOTS]; + u64 loaded_tc_count; + u64 loaded_tc[TXM_C_TRACK_SLOTS]; + u64 secure_chan_dva; + u64 secure_chan_reserved_pa; +}; + +/* Frame types (XNU's per-frame state). */ +#define SPTM_KERNEL_ROOT_TABLE 8 +#define SPTM_PAGE_TABLE 9 +#define XNU_DEFAULT 11 +#define XNU_USER_ROOT_TABLE 18 +#define XNU_SHARED_ROOT_TABLE 19 +#define XNU_PAGE_TABLE 20 +#define XNU_PAGE_TABLE_SHARED 21 +#define XNU_PAGE_TABLE_ROZONE 22 +#define XNU_PAGE_TABLE_COMMPAGE 23 +#define XNU_PAGE_TABLE_ALT 24 +#define XNU_IO 27 +#define XNU_PROTECTED_IO 28 +#define XNU_COPROCESSOR_RO_IO 29 +#define XNU_STAGE2_ROOT_TABLE 33 +#define XNU_STAGE2_PAGE_TABLE 34 +#define XNU_SUBPAGE_USER_ROOT_TABLES 40 +#define TXM_SEP_SECURE_CHANNEL 61 + +#define SPTM_PT_GEOMETRY_16K 0 +#define SPTM_PT_GEOMETRY_4K 1 +#define SPTM_PT_GEOMETRY_16K_KERN 2 +#define ROOT_GEOM_UNKNOWN 0xff +#define TTBR_ASID_SHIFT 48 +#define TTBR_BADDR_MASK 0x0000fffffffffffeUL +#define ROOT_GEOM_SLOTS 64 + +#define PAGE_SIZE_GUEST_EMUL 0x4000UL +#define PAGE_SIZE_4K_EMUL 0x1000UL +#define ARM_TTE_TABLE_MASK 0x0000fffffffff000UL +#define ARM_PTE_PAGE_MASK 0x0000ffffffffc000UL +#define ARM_PTE_PAGE_MASK_4K 0x0000fffffffff000UL +#define ARM_PTE_AP_MASK (3UL << 6) +#define ARM_PTE_ATTRINDX_MASK (7UL << 2) +#define ARM_PTE_SH_MASK (3UL << 8) +#define ARM_PTE_AF (1UL << 10) +#define ARM_PTE_NG (1UL << 11) +#define ARM_PTE_NS (1UL << 5) +#define ARM_PTE_AP_RO (2UL << 6) +#define ARM_PTE_GP (1UL << 50) +#define ARM_PTE_PNX (1UL << 53) +#define ARM_PTE_NX (1UL << 54) +#define ARM_PTE_WIRED (1UL << 58) +#define ARM_PTE_WRITEABLE (1UL << 59) +#define FRAME_PAGE_MASK 0x0000ffffffffc000UL +#define FT_KIND5_REFCNT_OFF 0x04UL +#define FT_TABLE_MAPPING_REFCNT_OFF 0x06UL +#define FT_TABLE_NESTED_REFCNT_OFF 0x08UL +#define FT_DATA_RO_REFCNT_OFF 0x08UL +#define FT_DATA_WX_REFCNT_OFF 0x0cUL +#define DISJOINT_OP_SIZE 24UL +#define SURT_SUBPAGE_SIZE 128UL +#define TTE_CONDEMNED_BIT 0x4UL +#define UINT64_MAX_VALUE 0xffffffffffffffffUL + +/* PT walk indices for XNU's 16K and 4K pmap geometries. */ +static inline u64 va_16k_l1_idx(u64 va) { return (va >> 36) & 0x7ff; } +static inline u64 va_16k_l2_idx(u64 va) { return (va >> 25) & 0x7ff; } +static inline u64 va_16k_l3_idx(u64 va) { return (va >> 14) & 0x7ff; } +static inline u64 va_4k_l0_idx(u64 va) { return (va >> 39) & 0x1ff; } +static inline u64 va_4k_l1_idx(u64 va) { return (va >> 30) & 0x1ff; } +static inline u64 va_4k_l2_idx(u64 va) { return (va >> 21) & 0x1ff; } +static inline u64 va_4k_l3_idx(u64 va) { return (va >> 12) & 0x1ff; } + +static inline u8 ft_get_type(u64 pa); +static inline u8 root_geom_lookup(u64 root_pa); +static inline bool_t root_uses_4k(u64 root_pa, u32 target_level); +static inline bool_t is_pt_type(u8 t); + +/* Translate xnu's PTE template to a plain ARM PTE we can actually install. + * Preserve the architectural leaf fields XNU controls plus the software bits + * XNU later compares in prev-PTE results; force only the valid leaf type. */ +__attribute__((unused)) +static inline u64 xnu_pte_to_arm_geom(u64 xnu_pte, bool_t geom4k) { + if ((xnu_pte & 3) == 0) return 0; + u64 pa = xnu_pte & (geom4k ? ARM_PTE_PAGE_MASK_4K : ARM_PTE_PAGE_MASK); + u64 leaf = xnu_pte & (ARM_PTE_ATTRINDX_MASK | ARM_PTE_SH_MASK | + ARM_PTE_AP_MASK | ARM_PTE_AF | ARM_PTE_NG | + ARM_PTE_NS | ARM_PTE_GP | ARM_PTE_PNX | + ARM_PTE_NX | ARM_PTE_WIRED | + ARM_PTE_WRITEABLE); + /* Preserve Apple-impdef bits (49=SK, 60-62=SPRR idx) so XNU's bookkeeping + * reads see what it wrote. Our MMU ignores the SPRR index bits. */ + u64 apple_bits = xnu_pte & ((1UL << 49) | (1UL << 60) | (1UL << 61) | + (1UL << 62)); + return pa | leaf | apple_bits | 3UL; +} + +/* + * UPDATE_REGION/DISJOINT receive masked attribute templates. XNU may pass only + * the bits selected by options, with valid/type and PA fields clear, such as + * permissions-only templates. Real SPTM still applies those selected fields to + * the existing PTE; do not collapse partial templates to zero just because + * ARM_PTE_TYPE_VALID is absent. + */ +__attribute__((unused)) +static inline u64 xnu_pte_update_template_to_arm_geom(u64 xnu_pte, bool_t geom4k) { + u64 leaf = xnu_pte & (ARM_PTE_ATTRINDX_MASK | ARM_PTE_SH_MASK | + ARM_PTE_AP_MASK | ARM_PTE_AF | ARM_PTE_NG | + ARM_PTE_NS | ARM_PTE_GP | ARM_PTE_PNX | + ARM_PTE_NX | ARM_PTE_WIRED | + ARM_PTE_WRITEABLE); + u64 apple_bits = xnu_pte & ((1UL << 49) | (1UL << 60) | (1UL << 61) | + (1UL << 62)); + + if ((xnu_pte & 3) == 0) + return leaf | apple_bits; + + u64 pa = xnu_pte & (geom4k ? ARM_PTE_PAGE_MASK_4K : ARM_PTE_PAGE_MASK); + return pa | leaf | apple_bits | 3UL; +} + +__attribute__((unused)) +static inline u64 xnu_pte_to_arm(u64 xnu_pte) { + return xnu_pte_to_arm_geom(xnu_pte, 0); +} +static inline u64 xnu_tte_to_arm(u64 xnu_tte) { + if ((xnu_tte & 3) == 0) return 0; + return (xnu_tte & ARM_TTE_TABLE_MASK) | 3UL; +} + +static inline u64 merge_pte_with_mask_geom(u64 existing, u64 new_template, + u64 mask, bool_t geom4k) { + if (mask == SPTM_UPDATE_MASK || mask == 0) + return new_template; + + u64 pa_mask = geom4k ? ARM_PTE_PAGE_MASK_4K : ARM_PTE_PAGE_MASK; + u64 pa_bits = existing & pa_mask; + u64 out = existing; + + if (mask & SPTM_UPDATE_SW_WIRED) + out = (out & ~ARM_PTE_WIRED) | (new_template & ARM_PTE_WIRED); + if (mask & SPTM_UPDATE_PERMS_AND_WAS_WRITABLE) { + u64 clr = ARM_PTE_PNX | ARM_PTE_NX | ARM_PTE_AP_MASK | + ARM_PTE_WRITEABLE; + out = (out & ~clr) | (new_template & clr); + } + if (mask & SPTM_UPDATE_NG) + out = (out & ~ARM_PTE_NG) | (new_template & ARM_PTE_NG); + if (mask & SPTM_UPDATE_AF) + out = (out & ~ARM_PTE_AF) | (new_template & ARM_PTE_AF); + if (mask & SPTM_UPDATE_SH) + out = (out & ~ARM_PTE_SH_MASK) | (new_template & ARM_PTE_SH_MASK); + if (mask & SPTM_UPDATE_MAIR) + out = (out & ~ARM_PTE_ATTRINDX_MASK) | + (new_template & ARM_PTE_ATTRINDX_MASK); + + return (out & ~pa_mask) | pa_bits; +} + +__attribute__((unused)) +static inline u64 merge_pte_with_mask(u64 existing, u64 new_template, u64 mask) { + return merge_pte_with_mask_geom(existing, new_template, mask, 0); +} + +static inline volatile u64 *pt_u64_ptr(u64 pa) { + return (volatile u64 *)pa; +} + +static inline volatile u64 *pt_slot_ptr(u64 table_pa, u64 idx) { + return pt_u64_ptr(table_pa + idx * 8); +} + +static inline u64 pt_read_slot_poc(volatile u64 *slot) { + return *slot; +} + +static inline u64 walk_to_l3_16k(u64 root_pa, u64 va, u64 *out_idx) { + volatile u64 *l1 = pt_u64_ptr(root_pa); + u64 l1e = pt_read_slot_poc(&l1[va_16k_l1_idx(va)]); + if ((l1e & 3) != 3) return 0; + volatile u64 *l2 = pt_u64_ptr(l1e & ARM_TTE_TABLE_MASK); + u64 l2e = pt_read_slot_poc(&l2[va_16k_l2_idx(va)]); + if ((l2e & 3) != 3 || (l2e & TTE_CONDEMNED_BIT)) return 0; + *out_idx = va_16k_l3_idx(va); + return l2e & ARM_TTE_TABLE_MASK; +} + +static inline u64 walk_to_l3_4k(u64 root_pa, u64 va, u64 *out_idx) { + volatile u64 *l0 = pt_u64_ptr(root_pa); + u64 l0e = pt_read_slot_poc(&l0[va_4k_l0_idx(va)]); + if ((l0e & 3) != 3) return 0; + volatile u64 *l1 = pt_u64_ptr(l0e & ARM_TTE_TABLE_MASK); + u64 l1e = pt_read_slot_poc(&l1[va_4k_l1_idx(va)]); + if ((l1e & 3) != 3) return 0; + volatile u64 *l2 = pt_u64_ptr(l1e & ARM_TTE_TABLE_MASK); + u64 l2e = pt_read_slot_poc(&l2[va_4k_l2_idx(va)]); + if ((l2e & 3) != 3 || (l2e & TTE_CONDEMNED_BIT)) return 0; + *out_idx = va_4k_l3_idx(va); + return l2e & ARM_TTE_TABLE_MASK; +} + +/* Walk to the leaf PTE table for either geometry. */ +__attribute__((unused)) +static inline u64 walk_to_l3(u64 root_pa, u64 va, u64 *out_idx) { + return root_uses_4k(root_pa, 3) ? + walk_to_l3_4k(root_pa, va, out_idx) : + walk_to_l3_16k(root_pa, va, out_idx); +} + +/* Walk xnu's pmap geometry to find the parent table at `target_level`. + * 16K roots start at L1; 4K roots can start at L0. Real SPTM gets this from + * root metadata; this emulator records the geometry from sptm_retype params. */ +static inline u64 walk_to_level(u64 root_pa, u64 va, u32 target_level, + u64 *out_idx) { + if (root_uses_4k(root_pa, target_level)) { + if (target_level == 0) { *out_idx = va_4k_l0_idx(va); return root_pa; } + volatile u64 *l0 = pt_u64_ptr(root_pa); + u64 l0e = pt_read_slot_poc(&l0[va_4k_l0_idx(va)]); + if ((l0e & 3) != 3) return 0; + if (target_level == 1) { + *out_idx = va_4k_l1_idx(va); + return l0e & ARM_TTE_TABLE_MASK; + } + volatile u64 *l1 = pt_u64_ptr(l0e & ARM_TTE_TABLE_MASK); + u64 l1e = pt_read_slot_poc(&l1[va_4k_l1_idx(va)]); + if ((l1e & 3) != 3) return 0; + if (target_level == 2) { + *out_idx = va_4k_l2_idx(va); + return l1e & ARM_TTE_TABLE_MASK; + } + volatile u64 *l2 = pt_u64_ptr(l1e & ARM_TTE_TABLE_MASK); + u64 l2e = pt_read_slot_poc(&l2[va_4k_l2_idx(va)]); + if ((l2e & 3) != 3) return 0; + *out_idx = va_4k_l3_idx(va); + return l2e & ARM_TTE_TABLE_MASK; + } + + if (target_level <= 1) { *out_idx = va_16k_l1_idx(va); return root_pa; } + volatile u64 *l1 = pt_u64_ptr(root_pa); + u64 l1e = pt_read_slot_poc(&l1[va_16k_l1_idx(va)]); + if ((l1e & 3) != 3) return 0; + if (target_level == 2) { + *out_idx = va_16k_l2_idx(va); + return l1e & ARM_TTE_TABLE_MASK; + } + volatile u64 *l2 = pt_u64_ptr(l1e & ARM_TTE_TABLE_MASK); + u64 l2e = pt_read_slot_poc(&l2[va_16k_l2_idx(va)]); + if ((l2e & 3) != 3) return 0; + *out_idx = va_16k_l3_idx(va); + return l2e & ARM_TTE_TABLE_MASK; +} + +static inline void barrier(void) { + __asm__ volatile("dsb ish; isb" ::: "memory"); +} + +static inline void tlbi_vmalle1is(void) { + __asm__ volatile( + "dsb ish\n\t" + "tlbi vmalle1is\n\t" + "dsb ish\n\t" + "isb\n\t" + : : : "memory"); +} + +static inline void tlbi_vmalle1os(void) { + __asm__ volatile( + "dsb oshst\n\t" + "sys #0, c8, c1, #0\n\t" // TLBI VMALLE1OS; mnemonic requires tlb-rmi in clang. + "dsb osh\n\t" + "isb\n\t" + : : : "memory"); +} + +static inline void tlbi_aside1os(u64 asid) +{ + __asm__ volatile( + "dsb oshst\n\t" + "sys #0, c8, c1, #2, %0\n\t" // TLBI ASIDE1OS; operand is ASID << 48. + "dsb osh\n\t" + "isb\n\t" + : : "r"(asid) : "memory"); +} + +static inline void tlbi_vmalls12e1is(void) { + __asm__ volatile( + "dsb ish\n\t" + "tlbi vmalls12e1is\n\t" + "dsb ish\n\t" + "isb\n\t" + : : : "memory"); +} + +static inline u64 ttbr1_el12_root_pa(void) { + u64 ttbr; + __asm__ volatile("mrs %0, TTBR1_EL12" : "=r"(ttbr)); + return ttbr & TTBR_BADDR_MASK; +} + +static inline void clean_range(void *ptr, u64 size) { + u64 p = (u64)ptr; + u64 end = p + size; + p &= ~63UL; + while (p < end) { + __asm__ volatile("dc cvau, %0" : : "r"(p) : "memory"); + p += 64; + } + __asm__ volatile("dsb ish" ::: "memory"); +} + +static inline void clean_range_poc(void *ptr, u64 size) { + u64 p = (u64)ptr; + u64 end = p + size; + p &= ~63UL; + while (p < end) { + __asm__ volatile("dc cvac, %0" : : "r"(p) : "memory"); + p += 64; + } + __asm__ volatile("dsb sy" ::: "memory"); +} + +static inline void sptm_coproc_cache_maint_page(u64 pa) { + /* + * M4/A18 SPTM uses guarded implementation-defined ops such as 0x00201401 + * for coprocessor/IOMMU page-cache maintenance. We cannot issue those from + * the emulator context, but the M3 SPTM path uses this older CRn=7 pair for + * the same by-page maintenance, and live probing on A18 Pro shows it does + * not fault from our EL2 context. + */ + register u64 x8 __asm__("x8") = pa & FRAME_PAGE_MASK; + __asm__ volatile( + /* + * LEADING outer-shareable barrier. Real SPTM brackets its per-page + * coprocessor maintenance with an outer-shareable barrier on BOTH sides + * (dsb oshnxs/oshst before, dsb oshnxs/dmb osh after). The caller's + * preceding dc_civac_range (sptm_frame_clean_page) emits no barrier, so + * without this the coprocessor invalidate/refetch below can race AHEAD + * of the CPU's clean reaching the PoC -> the GPU refetches stale page + * contents. The trailing dsb osh alone cannot fix an ordering that has + * already gone wrong before the sys ops execute. + */ + "dsb osh\n\t" + "sys #3, c7, c3, #4, x8\n\t" + "sys #3, c7, c3, #5, x8\n\t" + /* + * The AGX GPU coprocessor is an OUTER-SHAREABLE observer. Real SPTM + * drains its per-page coprocessor maintenance with outer-shareable + * barriers (dsb oshnxs / dmb osh) at every flush site. A `dsb nsh` + * here only guarantees completion for the local PE, so a freshly + * dc_civac'd GPU data page (drained only by this barrier, since + * dc_civac_range emits none) is never published to the GPU coherency + * point -> the GPU reads stale command/ring data, never advances its + * completion stamp, and agx_scheduler spins with faulted=0. + */ + "dsb osh\n\t" + "isb\n\t" + : "+r"(x8) : : "memory"); +} + +static inline void sptm_coproc_cache_maint_range(const void *ptr, u64 size) { + u64 p = (u64)ptr & FRAME_PAGE_MASK; + u64 end = ((u64)ptr + size + PAGE_SIZE_GUEST_EMUL - 1UL) & + FRAME_PAGE_MASK; + + while (p < end) { + sptm_coproc_cache_maint_page(p); + p += PAGE_SIZE_GUEST_EMUL; + } +} + +static inline void sptm_frame_clean_page(u64 pa) { + u64 page = pa & ARM_PTE_PAGE_MASK; + + dc_civac_range((void *)page, PAGE_SIZE_GUEST_EMUL); +} + +static inline void sptm_coproc_cache_maint_data_page(u64 pa) { + u64 page = pa & ARM_PTE_PAGE_MASK; + + sptm_frame_clean_page(page); + sptm_coproc_cache_maint_page(page); +} + +static inline void clean_pt_range_poc(volatile u64 *ptr, u64 size) { + clean_range_poc((void *)ptr, size); + sptm_coproc_cache_maint_range((const void *)ptr, size); + __asm__ volatile("isb" ::: "memory"); +} + +static inline void clean_pt_slot_poc(volatile u64 *slot) { + clean_pt_range_poc(slot, 8); +} + +static inline void zero_16k(u64 pa) { + volatile u64 *q = pt_u64_ptr(pa); + for (u32 i = 0; i < (PAGE_SIZE_GUEST_EMUL / 8); i++) + q[i] = 0; + sptm_coproc_cache_maint_data_page(pa); +} + +static inline void alt_ptd_set_el2_cacheable(u64 pa, bool_t cacheable) { + /* + * Type 24 is firmware kind 5, not an ordinary CPU page-table kind. XNU/PMP + * later read these PTD/ALT frames with non-coherent transactions; if the + * emulator keeps touching them through m1n1's Normal-WB identity alias, AMCC + * can see a stale SLC directory tag and raise UNEXP_RT_HIT_DIR. Mirror the + * /vram fix for m1n1's own alias only: clean the old identity mapping, then + * make future EL2 identity accesses Normal-NC while the frame is type 24. + */ + u64 page = pa & FRAME_PAGE_MASK; + u8 mair = cacheable ? SPTM_EL2_MAIR_NORMAL : SPTM_EL2_MAIR_NORMAL_NC; + + dc_civac_range((void *)page, PAGE_SIZE_GUEST_EMUL); + mmu_add_mapping(page, page, PAGE_SIZE_GUEST_EMUL, mair, SPTM_EL2_PERM_RWX); +} + +/* SEV — wake any guest CPU stuck in WFE waiting on us to complete an op. + * Real SPTM implicitly posts events; we mirror that for every fast-path + * return so xnu's spinlocks don't deadlock waiting for a TLB sync, etc. */ +static inline void sev(void) { + __asm__ volatile("sev" ::: "memory"); +} + +/* Refcount helpers ft_inc_refcount / ft_dec_refcount are defined below the + * g_state extern declaration. */ + +/* ---------------------------------------------------------------------- + * Shared state struct. Python reads counters here for telemetry. + * MUST match proxyclient/tools/sptm/seed.py. + * ---------------------------------------------------------------------- */ +struct sptm_emul_state { + u64 total_calls; /* +0x000 — every HVC the C emulator saw */ + u64 fast_path_calls; /* +0x008 — handled in C without falling through */ + u64 fallthrough_calls; /* +0x010 — fell through to host */ + u64 unknown_calls; /* +0x018 — unrecognized endpoint, fell through */ + u64 last_x16; /* +0x020 — last x16 seen (for debug) */ + u64 last_endpoint; /* +0x028 — last endpoint we saw */ + u64 last_elr; /* +0x030 — last ELR we saw */ + /* Per-endpoint counters: index by endpoint id, 0..63. */ + u64 ep_count[64]; /* +0x038 */ + /* Frame-table state: written at install time by the Python loader. + * RETYPE handler updates frame_table[pa_index(pa)] = new_type so + * xnu's direct frame_table reads (no HVC) stay coherent with our + * emul. Zero frame_table_base = disabled (RETYPE just acks). */ + u64 frame_table_base; /* +0x238 */ + u64 vm_first_phys; /* +0x240 */ + u64 frame_table_size; /* +0x248 — bytes; bounds-check */ + /* AUDIT FIX #1: prev_ptes scratch PA (per-CPU; CPU0 only here) and + * physmap_va base used to compute PAPT VA for sptm_map_page_output_t. + * papt_va = first_papt + (pa - vm_first_phys). xnu reads scratch+0 + * = prev_pte and scratch+8 = ptep PAPT VA. */ + u64 scratch_pa; /* +0x250: base of per-CPU prev_ptes pages */ + u64 first_papt; /* +0x258 */ + /* Circular buffer of LAST 128 calls' (x16, x0, x1, x2, x3, esr, elr, x30) + * for post-hang inspection. ring_idx is the next slot to write. + * Total size 128 * 8 * 8 = 8192 bytes. */ + u64 ring_idx; /* +0x260 */ + u64 ring[128][8]; /* +0x268 — x16, x0, x1, x2, x3, esr, elr, x30 */ + /* SPRR fast-path state. Set by Python at install time. When kc_vmin + * is non-zero, the C handler intercepts EL1t_sync HVCs whose ELR_EL12 + * is in [kc_vmin, kc_vmax) and decodes the trapping instruction by + * reading PA = kc_guest_base + (elr12 - kc_vmin). Avoids a UART round + * trip per pmap_ro_zone_memcpy_internal SPRR write. */ + u64 kc_vmin; /* +0x2268 — kernelcache lowest VA */ + u64 kc_vmax; /* +0x2270 — kernelcache highest VA (exclusive) */ + u64 kc_guest_base; /* +0x2278 — PA where kernelcache is loaded */ + u64 sprr_fast_count; /* +0x2280 — # of SPRR sysreg traps fast-pathed */ + u64 sprr_perm_el1_cache; /* +0x2288 — last MSR'd SPRR_PERM_EL1 value */ + u64 sprr_perm_el0_cache; /* +0x2290 — last MSR'd SPRR_PERM_EL0 value */ + u64 sprr_config_el1_cache;/* +0x2298 — last MSR'd SPRR_CONFIG_EL1 value */ + /* Ring buffer of pmap_ro_zone_memcpy_internal call args. The C + * SPRR fast-path detects when ELR_EL12 is inside the function range + * [pmap_rzm_lo, pmap_rzm_hi) and pushes (zid, va, offset, src, + * size) into this ring. Captured at the SPRR write inside the + * function — by then the args have been moved to x19..x24 by + * the prologue and are stable. Ring size 128 entries × 6 u64s. */ + u64 pmap_rzm_lo; /* +0x22a0 — function start VA */ + u64 pmap_rzm_hi; /* +0x22a8 — function end VA (exclusive) */ + u64 pmap_rzm_idx; /* +0x22b0 — next ring slot to write */ + u64 pmap_rzm_count; /* +0x22b8 — total pushes */ + u64 pmap_rzm_ring[128][6];/* +0x22c0 — (elr, zid, va, offset, src, size) */ + /* Wider-context ring capturing FULL register snapshot at each + * pmap_rzm fast-path. Same length and indexing as pmap_rzm_ring; + * use pmap_rzm_idx to find the matching slot. Captures x0..x31 + * (33 u64 = 264 B per slot, 128 * 264 = 33 KB). */ + u64 pmap_rzm_full[128][33];/* +0x3ac0 — full reg snapshot per slot (128*264B = 33KB) */ + /* DEBUG (2026-05-09): full register snapshot at every SPTM dispatch + * entry — saves ALL 31 GP regs BEFORE any C handler can touch them. + * Used to bisect register-clobber bugs introduced by new C handlers. + * 128 slots * 31 u64 = ~32 KB. */ + u64 sptm_in_idx; /* +0xbec0 — next slot to write */ + u64 sptm_in_regs[128][31];/* +0xbec8 — x0..x30 verbatim from ctx->regs */ + /* Last table/refcount ops. These sit after sptm_in_regs at +0x13ac8. + * Python dumps them on panic to correlate libsptm refcount failures. */ + u64 last_retype[8]; /* pa,current,new,old_type,page_ref,table_ref,idx,flags */ + u64 last_map_table[10]; /* root,va,level,parent,idx,existing,new_tte,arm_tte,child_pa,table_ref */ + u64 last_unmap_table[10]; /* root,va,level,parent,idx,existing,freed_pa,type,table_ref,ft_idx */ + u64 root_geom_pa[ROOT_GEOM_SLOTS]; /* roots/SURT slots with explicit pmap geometry */ + u64 root_geom_attr[ROOT_GEOM_SLOTS]; /* bits[7:0]=SPTM_PT_GEOMETRY_*, bits[31:16]=ASID */ + u64 last_map_page[14]; /* root,va,l3,idx,ptep_pa,ptep_papt,existing,new_pte,arm_pte,target_pa,target_type,page_ref,ptep_table_ref,ptep_ft_idx */ + /* C-side DART object recovery cache. Endpoint 5 used to walk the guest + * frame chain over UART, which is slow enough to wedge boot. Do the same + * verifier-shaped AppleT8110DART object recovery in EL2 and publish the + * decoded object for diagnostics. */ + u64 dart_obj_cache_idx; /* +0x14018 */ + u64 dart_obj_cache_hits; /* +0x14020 */ + u64 dart_obj_cache_misses; /* +0x14028 */ + u64 dart_obj_cache_id[DART_OBJ_CACHE_SLOTS]; /* +0x14030 */ + u64 dart_obj_cache_va[DART_OBJ_CACHE_SLOTS]; /* +0x14230 */ + u64 dart_obj_last[8]; /* +0x14430: table,ep,dart_id,obj,source,gapf_count,inst_count,state */ + u64 dart_obj_last_base_va[16]; /* +0x14470: XNU state MMIO VA per instance */ + u64 dart_obj_last_base_pa[16]; /* +0x144f0: translated MMIO PA per instance */ + struct nvme_c_state nvme; /* +0x14570: C-side NVMe table-6 emulator */ + struct dart_c_state dart; /* C-side T8110 DART table-3/4 emulator */ + /* Appended after the fixed-offset telemetry region so old Python offsets + * remain stable. Real SPTM assigns logical CPU IDs at register_cpu time; + * XNU then uses sptm_prev_ptes + PAGE_SIZE * sptm_cpu_id(phys_id). */ + u64 cpu_count; + u64 cpu_phys_ids[SPTM_MAX_CPUS_EMUL]; + u64 cpu_last_mpidr; + u64 cpu_last_id; + u64 cpu_unknown_mpidr; + struct txm_c_state txm; /* C-side TXM domain=2/table=0 emulator */ + struct sart_c_state sart; /* C-side SART table=5 emulator */ + /* + * Real SPTM's register_cpu endpoint records each CPU's cpu-impl-reg, + * acc-impl-reg, and cpm-impl-reg windows from ADT and creates private SPTM + * mappings for them. Keep the same discovered metadata here. This is + * Mostly bookkeeping; on T8140 the first reserved word can hold a private + * backing page for E-core cpu-impl-reg leaves that fault as locked PIO when + * exposed directly to XNU. + */ + struct sptm_cpu_pio_state cpu_pio; + struct papt_update_state papt_update; + u64 papt_span; + struct uat_c_state uat; /* C-side UAT table=7 emulator */ + struct alt_ptd_trace_state alt_ptd_trace; +}; + +typedef char sptm_emul_state_fits_in_scratch[ + (sizeof(struct sptm_emul_state) <= 0x100000UL) ? 1 : -1]; + +/* The single state instance exported for Python-side boot seeding. Keep this + * 1MiB even though struct sptm_emul_state is smaller: a few emulated SPTM + * devices use the tail as private scratch, matching the old bringup layout. */ +extern u8 g_sptm_state[0x100000]; +#define g_state (*(struct sptm_emul_state *)g_sptm_state) + +static inline bool_t finish_hvc_success(void *ctx, u64 *regs) { + u64 elr2; + __asm__ volatile("mrs %0, elr_el2" : "=r"(elr2)); + u64 *ctx_elr = (u64 *)((unsigned char *)ctx + 0x108); + *ctx_elr = elr2; + regs[0] = SPTM_SUCCESS; + g_state.fast_path_calls++; + sev(); + return 1; +} + +static inline bool_t compat_void_non_xnu_table(u32 table) { + /* + * Temporary parity shim for the removed Python fallback. After TXM, + * DART, SART, NVMe, UAT, and CPUTRACE had first crack, the mixed + * Python/C emulator acknowledged remaining non-XNU dispatch tables with + * x0=0. Keep that behavior only for known auxiliary tables so C-only + * bringup reaches the same wall without hiding real DART/NVMe/UAT + * state-machine failures. + */ + return table == SPTM_TABLE_TXM_BOOTSTRAP || + table == SPTM_TABLE_SK_BOOTSTRAP || + table == SPTM_TABLE_SHART || + table == SPTM_TABLE_HIB; +} + +static inline u64 current_mpidr_phys(void) { + u64 mpidr; + __asm__ volatile("mrs %0, mpidr_el1" : "=r"(mpidr)); + return mpidr & 0xffffffffUL; +} + +static inline u64 cpu_phys_compact(u64 phys) { + return phys & 0x00ffffffUL; +} + +static inline u64 cpu_phys_adt_reg(u64 phys) { + return phys & 0x0000ffffUL; +} + +static inline bool_t cpu_phys_matches(u64 a, u64 b) { + if (a == b) + return 1; + if ((a & 0xffffffffUL) == (b & 0xffffffffUL)) + return 1; + if (cpu_phys_compact(a) == cpu_phys_compact(b)) + return 1; + if (cpu_phys_adt_reg(a) == cpu_phys_adt_reg(b)) + return 1; + return 0; +} + +static inline void sptm_cpu_pio_note_registered(u64 phys, u64 logical_id) { + struct sptm_cpu_pio_state *st = &g_state.cpu_pio; + u64 count = st->seeded_count; + u64 flags = SPTM_CPU_PIO_F_MISSING; + + if (count > SPTM_MAX_CPUS_EMUL) + count = SPTM_MAX_CPUS_EMUL; + + st->last_phys = phys; + st->last_slot = UINT64_MAX_VALUE; + + for (u64 i = 0; i < count; i++) { + struct sptm_cpu_pio_entry *ent = &st->entries[i]; + + if (!cpu_phys_matches(ent->phys_id, phys)) + continue; + + flags = ent->flags | SPTM_CPU_PIO_F_REGISTERED; + if (!ent->registered) + st->registered_count++; + ent->registered++; + ent->logical_id = logical_id; + ent->flags = flags; + st->last_slot = i; + break; + } + + st->last_flags = flags; +} + +#define T8140_CPU_IMPL_LOCKED_REG_OFF 0x9000UL +#define T8140_CPM_DPE_LOCKED_REG_OFF 0x40f8UL + +static inline u64 t8140_cpu_pio_shadow_pa(void) { + return g_state.cpu_pio.reserved[0] & FRAME_PAGE_MASK; +} + +static inline bool_t t8140_cpu_pio_shadow_target(u64 pa, u64 page_mask) { + struct sptm_cpu_pio_state *st = &g_state.cpu_pio; + u64 count = st->seeded_count; + + if (count > SPTM_MAX_CPUS_EMUL) + count = SPTM_MAX_CPUS_EMUL; + + pa &= page_mask; + for (u64 i = 0; i < count; i++) { + struct sptm_cpu_pio_entry *ent = &st->entries[i]; + + if (ent->cpu_base && ent->cpu_size > T8140_CPU_IMPL_LOCKED_REG_OFF) { + u64 locked_page = + (ent->cpu_base + T8140_CPU_IMPL_LOCKED_REG_OFF) & page_mask; + if (pa == locked_page) + return 1; + } + + if (ent->cpm_base && ent->cpm_size > T8140_CPM_DPE_LOCKED_REG_OFF) { + u64 locked_page = + (ent->cpm_base + T8140_CPM_DPE_LOCKED_REG_OFF) & page_mask; + if (pa == locked_page) + return 1; + } + } + + return 0; +} + +static inline u64 t8140_cpu_pio_shadow_pte(u64 arm_pte, bool_t geom4k) { + if ((arm_pte & 3) != 3) + return arm_pte; + + u64 shadow_pa = t8140_cpu_pio_shadow_pa(); + if (!shadow_pa) + return arm_pte; + + u64 page_mask = geom4k ? ARM_PTE_PAGE_MASK_4K : ARM_PTE_PAGE_MASK; + u64 target_pa = arm_pte & page_mask; + if (!t8140_cpu_pio_shadow_target(target_pa, page_mask)) + return arm_pte; + + u64 shadow_target_pa = + shadow_pa + (target_pa & (PAGE_SIZE_GUEST_EMUL - 1UL)); + return (arm_pte & ~page_mask) | (shadow_target_pa & page_mask); +} + +static inline u64 cpu_id_for_phys(u64 phys) { + u64 count = g_state.cpu_count; + if (count > SPTM_MAX_CPUS_EMUL) + count = SPTM_MAX_CPUS_EMUL; + for (u64 i = 0; i < count; i++) { + if (cpu_phys_matches(g_state.cpu_phys_ids[i], phys)) + return i; + } + g_state.cpu_unknown_mpidr = phys; + return 0; +} + +static inline u64 current_sptm_cpu_id(void) { + u64 mpidr = current_mpidr_phys(); + u64 id = cpu_id_for_phys(mpidr); + g_state.cpu_last_mpidr = mpidr; + g_state.cpu_last_id = id; + return id; +} + +static inline volatile u64 *scratch_for_current_cpu(void) { + if (g_state.scratch_pa == 0) + return (volatile u64 *)0; + u64 id = current_sptm_cpu_id(); + if (id >= SPTM_MAX_CPUS_EMUL) + id = 0; + return (volatile u64 *)(g_state.scratch_pa + id * PAGE_SIZE_GUEST_EMUL); +} + +static inline bool_t ft_get_idx(u64 pa, u64 *idx_out) { + pa &= FRAME_PAGE_MASK; + if (g_state.frame_table_base == 0 || pa < g_state.vm_first_phys) return 0; + u64 idx = (pa - g_state.vm_first_phys) >> 14; + if ((idx * 16UL) >= g_state.frame_table_size) return 0; + *idx_out = idx; + return 1; +} + +static inline bool_t papt_va_for_managed_pa(u64 pa, u64 *papt_va_out) { + u64 idx; + + pa &= FRAME_PAGE_MASK; + if (g_state.first_papt == 0) + return 0; + if (!ft_get_idx(pa, &idx)) + return 0; + if (g_state.papt_span && idx * PAGE_SIZE_GUEST_EMUL >= g_state.papt_span) + return 0; + + *papt_va_out = g_state.first_papt + idx * PAGE_SIZE_GUEST_EMUL; + return 1; +} + +static inline void note_papt_nvme_hit_class(u64 cls, u64 rc) { + if (cls >= PAPT_NVME_CLASS_COUNT) + return; + g_state.papt_update.nvme_hits[cls]++; + if (rc == PAPT_UPDATE_RC_WRITE) + g_state.papt_update.nvme_writes[cls]++; +} + +static inline bool_t papt_pa_overlaps(u64 pa, u64 start, u64 size) { + if (start == 0 || start == UINT64_MAX_VALUE || size == 0) + return 0; + + u64 page = pa & FRAME_PAGE_MASK; + u64 first = start & FRAME_PAGE_MASK; + u64 end = (start + size + PAGE_SIZE_GUEST_EMUL - 1UL) & FRAME_PAGE_MASK; + if (end < start) + end = UINT64_MAX_VALUE; + return first <= page && page < end; +} + +static inline void note_papt_nvme_hits(u64 rc, u64 pa) { + u32 queue_count = g_state.nvme.queue_count; + if (queue_count == 0) + queue_count = 1; + if (queue_count > NVME_C_MAX_CID) + queue_count = NVME_C_MAX_CID; + + u64 admin_span = (((u64)queue_count * 0x80UL) + 0xfffUL) & ~0xfffUL; + u64 cq_span = (((u64)queue_count * 0x10UL) + 0xfffUL) & ~0xfffUL; + + if (papt_pa_overlaps(pa, g_state.nvme.admin_asq_kva, admin_span)) + note_papt_nvme_hit_class(PAPT_NVME_SEED_ASQ, rc); + if (papt_pa_overlaps(pa, g_state.nvme.admin_acq_kva, admin_span)) + note_papt_nvme_hit_class(PAPT_NVME_SEED_ACQ, rc); + if (papt_pa_overlaps(pa, g_state.nvme.prp_list_kva, + (u64)queue_count * 0x800UL)) + note_papt_nvme_hit_class(PAPT_NVME_PRP_LIST, rc); + if (papt_pa_overlaps(pa, g_state.nvme.asq_pa_cached, admin_span)) + note_papt_nvme_hit_class(PAPT_NVME_XNU_ASQ, rc); + if (papt_pa_overlaps(pa, g_state.nvme.acq_pa_cached, cq_span)) + note_papt_nvme_hit_class(PAPT_NVME_XNU_ACQ, rc); + if (papt_pa_overlaps(pa, g_state.nvme.iosq_pa_cached, admin_span)) + note_papt_nvme_hit_class(PAPT_NVME_XNU_IOSQ, rc); + if (papt_pa_overlaps(pa, g_state.nvme.iocq_pa_cached, cq_span)) + note_papt_nvme_hit_class(PAPT_NVME_XNU_IOCQ, rc); + + for (u32 cid = 0; cid < queue_count && cid < NVME_C_MAX_CID; cid++) { + if (g_state.nvme.cid_mode[cid] == NVME_CID_AVAILABLE || + g_state.nvme.cid_mode[cid] == NVME_CID_BUSY) + continue; + if (papt_pa_overlaps(pa, g_state.nvme.tcb_prp1[cid], + PAGE_SIZE_GUEST_EMUL) || + papt_pa_overlaps(pa, g_state.nvme.tcb_prp2[cid], + PAGE_SIZE_GUEST_EMUL)) { + note_papt_nvme_hit_class(PAPT_NVME_ACTIVE_PRP, rc); + break; + } + } +} + +static inline u64 sptm_ctx_elr(void *ctx) { + if (!ctx) + return 0; + return *(volatile u64 *)((unsigned char *)ctx + 0x108); +} + +static inline void note_papt_update(u64 rc, u64 pa, u32 options, u64 papt_va, + u64 root_pa, u64 template_pte, + u64 existing_pte, u64 new_pte, + u64 caller_elr, u64 inner_root_pa, + u64 inner_va, u64 inner_template_pte) { + struct papt_update_state *st = &g_state.papt_update; + u64 slot; + + note_papt_nvme_hits(rc, pa); + st->calls++; + switch (rc) { + case PAPT_UPDATE_RC_WRITE: + st->candidates++; + st->writes++; + break; + case PAPT_UPDATE_RC_SKIP_PAPT: + st->skip_skip_papt++; + break; + case PAPT_UPDATE_RC_NO_ATTR: + st->skip_no_attr++; + break; + case PAPT_UPDATE_RC_UNMANAGED: + st->candidates++; + st->skip_unmanaged++; + break; + case PAPT_UPDATE_RC_BAD_TEMPLATE: + st->candidates++; + st->skip_bad_template++; + break; + case PAPT_UPDATE_RC_BAD_ROOT: + st->candidates++; + st->skip_bad_root++; + break; + case PAPT_UPDATE_RC_BAD_WALK: + st->candidates++; + st->skip_bad_walk++; + break; + case PAPT_UPDATE_RC_BAD_EXISTING: + st->candidates++; + st->skip_bad_existing++; + break; + case PAPT_UPDATE_RC_SAME_ATTRS: + st->candidates++; + st->skip_same_attrs++; + break; + default: + break; + } + + slot = st->ring_idx++ & (PAPT_UPDATE_RING_SLOTS - 1); + st->ring[slot].rc = rc; + st->ring[slot].pa = pa; + st->ring[slot].options = options; + st->ring[slot].papt_va = papt_va; + st->ring[slot].root_pa = root_pa; + st->ring[slot].template_pte = template_pte; + st->ring[slot].existing_pte = existing_pte; + st->ring[slot].new_pte = new_pte; + st->ring[slot].caller_elr = caller_elr; + st->ring[slot].inner_root_pa = inner_root_pa; + st->ring[slot].inner_va = inner_va; + st->ring[slot].inner_template_pte = inner_template_pte; +} + +static inline bool_t papt_attrs_are_rt_nc(u64 pte) { + const u64 attr_mask = ARM_PTE_ATTRINDX_MASK | ARM_PTE_SH_MASK; + + return (pte & attr_mask) == (1UL << 2); +} + +static inline bool_t mirror_stage2_for_papt_attrs(u64 pa, u64 old_pte, + u64 new_pte) { + bool_t old_rt = papt_attrs_are_rt_nc(old_pte); + bool_t new_rt = papt_attrs_are_rt_nc(new_pte); + u64 page = pa & ARM_PTE_PAGE_MASK; + u64 attrs; + + if (old_rt == new_rt) + return 0; + + attrs = new_rt ? HV_S2_PTE_NORMAL_NC : HV_S2_PTE_NORMAL_WB; + return hv_map(page, page | attrs, PAGE_SIZE_GUEST_EMUL, 1) == 0; +} + +static inline bool_t update_kernel_papt_attrs(u64 pa, u64 papt_template, + u32 options, u64 caller_elr, + u64 inner_root_pa, u64 inner_va, + u64 inner_template_pte) { + u64 papt_va; + u64 idx; + u64 root_pa; + u64 l3; + volatile u64 *slot; + u64 existing; + u64 template_pte; + u64 new_pte; + const u64 attr_mask = (7UL << 2) | (3UL << 8); + + if (options & SPTM_UPDATE_SKIP_PAPT) { + note_papt_update(PAPT_UPDATE_RC_SKIP_PAPT, pa, options, 0, 0, 0, 0, 0, + caller_elr, inner_root_pa, inner_va, inner_template_pte); + return 0; + } + if ((options & (SPTM_UPDATE_SH | SPTM_UPDATE_MAIR)) == 0) { + note_papt_update(PAPT_UPDATE_RC_NO_ATTR, pa, options, 0, 0, 0, 0, 0, + caller_elr, inner_root_pa, inner_va, inner_template_pte); + return 0; + } + if (!papt_va_for_managed_pa(pa, &papt_va)) { + note_papt_update(PAPT_UPDATE_RC_UNMANAGED, pa, options, 0, 0, 0, 0, 0, + caller_elr, inner_root_pa, inner_va, inner_template_pte); + return 0; + } + + template_pte = xnu_pte_to_arm_geom(papt_template, 0); + if ((template_pte & 3) != 3) { + note_papt_update(PAPT_UPDATE_RC_BAD_TEMPLATE, pa, options, papt_va, 0, + template_pte, 0, 0, caller_elr, inner_root_pa, + inner_va, inner_template_pte); + return 0; + } + if ((template_pte & ARM_PTE_PAGE_MASK) != (pa & ARM_PTE_PAGE_MASK)) { + note_papt_update(PAPT_UPDATE_RC_BAD_TEMPLATE, pa, options, papt_va, 0, + template_pte, 0, 0, caller_elr, inner_root_pa, + inner_va, inner_template_pte); + return 0; + } + + root_pa = ttbr1_el12_root_pa(); + if (root_pa == 0) { + note_papt_update(PAPT_UPDATE_RC_BAD_ROOT, pa, options, papt_va, root_pa, + template_pte, 0, 0, caller_elr, inner_root_pa, + inner_va, inner_template_pte); + return 0; + } + + l3 = walk_to_l3_16k(root_pa, papt_va, &idx); + if (l3 == 0) { + note_papt_update(PAPT_UPDATE_RC_BAD_WALK, pa, options, papt_va, root_pa, + template_pte, 0, 0, caller_elr, inner_root_pa, + inner_va, inner_template_pte); + return 0; + } + + slot = pt_slot_ptr(l3, idx); + existing = pt_read_slot_poc(slot); + if ((existing & 3) != 3) { + note_papt_update(PAPT_UPDATE_RC_BAD_EXISTING, pa, options, papt_va, + root_pa, template_pte, existing, 0, caller_elr, + inner_root_pa, inner_va, inner_template_pte); + return 0; + } + if ((existing & ARM_PTE_PAGE_MASK) != (pa & ARM_PTE_PAGE_MASK)) { + note_papt_update(PAPT_UPDATE_RC_BAD_EXISTING, pa, options, papt_va, + root_pa, template_pte, existing, 0, caller_elr, + inner_root_pa, inner_va, inner_template_pte); + return 0; + } + + new_pte = existing; + if (options & SPTM_UPDATE_SH) + new_pte = (new_pte & ~(3UL << 8)) | (template_pte & (3UL << 8)); + if (options & SPTM_UPDATE_MAIR) + new_pte = (new_pte & ~(7UL << 2)) | (template_pte & (7UL << 2)); + + if ((new_pte & attr_mask) == (existing & attr_mask)) { + note_papt_update(PAPT_UPDATE_RC_SAME_ATTRS, pa, options, papt_va, + root_pa, template_pte, existing, new_pte, caller_elr, + inner_root_pa, inner_va, inner_template_pte); + return 0; + } + + /* + * XNU may be moving a managed DRAM page from cacheable to RT/NC + * semantics. Clean+invalidate the data page while our identity alias is + * still cacheable so the downstream RT mapping cannot hit stale cache/SLC + * directory state. + */ + sptm_coproc_cache_maint_data_page(pa); + mirror_stage2_for_papt_attrs(pa, existing, new_pte); + *slot = new_pte; + clean_pt_slot_poc(slot); + note_papt_update(PAPT_UPDATE_RC_WRITE, pa, options, papt_va, root_pa, + template_pte, existing, new_pte, caller_elr, inner_root_pa, + inner_va, inner_template_pte); + return 1; +} + +static inline bool_t set_kernel_papt_cacheable_for_managed_pa(u64 pa, + bool_t cacheable, + u64 caller_elr) { + u64 papt_va; + u64 root_pa; + u64 idx; + u64 l3; + volatile u64 *slot; + u64 existing; + u64 new_pte; + const u64 attr_mask = ARM_PTE_ATTRINDX_MASK | ARM_PTE_SH_MASK; + u32 options = cacheable ? SPTM_PAPT_OPT_ALT_TO_WB : + SPTM_PAPT_OPT_ALT_TO_NC; + + if (!papt_va_for_managed_pa(pa, &papt_va)) { + note_papt_update(PAPT_UPDATE_RC_UNMANAGED, pa, options, 0, 0, 0, 0, 0, + caller_elr, 0, 0, 0); + return 0; + } + + root_pa = ttbr1_el12_root_pa(); + if (root_pa == 0) { + note_papt_update(PAPT_UPDATE_RC_BAD_ROOT, pa, options, papt_va, root_pa, + 0, 0, 0, caller_elr, 0, 0, 0); + return 0; + } + + l3 = walk_to_l3_16k(root_pa, papt_va, &idx); + if (l3 == 0) { + note_papt_update(PAPT_UPDATE_RC_BAD_WALK, pa, options, papt_va, root_pa, + 0, 0, 0, caller_elr, 0, 0, 0); + return 0; + } + + slot = pt_slot_ptr(l3, idx); + existing = pt_read_slot_poc(slot); + if ((existing & 3) != 3 || + ((existing & ARM_PTE_PAGE_MASK) != (pa & ARM_PTE_PAGE_MASK))) { + note_papt_update(PAPT_UPDATE_RC_BAD_EXISTING, pa, options, papt_va, + root_pa, 0, existing, 0, caller_elr, 0, 0, 0); + return 0; + } + + new_pte = (existing & ~attr_mask) | + (cacheable ? SPTM_PAPT_ATTR_NORMAL_WB : SPTM_PAPT_ATTR_RT_NC); + if ((new_pte & attr_mask) == (existing & attr_mask)) { + note_papt_update(PAPT_UPDATE_RC_SAME_ATTRS, pa, options, papt_va, + root_pa, new_pte, existing, new_pte, caller_elr, + 0, 0, 0); + return 0; + } + + /* + * Real SPTM routes type-24/kind-5 frames through the coprocessor/IOMMU + * coherency path. We cannot issue that GXF/SLC operation here, so prevent + * XNU's PAPT/direct-map alias from creating the cacheable tag in the first + * place while the frame is a coprocessor-visible ALT/PTD page. + */ + sptm_coproc_cache_maint_data_page(pa); + mirror_stage2_for_papt_attrs(pa, existing, new_pte); + *slot = new_pte; + clean_pt_slot_poc(slot); + note_papt_update(PAPT_UPDATE_RC_WRITE, pa, options, papt_va, root_pa, + new_pte, existing, new_pte, caller_elr, 0, 0, 0); + return 1; +} + +static inline u16 ft_read_u16(u64 pa, u64 off) { + u64 idx; + if (!ft_get_idx(pa, &idx)) return 0; + volatile u16 *cnt = (volatile u16 *)(g_state.frame_table_base + idx * 16 + off); + return *cnt; +} + +static inline void ft_clean_entry_idx(u64 idx) { + clean_range_poc((void *)(g_state.frame_table_base + idx * 16), 16); +} + +static inline void ft_write_u16(u64 pa, u64 off, u16 val) { + u64 idx; + if (!ft_get_idx(pa, &idx)) return; + volatile u16 *cnt = (volatile u16 *)(g_state.frame_table_base + idx * 16 + off); + *cnt = val; + ft_clean_entry_idx(idx); +} + +static inline void ft_inc_u16(u64 pa, u64 off) { + u64 idx; + if (!ft_get_idx(pa, &idx)) return; + volatile u16 *cnt = (volatile u16 *)(g_state.frame_table_base + idx * 16 + off); + u16 v = *cnt; + if (v < 0xFFFF) { + *cnt = v + 1; + ft_clean_entry_idx(idx); + } +} + +static inline void ft_dec_u16(u64 pa, u64 off) { + u64 idx; + if (!ft_get_idx(pa, &idx)) return; + volatile u16 *cnt = (volatile u16 *)(g_state.frame_table_base + idx * 16 + off); + u16 v = *cnt; + if (v > 0) { + *cnt = v - 1; + ft_clean_entry_idx(idx); + } +} + +static inline void ft_inc_u32(u64 pa, u64 off) { + u64 idx; + if (!ft_get_idx(pa, &idx)) return; + volatile u32 *cnt = (volatile u32 *)(g_state.frame_table_base + idx * 16 + off); + u32 v = *cnt; + if (v < 0xffffffffU) { + *cnt = v + 1; + ft_clean_entry_idx(idx); + } +} + +static inline void ft_dec_u32(u64 pa, u64 off) { + u64 idx; + if (!ft_get_idx(pa, &idx)) return; + volatile u32 *cnt = (volatile u32 *)(g_state.frame_table_base + idx * 16 + off); + u32 v = *cnt; + if (v > 0) { + *cnt = v - 1; + ft_clean_entry_idx(idx); + } +} + +static inline u32 ft_read_u32(u64 pa, u64 off) { + u64 idx; + if (!ft_get_idx(pa, &idx)) return 0; + volatile u32 *cnt = (volatile u32 *)(g_state.frame_table_base + idx * 16 + off); + return *cnt; +} + +/* Refcount maintenance. + * + * Firmware-visible fields: + * - kind=5 secondary pages: u16 refcount at FTE+0x04. + * - page-table frames: u16 parent/table-link count at FTE+0x06. + * - page-table frames: u16 live child/leaf count at FTE+0x08. + * - kind=3 data pages: u32 ro_refcnt at FTE+0x08, wx_refcnt at FTE+0x0c. + */ +static inline bool_t pte_uses_wx_refcount(u64 pte) { + bool_t writable = (pte & ARM_PTE_WRITEABLE) || + ((pte & ARM_PTE_AP_RO) == 0); + bool_t executable = (pte & (ARM_PTE_PNX | ARM_PTE_NX)) != + (ARM_PTE_PNX | ARM_PTE_NX); + return writable || executable; +} + +static inline void ft_inc_page_refcount_for_pte(u64 pa, u64 pte) { + u8 type = ft_get_type(pa); + if (type == XNU_PAGE_TABLE_ALT) { + ft_inc_u16(pa, FT_KIND5_REFCNT_OFF); + return; + } + if (is_pt_type(type)) + return; + if (pte_uses_wx_refcount(pte)) + ft_inc_u32(pa, FT_DATA_WX_REFCNT_OFF); + else + ft_inc_u32(pa, FT_DATA_RO_REFCNT_OFF); +} + +static inline void ft_dec_page_refcount_for_pte(u64 pa, u64 pte) { + u8 type = ft_get_type(pa); + if (type == XNU_PAGE_TABLE_ALT) { + ft_dec_u16(pa, FT_KIND5_REFCNT_OFF); + return; + } + if (is_pt_type(type)) + return; + if (pte_uses_wx_refcount(pte)) + ft_dec_u32(pa, FT_DATA_WX_REFCNT_OFF); + else + ft_dec_u32(pa, FT_DATA_RO_REFCNT_OFF); +} + +static inline void ft_inc_table_mapping_refcount(u64 pa) { + ft_inc_u16(pa, FT_TABLE_MAPPING_REFCNT_OFF); +} + +static inline void ft_dec_table_mapping_refcount(u64 pa) { + ft_dec_u16(pa, FT_TABLE_MAPPING_REFCNT_OFF); +} + +static inline void ft_inc_table_nested_refcount(u64 pa) { + ft_inc_u16(pa, FT_TABLE_NESTED_REFCNT_OFF); +} + +static inline void ft_dec_table_nested_refcount(u64 pa) { + ft_dec_u16(pa, FT_TABLE_NESTED_REFCNT_OFF); +} + +static inline u32 ft_read_page_refcount(u64 pa) { + if (ft_get_type(pa) == XNU_PAGE_TABLE_ALT) + return ft_read_u16(pa, FT_KIND5_REFCNT_OFF); + return ft_read_u32(pa, FT_DATA_RO_REFCNT_OFF) + + ft_read_u32(pa, FT_DATA_WX_REFCNT_OFF); +} + +static inline u16 ft_read_table_refcount(u64 pa) { + return ft_read_u16(pa, FT_TABLE_NESTED_REFCNT_OFF); +} + +static inline bool_t is_kernel_va(u64 va); +static inline bool_t guest_read64(u64 va, u64 *out); + +static inline void alt_ptd_trace_add_bt(struct alt_ptd_trace_event *ev, + u64 value) { + if (!is_kernel_va(value)) + return; + + for (u64 i = 0; i < ev->bt_count && i < 12; i++) { + if (ev->bt[i] == value) + return; + } + + if (ev->bt_count < 12) + ev->bt[ev->bt_count++] = value; +} + +static inline void alt_ptd_trace_walk_frame(struct alt_ptd_trace_event *ev, + u64 fp) { + for (u32 depth = 0; depth < 8; depth++) { + u64 saved_fp; + u64 saved_lr; + + if (!is_kernel_va(fp)) + break; + if (!guest_read64(fp, &saved_fp) || + !guest_read64(fp + 8, &saved_lr)) + break; + + alt_ptd_trace_add_bt(ev, saved_lr); + + if (saved_fp == fp || saved_fp < fp || saved_fp - fp > 0x200000UL) + break; + fp = saved_fp; + } +} + +static inline void alt_ptd_trace_scan_stack(struct alt_ptd_trace_event *ev, + u64 sp) { + if (!is_kernel_va(sp)) + return; + if (!ev->bt_base) + ev->bt_base = sp; + + for (u32 off = 0; off < 0x180; off += 8) { + u64 value; + + if (!guest_read64(sp + off, &value)) + break; + alt_ptd_trace_add_bt(ev, value); + } +} + +static inline void note_alt_ptd_retype(u64 pa, u64 caller_elr, u64 caller_lr, + u64 caller_fp, u64 caller_sp0, + u64 caller_sp1, u8 current_type, + u8 existing_type, u8 new_type, + u64 retype_params, + u64 page_ref_before, + u64 table_ref_before, + u64 page_ref_after, + u64 table_ref_after, u64 ft_idx, + u64 papt_ring_before, u64 flags) { + struct alt_ptd_trace_state *st = &g_state.alt_ptd_trace; + u64 seq = st->ring_idx++; + struct alt_ptd_trace_event *ev = + &st->ring[seq & (ALT_PTD_TRACE_RING_SLOTS - 1)]; + u64 papt_ring_after = g_state.papt_update.ring_idx; + u64 papt_rc = 0; + u64 papt_existing = 0; + u64 papt_new = 0; + u64 papt_va = 0; + + if (papt_ring_after > papt_ring_before) { + struct papt_update_event *pev = + &g_state.papt_update.ring[(papt_ring_after - 1) & + (PAPT_UPDATE_RING_SLOTS - 1)]; + if ((pev->pa & FRAME_PAGE_MASK) == (pa & FRAME_PAGE_MASK)) { + papt_rc = pev->rc; + papt_existing = pev->existing_pte; + papt_new = pev->new_pte; + papt_va = pev->papt_va; + } + } + + ev->seq = seq; + ev->pa = pa & FRAME_PAGE_MASK; + ev->caller_elr = caller_elr; + ev->caller_lr = caller_lr; + ev->caller_fp = caller_fp; + ev->types_flags = ((u64)current_type) | + ((u64)existing_type << 8) | + ((u64)new_type << 16) | + (flags << 32); + ev->retype_params = retype_params; + ev->ft_idx = ft_idx; + ev->refs_before = (page_ref_before & 0xffffffffUL) | + ((table_ref_before & 0xffffffffUL) << 32); + ev->refs_after = (page_ref_after & 0xffffffffUL) | + ((table_ref_after & 0xffffffffUL) << 32); + ev->papt_ring_before = papt_ring_before; + ev->papt_ring_after = papt_ring_after; + ev->papt_rc = papt_rc; + ev->papt_existing_pte = papt_existing; + ev->papt_new_pte = papt_new; + ev->papt_va = papt_va; + ev->caller_sp0 = caller_sp0; + ev->caller_sp1 = caller_sp1; + ev->bt_count = 0; + ev->bt_base = 0; + for (u32 i = 0; i < 12; i++) + ev->bt[i] = 0; + + alt_ptd_trace_add_bt(ev, caller_lr); + alt_ptd_trace_walk_frame(ev, caller_fp); + alt_ptd_trace_scan_stack(ev, caller_sp0); + alt_ptd_trace_scan_stack(ev, caller_sp1); +} + +static inline u64 ft_idx_or_bad(u64 pa) { + u64 idx; + return ft_get_idx(pa, &idx) ? idx : UINT64_MAX_VALUE; +} + +static inline void ft_inc_refcount(u64 pa) { + ft_inc_page_refcount_for_pte(pa, 0); +} + +static inline void ft_dec_refcount(u64 pa) { + ft_dec_page_refcount_for_pte(pa, 0); +} + +#if 0 +/* Kept disabled to document the old single-refcount model. It was wrong for + * page-table frames because libsptm_get_table_mapping_count only accepts + * frame_type_params kind 1/2, not kind 5. */ +static inline void ft_inc_refcount_old(u64 pa) { + if (g_state.frame_table_base == 0 || pa < g_state.vm_first_phys) return; + u64 idx = (pa - g_state.vm_first_phys) >> 14; + if (idx >= g_state.frame_table_size) return; + volatile u16 *cnt = (volatile u16 *)(g_state.frame_table_base + idx * 16 + 4); + u16 v = *cnt; + if (v < 0xFFFF) *cnt = v + 1; +} + +static inline void ft_dec_refcount_old(u64 pa) { + if (g_state.frame_table_base == 0 || pa < g_state.vm_first_phys) return; + u64 idx = (pa - g_state.vm_first_phys) >> 14; + if (idx >= g_state.frame_table_size) return; + volatile u16 *cnt = (volatile u16 *)(g_state.frame_table_base + idx * 16 + 4); + u16 v = *cnt; + if (v > 0) *cnt = v - 1; +} +#endif + +static inline u8 ft_get_type(u64 pa) { + u64 idx; + if (!ft_get_idx(pa, &idx)) return XNU_DEFAULT; + volatile u8 *ft = (volatile u8 *)g_state.frame_table_base; + return ft[idx * 16 + 2]; +} + +static inline bool_t is_root_type(u8 t) { + return t == SPTM_KERNEL_ROOT_TABLE || + t == XNU_USER_ROOT_TABLE || + t == XNU_SHARED_ROOT_TABLE || + t == XNU_STAGE2_ROOT_TABLE || + t == XNU_SUBPAGE_USER_ROOT_TABLES; +} + +static inline u8 retype_params_attr_idx(u64 params) { + return (u8)(params & 0xff); +} + +static inline u16 retype_params_flags(u64 params) { + return (u16)((params >> 16) & 0xffff); +} + +static inline u16 retype_params_asid(u64 params) { + return (u16)((params >> 32) & 0xffff); +} + +static inline u8 root_geom_lookup(u64 root_pa) { + for (u32 i = 0; i < ROOT_GEOM_SLOTS; i++) { + if (g_state.root_geom_pa[i] == root_pa) + return (u8)(g_state.root_geom_attr[i] & 0xff); + } + + if (ft_get_type(root_pa) == XNU_SUBPAGE_USER_ROOT_TABLES) + return SPTM_PT_GEOMETRY_4K; + return ROOT_GEOM_UNKNOWN; +} + +static inline u16 root_asid_lookup(u64 root_pa) { + for (u32 i = 0; i < ROOT_GEOM_SLOTS; i++) { + if (g_state.root_geom_pa[i] == root_pa) + return (u16)((g_state.root_geom_attr[i] >> 16) & 0xffff); + } + return 0; +} + +static inline void root_geom_set_meta(u64 root_pa, u8 geom, u16 asid) { + u32 empty = ROOT_GEOM_SLOTS; + u64 packed = (u64)geom | ((u64)asid << 16); + + for (u32 i = 0; i < ROOT_GEOM_SLOTS; i++) { + if (g_state.root_geom_pa[i] == root_pa) { + g_state.root_geom_attr[i] = packed; + return; + } + if (empty == ROOT_GEOM_SLOTS && g_state.root_geom_pa[i] == 0) + empty = i; + } + + if (empty == ROOT_GEOM_SLOTS) + empty = (u32)(((root_pa >> 14) ^ (root_pa >> 6)) & (ROOT_GEOM_SLOTS - 1)); + g_state.root_geom_pa[empty] = root_pa; + g_state.root_geom_attr[empty] = packed; +} + +static inline void root_geom_set(u64 root_pa, u8 geom) { + root_geom_set_meta(root_pa, geom, root_asid_lookup(root_pa)); +} + +static inline bool_t root_uses_4k(u64 root_pa, u32 target_level) { + if (target_level == 0) { + /* XNU only calls sptm_map_table(..., level=0, ...) for roots whose + * walk really starts at L0. Record that runtime fact so the next + * level=1/2 operation for this same root descends through the table we + * just installed instead of treating root[0] as an already-present + * 16K L1 entry. */ + root_geom_set(root_pa, SPTM_PT_GEOMETRY_4K); + return 1; + } + + u8 geom = root_geom_lookup(root_pa); + if (geom != ROOT_GEOM_UNKNOWN) + return geom == SPTM_PT_GEOMETRY_4K; + + /* Do not infer 4K from XNU_USER_ROOT_TABLE. Apple Silicon supports mixed + * user geometries, and the normal ARM64 bringup path here uses 16K user + * roots. The earlier frame-type heuristic returned TABLE_NOT_PRESENT for + * a valid 16K user root and sent xnu into pmap_expand()'s WFE loop. */ + return 0; +} + +static inline u32 table_group_count(bool_t geom4k) { + return geom4k ? 4U : 1U; +} + +static inline u64 table_group_base_idx(u64 idx, bool_t geom4k) { + return geom4k ? (idx & ~3UL) : idx; +} + +static inline u64 table_group_child_pa(u64 child_pa, u32 n, bool_t geom4k) { + if (!geom4k) + return child_pa & ARM_TTE_TABLE_MASK; + return (child_pa & FRAME_PAGE_MASK) + ((u64)n * PAGE_SIZE_4K_EMUL); +} + +static inline void ft_set_type(u64 pa, u8 type) { + u64 idx; + if (!ft_get_idx(pa, &idx)) return; + volatile u8 *ft = (volatile u8 *)g_state.frame_table_base; + ft[idx * 16 + 2] = type; + ft_clean_entry_idx(idx); +} + +static inline bool_t is_pt_type(u8 t) { + return t == SPTM_KERNEL_ROOT_TABLE || + t == SPTM_PAGE_TABLE || + t == XNU_USER_ROOT_TABLE || + t == XNU_SHARED_ROOT_TABLE || + t == XNU_PAGE_TABLE || + t == XNU_PAGE_TABLE_SHARED || + t == XNU_PAGE_TABLE_ROZONE || + t == XNU_PAGE_TABLE_COMMPAGE || + t == XNU_STAGE2_ROOT_TABLE || + t == XNU_STAGE2_PAGE_TABLE || + t == XNU_SUBPAGE_USER_ROOT_TABLES; +} + +static inline bool_t is_io_type(u8 t) { + return t == XNU_IO || t == XNU_PROTECTED_IO || t == XNU_COPROCESSOR_RO_IO; +} + +static inline void update_refcounts_for_pte_change_geom(u64 existing, u64 new_pte, + bool_t geom4k) { + bool_t old_valid = ((existing & 3) == 3); + bool_t new_valid = ((new_pte & 3) == 3); + u64 page_mask = geom4k ? ARM_PTE_PAGE_MASK_4K : ARM_PTE_PAGE_MASK; + u64 old_pa = existing & page_mask; + u64 new_pa = new_pte & page_mask; + + if (old_valid && !new_valid) { + ft_dec_page_refcount_for_pte(old_pa, existing); + } else if (!old_valid && new_valid) { + ft_inc_page_refcount_for_pte(new_pa, new_pte); + } else if (old_valid && new_valid && old_pa != new_pa) { + ft_dec_page_refcount_for_pte(old_pa, existing); + ft_inc_page_refcount_for_pte(new_pa, new_pte); + } +} + +static inline void update_refcounts_for_leaf_pte_change(u64 table_pa, + u64 existing, + u64 new_pte, + bool_t geom4k) { + bool_t old_valid = ((existing & 3) == 3); + bool_t new_valid = ((new_pte & 3) == 3); + + update_refcounts_for_pte_change_geom(existing, new_pte, geom4k); + + table_pa &= FRAME_PAGE_MASK; + if (old_valid && !new_valid) + ft_dec_table_nested_refcount(table_pa); + else if (!old_valid && new_valid) + ft_inc_table_nested_refcount(table_pa); +} + +static inline bool_t maybe_map_t8140_pmgr_cpustart_window(u64 root_pa, u64 va, + u64 arm_pte, + bool_t geom4k, + u64 known_l3, + u64 known_idx) { + /* + * T8140 ApplePMGR maps PMGR reg[24] at 0x300730000, but the CPUSTART + * write lands at offset +0x4008 inside that regmap. XNU may install this + * mapping through MAP_PAGE or a region/disjoint update, and it may hand us + * any page inside the 0x14000-byte ADT window. If we leave holes in the + * same real regmap, XNU faults in ApplePMGR::writeReg32 before the + * hypervisor's physical CPUSTART hook can see the access. + * + * This is not generic MMIO expansion: it preserves only this exact ADT + * PMGR regmap window, with the same PTE attributes XNU requested. + */ + const u64 pmgr_base = 0x300730000UL; + const u64 pmgr_size = 0x14000UL; + u64 page_size = geom4k ? PAGE_SIZE_4K_EMUL : PAGE_SIZE_GUEST_EMUL; + u64 page_mask = geom4k ? ARM_PTE_PAGE_MASK_4K : ARM_PTE_PAGE_MASK; + u64 mapped_pa = arm_pte & page_mask; + if ((arm_pte & 3) != 3) + return 0; + if (mapped_pa < pmgr_base || mapped_pa >= (pmgr_base + pmgr_size)) + return 0; + + u64 mapped_off = mapped_pa - pmgr_base; + u64 window_va = va - mapped_off; + bool_t changed = 0; + u64 table_entries = geom4k ? 512UL : 2048UL; + u64 mapped_page_idx = mapped_off / page_size; + bool_t can_fill_direct = + known_l3 != 0 && mapped_off == mapped_page_idx * page_size && + known_idx >= mapped_page_idx; + u64 direct_base_idx = can_fill_direct ? known_idx - mapped_page_idx : 0; + + for (u64 off = 0; off < pmgr_size; off += page_size) { + u64 idx = 0; + u64 l3 = 0; + if (can_fill_direct && direct_base_idx + off / page_size < table_entries) { + l3 = known_l3; + idx = direct_base_idx + off / page_size; + } else { + l3 = walk_to_l3(root_pa, window_va + off, &idx); + if (!l3) + continue; + } + + volatile u64 *slot = pt_slot_ptr(l3, idx); + u64 existing = pt_read_slot_poc(slot); + if ((existing & 3) == 3) + continue; + + u64 extra_pte = (arm_pte & ~page_mask) | ((pmgr_base + off) & page_mask); + *slot = extra_pte; + clean_pt_slot_poc(slot); + update_refcounts_for_leaf_pte_change(l3, existing, extra_pte, geom4k); + changed = 1; + } + + if (changed) + tlbi_vmalle1is(); + return changed; +} + +static inline void write_prev_pte(volatile u64 *prev_out, u32 *prev_count, + u64 existing) { + if (prev_out != (volatile u64 *)0 && *prev_count < 2048) + prev_out[*prev_count] = existing; + *prev_count = *prev_count + 1; +} + +/* Hardcoded offset of regs[] in m1n1's struct exc_info (always 0). */ +#define REGS_OFFSET 0 +#define ESR_OFFSET 0x110 + +static inline bool_t is_dispatch_hvc(u64 esr, u64 x16) { + if ((esr & ESR_ISS_MASK) != SPTM_GENTER_DISPATCH_CALL) return 0; + if (x16 & SPTM_DISPATCH_RESERVED_MASK) return 0; + if (((x16 >> 48) & 0xff) > 4) return 0; + if (((x16 >> 32) & 0xff) > 11) return 0; + return 1; +} + +#define KERNEL_VA_BASE 0xfffffe0000000000UL +#define DART_OBJ_MODE_OFF 0x0b8UL +#define DART_OBJ_STATE_OFF 0x0c50UL +#define DART_OBJ_INST_COUNT_OFF 0x14d8UL +#define DART_OBJ_GAPF_COUNT_OFF 0x17b0UL + +static inline bool_t is_kernel_va(u64 va) { + return (va & KERNEL_VA_BASE) == KERNEL_VA_BASE; +} + +static inline bool_t guest_va_to_pa(u64 va, bool_t write, u64 *pa_out) { + u64 par; + if (write) { + __asm__ volatile( + "at s1e1w, %1\n\t" + "isb\n\t" + "mrs %0, par_el1\n\t" + : "=r"(par) : "r"(va) : "memory"); + } else { + __asm__ volatile( + "at s1e1r, %1\n\t" + "isb\n\t" + "mrs %0, par_el1\n\t" + : "=r"(par) : "r"(va) : "memory"); + } + if (par & 1) + return 0; + *pa_out = (par & 0xfffffffffc000UL) | (va & 0x3fffUL); + return 1; +} + +static inline bool_t guest_read32(u64 va, u32 *out) { + u64 pa; + if (!is_kernel_va(va) || !guest_va_to_pa(va, 0, &pa)) + return 0; + *out = *(volatile u32 *)pa; + return 1; +} + +static inline bool_t guest_read64(u64 va, u64 *out) { + u64 pa; + if (!is_kernel_va(va) || !guest_va_to_pa(va, 0, &pa)) + return 0; + *out = *(volatile u64 *)pa; + return 1; +} + +static inline bool_t guest_write64(u64 va, u64 value) { + u64 pa; + if (!is_kernel_va(va) || !guest_va_to_pa(va, 1, &pa)) + return 0; + *(volatile u64 *)pa = value; + clean_range((void *)pa, 8); + return 1; +} + + +/* Table/module entry points. */ +bool_t dart_handle_table(u32 table, u32 endpoint, u64 *regs); +void dart_cache_xnu_object_from_regs(u32 table, u32 endpoint, u64 *regs); +bool_t nvme_handle_table6(u32 endpoint, u64 *regs); +bool_t sart_handle_table5(u32 endpoint, u64 *regs); +bool_t txm_handle_selector(u32 selector, u64 x16, u64 *regs); +bool_t uat_handle_table7(u32 endpoint, u64 *regs); +bool_t xnu_handle_table0(void *ctx, u32 endpoint, u64 *regs); +bool_t sptm_handler(void *ctx); + +#endif /* SPTM_PRIVATE_H */ diff --git a/src/sptm/sart.c b/src/sptm/sart.c new file mode 100644 index 000000000..be37a18a7 --- /dev/null +++ b/src/sptm/sart.c @@ -0,0 +1,277 @@ +/* SPDX-License-Identifier: MIT */ +#include "private.h" + +int debug_printf(const char *fmt, ...); + +static u32 sart_config_off(u32 idx) { + return idx * 4U; +} + +static u32 sart_paddr_off(u32 idx) { + return 0x40U + idx * 4U; +} + +static u32 sart_size_off(u32 idx) { + return 0x80U + idx * 4U; +} + +static bool_t sart_valid_index(u32 idx) { + return idx < SART_C_MAX_ENTRIES; +} + +static u32 sart_flags_for_perm(u32 perm) { + if (perm > 1) + return 0; + + if (g_state.sart.version == 2 || g_state.sart.version == 3) { + u32 access = perm ? 3U : 2U; + return 0xc0U | access | (access << 2) | (access << 4); + } + + return g_state.sart.flags_allow; +} + +static struct sart_c_entry sart_live_entry(u32 idx) { + struct sart_c_entry ent; + ent.paddr = 0; + ent.size = 0; + ent.flags = 0; + ent.guard = 0; + if (!g_state.sart.base_pa || !sart_valid_index(idx)) + return ent; + + u32 cfg = *(volatile u32 *)(g_state.sart.base_pa + sart_config_off(idx)); + u32 paddr_raw = *(volatile u32 *)(g_state.sart.base_pa + sart_paddr_off(idx)); + u32 size_raw; + + if (g_state.sart.version == 3) { + ent.flags = cfg & 0xffU; + size_raw = *(volatile u32 *)(g_state.sart.base_pa + sart_size_off(idx)); + } else if (g_state.sart.version == 2) { + ent.flags = (cfg >> 24) & 0xffU; + size_raw = cfg & ((1U << 24) - 1U); + } else { + ent.flags = (cfg >> 24) & 0x1fU; + size_raw = cfg & ((1U << 19) - 1U); + } + if (ent.flags && !g_state.sart.exclusive_bounds) + size_raw++; + + ent.paddr = (u64)paddr_raw << g_state.sart.paddr_shift; + ent.size = (u64)size_raw << g_state.sart.size_shift; + return ent; +} + +static bool_t sart_write_entry(u32 idx, u64 paddr, u64 size, u32 flags) { + if (!g_state.sart.base_pa || !sart_valid_index(idx)) + return 0; + if ((paddr & ((1UL << g_state.sart.paddr_shift) - 1UL)) || + (size & ((1UL << g_state.sart.size_shift) - 1UL))) + return 0; + + u32 paddr_raw = (u32)(paddr >> g_state.sart.paddr_shift); + u32 size_raw = (u32)(size >> g_state.sart.size_shift); + if (size_raw > g_state.sart.size_max) + return 0; + if (flags && !g_state.sart.exclusive_bounds) { + if (size_raw == 0) + return 0; + size_raw--; + } + + if (g_state.sart.version == 3) { + *(volatile u32 *)(g_state.sart.base_pa + sart_paddr_off(idx)) = paddr_raw; + *(volatile u32 *)(g_state.sart.base_pa + sart_size_off(idx)) = size_raw; + *(volatile u32 *)(g_state.sart.base_pa + sart_config_off(idx)) = + flags & g_state.sart.flags_allow; + } else { + u32 cfg = ((flags & g_state.sart.flags_allow) << 24) | size_raw; + *(volatile u32 *)(g_state.sart.base_pa + sart_paddr_off(idx)) = paddr_raw; + *(volatile u32 *)(g_state.sart.base_pa + sart_config_off(idx)) = cfg; + } + barrier(); + return 1; +} + +static bool_t sart_power_canary_acquire(u32 guard) { + if (guard != 1 || g_state.sart.power_canary_pa == 0) + return 1; + + if (g_state.sart.power_canary_count == 0) + *(volatile u32 *)(g_state.sart.power_canary_pa + + g_state.sart.power_canary_offset) = 0xabfedeedU; + g_state.sart.power_canary_count++; + if (g_state.sart.power_canary_count == 0) + return 0; + return 1; +} + +static bool_t sart_power_canary_release(u32 guard) { + if (guard != 1 || g_state.sart.power_canary_pa == 0) + return 1; + if (g_state.sart.power_canary_count == 0) + return 0; + if (*(volatile u32 *)(g_state.sart.power_canary_pa + + g_state.sart.power_canary_offset) != 0xabfedeedU) + return 0; + g_state.sart.power_canary_count--; + return 1; +} + +static bool_t sart_validate_range(u64 paddr, u64 size) { + if (size == 0) + return 0; + if (size & ((1UL << g_state.sart.size_shift) - 1UL)) + return 0; + if (paddr & ((1UL << g_state.sart.paddr_shift) - 1UL)) + return 0; + if ((paddr + size) < paddr) + return 0; + if ((size >> g_state.sart.size_shift) > g_state.sart.size_max) + return 0; + return 1; +} + +static int sart_find_exact(u64 paddr, u64 size, struct sart_c_entry *out) { + for (u32 i = 0; i < SART_C_MAX_ENTRIES; i++) { + struct sart_c_entry ent = g_state.sart.shadow[i]; + if (!ent.flags) + ent = sart_live_entry(i); + if (ent.flags && ent.paddr == paddr && ent.size == size) { + if (out) + *out = ent; + return (int)i; + } + } + return -1; +} + +static int sart_find_free(void) { + for (u32 i = 0; i < SART_C_MAX_ENTRIES; i++) { + if (g_state.sart.protected_mask & (1U << i)) + continue; + if (g_state.sart.shadow[i].flags) + continue; + struct sart_c_entry ent = sart_live_entry(i); + if (!ent.flags) + return (int)i; + } + return -1; +} + +static u64 sart_map_region(u64 *regs) { + u64 paddr = regs[0]; + u64 size = regs[1]; + u32 perm = (u32)regs[2]; + u32 guard = (u32)regs[3]; + + if (!sart_validate_range(paddr, size)) + return 1; + if (perm > 1 || guard > 1) + return 1; + + u32 flags = sart_flags_for_perm(perm); + if (flags == 0) + return 1; + + int idx = sart_find_free(); + if (idx < 0) + return 1; + + if (!sart_power_canary_acquire(guard)) + return 1; + if (!sart_write_entry((u32)idx, paddr, size, flags)) { + sart_power_canary_release(guard); + return 1; + } + + g_state.sart.shadow[idx].paddr = paddr; + g_state.sart.shadow[idx].size = size; + g_state.sart.shadow[idx].flags = flags; + g_state.sart.shadow[idx].guard = guard; + if (g_state.nvme.debug_flags & NVME_DBG_PRINT_CALLS) { + u64 ring = (g_state.ring_idx - 1) & 0x7fUL; + debug_printf("C-SART map idx=%d pa=0x%lx size=0x%lx perm=%u guard=%u flags=0x%x pc=0x%lx lr=0x%lx\n", + idx, paddr, size, perm, guard, flags, + g_state.ring[ring][6], g_state.ring[ring][7]); + } + return SPTM_SUCCESS; +} + +static u64 sart_unmap_region(u64 *regs) { + u64 paddr = regs[0]; + u64 size = regs[1]; + struct sart_c_entry ent; + + if (!sart_validate_range(paddr, size)) + return 1; + + int idx = sart_find_exact(paddr, size, &ent); + if (idx < 0) + return 1; + + if (g_state.sart.protected_mask & (1U << (u32)idx)) + return SPTM_SUCCESS; + + u32 guard = g_state.sart.shadow[idx].guard; + + if (!sart_write_entry((u32)idx, 0, 0, 0)) + return 1; + if (!sart_power_canary_release(guard)) + return 1; + g_state.sart.shadow[idx].paddr = 0; + g_state.sart.shadow[idx].size = 0; + g_state.sart.shadow[idx].flags = 0; + g_state.sart.shadow[idx].guard = 0; + if (g_state.nvme.debug_flags & NVME_DBG_PRINT_CALLS) { + u64 ring = (g_state.ring_idx - 1) & 0x7fUL; + debug_printf("C-SART unmap idx=%d pa=0x%lx size=0x%lx guard=%u pc=0x%lx lr=0x%lx\n", + idx, paddr, size, guard, + g_state.ring[ring][6], g_state.ring[ring][7]); + } + return SPTM_SUCCESS; +} + +bool_t sart_handle_table5(u32 endpoint, u64 *regs) { + u64 rc = 1; + + if (!g_state.sart.enabled) + return 0; + if (endpoint >= SART_C_MAX_ENDPOINTS) + return 0; + + g_state.sart.total_calls++; + g_state.sart.ep_count[endpoint]++; + for (u32 i = 0; i < 8; i++) + g_state.sart.last[i] = regs[i]; + + switch (endpoint) { + case 0: + /* sptm_sart_set_state(bool). XNU reaches this through + * sart_ioctl(cmd=0x5ab7), leaving cmd/len in x1/x3. */ + g_state.sart.active = (u32)(regs[0] & 1UL); + rc = SPTM_SUCCESS; + if (g_state.nvme.debug_flags & NVME_DBG_PRINT_CALLS) { + u64 ring = (g_state.ring_idx - 1) & 0x7fUL; + debug_printf("C-SART state active=%u arg=0x%lx pc=0x%lx lr=0x%lx\n", + g_state.sart.active, regs[0], + g_state.ring[ring][6], g_state.ring[ring][7]); + } + break; + case 1: + rc = sart_map_region(regs); + break; + case 2: + rc = sart_unmap_region(regs); + break; + } + + g_state.sart.last_rc = (u32)rc; + if (rc != SPTM_SUCCESS) + g_state.sart.violations++; + regs[0] = rc; + g_state.sart.fast_path_calls++; + g_state.fast_path_calls++; + sev(); + return 1; +} diff --git a/src/sptm/txm.c b/src/sptm/txm.c new file mode 100644 index 000000000..edc0374bf --- /dev/null +++ b/src/sptm/txm.c @@ -0,0 +1,383 @@ +/* SPDX-License-Identifier: MIT */ +#include "private.h" + +#define TXM_SYNTH_RESERVED_TOP 0x200UL + +static inline bool_t txm_synth_pa_for_va(u64 va, u64 *pa_out) { + if (g_state.txm.synth_base_va == 0 || g_state.txm.synth_base_pa == 0) + return 0; + if (va < g_state.txm.synth_base_va) + return 0; + u64 off = va - g_state.txm.synth_base_va; + if (off >= g_state.txm.synth_size) + return 0; + *pa_out = g_state.txm.synth_base_pa + off; + return 1; +} + +static inline void txm_synth_write8(u64 va, u8 value) { + u64 pa; + if (!txm_synth_pa_for_va(va, &pa)) + return; + *(volatile u8 *)pa = value; + clean_range((void *)(pa & ~63UL), 64); +} + +static inline void txm_synth_write16(u64 va, u16 value) { + u64 pa; + if (!txm_synth_pa_for_va(va, &pa)) + return; + *(volatile u16 *)pa = value; + clean_range((void *)(pa & ~63UL), 64); +} + +static inline void txm_synth_write32(u64 va, u32 value) { + u64 pa; + if (!txm_synth_pa_for_va(va, &pa)) + return; + *(volatile u32 *)pa = value; + clean_range((void *)(pa & ~63UL), 64); +} + +static inline void txm_synth_write64(u64 va, u64 value) { + u64 pa; + if (!txm_synth_pa_for_va(va, &pa)) + return; + *(volatile u64 *)pa = value; + clean_range((void *)(pa & ~63UL), 64); +} + +static inline u64 txm_alloc_synth(u64 size, u64 fallback_va) { + if (g_state.txm.synth_base_va == 0 || g_state.txm.synth_size == 0) + return fallback_va; + + size = (size + (TXM_C_SYNTH_ALIGN - 1)) & ~(TXM_C_SYNTH_ALIGN - 1); + u64 off = (g_state.txm.synth_next_off + (TXM_C_SYNTH_ALIGN - 1)) & + ~(TXM_C_SYNTH_ALIGN - 1); + u64 limit = g_state.txm.synth_size; + if (limit > TXM_SYNTH_RESERVED_TOP) + limit -= TXM_SYNTH_RESERVED_TOP; + if (off + size > limit) + return fallback_va ? fallback_va : g_state.txm.synth_base_va; + + g_state.txm.synth_next_off = off + size; + return g_state.txm.synth_base_va + off; +} + +/* Offsets verified against KDK 26.5_25F71 private TXM/libCodeSignature headers. */ +#define TXM_SYNTH_TRUST_CACHE_OFF 0x200UL +#define TXM_SYNTH_TRUST_CACHE_SIZE 0x200UL +#define TXM_SYNTH_TC_OBJECT_OFF 0x000UL +#define TXM_SYNTH_TC_MODULE_OFF 0x080UL +#define TXM_SYNTH_TC_ENTRY_OFF 0x098UL +#define TXM_TRUSTCACHE_TYPE_OFF 0x008UL +#define TXM_TRUSTCACHE_MODULE_SIZE_OFF 0x010UL +#define TXM_TRUSTCACHE_MODULE_OFF 0x018UL +#define TXM_MODULE1_NUM_ENTRIES_OFF 0x014UL +#define TXM_MODULE1_ENTRIES_OFF 0x018UL +#define TXM_ENTRY1_HASH_TYPE_OFF 0x014UL +#define TXM_ENTRY1_FLAGS_OFF 0x015UL +#define TXM_CODE_SIGNATURE_SIZE 0x180UL +#define TXM_CODE_SIGNATURE_TRUST_OFF 0x132UL +#define TXM_ASPACE_MAIN_REGION_OFF 0x040UL +#define TXM_ASPACE_SYNTH_REGION_SIZE 0x080UL +#define TXM_CS_TRUST_STATIC_TC 9U +#define TXM_TC_TYPE_STATIC 0U +#define TXM_TC_TYPE_LTRS 4U +#define TXM_TC_QUERY_LOADABLE 2U +#define TXM_TC_HASH_SHA256_TRUNCATED 3U +#define TXM_TC_CAP_HASH_TYPE (1UL << 0) +#define TXM_TC_CAP_FLAGS (1UL << 1) + +static bool_t txm_fake_trust_cache_token(u64 query_type, u64 *tc_va, u64 *entry_va) { + if (g_state.txm.synth_base_va == 0 || + g_state.txm.synth_size < TXM_SYNTH_TRUST_CACHE_SIZE) + return 0; + + u64 base = g_state.txm.synth_base_va + g_state.txm.synth_size - + TXM_SYNTH_TRUST_CACHE_OFF; + u64 tc = base + TXM_SYNTH_TC_OBJECT_OFF; + u64 module = base + TXM_SYNTH_TC_MODULE_OFF; + u64 entry = base + TXM_SYNTH_TC_ENTRY_OFF; + u8 tc_type = (query_type == TXM_TC_QUERY_LOADABLE) ? TXM_TC_TYPE_LTRS : + TXM_TC_TYPE_STATIC; + + /* TrustCache_t: next=0, type, tombstoned=0, moduleSize, module pointer. */ + txm_synth_write64(tc + 0x00, 0); + txm_synth_write8(tc + TXM_TRUSTCACHE_TYPE_OFF, tc_type); + txm_synth_write8(tc + TXM_TRUSTCACHE_TYPE_OFF + 1, 0); + txm_synth_write64(tc + TXM_TRUSTCACHE_MODULE_SIZE_OFF, 0x18 + 0x16); + txm_synth_write64(tc + TXM_TRUSTCACHE_MODULE_OFF, module); + + /* TrustCacheModule1_t with one synthetic entry. */ + txm_synth_write32(module + 0x00, 1); + txm_synth_write32(module + TXM_MODULE1_NUM_ENTRIES_OFF, 1); + txm_synth_write8(entry + TXM_ENTRY1_HASH_TYPE_OFF, + TXM_TC_HASH_SHA256_TRUNCATED); + txm_synth_write8(entry + TXM_ENTRY1_FLAGS_OFF, 0); + + *tc_va = tc; + *entry_va = entry; + return 1; +} + +static inline void txm_record_call(u32 selector, u64 x16, u64 *regs) { + g_state.txm.total_calls++; + g_state.total_calls++; + g_state.last_x16 = x16; + g_state.last_endpoint = selector; + g_state.txm.last_selector = selector; + g_state.txm.last_stack_pa = regs[0]; + g_state.txm.last_x16 = x16; + for (u32 i = 0; i < 8; i++) + g_state.txm.last_args[i] = regs[i]; + if (selector < TXM_C_MAX_ENDPOINTS) + g_state.txm.ep_count[selector]++; +} + +static bool_t txm_write_stack_return(u64 stack_pa, u64 rc, + const u64 *words, u64 nwords) { + if (stack_pa == 0 || nwords > TXM_STACK_RETURN_WORDS) + return 0; + + u64 shared = stack_pa + TXM_THREAD_STACK_SHARED_OFF; + *(volatile u64 *)(shared + TXM_SHARED_RETURN_CODE_OFF) = rc; + *(volatile u8 *)(shared + TXM_SHARED_RETURN_TYPE_OFF) = 0; /* words */ + *(volatile u64 *)(shared + TXM_SHARED_NUM_WORDS_OFF) = nwords; + for (u64 i = 0; i < TXM_STACK_RETURN_WORDS; i++) { + u64 value = (i < nwords) ? words[i] : 0; + *(volatile u64 *)(shared + TXM_SHARED_WORDS_OFF + i * 8UL) = value; + g_state.txm.last_words[i] = value; + } + + g_state.txm.last_rc = rc; + clean_range((void *)shared, 0x80); + barrier(); + return 1; +} + +static bool_t txm_finish(u32 selector, u64 *regs, u64 rc, + const u64 *words, u64 nwords) { + if (!txm_write_stack_return(regs[0], rc, words, nwords)) { + g_state.txm.violations++; + return 0; + } + regs[0] = rc; + g_state.txm.fast_path_calls++; + g_state.fast_path_calls++; + sev(); + (void)selector; + return 1; +} + +static bool_t txm_success0(u32 selector, u64 *regs) { + u64 words[TXM_STACK_RETURN_WORDS] = {0, 0, 0, 0, 0, 0}; + return txm_finish(selector, regs, TXM_SUCCESS, words, 0); +} + +static bool_t txm_failure(u32 selector, u64 *regs, u64 rc) { + u64 words[TXM_STACK_RETURN_WORDS] = {0, 0, 0, 0, 0, 0}; + return txm_finish(selector, regs, rc, words, 0); +} + +bool_t txm_handle_selector(u32 selector, u64 x16, u64 *regs) { + u64 words[TXM_STACK_RETURN_WORDS] = {0, 0, 0, 0, 0, 0}; + + if (!g_state.txm.enabled) + return 0; + + txm_record_call(selector, x16, regs); + + switch (selector) { + case 0: + /* Selector 0 belongs to the SPTM-facing TXM table, not XNU table 0. */ + return txm_failure(selector, regs, TXM_RETURN_GENERIC); + + case TXM_SEL_GET_LOG_INFO: + words[0] = g_state.txm.log_page_va; + words[1] = g_state.txm.log_head_va; + words[2] = g_state.txm.log_sync_va; + return txm_finish(selector, regs, TXM_SUCCESS, words, 3); + + case TXM_SEL_GET_CODE_SIGNING_INFO: + words[0] = g_state.txm.txm_rw_data_va + 0x690UL; + words[1] = g_state.txm.developer_mode_flag_va; + words[2] = g_state.txm.txm_rw_data_va + 0x918UL; + words[3] = g_state.txm.managed_signature_size; + words[4] = g_state.txm.txm_ro_data_va + 0x190UL; + words[5] = g_state.txm.txm_ro_data_va; + return txm_finish(selector, regs, TXM_SUCCESS, words, 6); + + case TXM_SEL_GET_TRUST_CACHE_INFO: + words[0] = g_state.txm.trust_cache_runtime_va; + words[1] = g_state.txm.trust_cache_static_count; + words[2] = g_state.txm.trust_cache_caps0; + words[3] = g_state.txm.trust_cache_caps1; + return txm_finish(selector, regs, TXM_SUCCESS, words, 4); + + case 4: /* GetBuildVariant */ + words[0] = 0; /* release */ + return txm_finish(selector, regs, TXM_SUCCESS, words, 1); + + case 5: /* EnterLockdownMode */ + case 6: /* EnableRestrictedMode */ + case 8: /* UpdateDeviceState */ + case 9: /* CompleteSecurityBootMode */ + return txm_success0(selector, regs); + + case TXM_SEL_GET_SECURE_CHANNEL: + words[0] = g_state.txm.secure_chan_pa; + words[1] = g_state.txm.secure_chan_size; + return txm_finish(selector, regs, TXM_SUCCESS, words, 2); + + case TXM_SEL_ADD_FREE_LIST_PAGE: + if (regs[1] == 0) + return txm_failure(selector, regs, TXM_RETURN_GENERIC); + if (g_state.txm.free_count < TXM_C_TRACK_SLOTS) + g_state.txm.free_pages[g_state.txm.free_count++] = regs[1]; + return txm_success0(selector, regs); + + case TXM_SEL_GET_FREE_LIST_PAGE: + if (g_state.txm.free_count == 0) + return txm_failure(selector, regs, TXM_RETURN_NOT_FOUND); + words[0] = g_state.txm.free_pages[--g_state.txm.free_count]; + return txm_finish(selector, regs, TXM_SUCCESS, words, 1); + + case TXM_SEL_LOAD_TRUST_CACHE: + if (g_state.txm.loaded_tc_count < TXM_C_TRACK_SLOTS) + g_state.txm.loaded_tc[g_state.txm.loaded_tc_count++] = regs[1]; + /* Return payload VA so xnu can reclaim its temporary copy. */ + words[0] = regs[1]; + return txm_finish(selector, regs, TXM_SUCCESS, words, 1); + + case 13: /* UnloadTrustCache */ + return txm_success0(selector, regs); + + case TXM_SEL_QUERY_TRUST_CACHE: + if (!txm_fake_trust_cache_token(regs[1], &words[0], &words[1])) + return txm_failure(selector, regs, TXM_RETURN_NOT_FOUND); + return txm_finish(selector, regs, TXM_SUCCESS, words, 2); + + case TXM_SEL_QUERY_TC_REM: + words[0] = 0; + return txm_finish(selector, regs, TXM_SUCCESS, words, 1); + + case TXM_SEL_CHECK_TC_UUID: + return txm_failure(selector, regs, TXM_RETURN_NOT_FOUND); + + case TXM_SEL_REGISTER_PROFILE: + words[0] = txm_alloc_synth(0x100, g_state.txm.default_profile_va); + return txm_finish(selector, regs, TXM_SUCCESS, words, 1); + + case 18: /* TrustProvisioningProfile */ + case 20: /* AssociateProvisioningProfile */ + case 21: /* DisassociateProvisioningProfile */ + return txm_success0(selector, regs); + + case TXM_SEL_UNREGISTER_PROFILE: + words[0] = 0; + words[1] = 0; + return txm_finish(selector, regs, TXM_SUCCESS, words, 2); + + case TXM_SEL_REGISTER_SIGNATURE: { + u64 obj = txm_alloc_synth(TXM_CODE_SIGNATURE_SIZE, + g_state.txm.default_signature_va); + txm_synth_write8(obj + TXM_CODE_SIGNATURE_TRUST_OFF, + TXM_CS_TRUST_STATIC_TC); + words[0] = obj; + words[1] = regs[1]; + return txm_finish(selector, regs, TXM_SUCCESS, words, 2); + } + + case TXM_SEL_UNREGISTER_SIGNATURE: + case TXM_SEL_RECONSTITUTE_SIG: + words[0] = 0; + words[1] = 0; + return txm_finish(selector, regs, TXM_SUCCESS, words, 2); + + case 26: /* SetLocalSigningPublicKey */ + case 28: /* AuthorizeLocalSigningCDHash */ + case 29: /* AuthorizeCompilationServiceCDHash */ + case 31: /* DeveloperModeToggle */ + case 33: /* AssociateKernelEntitlements */ + case 36: /* UnregisterAddressSpace */ + case 37: /* SetupNestedAddressSpace */ + case 39: /* AllowJITRegion */ + case 40: /* AssociateJITRegion */ + case 42: /* AssociateDebugRegion */ + return txm_success0(selector, regs); + + case 24: /* ValidateCodeSignature */ + txm_synth_write8(regs[1] + TXM_CODE_SIGNATURE_TRUST_OFF, + TXM_CS_TRUST_STATIC_TC); + return txm_success0(selector, regs); + + case 38: { /* AssociateCodeSignature */ + u64 region = txm_alloc_synth(TXM_ASPACE_SYNTH_REGION_SIZE, regs[2]); + txm_synth_write64(regs[1] + TXM_ASPACE_MAIN_REGION_OFF, region); + return txm_success0(selector, regs); + } + + case TXM_SEL_GET_LOCAL_PUBKEY: + words[0] = g_state.txm.local_signing_pubkey_va; + return txm_finish(selector, regs, TXM_SUCCESS, words, 1); + + case TXM_SEL_MATCH_COMPILATION: + return txm_failure(selector, regs, TXM_RETURN_NOT_FOUND); + + case TXM_SEL_ACQUIRE_SIGNING_ID: + words[0] = g_state.txm.signing_id_va; + return txm_finish(selector, regs, TXM_SUCCESS, words, 1); + + case TXM_SEL_ACCEL_ENTITLEMENTS: + /* Route entitlement evaluation to XNU's non-monitor path. Returning + * NOT_SUPPORTED (what csm_enabled()==false yields) makes + * accelerate_entitlement_queries() (ubc_subr.c:4441) take the + * adjustContextWithoutMonitor branch, which parses the on-disk + * csb_entitlements_blob directly. Previously we returned a NULL + * monitor ce_ctx, which made every per-binary entitlement query + * (incl. com.apple.security.get-movable-control-port) read as absent, + * forcing IMMOVABLE_HARD control ports system-wide and breaking the + * mode-3 cross-task task-port send. See docs/mode3_txm_stub_audit.md. */ + return txm_failure(selector, regs, TXM_RETURN_NOT_SUPPORTED); + + case TXM_SEL_GET_ENTITLEMENTS_CTX: + words[0] = g_state.txm.ce_ctx_va; + return txm_finish(selector, regs, TXM_SUCCESS, words, 1); + + case TXM_SEL_REGISTER_ASPACE: { + u64 obj = txm_alloc_synth(0x100, g_state.txm.default_address_space_va); + /* Offsets verified from KDK 26.5 TXMAddressSpace_t. */ + txm_synth_write64(obj + 0x18, regs[2]); /* addrSpaceFlags */ + txm_synth_write16(obj + 0x24, (u16)regs[1]); /* identifier */ + txm_synth_write16(obj + 0x26, 0); /* type=AddressSpace */ + words[0] = obj; + return txm_finish(selector, regs, TXM_SUCCESS, words, 1); + } + + case 41: /* AllowInvalidCode */ + txm_synth_write8(regs[1] + 0x30, 1); /* allowsInvalidCode */ + return txm_success0(selector, regs); + + case TXM_SEL_RESOLVE_KENT_ASPACE: + return txm_failure(selector, regs, TXM_RETURN_NOT_FOUND); + + case TXM_SEL_IMAGE4_DISPATCH: + return txm_success0(selector, regs); + + case TXM_SEL_IMAGE4_GET_EXPORTS: + case 47: /* Image4SetReleaseType */ + case 48: /* Image4SetBootNonceShadow */ + case 49: /* Image4SetNonce */ + case 50: /* Image4RollNonce */ + case TXM_SEL_IMAGE4_GET_NONCE: + /* The mixed Python/C reference matched the T8140 TXM binary here: + * selectors 46..51 go through the default-fail trampoline and + * return kTXMReturnNotPermitted, not success or synthetic pointers. + */ + return txm_failure(selector, regs, TXM_RETURN_NOT_PERMITTED); + + default: + return txm_failure(selector, regs, TXM_RETURN_NOT_SUPPORTED); + } +} diff --git a/src/sptm/uat.c b/src/sptm/uat.c new file mode 100644 index 000000000..84060a1c6 --- /dev/null +++ b/src/sptm/uat.c @@ -0,0 +1,1323 @@ +/* SPDX-License-Identifier: MIT */ +#include "private.h" + +int debug_printf(const char *fmt, ...); + +#define UAT_TTE_ADDR_MASK 0x000001fffffff000UL +#define UAT_PTE_ADDR_MASK 0x0000fffffffff000UL +#define UAT_TABLE_DESC 3UL +#define UAT_VALID_LEAF_BITS 0x0080000000000400UL +#define UAT_OPTIONS_INVALID_MASK 0xfff0fcf0UL +#define UAT_FW_OWNED_BIT 8UL +#define UAT_DEFAULT_VADDR_SHIFT 0x27UL +#define UAT_MAX_VA_PAGES (1UL << 50) +#define UAT_STATE_ROOT0_OFF 0x08UL +#define UAT_STATE_ROOT1_OFF 0x10UL +#define UAT_STATE_CTX_ID_OFF 0x18UL +#define UAT_STATE_GUARD_OFF 0x1aUL +#define UAT_STATE_WORK0_OFF 0x20UL +#define UAT_STATE_WORK1_OFF 0x28UL +#define UAT_STATE_WORK2_OFF 0x30UL +#define UAT_STATE_WORK3_OFF 0x38UL +#define UAT_STATE_OPTIONS_OFF 0x40UL +#define UAT_STATE_FLUSH_LEVEL_OFF 0x44UL +#define UAT_STATE_MAP_SEGS_OFF 0x48UL +#define UAT_STATE_CTX_FLUSH_OFF 0x248UL +#define UAT_STATE_UNMAP_SEGS_OFF 0x250UL +#define UAT_SAPT_ALIAS_COUNT 12 +#define UAT_SAPT_LAST_PA 13 +#define UAT_SAPT_LAST_OLD 14 +#define UAT_SAPT_LAST_NEW 15 +#define UAT_HANDOFF_MAGIC 0x4b1d000000000002UL +#define UAT_HANDOFF_MAGIC_FW_OFF 0x8UL +#define UAT_HANDOFF_SLOT_BASE 0x20UL +#define UAT_HANDOFF_SLOT_STRIDE 0x18UL +#define UAT_HANDOFF_CUR_CTX_OFF 0x18UL +#define UAT_HANDOFF_READY_OFF 0x638UL +#define UAT_HANDOFF_DEAD_PREFIX 0xdead000000000000UL +#define UAT_HANDOFF_ADDR_MASK 0x0000ffffffffffffUL + +static const u8 uat_attr_table[16] = { + 0, 8, 9, 10, 4, 1, 0xff, 0xff, + 5, 0xff, 2, 0xff, 6, 7, 0xff, 3, +}; + +static inline u8 uat_sapt_field_for_options(u64 options) +{ + switch (options & 3UL) { + case 1: + return 1; + case 3: + return 0; + default: + return 3; + } +} + +static inline bool_t uat_looks_like_pa(u64 pa) +{ + return !is_kernel_va(pa) && pa >= 0x100000000UL && pa < 0x20000000000UL; +} + +static inline bool_t uat_resolve_data_pa(u64 ptr, bool_t write, u64 *pa_out) +{ + if (is_kernel_va(ptr)) + return guest_va_to_pa(ptr, write, pa_out); + if (uat_looks_like_pa(ptr)) { + *pa_out = ptr; + return 1; + } + return 0; +} + +static inline bool_t uat_read64_ptr(u64 ptr, u64 *out) +{ + u64 pa; + if (!uat_resolve_data_pa(ptr, 0, &pa)) + return 0; + *out = *(volatile u64 *)pa; + return 1; +} + +static inline void uat_clean_state_range_poc(u64 pa, u64 size) +{ + /* + * AGX/uPPL observes UAT state objects as a coprocessor-visible protocol, + * not as CPU-local data. The real SPTM runs in the hardware domain that + * makes these stores visible before the guard changes; our EL2 emulator + * must publish them explicitly to PoC plus the coprocessor cache path. + */ + clean_pt_range_poc((volatile u64 *)pa, size); +} + +static inline bool_t uat_sapt_byte_for_pa(u64 pa, u64 *byte_pa, u8 *shift) +{ + u64 base = g_state.uat.sapt_base; + u64 entries = g_state.uat.sapt_entries; + u64 dram_base = g_state.uat.sapt_dram_base; + u64 dram_end = g_state.uat.sapt_dram_end; + u64 page = pa & ~(UAT_C_PAGE_SIZE - 1UL); + u64 offset; + u64 index; + + if (!base || !entries || !dram_base || page < dram_base || page >= dram_end) + return 0; + + offset = page - dram_base; + index = offset >> 16; + if (index >= entries) + return 0; + + *byte_pa = base + index; + *shift = (u8)((offset >> 13) & 6UL); + return 1; +} + +static inline void uat_sapt_set_page(u64 pa, u8 field, bool_t expect_released) +{ + u64 byte_pa; + u8 shift; + u8 mask; + u8 old; + u8 old_field; + u8 new_value; + + if (!uat_sapt_byte_for_pa(pa, &byte_pa, &shift)) + return; + + mask = (u8)(3U << shift); + old = *(volatile u8 *)byte_pa; + old_field = (old >> shift) & 3U; + if (expect_released && old_field != 3U) { + g_state.uat.violations++; + g_state.uat.last[UAT_SAPT_ALIAS_COUNT]++; + g_state.uat.last[UAT_SAPT_LAST_PA] = pa & ~(UAT_C_PAGE_SIZE - 1UL); + g_state.uat.last[UAT_SAPT_LAST_OLD] = old_field; + g_state.uat.last[UAT_SAPT_LAST_NEW] = field & 3U; + } + + new_value = (u8)((old & ~mask) | ((field & 3U) << shift)); + *(volatile u8 *)byte_pa = new_value; + uat_clean_state_range_poc(byte_pa, 1); +} + +static inline void uat_sapt_grant_page(u64 pa, u64 options) +{ + uat_sapt_set_page(pa, uat_sapt_field_for_options(options), 1); +} + +static inline void uat_sapt_release_page(u64 pa) +{ + uat_sapt_set_page(pa, 3, 0); +} + +static u64 uat_publish_fw_unmap_handoff(struct uat_c_root_state *root, + u64 addr, u64 size) +{ + u64 handoff = g_state.uat.gfx_handoff_pa; + u64 slot_pa; + u32 slot_state; + u32 cur_ctx; + u8 ready; + bool_t dead; + + if (root->ctx_id >= 0x41U) + return 0; + + if (!uat_looks_like_pa(handoff) || + g_state.uat.gfx_handoff_size < UAT_HANDOFF_READY_OFF + 1UL) { + g_state.uat.violations++; + return 0; + } + + slot_pa = handoff + UAT_HANDOFF_SLOT_BASE + + (u64)root->ctx_id * UAT_HANDOFF_SLOT_STRIDE; + slot_state = *(volatile u32 *)slot_pa; + if (slot_state != 0) { + g_state.uat.violations++; + root->last[6] = slot_pa; + root->last[7] = slot_state; + return 0; + } + + *(volatile u64 *)(slot_pa + 8) = addr; + *(volatile u64 *)(slot_pa + 0x10) = size; + + cur_ctx = *(volatile u32 *)(handoff + UAT_HANDOFF_CUR_CTX_OFF); + ready = *(volatile u8 *)(handoff + UAT_HANDOFF_READY_OFF); + dead = (ready & 1U) != 0 || + (((root->type & 5U) != 0) && cur_ctx != (u32)root->ctx_id); + + if (dead) { + addr = UAT_HANDOFF_DEAD_PREFIX | (addr & UAT_HANDOFF_ADDR_MASK); + *(volatile u64 *)(slot_pa + 8) = addr; + slot_state = 2; + } else { + slot_state = 1; + } + + /* + * Real SPTM writes addr/size first, publishes state last, then orders the + * handoff with an outer-shareable barrier. In EL2 we also clean to PoC so + * the AGX firmware and UAT hardware see the slot update. + */ + *(volatile u32 *)slot_pa = slot_state; + clean_pt_range_poc((volatile u64 *)slot_pa, UAT_HANDOFF_SLOT_STRIDE); + __asm__ volatile("dsb sy" ::: "memory"); + + root->last[6] = slot_pa; + root->last[7] = ((u64)slot_state << 32) | ready; + return dead ? 0 : 2; +} + +static bool_t uat_handoff_magic_valid(struct uat_c_root_state *root) +{ + u64 handoff = g_state.uat.gfx_handoff_pa; + u64 magic; + + if (root->ctx_id >= 0x41U) + return 1; + + if (!uat_looks_like_pa(handoff) || + g_state.uat.gfx_handoff_size < UAT_HANDOFF_MAGIC_FW_OFF + 8UL) { + g_state.uat.violations++; + root->last[6] = handoff; + root->last[7] = g_state.uat.gfx_handoff_size; + return 0; + } + + magic = *(volatile u64 *)(handoff + UAT_HANDOFF_MAGIC_FW_OFF); + if (magic != UAT_HANDOFF_MAGIC) { + g_state.uat.violations++; + root->last[6] = handoff + UAT_HANDOFF_MAGIC_FW_OFF; + root->last[7] = magic; + return 0; + } + + return 1; +} + +static inline bool_t uat_write64_ptr(u64 ptr, u64 value) +{ + u64 pa; + if (!uat_resolve_data_pa(ptr, 1, &pa)) + return 0; + *(volatile u64 *)pa = value; + uat_clean_state_range_poc(pa, 8); + return 1; +} + +static inline bool_t uat_read16_ptr(u64 ptr, u16 *out) +{ + u64 pa; + if (!uat_resolve_data_pa(ptr, 0, &pa)) + return 0; + *out = *(volatile u16 *)pa; + return 1; +} + +static inline bool_t uat_read8_ptr(u64 ptr, u8 *out) +{ + u64 pa; + if (!uat_resolve_data_pa(ptr, 0, &pa)) + return 0; + *out = *(volatile u8 *)pa; + return 1; +} + +static inline bool_t uat_write16_ptr(u64 ptr, u16 value) +{ + u64 pa; + if (!uat_resolve_data_pa(ptr, 1, &pa)) + return 0; + *(volatile u16 *)pa = value; + uat_clean_state_range_poc(pa, 2); + return 1; +} + +static inline bool_t uat_write8_ptr(u64 ptr, u8 value) +{ + u64 pa; + if (!uat_resolve_data_pa(ptr, 1, &pa)) + return 0; + *(volatile u8 *)pa = value; + uat_clean_state_range_poc(pa, 1); + return 1; +} + +static inline u8 uat_read8_pa_default(u64 pa, u8 def) +{ + if (!uat_looks_like_pa(pa)) + return def; + return *(volatile u8 *)pa; +} + +static inline u64 uat_default_l1_mask(u64 shift) +{ + if (shift <= 36) + return 0; + if (shift >= 47) + return 0x7ff000000000UL; + return (((1UL << (shift - 36)) - 1UL) << 36) & 0x7ff000000000UL; +} + +static inline void uat_set_defaults(void) +{ + struct uat_c_state *st = &g_state.uat; + + if (!st->vaddr_shift) + st->vaddr_shift = UAT_DEFAULT_VADDR_SHIFT; + if (!st->l1_index_mask) + st->l1_index_mask = uat_default_l1_mask(st->vaddr_shift); + if (!st->segment_limit) + st->segment_limit = 0x40; + if (!st->mapping_limit) + st->mapping_limit = 0x100; + if (!st->state_object_size) + st->state_object_size = st->segment_limit * 0x10 + 0x250; + if (!st->gfx_shared_l2_pa) + st->gfx_shared_l2_pa = st->gfx_shared_region_pa; + if (!st->raw_gfx_shared_l2_pa) + st->raw_gfx_shared_l2_pa = st->gfx_shared_l2_pa; + st->enabled = 1; +} + +static inline u64 uat_selector2_state_pa(void) +{ + if (g_state.uat.mode != 1) + return 0; + if (!uat_looks_like_pa(g_state.uat.selector2_pa) || + !uat_looks_like_pa(g_state.uat.selector2_root_pa)) { + g_state.uat.violations++; + return 0; + } + return g_state.uat.selector2_pa; +} + +static inline u64 uat_selector9_papt_va(void) +{ + u64 pa = 0; + + if (!is_kernel_va(g_state.uat.selector9_pa) || + !g_state.uat.raw_gfx_shared_l2_pa || + !guest_va_to_pa(g_state.uat.selector9_pa, 0, &pa) || + (pa & UAT_PTE_ADDR_MASK) != (g_state.uat.raw_gfx_shared_l2_pa & UAT_PTE_ADDR_MASK)) { + g_state.uat.violations++; + return 0; + } + return g_state.uat.selector9_pa; +} + +static inline void uat_refresh_state_object(struct uat_c_root_state *root) +{ + u64 value; + + if (uat_read64_ptr(root->handle + UAT_STATE_ROOT0_OFF, &value)) + root->root0_pa = value; + if (uat_read64_ptr(root->handle + UAT_STATE_ROOT1_OFF, &value)) + root->root1_pa = value; + if (uat_read16_ptr(root->handle + UAT_STATE_CTX_ID_OFF, &root->ctx_id) && + uat_read8_ptr(root->handle, &root->type)) + return; +} + +static inline void uat_record(u32 endpoint, u64 *regs, u64 root_pa, u64 slot_pa, + u64 before, u64 after, u64 rc) +{ + struct uat_c_state *st = &g_state.uat; + u64 idx = st->recent_idx & (UAT_C_RECENT_SLOTS - 1); + struct uat_c_recent_event *ev = &st->recent[idx]; + + ev->seq = st->recent_idx; + ev->endpoint = endpoint; + ev->handle = regs[0]; + for (u32 i = 0; i < 5; i++) + ev->regs[i] = regs[i]; + ev->root_pa = root_pa; + ev->slot_pa = slot_pa; + ev->before = before; + ev->after = after; + ev->rc = rc; + st->recent_idx++; + st->last_rc = rc; +} + +static struct uat_c_root_state *uat_find_state(u64 handle, bool_t alloc) +{ + struct uat_c_root_state *free_slot = (struct uat_c_root_state *)0; + bool_t selector2_shadow; + + if (!handle) + return (struct uat_c_root_state *)0; + + selector2_shadow = handle == g_state.uat.selector2_pa && + uat_looks_like_pa(g_state.uat.selector2_pa) && + uat_looks_like_pa(g_state.uat.selector2_root_pa); + + for (u32 i = 0; i < UAT_C_MAX_STATES; i++) { + struct uat_c_root_state *slot = &g_state.uat.roots[i]; + if (slot->used && slot->handle == handle) { + uat_refresh_state_object(slot); + return slot; + } + if (!slot->used && free_slot == (struct uat_c_root_state *)0) + free_slot = slot; + } + + /* + * Selector 2 returns a firmware-owned UAT state object seeded before any + * table-7 init_state call. Real SPTM can use that object immediately for + * map_table/map/unmap; mirror that by importing it into the shadow root + * table on first use instead of treating map_table as a no-op success. + */ + if ((!alloc && !selector2_shadow) || free_slot == (struct uat_c_root_state *)0) + return (struct uat_c_root_state *)0; + + for (u32 i = 0; i < sizeof(*free_slot); i++) + ((volatile u8 *)free_slot)[i] = 0; + free_slot->used = 1; + free_slot->handle = handle; + free_slot->ctx_id = UAT_C_CTX_NONE; + uat_refresh_state_object(free_slot); + if (selector2_shadow) { + if (!free_slot->root1_pa) + free_slot->root1_pa = g_state.uat.selector2_root_pa; + if (!free_slot->type) + free_slot->type = uat_read8_pa_default(handle, 8); + } + return free_slot; +} + +static inline u64 uat_root_for_va(struct uat_c_root_state *root, u64 va) +{ + u64 shift = g_state.uat.vaddr_shift ? g_state.uat.vaddr_shift : + UAT_DEFAULT_VADDR_SHIFT; + if (((va >> shift) & 1UL) && root->root1_pa) + return root->root1_pa; + return root->root0_pa; +} + +static inline u64 uat_l1_idx(u64 va) +{ + u64 mask = g_state.uat.l1_index_mask; + if (!mask) + mask = uat_default_l1_mask(g_state.uat.vaddr_shift ? + g_state.uat.vaddr_shift : + UAT_DEFAULT_VADDR_SHIFT); + if (mask) + return (va & mask) >> 36; + return (va >> 36) & 0x7ffUL; +} + +static inline u64 uat_l2_idx(u64 va) { return (va >> 25) & 0x7ffUL; } +static inline u64 uat_l3_idx(u64 va) { return (va >> 14) & 0x7ffUL; } + +static bool_t uat_walk_slot(u64 root_pa, u64 va, u32 level, + u64 *slot_pa_out, u64 *before_out) +{ + volatile u64 *l1; + volatile u64 *l2; + u64 l1e, l2e; + u64 slot_pa; + + if (!uat_looks_like_pa(root_pa)) + return 0; + + if (level == 1) { + slot_pa = root_pa + uat_l1_idx(va) * 8; + *slot_pa_out = slot_pa; + *before_out = *(volatile u64 *)slot_pa; + return 1; + } + + l1 = (volatile u64 *)root_pa; + l1e = l1[uat_l1_idx(va)]; + if ((l1e & 3UL) != UAT_TABLE_DESC) + return 0; + + if (level == 2) { + slot_pa = (l1e & UAT_TTE_ADDR_MASK) + uat_l2_idx(va) * 8; + *slot_pa_out = slot_pa; + *before_out = *(volatile u64 *)slot_pa; + return 1; + } + + l2 = (volatile u64 *)(l1e & UAT_TTE_ADDR_MASK); + l2e = l2[uat_l2_idx(va)]; + if ((l2e & 3UL) != UAT_TABLE_DESC) + return 0; + + slot_pa = (l2e & UAT_TTE_ADDR_MASK) + uat_l3_idx(va) * 8; + *slot_pa_out = slot_pa; + *before_out = *(volatile u64 *)slot_pa; + return 1; +} + +static inline void uat_write_slot(u64 slot_pa, u64 value) +{ + *(volatile u64 *)slot_pa = value; + clean_pt_range_poc((volatile u64 *)slot_pa, 8); +} + +static inline void uat_publish_mapped_data_page(struct uat_c_root_state *root, + u64 pa) +{ + u64 page = pa & ARM_PTE_PAGE_MASK; + u64 idx; + + /* + * Real SPTM's UAT map path performs per-mapped-frame side effects in + * addition to installing the leaf PTE. The important correctness effect for + * this emulator is that data the AGX firmware will read is pushed through + * the coprocessor-visible cache path, not just the 8-byte UAT PTE slot. + * + * Some UAT operations carry low/sentinel/non-DRAM values through the same + * segment format. Do not run data-cache maintenance on those addresses: + * EL2 can fault on e.g. PA 0 before XNU even boots. + */ + if (!uat_looks_like_pa(page) || !ft_get_idx(page, &idx)) { + root->last[6] = 0; + root->last[7] = page; + return; + } + + sptm_coproc_cache_maint_data_page(page); + uat_write8_ptr(root->handle + UAT_STATE_FLUSH_LEVEL_OFF, 1); + root->last[6]++; + root->last[7] = page; +} + +static inline void uat_tlbi_ctx_if_bound(struct uat_c_root_state *root) +{ + (void)root; +} + +static inline u64 uat_leaf_pte(struct uat_c_root_state *root, u64 pa, u64 options) +{ + u64 idx = ((options >> 8) & 2UL) | + ((options & 3UL) << 2) | + ((options >> 8) & 1UL); + u64 attr = uat_attr_table[idx & 0xf]; + u64 perm = 3UL; + u64 os_ng = ((root->type & 10U) != 0) ? 0 : 0x800UL; + + if (attr == 0xff) + attr = 0; + if (options & 8UL) + perm = 0xbUL; + if (options & 4UL) + perm = 7UL; + + return os_ng | + ((attr & 3UL) << 53) | + (pa & UAT_PTE_ADDR_MASK) | + perm | + (((attr >> 2) & 3UL) << 6) | + UAT_VALID_LEAF_BITS; +} + +static bool_t uat_read_segment(u64 seglist_pa, u64 index, u64 *first, u64 *pages) +{ + u64 base = seglist_pa + index * 0x10UL; + return uat_read64_ptr(base, first) && + uat_read64_ptr(base + 8, pages); +} + +static bool_t uat_copy_segments_to_state(struct uat_c_root_state *root, + u64 dst_off, u64 seglist_pa, + u64 num_segs) +{ + u64 state_size = g_state.uat.state_object_size ? + g_state.uat.state_object_size : + ((g_state.uat.segment_limit ? g_state.uat.segment_limit : 0x40) * + 0x10UL + 0x250UL); + + if (dst_off + num_segs * 0x10UL > state_size) + return 0; + + for (u64 i = 0; i < num_segs; i++) { + u64 first, pages; + if (!uat_read_segment(seglist_pa, i, &first, &pages) || + !uat_write64_ptr(root->handle + dst_off + i * 0x10UL, first) || + !uat_write64_ptr(root->handle + dst_off + i * 0x10UL + 8, pages)) + return 0; + } + return 1; +} + +static inline void uat_write_state_header(struct uat_c_root_state *root) +{ + u64 handle = root->handle; + + uat_write64_ptr(handle + UAT_STATE_ROOT0_OFF, root->root0_pa); + uat_write64_ptr(handle + UAT_STATE_ROOT1_OFF, root->root1_pa); + uat_write16_ptr(handle + UAT_STATE_CTX_ID_OFF, root->ctx_id); +} + +static inline void uat_release_state_guard(struct uat_c_root_state *root, u8 guard) +{ + uat_write8_ptr(root->handle + UAT_STATE_GUARD_OFF, guard); +} + +static inline void uat_write_idle_state(struct uat_c_root_state *root) +{ + uat_write_state_header(root); + uat_release_state_guard(root, 2); +} + +static void uat_write_map_state(struct uat_c_root_state *root, u8 guard) +{ + u64 handle = root->handle; + + uat_write_state_header(root); + uat_write64_ptr(handle + UAT_STATE_WORK0_OFF, root->current_va); + uat_write64_ptr(handle + UAT_STATE_WORK1_OFF, root->num_segs); + uat_write64_ptr(handle + UAT_STATE_WORK2_OFF, root->current_seg); + uat_write64_ptr(handle + UAT_STATE_WORK3_OFF, root->seg_offset); + uat_write64_ptr(handle + UAT_STATE_OPTIONS_OFF, root->options); + /* + * BUG FIX 2026-07-05: do NOT write CTX_FLUSH_OFF (0x248) here. The map + * seglist copy lives at MAP_SEGS_OFF (0x48); entry 32's first_pa is at + * 0x48 + 32*0x10 = 0x248 == CTX_FLUSH_OFF. For any map with >=33 segments + * this 0-write clobbered the 33rd segment's PA to 0 right after the copy, + * which our old zero-leaf branch then turned into a UAT hole -> GPU FW data + * abort at the corresponding VA (0xfffffc20c0130000). Real SPTM's map-begin + * writes no such field; this write was spurious (CTX_FLUSH_OFF is write-only + * in our emulator, never read). + */ + uat_release_state_guard(root, guard); +} + +static void uat_write_prepare_fw_unmap_state(struct uat_c_root_state *root, + u8 guard, u64 changed) +{ + u64 handle = root->handle; + + uat_write_state_header(root); + uat_write64_ptr(handle + UAT_STATE_WORK0_OFF, root->current_va); + uat_write64_ptr(handle + UAT_STATE_WORK1_OFF, root->current_seg); + uat_write64_ptr(handle + UAT_STATE_WORK2_OFF, root->num_segs); + uat_write64_ptr(handle + UAT_STATE_WORK3_OFF, changed ? 1 : 0); + uat_release_state_guard(root, guard); +} + +static void uat_write_unmap_state(struct uat_c_root_state *root, u8 guard, + u64 op_state) +{ + u64 handle = root->handle; + + uat_write_state_header(root); + uat_write64_ptr(handle + UAT_STATE_WORK0_OFF, op_state); + uat_write64_ptr(handle + UAT_STATE_WORK1_OFF, root->num_segs); + uat_write64_ptr(handle + UAT_STATE_WORK2_OFF, root->current_seg); + uat_write64_ptr(handle + UAT_STATE_WORK3_OFF, root->seg_offset); + uat_write64_ptr(handle + UAT_STATE_OPTIONS_OFF, root->options); + uat_release_state_guard(root, guard); +} + +static u64 uat_map_pending(struct uat_c_root_state *root) +{ + u64 limit = g_state.uat.mapping_limit ? g_state.uat.mapping_limit : 0x100; + u64 mapped = 0; + + while (root->current_seg < root->num_segs) { + u64 first, pages; + if (!uat_read_segment(root->seglist_pa, root->current_seg, &first, &pages)) { + g_state.uat.violations++; + root->state = 0; + return 0; + } + + while (root->seg_offset < pages) { + u64 pa = first + root->seg_offset * UAT_C_PAGE_SIZE; + u64 va = root->current_va; + u64 root_pa = uat_root_for_va(root, va); + u64 slot_pa = 0, before = 0, pte; + + if (!uat_walk_slot(root_pa, va, 3, &slot_pa, &before)) { + g_state.uat.violations++; + root->last[0] = root_pa; + root->last[1] = va; + root->last[2] = pa; + root->last[3] = 3; + root->state = 0; + uat_write_idle_state(root); + return 0; + } + + pte = uat_leaf_pte(root, pa, root->options); + if (before & 1UL) { + g_state.uat.violations++; + root->last[0] = root_pa; + root->last[1] = va; + root->last[2] = pa; + root->last[3] = slot_pa; + root->last[4] = before; + root->last[5] = pte; + root->state = 0; + uat_write_map_state(root, 2); + return 0; + } + if (!uat_looks_like_pa(pa)) { + /* + * XNU can describe holes in GPU queue VA space with a zero + * backing address. Treat those as invalid leaves. Turning PA 0 + * into a valid GPU translation makes AGX read physical low + * memory and trips SAPT at e.g. PA 0x600. + */ + uat_write_slot(slot_pa, 0); + root->last[0] = root_pa; + root->last[1] = va; + root->last[2] = pa; + root->last[3] = slot_pa; + root->last[4] = before; + root->last[5] = 0; + root->last[6]++; + root->last[7] = pa; + + root->current_va += UAT_C_PAGE_SIZE; + root->seg_offset++; + mapped++; + if (mapped >= limit) { + uat_tlbi_ctx_if_bound(root); + root->state = 3; + uat_write_map_state(root, 3); + return 1; + } + continue; + } + uat_write_slot(slot_pa, pte); + uat_sapt_grant_page(pa, root->options); + uat_publish_mapped_data_page(root, pa); + + root->last[0] = root_pa; + root->last[1] = va; + root->last[2] = pa; + root->last[3] = slot_pa; + root->last[4] = before; + root->last[5] = pte; + + root->current_va += UAT_C_PAGE_SIZE; + root->seg_offset++; + mapped++; + if (mapped >= limit) { + uat_tlbi_ctx_if_bound(root); + root->state = 3; + uat_write_map_state(root, 3); + return 1; + } + } + + root->current_seg++; + root->seg_offset = 0; + } + + root->state = 0; + uat_tlbi_ctx_if_bound(root); + uat_write_map_state(root, 2); + return 0; +} + +static inline bool_t uat_fw_unmap_leaf_qualifies(u64 pte) +{ + u32 perm_bits = ((u32)(pte >> 4)) & 0xcU; + u32 attr_hi = (u32)(pte >> 53); + u32 attr_code = perm_bits | (attr_hi & 3U); + + if ((pte & 0x1cUL) != 0) + return 0; + return (perm_bits | (attr_hi & 2U)) == 2U || + (attr_code >= 5U && attr_code < 8U); +} + +static u64 uat_prepare_fw_unmap_pending(struct uat_c_root_state *root) +{ + u64 limit = g_state.uat.mapping_limit ? g_state.uat.mapping_limit : 0x100; + u64 walked = 0; + u64 changed = root->options & 1UL; + + if (!uat_handoff_magic_valid(root)) { + root->state = 0; + root->options = 0; + uat_write_prepare_fw_unmap_state(root, 2, changed); + return 0; + } + + while (root->current_seg < root->num_segs) { + u64 cur = root->current_va + root->current_seg * UAT_C_PAGE_SIZE; + u64 root_pa = uat_root_for_va(root, cur); + u64 slot_pa = 0, before = 0, after = 0; + + if (!uat_walk_slot(root_pa, cur, 3, &slot_pa, &before)) { + g_state.uat.violations++; + root->last[0] = root_pa; + root->last[1] = cur; + root->last[2] = slot_pa; + root->last[3] = before; + root->state = 0; + root->options = 0; + uat_write_prepare_fw_unmap_state(root, 2, changed); + return 0; + } + + after = before; + if ((before & 3UL) == 3UL && uat_fw_unmap_leaf_qualifies(before)) { + after = before | UAT_FW_OWNED_BIT; + uat_write_slot(slot_pa, after); + changed = 1; + } + + root->last[0] = root_pa; + root->last[1] = cur; + root->last[2] = slot_pa; + root->last[3] = before; + root->last[4] = after; + root->last[5] = changed; + + root->current_seg++; + walked++; + if (root->current_seg != root->num_segs && walked >= limit) { + root->state = 4; + root->options = changed; + uat_write_prepare_fw_unmap_state(root, 4, changed); + return 1; + } + } + + root->state = 0; + root->options = 0; + uat_write_prepare_fw_unmap_state(root, 2, changed); + if (!changed) + return 0; + + __asm__ volatile("dsb oshst" ::: "memory"); + tlbi_vmalle1os(); + return uat_publish_fw_unmap_handoff(root, root->current_va, + root->num_segs * UAT_C_PAGE_SIZE); +} + +static u64 uat_unmap_pending(struct uat_c_root_state *root) +{ + u64 limit = g_state.uat.mapping_limit ? g_state.uat.mapping_limit : 0x100; + u64 unmapped = 0; + + while (root->current_seg < root->num_segs) { + u64 va, pages; + if (!uat_read_segment(root->seglist_pa, root->current_seg, &va, &pages)) { + g_state.uat.violations++; + root->state = 0; + uat_write_unmap_state(root, 2, 0); + return 0; + } + + while (root->seg_offset < pages) { + u64 cur = va + root->seg_offset * UAT_C_PAGE_SIZE; + u64 root_pa = uat_root_for_va(root, cur); + u64 slot_pa = 0, before = 0; + + if (!uat_walk_slot(root_pa, cur, 3, &slot_pa, &before) || + !(before & 1UL)) { + g_state.uat.violations++; + root->last[0] = root_pa; + root->last[1] = cur; + root->last[2] = slot_pa; + root->last[3] = before; + root->state = 0; + uat_write_unmap_state(root, 2, 0); + return 0; + } + uat_sapt_release_page(before & UAT_PTE_ADDR_MASK); + uat_write_slot(slot_pa, 0); + + root->last[0] = root_pa; + root->last[1] = cur; + root->last[2] = slot_pa; + root->last[3] = before; + + root->seg_offset++; + unmapped++; + if (unmapped >= limit) { + root->state = 5; + uat_write_unmap_state(root, 5, 1); + tlbi_vmalle1os(); + return 1; + } + } + + root->current_seg++; + root->seg_offset = 0; + } + + root->state = 0; + uat_write_unmap_state(root, 2, 0); + tlbi_vmalle1os(); + return 0; +} + +static u64 uat_unmap_segments(struct uat_c_root_state *root, u64 seglist_pa, + u64 num_segs) +{ + if (num_segs == 0 || + num_segs > (g_state.uat.segment_limit ? g_state.uat.segment_limit : 0x40) || + !uat_copy_segments_to_state(root, UAT_STATE_UNMAP_SEGS_OFF, + seglist_pa, num_segs)) { + g_state.uat.violations++; + return 0; + } + + root->seglist_pa = root->handle + UAT_STATE_UNMAP_SEGS_OFF; + root->num_segs = num_segs; + root->current_seg = 0; + root->seg_offset = 0; + root->options = 0; + root->state = 5; + uat_write_unmap_state(root, 5, 1); + return uat_unmap_pending(root); +} + +static u64 uat_ep_init(u64 *regs) +{ + struct uat_c_root_state *root = uat_find_state(regs[0], 1); + u64 root1 = 0xffffffffUL; + + if (root == (struct uat_c_root_state *)0) { + g_state.uat.violations++; + return 0; + } + + root->type = uat_read8_pa_default(root->handle, 0); + if ((root->type & 5U) == 0) { + if (uat_looks_like_pa(regs[2])) + root->type = 4; + else if (regs[2] == 0xffffffffUL) + root->type = 1; + if (root->type & 5U) + uat_write8_ptr(root->handle, root->type); + } + if ((root->type & 5U) == 0 || + !uat_looks_like_pa(regs[1]) || + (root->root0_pa != 0 && root->root0_pa != 0xffffffffUL) || + (root->type == 1 && regs[2] != 0xffffffffUL) || + (root->type == 4 && !uat_looks_like_pa(regs[2]))) { + g_state.uat.violations++; + return 0; + } + + if (root->type == 4) { + if (root->root1_pa != 0 && root->root1_pa != 0xffffffffUL) { + g_state.uat.violations++; + return 0; + } + root1 = regs[2]; + } + + root->root0_pa = regs[1]; + root->root1_pa = root1; + root->ctx_id = UAT_C_CTX_NONE; + root->state = 0; + uat_write_idle_state(root); + return 0; +} + +static u64 uat_ep_destroy(u64 *regs) +{ + struct uat_c_root_state *root = uat_find_state(regs[0], 0); + if (root == (struct uat_c_root_state *)0) + return 0; + if (root->ctx_id != UAT_C_CTX_NONE) { + g_state.uat.violations++; + return 0; + } + for (u32 i = 0; i < sizeof(*root); i++) + ((volatile u8 *)root)[i] = 0; + tlbi_vmalle1os(); + return 0; +} + +static u64 uat_ep_map_table(u64 *regs, u64 *root_out, u64 *slot_out, + u64 *before_out, u64 *after_out) +{ + struct uat_c_root_state *root = uat_find_state(regs[0], 0); + u64 va = regs[1]; + u32 level = (u32)regs[2]; + u64 table_pa = regs[3]; + u64 root_pa, slot_pa = 0, before = 0, after = 0; + + if (root == (struct uat_c_root_state *)0 || (level != 1 && level != 2) || + !uat_looks_like_pa(table_pa)) { + g_state.uat.violations++; + return 0; + } + + root_pa = uat_root_for_va(root, va); + if (!uat_walk_slot(root_pa, va, level, &slot_pa, &before)) { + g_state.uat.violations++; + return 0; + } + after = (table_pa & UAT_TTE_ADDR_MASK) | UAT_TABLE_DESC; + if (before != 0) { + g_state.uat.violations++; + root->last[0] = root_pa; + root->last[1] = va; + root->last[2] = level; + root->last[3] = slot_pa; + root->last[4] = before; + root->last[5] = after; + *root_out = root_pa; + *slot_out = slot_pa; + *before_out = before; + *after_out = after; + return 0; + } + uat_write_slot(slot_pa, after); + + root->last[0] = root_pa; + root->last[1] = va; + root->last[2] = level; + root->last[3] = slot_pa; + root->last[4] = before; + root->last[5] = after; + *root_out = root_pa; + *slot_out = slot_pa; + *before_out = before; + *after_out = after; + return 0; +} + +static u64 uat_ep_unmap_table(u64 *regs, u64 *root_out, u64 *slot_out, + u64 *before_out) +{ + struct uat_c_root_state *root = uat_find_state(regs[0], 0); + u64 va = regs[1]; + u32 level = (u32)regs[2]; + u64 root_pa, slot_pa = 0, before = 0; + + if (root == (struct uat_c_root_state *)0 || (level != 1 && level != 2)) { + g_state.uat.violations++; + return 0; + } + + root_pa = uat_root_for_va(root, va); + if (!uat_walk_slot(root_pa, va, level, &slot_pa, &before)) { + g_state.uat.violations++; + return 0; + } + if ((before & 3UL) != UAT_TABLE_DESC) { + g_state.uat.violations++; + *root_out = root_pa; + *slot_out = slot_pa; + *before_out = before; + return 0; + } + uat_write_slot(slot_pa, 0); + tlbi_vmalle1os(); + + root->last[0] = root_pa; + root->last[1] = va; + root->last[2] = level; + root->last[3] = slot_pa; + root->last[4] = before; + *root_out = root_pa; + *slot_out = slot_pa; + *before_out = before; + return 0; +} + +static u64 uat_ep_map_begin(u64 *regs) +{ + struct uat_c_root_state *root = uat_find_state(regs[0], 0); + u64 segs_pa = regs[2]; + u64 num_segs = regs[3]; + u64 options = regs[4]; + u64 idx = ((options >> 8) & 2UL) | + ((options & 3UL) << 2) | + ((options >> 8) & 1UL); + + if (root == (struct uat_c_root_state *)0 || + num_segs == 0 || + num_segs > (g_state.uat.segment_limit ? g_state.uat.segment_limit : 0x40) || + (options & UAT_OPTIONS_INVALID_MASK) || + uat_attr_table[idx & 0xf] == 0xff) { + g_state.uat.violations++; + if (root == (struct uat_c_root_state *)0) + return 0; + } + if (!uat_copy_segments_to_state(root, UAT_STATE_MAP_SEGS_OFF, + segs_pa, num_segs)) { + g_state.uat.violations++; + return 0; + } + + root->current_va = regs[1]; + root->seglist_pa = root->handle + UAT_STATE_MAP_SEGS_OFF; + root->num_segs = num_segs; + root->current_seg = 0; + root->seg_offset = 0; + root->options = options; + root->state = 3; + uat_write8_ptr(root->handle + UAT_STATE_FLUSH_LEVEL_OFF, 0); + uat_write_map_state(root, 3); + return uat_map_pending(root); +} + +static u64 uat_ep_prepare_fw_unmap_begin(u64 *regs) +{ + struct uat_c_root_state *root = uat_find_state(regs[0], 0); + u64 va = regs[1]; + u64 pages = regs[2]; + + if (root == (struct uat_c_root_state *)0 || + (va & (UAT_C_PAGE_SIZE - 1)) != 0 || + pages == 0 || pages >= UAT_MAX_VA_PAGES) { + g_state.uat.violations++; + return 0; + } + + if (!uat_handoff_magic_valid(root)) + return 0; + + root->current_va = va; + root->current_seg = 0; + root->num_segs = pages; + root->seg_offset = 0; + root->options = 0; + root->state = 4; + uat_write_prepare_fw_unmap_state(root, 4, 0); + return uat_prepare_fw_unmap_pending(root); +} + +static u64 uat_ep_set_ctx_id(u64 *regs) +{ + struct uat_c_root_state *root = uat_find_state(regs[0], 0); + u64 ctx = regs[1]; + u64 base, root1; + u64 ttbr0, ttbr1; + + if (root == (struct uat_c_root_state *)0 || ctx >= 64 || + root->ctx_id != UAT_C_CTX_NONE || + !g_state.uat.gpu_region_pa) { + g_state.uat.violations++; + return 0; + } + + root1 = root->root1_pa; + base = g_state.uat.gpu_region_pa + ctx * 16UL; + if ((root->type & 0xeU) == 0 || + !uat_looks_like_pa(root->root0_pa) || !uat_looks_like_pa(root1) || + *(volatile u64 *)base != 0 || *(volatile u64 *)(base + 8) != 0) { + g_state.uat.violations++; + return 0; + } + ttbr0 = (root->root0_pa & 0x3ffffffc000UL) | + ((ctx & 0x7fffUL) << 48) | 1UL; + ttbr1 = (root1 & 0x3ffffffc000UL) | + ((ctx & 0xffffUL) << 48) | 1UL; + + uat_write_slot(base, ttbr0); + uat_write_slot(base + 8, ttbr1); + root->ctx_id = (u16)ctx; + uat_write_idle_state(root); + tlbi_vmalle1os(); + return 0; +} + +static u64 uat_ep_remove_ctx_id(u64 *regs) +{ + struct uat_c_root_state *root = uat_find_state(regs[0], 0); + if (root == (struct uat_c_root_state *)0) + return 0; + if (root->ctx_id == UAT_C_CTX_NONE) { + g_state.uat.violations++; + return 0; + } + if (root->ctx_id != UAT_C_CTX_NONE && g_state.uat.gpu_region_pa) { + u64 base = g_state.uat.gpu_region_pa + (u64)root->ctx_id * 16UL; + uat_write_slot(base, *(volatile u64 *)base & ~1UL); + uat_write_slot(base + 8, *(volatile u64 *)(base + 8) & ~1UL); + } + root->ctx_id = UAT_C_CTX_NONE; + uat_write_idle_state(root); + tlbi_vmalle1os(); + return 0; +} + +static u64 uat_ep_get_info(u64 *regs) +{ + u64 selector = regs[0]; + + switch (selector) { + case 0: + return g_state.uat.mode; + case 1: + if (g_state.uat.mode == 0) + return uat_looks_like_pa(g_state.uat.selector2_pa) ? + g_state.uat.selector2_pa : 0; + return 0; + case 2: + return uat_selector2_state_pa(); + case 3: + return g_state.uat.vaddr_shift ? g_state.uat.vaddr_shift : + UAT_DEFAULT_VADDR_SHIFT; + case 4: + return g_state.uat.l1_index_mask ? g_state.uat.l1_index_mask : + uat_default_l1_mask(g_state.uat.vaddr_shift ? + g_state.uat.vaddr_shift : + UAT_DEFAULT_VADDR_SHIFT); + case 5: + return g_state.uat.segment_limit ? g_state.uat.segment_limit : 0x40; + case 6: + return 0x1000000000UL; + case 7: + return 0x7000000000UL; + case 8: + return g_state.uat.state_object_size ? g_state.uat.state_object_size : + ((g_state.uat.segment_limit ? g_state.uat.segment_limit : 0x40) * + 0x10UL + 0x250UL); + case 9: + return uat_selector9_papt_va(); + default: + g_state.uat.violations++; + return 0; + } +} + +bool_t uat_handle_table7(u32 endpoint, u64 *regs) +{ + u64 rc = 0; + u64 root_pa = 0, slot_pa = 0, before = 0, after = 0; + u64 saved_regs[5]; + struct uat_c_root_state *root; + + if (endpoint >= UAT_C_MAX_ENDPOINTS) { + g_state.uat.fallthrough_calls++; + return 0; + } + + uat_set_defaults(); + g_state.uat.total_calls++; + g_state.uat.ep_count[endpoint]++; + for (u32 i = 0; i < 5; i++) { + saved_regs[i] = regs[i]; + g_state.uat.last[i] = regs[i]; + } + g_state.uat.last[5] = endpoint; + + switch (endpoint) { + case UAT_FN_INIT_STATE: + rc = uat_ep_init(regs); + break; + case UAT_FN_DESTROY_STATE: + rc = uat_ep_destroy(regs); + break; + case UAT_FN_MAP_TABLE: + rc = uat_ep_map_table(regs, &root_pa, &slot_pa, &before, &after); + break; + case UAT_FN_UNMAP_TABLE: + rc = uat_ep_unmap_table(regs, &root_pa, &slot_pa, &before); + after = 0; + break; + case UAT_FN_MAP_BEGIN: + rc = uat_ep_map_begin(regs); + break; + case UAT_FN_MAP_CONTINUE: + root = uat_find_state(regs[0], 0); + rc = root ? uat_map_pending(root) : 0; + if (!root) + g_state.uat.violations++; + break; + case UAT_FN_PREP_FW_UNMAP_BEGIN: + rc = uat_ep_prepare_fw_unmap_begin(regs); + root = uat_find_state(saved_regs[0], 0); + if (root) { + root_pa = root->last[0]; + slot_pa = root->last[2]; + before = root->last[3]; + after = root->last[4]; + } + break; + case UAT_FN_PREP_FW_UNMAP_CONTINUE: + root = uat_find_state(regs[0], 0); + if (root && root->state == 4) { + rc = uat_prepare_fw_unmap_pending(root); + root_pa = root->last[0]; + slot_pa = root->last[2]; + before = root->last[3]; + after = root->last[4]; + } else { + g_state.uat.violations++; + } + break; + case UAT_FN_UNMAP_BEGIN: + root = uat_find_state(regs[0], 0); + if (root) + rc = uat_unmap_segments(root, regs[1], regs[2]); + else + g_state.uat.violations++; + break; + case UAT_FN_UNMAP_CONTINUE: + root = uat_find_state(regs[0], 0); + rc = root ? uat_unmap_pending(root) : 0; + if (!root) + g_state.uat.violations++; + break; + case UAT_FN_SET_CTX_ID: + rc = uat_ep_set_ctx_id(regs); + break; + case UAT_FN_REMOVE_CTX_ID: + rc = uat_ep_remove_ctx_id(regs); + break; + case UAT_FN_GET_INFO: + rc = uat_ep_get_info(regs); + break; + default: + g_state.uat.fallthrough_calls++; + return 0; + } + + regs[0] = rc; + g_state.uat.fast_path_calls++; + g_state.fast_path_calls++; + uat_record(endpoint, saved_regs, root_pa, slot_pa, before, after, rc); + sev(); + return 1; +} diff --git a/src/sptm/xnu.c b/src/sptm/xnu.c new file mode 100644 index 000000000..e64291d26 --- /dev/null +++ b/src/sptm/xnu.c @@ -0,0 +1,832 @@ +/* SPDX-License-Identifier: MIT */ +#include "private.h" + +int debug_printf(const char *fmt, ...); + +#define USER_POINTER_BATCH_LIMIT 64U +#define USER_POINTER_OP_SIZE 0x18UL +#define USER_POINTER_VALUE_OFF 0x00UL + +bool_t xnu_handle_table0(void *ctx, u32 endpoint, u64 *regs) +{ + if (endpoint < 64) g_state.ep_count[endpoint]++; + + switch (endpoint) { + case SPTM_FN_MAP_PAGE: { + /* (root_pa, va, new_pte) -> sptm_return_t */ + u64 root_pa = regs[0]; + u64 va = regs[1]; + u64 new_pte = regs[2]; + u64 idx = 0; + bool_t geom4k = root_uses_4k(root_pa, 3); + u64 l3 = walk_to_l3(root_pa, va, &idx); + if (l3 == 0) { + regs[0] = SPTM_TABLE_NOT_PRESENT; + g_state.fast_path_calls++; sev(); + return 1; + } + volatile u64 *slot = pt_slot_ptr(l3, idx); + u64 existing = pt_read_slot_poc(slot); + bool_t existing_valid = ((existing & 3) == 3); + u64 arm_pte = + t8140_cpu_pio_shadow_pte(xnu_pte_to_arm_geom(new_pte, geom4k), + geom4k); + u64 ptep_pa = l3 + idx * 8; + u64 ptep_frame_pa = ptep_pa & FRAME_PAGE_MASK; + u64 page_mask = geom4k ? ARM_PTE_PAGE_MASK_4K : ARM_PTE_PAGE_MASK; + u64 target_pa = new_pte & page_mask; + u64 ptep_papt_va = 0; + if (g_state.first_papt != 0 && ptep_pa >= g_state.vm_first_phys) + ptep_papt_va = g_state.first_papt + (ptep_pa - g_state.vm_first_phys); + g_state.last_map_page[0] = root_pa; + g_state.last_map_page[1] = va; + g_state.last_map_page[2] = l3; + g_state.last_map_page[3] = idx; + g_state.last_map_page[4] = ptep_pa; + g_state.last_map_page[5] = ptep_papt_va; + g_state.last_map_page[6] = existing; + g_state.last_map_page[7] = new_pte; + g_state.last_map_page[8] = arm_pte; + g_state.last_map_page[9] = target_pa; + g_state.last_map_page[10] = ft_get_type(target_pa); + g_state.last_map_page[11] = ft_read_page_refcount(target_pa); + g_state.last_map_page[12] = ft_read_table_refcount(ptep_frame_pa); + g_state.last_map_page[13] = ft_idx_or_bad(ptep_frame_pa); + /* AUDIT FIX #1: write sptm_map_page_output_t. + * Then DC CVAU + DSB to make scratch visible to xnu's read. */ + if (g_state.scratch_pa != 0) { + volatile u64 *out = scratch_for_current_cpu(); + out[0] = existing; + out[1] = ptep_papt_va; + __asm__ volatile( + "dc cvau, %0\n\t" + "dc cvau, %1\n\t" + "dsb ish\n\t" + : : "r"(&out[0]), "r"(&out[1]) : "memory"); + } + /* + * Existing same-PA mappings are upgraded in place. Different-PA + * replacements must go back through XNU's remove/retry path so the + * architecture's break-before-make sequence is preserved. + */ + if (existing_valid && ((existing & page_mask) != (arm_pte & page_mask))) { + regs[0] = SPTM_MAP_PADDR_CONFLICT; + g_state.fast_path_calls++; sev(); + return 1; + } + *slot = arm_pte; + clean_pt_slot_poc(slot); + update_refcounts_for_leaf_pte_change(l3, existing, arm_pte, geom4k); + maybe_map_t8140_pmgr_cpustart_window(root_pa, va, arm_pte, + geom4k, l3, idx); + if (existing_valid) + tlbi_vmalle1is(); + regs[0] = existing_valid ? SPTM_MAP_VALID : SPTM_SUCCESS; + g_state.fast_path_calls++; sev(); + return 1; + } + + case SPTM_FN_MAP_TABLE: { + /* (root_pa, va, target_level, new_tte) -> sptm_return_t */ + u64 root_pa = regs[0]; + u64 va = regs[1]; + u32 target_level = (u32)regs[2]; + u64 new_tte = regs[3]; + u64 idx = 0; + u64 parent = walk_to_level(root_pa, va, target_level, &idx); + if (parent == 0) { + regs[0] = SPTM_TABLE_NOT_PRESENT; + g_state.fast_path_calls++; sev(); + return 1; + } + bool_t geom4k = root_uses_4k(root_pa, target_level); + u64 base_idx = table_group_base_idx(idx, geom4k); + u32 group_count = table_group_count(geom4k); + volatile u64 *slot = pt_slot_ptr(parent, base_idx); + u64 existing = 0; + u64 arm_tte = xnu_tte_to_arm(new_tte); + u64 child_pa = new_tte & ARM_TTE_TABLE_MASK; + bool_t already_present = 0; + for (u32 n = 0; n < group_count; n++) { + u64 cur = pt_read_slot_poc(&slot[n]); + if ((cur & 3) == 3) { + if (!already_present) + existing = cur; + already_present = 1; + } + } + if (already_present) { + g_state.last_map_table[0] = root_pa; + g_state.last_map_table[1] = va; + g_state.last_map_table[2] = target_level; + g_state.last_map_table[3] = parent; + g_state.last_map_table[4] = base_idx; + g_state.last_map_table[5] = existing; + g_state.last_map_table[6] = new_tte; + g_state.last_map_table[7] = arm_tte; + g_state.last_map_table[8] = child_pa & FRAME_PAGE_MASK; + g_state.last_map_table[9] = ft_read_table_refcount(child_pa); + regs[0] = SPTM_TABLE_ALREADY_PRESENT; + g_state.fast_path_calls++; sev(); + return 1; + } + u64 desc_attrs = arm_tte & ~ARM_TTE_TABLE_MASK; + for (u32 n = 0; n < group_count; n++) { + u64 sub_pa = table_group_child_pa(child_pa, n, geom4k); + slot[n] = desc_attrs | (sub_pa & ARM_TTE_TABLE_MASK); + } + clean_pt_range_poc(slot, (u64)group_count * 8); + ft_inc_table_mapping_refcount(child_pa & FRAME_PAGE_MASK); + g_state.last_map_table[0] = root_pa; + g_state.last_map_table[1] = va; + g_state.last_map_table[2] = target_level; + g_state.last_map_table[3] = parent; + g_state.last_map_table[4] = base_idx; + g_state.last_map_table[5] = existing; + g_state.last_map_table[6] = new_tte; + g_state.last_map_table[7] = arm_tte; + g_state.last_map_table[8] = child_pa & FRAME_PAGE_MASK; + g_state.last_map_table[9] = ft_read_table_refcount(child_pa); + regs[0] = SPTM_SUCCESS; + g_state.fast_path_calls++; sev(); + return 1; + } + + case SPTM_FN_UNMAP_TABLE: { + /* (root_pa, va, target_level) -> void. + * Clear the parent TTE and leave the child frame type/counts + * alone. XNU drains leaf mappings first, queries the child table + * count, then performs the retype itself. */ + u64 root_pa = regs[0]; + u64 va = regs[1]; + u32 target_level = (u32)regs[2]; + u64 idx = 0; + u64 parent = walk_to_level(root_pa, va, target_level, &idx); + u64 existing = 0; + u64 freed_pa = 0; + if (parent != 0) { + bool_t geom4k = root_uses_4k(root_pa, target_level); + u64 base_idx = table_group_base_idx(idx, geom4k); + u32 group_count = table_group_count(geom4k); + volatile u64 *slot = pt_slot_ptr(parent, base_idx); + idx = base_idx; + for (u32 n = 0; n < group_count; n++) { + u64 cur = pt_read_slot_poc(&slot[n]); + if (existing == 0 && (cur & 3) == 3) + existing = cur; + slot[n] = 0; + } + clean_pt_range_poc(slot, (u64)group_count * 8); + if ((existing & 3) == 3) { + freed_pa = existing & FRAME_PAGE_MASK; + ft_dec_table_mapping_refcount(freed_pa); + } + } + g_state.last_unmap_table[0] = root_pa; + g_state.last_unmap_table[1] = va; + g_state.last_unmap_table[2] = target_level; + g_state.last_unmap_table[3] = parent; + g_state.last_unmap_table[4] = idx; + g_state.last_unmap_table[5] = existing; + g_state.last_unmap_table[6] = freed_pa; + g_state.last_unmap_table[7] = ft_get_type(freed_pa); + g_state.last_unmap_table[8] = ft_read_table_refcount(freed_pa); + g_state.last_unmap_table[9] = ft_idx_or_bad(freed_pa); + tlbi_vmalle1is(); + regs[0] = SPTM_SUCCESS; + g_state.fast_path_calls++; sev(); + return 1; + } + + case SPTM_FN_RETYPE: { + /* (pa, current_type, new_type, options) -> sptm_return_t. + * Real SPTM validates the old type and zeroes pages that become + * page tables. We keep the functional side effects in C; violations + * remain accepted in this permissive bringup model, but frame_table + * is the single source of truth for XNU/libsptm direct reads. */ + u64 pa = regs[0]; + u8 current_type = (u8)regs[1]; + u8 nt = (u8)regs[2]; + u8 existing_type = ft_get_type(pa); + bool_t was_alt_ptd = existing_type == XNU_PAGE_TABLE_ALT; + bool_t will_alt_ptd = nt == XNU_PAGE_TABLE_ALT; + bool_t alt_papt_attr_write = 0; + u64 page_ref_before = ft_read_page_refcount(pa); + u64 table_ref_before = ft_read_table_refcount(pa); + u64 papt_ring_before = g_state.papt_update.ring_idx; + u8 root_geom = ROOT_GEOM_UNKNOWN; + u16 root_asid = 0; + + if (was_alt_ptd && !will_alt_ptd) { + alt_papt_attr_write = + set_kernel_papt_cacheable_for_managed_pa(pa, 1, + sptm_ctx_elr(ctx)); + alt_ptd_set_el2_cacheable(pa, 1); + } else if (!was_alt_ptd && will_alt_ptd) { + alt_papt_attr_write = + set_kernel_papt_cacheable_for_managed_pa(pa, 0, + sptm_ctx_elr(ctx)); + alt_ptd_set_el2_cacheable(pa, 0); + } + + if (is_pt_type(nt)) { + zero_16k(pa); + ft_write_u16(pa, FT_TABLE_MAPPING_REFCNT_OFF, 0); + ft_write_u16(pa, FT_TABLE_NESTED_REFCNT_OFF, 0); + } + + if (is_pt_type(existing_type) && !is_pt_type(nt)) { + ft_write_u16(pa, FT_TABLE_MAPPING_REFCNT_OFF, 0); + ft_write_u16(pa, FT_TABLE_NESTED_REFCNT_OFF, 0); + } + + ft_set_type(pa, nt); + if (was_alt_ptd || will_alt_ptd) + sptm_coproc_cache_maint_data_page(pa); + if (is_root_type(nt)) { + root_geom = retype_params_attr_idx(regs[3]); + root_asid = retype_params_asid(regs[3]); + root_geom_set_meta(pa, root_geom, root_asid); + } + barrier(); + + g_state.last_retype[0] = pa; + g_state.last_retype[1] = current_type; + g_state.last_retype[2] = nt; + g_state.last_retype[3] = existing_type; + g_state.last_retype[4] = ft_read_page_refcount(pa); + g_state.last_retype[5] = ft_read_table_refcount(pa); + g_state.last_retype[6] = ft_idx_or_bad(pa); + g_state.last_retype[7] = (is_pt_type(existing_type) ? 1UL : 0UL) | + (is_pt_type(nt) ? 2UL : 0UL) | + ((u64)root_geom << 8) | + ((u64)root_asid << 16) | + ((u64)retype_params_flags(regs[3]) << 32); + if (was_alt_ptd || will_alt_ptd) { + u64 flags = (was_alt_ptd ? 1UL : 0UL) | + (will_alt_ptd ? 2UL : 0UL) | + (alt_papt_attr_write ? 4UL : 0UL) | + (is_pt_type(existing_type) ? 8UL : 0UL) | + (is_pt_type(nt) ? 16UL : 0UL); + u64 sp0 = ctx ? *(volatile u64 *)((unsigned char *)ctx + 0x158) : 0; + u64 sp1 = ctx ? *(volatile u64 *)((unsigned char *)ctx + 0x160) : 0; + note_alt_ptd_retype(pa, sptm_ctx_elr(ctx), regs[30], regs[29], + sp0, sp1, current_type, existing_type, nt, + regs[3], + page_ref_before, table_ref_before, + ft_read_page_refcount(pa), + ft_read_table_refcount(pa), + ft_idx_or_bad(pa), papt_ring_before, + flags); + } + + if (alt_papt_attr_write) + tlbi_vmalls12e1is(); + else if (is_pt_type(nt) || is_io_type(nt) || + is_pt_type(existing_type) || is_pt_type(current_type)) + tlbi_vmalle1is(); + + regs[0] = SPTM_SUCCESS; + g_state.fast_path_calls++; sev(); + return 1; + } + + case SPTM_FN_UPDATE_REGION: { + u64 root_pa = regs[0]; + u64 start_va = regs[1]; + u32 num = (u32)(regs[2] & 0xffffffff); + u64 templates_pa = regs[3]; + u64 options = regs[4]; + u64 mask = options & SPTM_UPDATE_MASK; + bool_t geom4k = root_uses_4k(root_pa, 3); + u64 page_size = geom4k ? PAGE_SIZE_4K_EMUL : PAGE_SIZE_GUEST_EMUL; + volatile u64 *prev_out = scratch_for_current_cpu(); + u32 prev_count = 0; + if (num > 2048) num = 2048; + for (u32 i = 0; i < num; i++) { + u64 va = start_va + (u64)i * page_size; + u64 idx; + u64 l3 = walk_to_l3(root_pa, va, &idx); + u64 existing = 0; + if (l3) { + volatile u64 *slot = pt_slot_ptr(l3, idx); + existing = pt_read_slot_poc(slot); + u64 tmpl = ((volatile u64 *)templates_pa)[i]; + u64 arm_template = + xnu_pte_update_template_to_arm_geom(tmpl, geom4k); + u64 arm_pte = + t8140_cpu_pio_shadow_pte( + merge_pte_with_mask_geom(existing, arm_template, + mask, geom4k), + geom4k); + *slot = arm_pte; + clean_pt_slot_poc(slot); + update_refcounts_for_leaf_pte_change(l3, existing, arm_pte, + geom4k); + maybe_map_t8140_pmgr_cpustart_window(root_pa, va, arm_pte, + geom4k, l3, idx); + } + write_prev_pte(prev_out, &prev_count, existing); + } + if (prev_out) clean_range((void *)prev_out, (u64)prev_count * 8); + if (!(options & 0x100)) + tlbi_vmalle1is(); + else + barrier(); + regs[0] = (options & 0x100) ? SPTM_UPDATE_DELAYED_TLBI : SPTM_SUCCESS; + g_state.fast_path_calls++; sev(); + return 1; + } + + case SPTM_FN_UPDATE_DISJOINT: { + u64 ops_pa = regs[1]; + u32 num = (u32)(regs[2] & 0xffffffff); + u64 options = regs[3]; + u64 mask = options & SPTM_UPDATE_MASK; + volatile u64 *prev_out = scratch_for_current_cpu(); + u32 prev_count = 0; + if (num > 2048) num = 2048; + for (u32 i = 0; i < num; i++) { + volatile u64 *op = (volatile u64 *)(ops_pa + (u64)i * DISJOINT_OP_SIZE); + u64 root_pa = op[0]; + u64 va = op[1]; + u64 tmpl = op[2]; + u64 idx; + bool_t geom4k = root_uses_4k(root_pa, 3); + u64 l3 = walk_to_l3(root_pa, va, &idx); + u64 existing = 0; + if (l3) { + volatile u64 *slot = pt_slot_ptr(l3, idx); + existing = pt_read_slot_poc(slot); + u64 arm_template = + xnu_pte_update_template_to_arm_geom(tmpl, geom4k); + u64 arm_pte = + t8140_cpu_pio_shadow_pte( + merge_pte_with_mask_geom(existing, arm_template, + mask, geom4k), + geom4k); + *slot = arm_pte; + clean_pt_slot_poc(slot); + update_refcounts_for_leaf_pte_change(l3, existing, arm_pte, + geom4k); + maybe_map_t8140_pmgr_cpustart_window(root_pa, va, arm_pte, + geom4k, l3, idx); + } + write_prev_pte(prev_out, &prev_count, existing); + } + if (prev_out) clean_range((void *)prev_out, (u64)prev_count * 8); + if (!(options & 0x100)) + tlbi_vmalle1is(); + else + barrier(); + regs[0] = (options & 0x100) ? SPTM_UPDATE_DELAYED_TLBI : SPTM_SUCCESS; + g_state.fast_path_calls++; sev(); + return 1; + } + + case SPTM_FN_UPDATE_DISJOINT_MULTIPAGE: { + u64 ops_pa = regs[0]; + u32 num_entries = (u32)(regs[1] & 0xffffffff); + volatile u64 *prev_out = scratch_for_current_cpu(); + u32 prev_count = 0; + bool_t any_deferred = 0; + bool_t any_papt_attr_write = 0; + u32 i = 0; + if (num_entries > 2048) num_entries = 2048; + while (i < num_entries) { + u64 base = ops_pa + (u64)i * DISJOINT_OP_SIZE; + u64 paddr = *(volatile u64 *)base; + u64 papt_template = *(volatile u64 *)(base + 8); + u32 inner_n = *(volatile u32 *)(base + 16); + u32 opts = *(volatile u32 *)(base + 20); + u64 inner_root_pa = 0; + u64 inner_va = 0; + u64 inner_template = 0; + i++; + if (inner_n != 0 && i < num_entries) { + volatile u64 *first_inner = + (volatile u64 *)(ops_pa + (u64)i * DISJOINT_OP_SIZE); + inner_root_pa = first_inner[0]; + inner_va = first_inner[1]; + inner_template = first_inner[2]; + } + if (opts & SPTM_UPDATE_DEFER_TLBI) + any_deferred = 1; + if (update_kernel_papt_attrs(paddr, papt_template, opts, + sptm_ctx_elr(ctx), inner_root_pa, + inner_va, inner_template)) + any_papt_attr_write = 1; + for (u32 j = 0; j < inner_n && i < num_entries; j++, i++) { + volatile u64 *op = (volatile u64 *)(ops_pa + (u64)i * DISJOINT_OP_SIZE); + u64 root_pa = op[0]; + u64 va = op[1]; + u64 tmpl = op[2]; + u64 idx; + bool_t geom4k = root_uses_4k(root_pa, 3); + u64 l3 = walk_to_l3(root_pa, va, &idx); + u64 existing = 0; + if (l3) { + volatile u64 *slot = pt_slot_ptr(l3, idx); + existing = pt_read_slot_poc(slot); + u64 arm_template = + xnu_pte_update_template_to_arm_geom(tmpl, geom4k); + u64 arm_pte = + t8140_cpu_pio_shadow_pte( + merge_pte_with_mask_geom( + existing, arm_template, + opts & SPTM_UPDATE_MASK, geom4k), + geom4k); + *slot = arm_pte; + clean_pt_slot_poc(slot); + update_refcounts_for_leaf_pte_change(l3, existing, arm_pte, + geom4k); + maybe_map_t8140_pmgr_cpustart_window(root_pa, va, arm_pte, + geom4k, l3, idx); + } + write_prev_pte(prev_out, &prev_count, existing); + } + } + if (prev_out) clean_range((void *)prev_out, (u64)prev_count * 8); + /* + * Deferred TLBI covers the guest-requested VA updates. The PAPT + * mirror is a private kernel alias we update on XNU's behalf, so + * flush it here when its cacheability attributes changed. + */ + if (any_papt_attr_write) + tlbi_vmalls12e1is(); + else if (!any_deferred) + tlbi_vmalle1is(); + else + barrier(); + regs[0] = any_deferred ? SPTM_UPDATE_DELAYED_TLBI : SPTM_SUCCESS; + g_state.fast_path_calls++; sev(); + return 1; + } + + case SPTM_FN_UNMAP_DISJOINT: { + u64 ops_pa = regs[1]; + u32 num = (u32)(regs[2] & 0xffffffff); + volatile u64 *prev_out = scratch_for_current_cpu(); + u32 prev_count = 0; + if (num > 2048) num = 2048; + for (u32 i = 0; i < num; i++) { + volatile u64 *op = (volatile u64 *)(ops_pa + (u64)i * DISJOINT_OP_SIZE); + u64 root_pa = op[0]; + u64 va = op[1]; + u64 tmpl = op[2]; + u64 idx; + bool_t geom4k = root_uses_4k(root_pa, 3); + u64 l3 = walk_to_l3(root_pa, va, &idx); + u64 existing = 0; + if (l3) { + volatile u64 *slot = pt_slot_ptr(l3, idx); + existing = pt_read_slot_poc(slot); + /* + * Disjoint unmap receives an XNU template, but leaving + * template PA/attribute bits in an invalid slot can make + * XNU later interpret the entry as a corrupt compressed + * PTE. Real pmap remove paths fault the slot; keep the + * old PTE only in the prev-PTE scratch output. + */ + (void)tmpl; + *slot = 0; + clean_pt_slot_poc(slot); + update_refcounts_for_leaf_pte_change(l3, existing, 0, geom4k); + } + write_prev_pte(prev_out, &prev_count, existing); + } + if (prev_out) clean_range((void *)prev_out, (u64)prev_count * 8); + tlbi_vmalle1is(); + regs[0] = SPTM_SUCCESS; + g_state.fast_path_calls++; sev(); + return 1; + } + + case SPTM_FN_SWITCH_ROOT: { + u64 root_pa = regs[0]; + if (ft_get_type(root_pa) == SPTM_KERNEL_ROOT_TABLE) { + u64 ttbr = root_pa & TTBR_BADDR_MASK; + __asm__ volatile("msr S3_5_C2_C0_1, %0" : : "r"(ttbr)); + } else { + u64 asid = root_asid_lookup(root_pa); + u64 ttbr = (root_pa & TTBR_BADDR_MASK) | (asid << TTBR_ASID_SHIFT); + __asm__ volatile("msr S3_5_C2_C0_0, %0" : : "r"(ttbr)); + } + tlbi_vmalle1is(); + regs[0] = SPTM_SUCCESS; + g_state.fast_path_calls++; sev(); + return 1; + } + + case SPTM_FN_CONFIGURE_SHAREDREGION: + ft_set_type(regs[0], XNU_SHARED_ROOT_TABLE); + regs[0] = SPTM_SUCCESS; + g_state.fast_path_calls++; sev(); + return 1; + + case SPTM_FN_SET_SHARED_REGION: + regs[0] = SPTM_SUCCESS; + g_state.fast_path_calls++; sev(); + return 1; + + case SPTM_FN_NEST_REGION: { + u64 user_root = regs[0]; + u64 shared_root = regs[1]; + u64 start_va = regs[2]; + u32 page_count = (u32)(regs[3] & 0xffffffff); + bool_t geom4k = root_uses_4k(user_root, 2); + u64 page_size = geom4k ? PAGE_SIZE_4K_EMUL : PAGE_SIZE_GUEST_EMUL; + u64 l2_coverage = (geom4k ? 0x200UL : 0x800UL) * page_size; + u64 end_va = start_va + (u64)page_count * page_size; + u64 va = start_va & ~(l2_coverage - 1); + while (va < end_va) { + u64 sidx = 0, uidx = 0; + u64 sparent = walk_to_level(shared_root, va, 2, &sidx); + u64 uparent = walk_to_level(user_root, va, 2, &uidx); + if (sparent != 0 && uparent != 0 && sidx == uidx) { + volatile u64 *uslot = pt_slot_ptr(uparent, uidx); + u64 tte = pt_read_slot_poc(pt_slot_ptr(sparent, sidx)); + if ((tte & 3) == 3) { + *uslot = tte; + clean_pt_slot_poc(uslot); + } + } + va += l2_coverage; + } + barrier(); + regs[0] = SPTM_SUCCESS; + g_state.fast_path_calls++; sev(); + return 1; + } + + case SPTM_FN_UNNEST_REGION: { + u64 user_root = regs[0]; + u64 start_va = regs[2]; + u32 page_count = (u32)(regs[3] & 0xffffffff); + bool_t geom4k = root_uses_4k(user_root, 2); + u64 page_size = geom4k ? PAGE_SIZE_4K_EMUL : PAGE_SIZE_GUEST_EMUL; + u64 l2_coverage = (geom4k ? 0x200UL : 0x800UL) * page_size; + u64 end_va = start_va + (u64)page_count * page_size; + u64 va = start_va & ~(l2_coverage - 1); + while (va < end_va) { + u64 idx = 0; + u64 parent = walk_to_level(user_root, va, 2, &idx); + if (parent != 0) { + volatile u64 *slot = pt_slot_ptr(parent, idx); + *slot = 0; + clean_pt_slot_poc(slot); + } + va += l2_coverage; + } + tlbi_vmalle1is(); + regs[0] = SPTM_SUCCESS; + g_state.fast_path_calls++; sev(); + return 1; + } + + /* C-only ack-success endpoints. + * + * Historical note: older m1n1 builds also added +4 to ELR for + * handled HVCs, which double-advanced these non-trampoline call + * sites and skipped one real instruction. Core m1n1 now leaves + * handled HVC ELR alone, so resume should use ELR_EL2 directly. */ + case SPTM_FN_FIXUPS_COMPLETE: + case SPTM_FN_CONFIGURE_ROOT: + case SPTM_FN_SLIDE_REGION: + case SPTM_FN_REG_WRITE: + case SPTM_FN_SERIAL_DISABLE: + case SPTM_FN_REGISTER_EXC_RETURN: + case SPTM_FN_DISABLE_KERNEL_MODE_CPA2: + case SPTM_FN_PROGRAM_IRGKEY: + case SPTM_FN_REG_SNAPSHOT: + case SPTM_FN_MAP_SK_DOMAIN: + case SPTM_FN_HIB_BEGIN: + case SPTM_FN_HIB_VERIFY_HASH_NON_WIRED: + case SPTM_FN_HIB_FINALIZE_NON_WIRED: + case SPTM_FN_GUEST_EXIT: + { + return finish_hvc_success(ctx, regs); + } + + case SPTM_FN_GUEST_STAGE1_TLBOP: + /* + * Real SPTM decodes the requested guest S1 operation and issues a + * scoped TLBI while running under the guest's stage-2 context. We + * do not model nested guest contexts yet, but ACK-only is wrong: + * callers use this endpoint specifically to retire stale S1 + * translations. A broadcast S1 invalidate is conservative and + * preserves the architectural effect until the op decoder exists. + */ + tlbi_vmalle1is(); + return finish_hvc_success(ctx, regs); + + case SPTM_FN_GUEST_STAGE2_TLBOP: + /* + * Same conservative treatment for guest stage-2 invalidates. The + * exact firmware path can narrow this to IPA/range later. + */ + tlbi_vmalls12e1is(); + return finish_hvc_success(ctx, regs); + + case SPTM_FN_REGISTER_CPU: { + u64 phys = regs[0]; + u64 count = g_state.cpu_count; + if (count > SPTM_MAX_CPUS_EMUL) + count = SPTM_MAX_CPUS_EMUL; + + u64 slot = SPTM_MAX_CPUS_EMUL; + for (u64 i = 0; i < count; i++) { + if (cpu_phys_matches(g_state.cpu_phys_ids[i], phys)) { + slot = i; + break; + } + } + + if (slot == SPTM_MAX_CPUS_EMUL && count < SPTM_MAX_CPUS_EMUL) { + slot = count; + g_state.cpu_phys_ids[slot] = phys; + g_state.cpu_count = count + 1; + } + + g_state.cpu_last_mpidr = current_mpidr_phys(); + g_state.cpu_last_id = (slot == SPTM_MAX_CPUS_EMUL) ? 0 : slot; + sptm_cpu_pio_note_registered(phys, g_state.cpu_last_id); + return finish_hvc_success(ctx, regs); + } + + case SPTM_FN_SURT_ALLOC: + case SPTM_FN_SURT_FREE: { + u64 surt_frame = regs[0]; + u64 surt_index = regs[1] & 0xff; + u8 attr_idx = (u8)(regs[2] & 0xff); + u64 slot_pa = surt_frame + surt_index * SURT_SUBPAGE_SIZE; + volatile u64 *q = (volatile u64 *)slot_pa; + for (u32 i = 0; i < (SURT_SUBPAGE_SIZE / 8); i++) + q[i] = 0; + clean_range_poc((void *)slot_pa, SURT_SUBPAGE_SIZE); + if (endpoint == SPTM_FN_SURT_ALLOC) + root_geom_set_meta(slot_pa, attr_idx, (u16)(regs[4] & 0xffff)); + regs[0] = SPTM_SUCCESS; + g_state.fast_path_calls++; sev(); + return 1; + } + + case SPTM_FN_REG_READ: + case SPTM_FN_SPTM_SYSCTL: + regs[0] = 0; + g_state.fast_path_calls++; sev(); + return 1; + + case SPTM_FN_SIGN_USER_POINTER: + case SPTM_FN_AUTH_USER_POINTER: + g_state.fast_path_calls++; sev(); + return 1; + + case SPTM_FN_BATCH_SIGN_USER_POINTER: { + u64 ops_pa = regs[0]; + u32 ops_count = (u32)(regs[1] & 0xffffffff); + volatile u64 *out = scratch_for_current_cpu(); + if (out) { + if (ops_count > USER_POINTER_BATCH_LIMIT) + ops_count = USER_POINTER_BATCH_LIMIT; + for (u32 i = 0; i < ops_count; i++) { + u64 op_pa = ops_pa + (u64)i * USER_POINTER_OP_SIZE; + out[i] = *(volatile u64 *)(op_pa + + USER_POINTER_VALUE_OFF); + } + clean_range((void *)out, (u64)ops_count * 8); + } + regs[0] = SPTM_SUCCESS; + g_state.fast_path_calls++; sev(); + return 1; + } + + case SPTM_FN_GUEST_VA_TO_IPA: + regs[0] = UINT64_MAX_VALUE; + g_state.fast_path_calls++; sev(); + return 1; + + case SPTM_FN_IOFILTER_PROTECTED_WRITE: { + /* + * No C model yet. Leave this unhandled so any future caller stops + * at the missing endpoint instead of silently touching guarded + * MMIO or reviving the removed Python fallback policy. + */ + return 0; + } + + case SPTM_FN_CONDEMN_LEAF_TABLE: { + u64 root_pa = regs[0]; + u64 va = regs[1]; + u64 idx = 0; + u64 parent = walk_to_level(root_pa, va, 2, &idx); + if (parent == 0) { + regs[0] = SPTM_TABLE_NOT_PRESENT; + } else { + volatile u64 *slot = pt_slot_ptr(parent, idx); + u64 l2e = pt_read_slot_poc(slot); + if ((l2e & 3) != 3 || (l2e & TTE_CONDEMNED_BIT)) + regs[0] = SPTM_TABLE_NOT_PRESENT; + else { + *slot = l2e | TTE_CONDEMNED_BIT; + clean_pt_slot_poc(slot); + regs[0] = SPTM_SUCCESS; + } + } + g_state.fast_path_calls++; sev(); + return 1; + } + + case SPTM_FN_UNCONDEMN_LEAF_TABLE: { + u64 root_pa = regs[0]; + u64 va = regs[1]; + u64 idx = 0; + u64 parent = walk_to_level(root_pa, va, 2, &idx); + if (parent != 0) { + volatile u64 *slot = pt_slot_ptr(parent, idx); + u64 l2e = pt_read_slot_poc(slot); + *slot = l2e & ~TTE_CONDEMNED_BIT; + clean_pt_slot_poc(slot); + } + regs[0] = SPTM_SUCCESS; + g_state.fast_path_calls++; sev(); + return 1; + } + + case SPTM_FN_SERIAL_PUTC: + /* Avoid per-character UART round trips. Panic diagnostics are + * captured through XNU_PANIC_BEGIN and framebuffer/serial KDP. */ + regs[0] = SPTM_SUCCESS; + g_state.fast_path_calls++; sev(); + return 1; + + case SPTM_FN_LOCKDOWN: + /* Our boot page tables already establish coarse kernel RX/RO/RW. + * Real SPTM would lock CTRR and retype text pages. */ + regs[0] = SPTM_SUCCESS; + g_state.fast_path_calls++; sev(); + return 1; + + /* UNMAP_REGION (ep=7) — port to C, ATTEMPT 3 (subagent fix). + * + * ROOT CAUSE of prior 2 attempts: incorrectly applied the ELR-=4 + * workaround. That workaround is ONLY needed for HVCs at NON- + * trampoline call sites (FIXUPS_COMPLETE etc.). UNMAP_REGION + * goes through the SPTM trampoline at 0xbcee880 — same as + * MAP_PAGE/MAP_TABLE/RETYPE — which has `hvc; retab; bti c`. + * m1n1's auto +4 lands on `bti c` (benign NOP); ELR-=4 made it + * land on `retab` instead, NOT a BTI landing pad → EC=0x1b BTI + * fail → corrupt PC → panic 2-3 calls later. + * + * Plus: real SPTM `f6180 sptm_pte_finalize_unmap` decrements + * the page-FTE refcount on each unmapped target. Mirror that. */ + case 7: { /* SPTM_FN_UNMAP_REGION */ + u64 root_pa = regs[0]; + u64 start_va = regs[1]; + u32 num = (u32)(regs[2] & 0xffffffff); + u64 options = regs[3]; + bool_t geom4k = root_uses_4k(root_pa, 3); + u64 page_size = geom4k ? PAGE_SIZE_4K_EMUL : PAGE_SIZE_GUEST_EMUL; + volatile u64 *prev_out = scratch_for_current_cpu(); + u32 prev_count = 0; + bool_t any_unmapped = 0; + if (num > 2048) num = 2048; + for (u32 i = 0; i < num; i++) { + u64 va = start_va + (u64)i * page_size; + u64 idx; + u64 l3 = walk_to_l3(root_pa, va, &idx); + u64 existing = 0; + if (l3) { + volatile u64 *slot = pt_slot_ptr(l3, idx); + existing = pt_read_slot_poc(slot); + *slot = 0; + clean_pt_slot_poc(slot); + if ((existing & 3) == 3) + any_unmapped = 1; + } + write_prev_pte(prev_out, &prev_count, existing); + update_refcounts_for_leaf_pte_change(l3, existing, 0, geom4k); + } + if (prev_out) clean_range((void *)prev_out, (u64)prev_count * 8); + if (any_unmapped && !(options & 0x100)) + tlbi_vmalle1is(); + else + barrier(); + regs[0] = (any_unmapped && (options & 0x100)) ? + SPTM_UPDATE_DELAYED_TLBI : SPTM_SUCCESS; + g_state.fast_path_calls++; sev(); + return 1; + } + + case SPTM_FN_CPU_ID: + regs[0] = cpu_id_for_phys(regs[0]); + g_state.fast_path_calls++; sev(); + return 1; + + default: + /* Unrecognized — host treats this as a hard diagnostic failure. */ + g_state.unknown_calls++; + g_state.fallthrough_calls++; + return 0; + } +} From f47d3085cb1ed0b19e616d21ce775baca38ec2cd Mon Sep 17 00:00:00 2001 From: Cody Ho Date: Thu, 9 Jul 2026 01:28:00 -0700 Subject: [PATCH 20/33] run_guest: macOS-under-HV boot arguments for M4-class SoCs For SPTM enabled devices, booting macOS requires some boot args. Set these by default to avoid users needing to specify them manually. Signed-off-by: Cody Ho --- proxyclient/tools/run_guest.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/proxyclient/tools/run_guest.py b/proxyclient/tools/run_guest.py index bb10e3cac..c039e5ed4 100755 --- a/proxyclient/tools/run_guest.py +++ b/proxyclient/tools/run_guest.py @@ -9,6 +9,26 @@ def volumespec(s): return tuple(s.split(":", 2)) + +def sptm_hv_boot_args(extra=()): + def boot_arg_key(arg): + return arg.split("=", 1)[0] + + words = list(extra) + defaults = [ + "-nobsdmgroot", # dodges a panic we otherwise hit on this boot path + "wdt=-1", # disables some internal XNU watchdog + "sprr_tpro=0", # disable XNU's TPRO boot policy; the commpage bit is patched separately + "sprr_tpro_pagers=0", # ... same, for pager mappings + "-v", # Optional: verbose boot + f"msgbuf={1024 * 1024}", # Optional: enlarge the kernel msgbuf + ] + for arg in defaults: + key = boot_arg_key(arg) + if not any(boot_arg_key(w) == key for w in words): + words.append(arg) + return " ".join(words) + parser = argparse.ArgumentParser(description='Run a Mach-O payload under the hypervisor') parser.add_argument('-s', '--symbols', type=pathlib.Path) parser.add_argument('-m', '--script', type=pathlib.Path, action='append', default=[]) @@ -78,7 +98,10 @@ def volumespec(s): if args.logfile: hv.set_logfile(args.logfile.open("w")) -if len(args.boot_args) > 0: +# macOS-under-HV needs a specific boot-arg set on the M4/macOS chip_ids +if not args.raw and u.adt["/chosen"].chip_id in (0x8132, 0x8140, 0x6040, 0x6041): + hv.set_bootargs(sptm_hv_boot_args(args.boot_args)) +elif len(args.boot_args) > 0: boot_args = " ".join(args.boot_args) hv.set_bootargs(boot_args) From 1e769c0e8d85978cbbd2ad1b00d2f9f6bd7a5279 Mon Sep 17 00:00:00 2001 From: Cody Ho Date: Thu, 9 Jul 2026 01:28:19 -0700 Subject: [PATCH 21/33] run_guest: drop exclave related ADT nodes The hypervisor doesn't yet support exclaves, so remove the stale adt nodes to not confuse XNU. Signed-off-by: Cody Ho --- proxyclient/tools/run_guest.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/proxyclient/tools/run_guest.py b/proxyclient/tools/run_guest.py index c039e5ed4..6ac199187 100755 --- a/proxyclient/tools/run_guest.py +++ b/proxyclient/tools/run_guest.py @@ -91,6 +91,24 @@ def boot_arg_key(arg): if args.debug_xnu: hv.adt["chosen"].debug_enabled = 1 +# Exclaves are not yet supported +if not args.raw and u.adt["/chosen"].chip_id in (0x8132, 0x8140, 0x6040, 0x6041): + for name in ("/arm-io/exdisplaypipe", "/arm-io/exdisplaypipe-s-proxy", + "/arm-io/dcp-exclave-ioreporting", "/arm-io/dcp-exclave-mailbox"): + try: + del hv.adt[name] + except KeyError: + pass + # The internal DCP's iop-dcp-nub binds its RTKit transport to the (now-gone) + # dcp-exclave-mailbox via routes=206; clearing routes makes the DCP firmware + # fall back to the plain ASC mailbox like the routeless external dcpext. + try: + nub = hv.adt["/arm-io/dcp/iop-dcp-nub"] + if getattr(nub, "routes", None) is not None: + del nub.routes + except KeyError: + pass + if args.volume: for path, tag in args.volume: hv.attach_virtio(Virtio9PTransport(root=path, tag=tag)) From d82f372b7bbadcf3d1d83f1b5061ff3640ffa5de Mon Sep 17 00:00:00 2001 From: Cody Ho Date: Fri, 10 Jul 2026 00:04:27 -0700 Subject: [PATCH 22/33] hv: configure SME state for M4 guests M4-class CPUs and above support SME, but XNU's native boot path normally runs with the privilege to program EL2 SME controls directly. Under m1n1 the macOS guest runs at EL1, so those architectural EL2 registers are not writable by the guest even though the hardware feature exists. Add an M4 feature flag, expose it through the proxy ABI, and have EL2 seed CPTR_EL2/FGT/SMPRIMAP_EL2 on the boot CPU and secondaries. Guest loaders can use the same SoC capability bit when deciding whether XNU's direct EL2 SME-control writes need to be neutralized. Signed-off-by: Cody Ho --- proxyclient/m1n1/proxy.py | 2 +- src/arm_cpu_regs.h | 15 +++++++++++++++ src/chickens.c | 1 + src/hv.c | 21 +++++++++++++++++++++ src/utils.h | 1 + 5 files changed, 39 insertions(+), 1 deletion(-) diff --git a/proxyclient/m1n1/proxy.py b/proxyclient/m1n1/proxy.py index b75f5268b..01cd22cbc 100644 --- a/proxyclient/m1n1/proxy.py +++ b/proxyclient/m1n1/proxy.py @@ -507,7 +507,7 @@ class GUARD(IntFlag): "amx" / bool_, "actlr_el2" / bool_, "counter_redirect" / bool_, - "padding" / Bytes(1), + "sme_enabled" / bool_, ) # Uses UartInterface.proxyreq() to send requests to M1N1 and process diff --git a/src/arm_cpu_regs.h b/src/arm_cpu_regs.h index a45868ec6..8cd45d88a 100644 --- a/src/arm_cpu_regs.h +++ b/src/arm_cpu_regs.h @@ -188,6 +188,21 @@ #define ID_AA64MMFR0_TGran4_2 GENMASK(43, 40) #define ID_AA64MMFR0_TGran64_2 GENMASK(39, 36) #define ID_AA64MMFR0_TGran16_2 GENMASK(35, 32) + +#define SYS_CPTR_EL2 sys_reg(3, 4, 1, 1, 2) +#define CPTR_EL2_SMEN GENMASK(25, 24) +#define CPTR_EL2_SMEN_NONE (0x3UL << 24) +#define CPTR_EL2_FPEN GENMASK(21, 20) +#define CPTR_EL2_FPEN_NONE (0x3UL << 20) +#define CPTR_EL2_ZEN GENMASK(17, 16) +#define CPTR_EL2_ZEN_NONE (0x3UL << 16) + +#define SYS_HFGRTR_EL2 sys_reg(3, 4, 1, 1, 4) +#define SYS_HFGWTR_EL2 sys_reg(3, 4, 1, 1, 5) +#define HFGxTR_EL2_nTPIDR2_EL0 BIT(55) +#define HFGxTR_EL2_nSMPRI_EL1 BIT(54) + +#define SYS_SMPRIMAP_EL2 sys_reg(3, 4, 1, 2, 5) #define ID_AA64MMFR0_TGran4 GENMASK(31, 28) #define ID_AA64MMFR0_TGran64 GENMASK(27, 24) #define ID_AA64MMFR0_TGran16 GENMASK(23, 20) diff --git a/src/chickens.c b/src/chickens.c index 0cf98c48d..0041c9c22 100644 --- a/src/chickens.c +++ b/src/chickens.c @@ -113,6 +113,7 @@ const struct midr_part_features features_m4 = { .sleep_mode = SLEEP_NONE, // XXX probably new mode required .fast_ipi = true, .actlr_el2 = true, + .sme_enabled = true, }; /* diff --git a/src/hv.c b/src/hv.c index 04b47f92f..7acc8fc4c 100644 --- a/src/hv.c +++ b/src/hv.c @@ -96,6 +96,25 @@ static void hv_restore_guest_el12_state(const struct hv_secondary_info_t *info) sysop("isb"); } +static void hv_config_sme(bool verbose) +{ + if (!cpu_features->sme_enabled) + return; + + reg_mask(SYS_CPTR_EL2, CPTR_EL2_SMEN | CPTR_EL2_FPEN | CPTR_EL2_ZEN, + CPTR_EL2_SMEN_NONE | CPTR_EL2_FPEN_NONE | CPTR_EL2_ZEN_NONE); + + if (FIELD_GET(ID_AA64MMFR0_FGT, mrs(ID_AA64MMFR0_EL1))) { + reg_set(SYS_HFGRTR_EL2, HFGxTR_EL2_nTPIDR2_EL0 | HFGxTR_EL2_nSMPRI_EL1); + reg_set(SYS_HFGWTR_EL2, HFGxTR_EL2_nTPIDR2_EL0 | HFGxTR_EL2_nSMPRI_EL1); + } + + msr(SYS_SMPRIMAP_EL2, 0); + + if (verbose) + printf("HV: SME enabled (CPTR_EL2=%lx)\n", mrs(SYS_CPTR_EL2)); +} + void hv_init(void) { pcie_shutdown(); @@ -118,6 +137,7 @@ void hv_init(void) HCR_RW | // AArch64 guest HCR_AMO | // Trap SError exceptions HCR_VM); // Enable stage 2 translation + hv_config_sme(true); // No guest vectors initially msr(VBAR_EL12, 0); @@ -249,6 +269,7 @@ static void hv_init_secondary(struct hv_secondary_info_t *info) msr(VBAR_EL1, _hv_vectors_start); msr(HCR_EL2, info->hcr); + hv_config_sme(false); msr(HACR_EL2, info->hacr); msr(VTCR_EL2, info->vtcr); msr(VTTBR_EL2, info->vttbr); diff --git a/src/utils.h b/src/utils.h index a7bf86aac..ff3f36bb0 100644 --- a/src/utils.h +++ b/src/utils.h @@ -510,6 +510,7 @@ struct midr_part_features { bool amx; bool actlr_el2; bool counter_redirect; + bool sme_enabled; }; extern bool is_mac; From 10d30d56a1c1fa8e71ebda7ab2b8c69aac634d99 Mon Sep 17 00:00:00 2001 From: Cody Ho Date: Thu, 9 Jul 2026 17:11:04 -0700 Subject: [PATCH 23/33] hv: add M4 XNU load hook patches Signed-off-by: Cody Ho --- proxyclient/m1n1/hv/__init__.py | 76 ++++++++++++++++++++++++++++++++- 1 file changed, 75 insertions(+), 1 deletion(-) diff --git a/proxyclient/m1n1/hv/__init__.py b/proxyclient/m1n1/hv/__init__.py index d662f40b4..d877303cd 100644 --- a/proxyclient/m1n1/hv/__init__.py +++ b/proxyclient/m1n1/hv/__init__.py @@ -1915,10 +1915,84 @@ def load_hook_m3(data, segname, size, fileoff, dest): print("Done.") return a.tobytes() - #image = macho.prepare_image(load_hook) + def load_hook_m4(data, segname, size, fileoff, dest): + if segname not in ("__TEXT_EXEC", "__TEXT_BOOT_EXEC"): + return data + + nop = 0xd503201f + print(f"Patching segment {segname}...") + + a = array.array("I", data) + seg_vmaddr = macho.vmin + dest + + # genter -> HVC to allow us to catch SPTM entry calls + p = 0 + while (p := data.find(b"\x20\x00", p)) != -1: + if (p & 3) != 2: + p += 1 + continue + opcode = a[p // 4] + if 0x00201420 <= opcode <= 0x00201424: # genter #imm (imm 0..4) + a[p // 4] = self.hvc(0) + p += 4 + + # sysreg neutralization: NOP writes to guarded impdef regs and + # SME control register + nop_msr = [ + 0xd51cf100, # msr KERNKEYLO_EL1 (s3_4_c15_c1_0) + 0xd51cf120, # msr KERNKEYHI_EL1 (s3_4_c15_c1_1) + 0xd51cfdc0, # msr s3_4_c15_c13_6 (Apple PAC key) + 0xd51cfde0, # msr s3_4_c15_c13_7 (Apple PAC key) + 0xd51c12a0, # msr SMPRIMAP_EL2 (s3_4_c1_c2_5) + ] + p = 0 + while (p := data.find(b"\x1c\xd5", p)) != -1: + if (p & 3) != 2: + p += 1 + continue + opcode = a[p // 4] & ~0x1f + if opcode in nop_msr: + if opcode == 0xd51c12a0: + print(f" 0x{seg_vmaddr + (p & ~3):x}: SMPRIMAP_EL2 -> noop") + a[p // 4] = nop + p += 4 + + # commpage: force XNU to publish VM-safe userspace feature policy + # bytes. The Apple timebase path and HW-TPRO/SPRR path use + # userspace-inaccessible Apple registers under this EL1 guest. + # + # hard coded kernelcache addresses is a hack we should find a way + # to avoid + off = 0xfffffe000b5a3330 - seg_vmaddr + if 0 <= off < size: + if a[off // 4] != 0x52800068: + raise RuntimeError("_COMM_PAGE_USER_TIMEBASE patch target mismatch") + a[off // 4] = 0x52800008 + print(f" 0xfffffe000b5a3330: _COMM_PAGE_USER_TIMEBASE=0") + + off = 0xfffffe000b5a3388 - seg_vmaddr + if 0 <= off < size: + if a[off // 4] != 0x52800028: + raise RuntimeError("_COMM_PAGE_CONT_HWCLOCK patch target mismatch") + a[off // 4] = 0x52800008 + print(f" 0xfffffe000b5a3388: _COMM_PAGE_CONT_HWCLOCK=0") + + off = 0xfffffe000b5a358c - seg_vmaddr + if 0 <= off < size: + if a[off // 4] != 0x39043128: + raise RuntimeError("_COMM_PAGE_HW_TPRO patch target mismatch") + a[off // 4] = 0x3904313f + print(f" 0xfffffe000b5a358c: _COMM_PAGE_HW_TPRO=0") + + print("Done.") + return a.tobytes() + chip_id = self.u.adt["/chosen"].chip_id if chip_id in (0x8122, 0x6030, 0x6031, 0x6032, 0x6034): image = macho.prepare_image(load_hook_m3) + elif chip_id in (0x8132, 0x8140, 0x6040, 0x6041): + # M4 / A18 Pro class: T8132, T8140, T6040 (M4 Pro), T6041 (M4 Max) + image = macho.prepare_image(load_hook_m4) else: image = macho.prepare_image() self.load_raw(image, entryoffset=(macho.entry - macho.vmin), use_xnu_symbols=self.xnu_mode and symfile is not None, vmin=macho.vmin) From f58c9ead5940ccf4dfee0b8b5fb77d22847ddc94 Mon Sep 17 00:00:00 2001 From: Cody Ho Date: Wed, 8 Jul 2026 19:19:16 -0700 Subject: [PATCH 24/33] hv: trap Apple impdef sysregs (HCR_EL2.TIDCP) on guarded SoCs On M4-class (guarded) SoCs the Apple impdef system registers are GL2-only and fault when executed from EL2, so the HV must intercept the guest's EL1 accesses and soft-model/shadow them (the KERNEL_CNTV timer and the SYSREG_SHADOW bank). Set HCR_EL2.TIDCP so those accesses trap to EL2. Gate it on !apple_sysregs_unlocked: on unlocked SoCs the guest accesses these registers directly at EL1, and trapping them would regress existing (M1-M3) virtualization. --- src/hv.c | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/src/hv.c b/src/hv.c index 7acc8fc4c..89f7ea464 100644 --- a/src/hv.c +++ b/src/hv.c @@ -130,13 +130,25 @@ void hv_init(void) hv_pt_init(); // Configure hypervisor defaults - hv_write_hcr(HCR_API | // Allow PAuth instructions - HCR_APK | // Allow PAuth key registers - HCR_TEA | // Trap external aborts - HCR_E2H | // VHE mode (forced) - HCR_RW | // AArch64 guest - HCR_AMO | // Trap SError exceptions - HCR_VM); // Enable stage 2 translation + u64 hcr = HCR_API | // Allow PAuth instructions + HCR_APK | // Allow PAuth key registers + HCR_TEA | // Trap external aborts + HCR_E2H | // VHE mode (forced) + HCR_RW | // AArch64 guest + HCR_AMO | // Trap SError exceptions + HCR_VM; // Enable stage 2 translation + + /* + * On guarded (M4-class) SoCs the Apple impdef sysregs are GL2-only and fault + * when executed from EL2, so trap the guest's EL1 accesses to us and soft- + * model/shadow them. Do NOT set TIDCP on unlocked SoCs: there the guest + * accesses these registers directly at EL1 and trapping them would regress + * existing (M1-M3) virtualization. + */ + if (!cpu_features->apple_sysregs_unlocked) + hcr |= HCR_TIDCP; + + hv_write_hcr(hcr); hv_config_sme(true); // No guest vectors initially From 83e1e01ecdedb65eec8d0c285e0844ed281e633a Mon Sep 17 00:00:00 2001 From: Cody Ho Date: Wed, 8 Jul 2026 19:02:27 -0700 Subject: [PATCH 25/33] hv: add SYSREG_SHADOW helper Alongside SYSREG_MAP and SYSREG_PASS, add SYSREG_SHADOW(sr, store): return the last value the guest wrote (or a seeded reset default on first read) and swallow the write, dropping the register hardware side effect. This is for guarded Apple impdef registers that fault at EL2 but whose value XNU reads back (RMW or context save/restore) Signed-off-by: Cody Ho --- src/hv_exc.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/hv_exc.c b/src/hv_exc.c index d4dd86d29..8704f6403 100644 --- a/src/hv_exc.c +++ b/src/hv_exc.c @@ -203,6 +203,21 @@ static void hv_update_fiq(void) _msr(sr_tkn(sr), regs[rt]); \ return true; +/* + * Soft-cache a guarded Apple impdef register: return the last value the guest + * wrote (or the seeded reset default on first read) and swallow the write. + * This drops the register's hardware side effect entirely. `store` is any + * lvalue: PERCPU(field) for per-CPU state, or a static global seeded with the + * reset value the guest expects to read back. + */ +#define SYSREG_SHADOW(sr, store) \ + case SYSREG_ISS(sr): \ + if (is_read) \ + regs[rt] = (store); \ + else \ + (store) = regs[rt]; \ + return true; + static bool hv_handle_msr_unlocked(struct exc_info *ctx, u64 iss) { u64 reg = iss & (ESR_ISS_MSR_OP0 | ESR_ISS_MSR_OP2 | ESR_ISS_MSR_OP1 | ESR_ISS_MSR_CRn | From b4892d59699eb52cba4bea6d7b0ebc5b828d015e Mon Sep 17 00:00:00 2001 From: Cody Ho Date: Wed, 8 Jul 2026 18:50:48 -0700 Subject: [PATCH 26/33] hv: soft-model the Apple KERNEL_CNTV kernel timer XNU on Apple Silicon schedules off Apple's impdef KERNEL_CNTV_* timer, not the architectural timer that m1n1 current virtualizes; its CTL/TVAL registers are write-locked at EL2 on M4-class SoCs, so they can't be passed through to hardware. Soft-model it: trap KERNEL_CNTV_CTL/TVAL, keep the deadline and ctl in a per-CPU shadow, read "now" from the counter (KERNEL_CNTVCTSS aliases the already-passed-through CNTVCT_ALIAS), and assert a virtual FIQ from hv_update_fiq() once the soft deadline elapses. Delivery currently rides the periodic HV tick; arming the tick to the exact deadline is a latency refinement. Signed-off-by: Cody Ho --- src/cpu_regs.h | 5 +++ src/hv_exc.c | 118 +++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 119 insertions(+), 4 deletions(-) diff --git a/src/cpu_regs.h b/src/cpu_regs.h index f40cc8a9e..0ba5b9073 100644 --- a/src/cpu_regs.h +++ b/src/cpu_regs.h @@ -19,6 +19,11 @@ #define AMX_CTL_EN_EL1 BIT(62) #define SYS_IMP_APL_CNTVCT_ALIAS_EL0 sys_reg(3, 4, 15, 10, 6) +/* Apple kernel deadline timer (XNU's local scheduler timer on Apple Silicon). + * CNTVCTSS aliases CNTVCT_ALIAS above (same encoding). */ +#define SYS_IMP_APL_KERNEL_CNTV_CTL_EL0 sys_reg(3, 1, 15, 0, 5) +#define SYS_IMP_APL_KERNEL_CNTV_TVAL_EL0 sys_reg(3, 1, 15, 15, 4) +#define SYS_IMP_APL_KERNEL_CNTVCT_EL0 sys_reg(3, 4, 15, 11, 7) /* HID registers */ #define SYS_IMP_APL_HID0 sys_reg(3, 0, 15, 0, 0) diff --git a/src/hv_exc.c b/src/hv_exc.c index 8704f6403..f2baf8324 100644 --- a/src/hv_exc.c +++ b/src/hv_exc.c @@ -27,13 +27,96 @@ struct hv_pcpu_data { u32 pmc_pending; u64 pmc_irq_mode; u64 exc_entry_pmcr0_cnt; + u64 kernel_cntv_cval; + u64 kernel_cntv_ctl; + bool kernel_cntv_valid; } ALIGNED(64); struct hv_pcpu_data pcpu[MAX_CPUS]; +/* + * Track EL2/proxy time hidden from the guest. The architectural counter and + * Apple CNTVCT alias run in different clock domains, so each needs its own + * accumulator. + */ +static u64 stolen_time = 0; +static u64 stolen_time_kernel_cntv = 0; + +/* + * M4-class SoCs schedule based off an Apple impdef timer that is locked in raw + * boot mode. We soft-model it: keep the deadline in a per-CPU shadow, read + * "now" from a counter (KERNEL_CNTVCTSS == CNTVCT_ALIAS, readable and + * already passed through), and deliver the timer as a virtual FIQ from + * hv_update_fiq() once the soft deadline elapses. + */ +static inline u64 hv_kernel_cntv_now(void) +{ + /* + * CNTVOFF_EL2 only offsets the architectural virtual counter. XNU's Apple + * kernel deadline timer reads CNTVCT_ALIAS instead, so track stolen time for + * that timer separately in CNTVCT_ALIAS units and subtract it here. + */ + return mrs(SYS_IMP_APL_CNTVCT_ALIAS_EL0) - stolen_time_kernel_cntv; +} + +static inline s64 hv_kernel_cntv_delta(void) +{ + return (s64)(PERCPU(kernel_cntv_cval) - hv_kernel_cntv_now()); +} + +static void hv_kernel_cntv_init(void) +{ + if (PERCPU(kernel_cntv_valid)) + return; + + /* CTL/TVAL are readable at EL2; capture XNU's initial deadline once. */ + u32 tval = (u32)mrs(SYS_IMP_APL_KERNEL_CNTV_TVAL_EL0); + PERCPU(kernel_cntv_ctl) = + mrs(SYS_IMP_APL_KERNEL_CNTV_CTL_EL0) & (CNTx_CTL_ENABLE | CNTx_CTL_IMASK); + PERCPU(kernel_cntv_cval) = hv_kernel_cntv_now() + (s64)(s32)tval; + PERCPU(kernel_cntv_valid) = true; +} + +static u64 hv_kernel_cntv_read_ctl(void) +{ + hv_kernel_cntv_init(); + + u64 ctl = PERCPU(kernel_cntv_ctl) & (CNTx_CTL_ENABLE | CNTx_CTL_IMASK); + if ((ctl & CNTx_CTL_ENABLE) && hv_kernel_cntv_delta() <= 0) + ctl |= CNTx_CTL_ISTATUS; + return ctl; +} + +static u64 hv_kernel_cntv_read_tval(void) +{ + hv_kernel_cntv_init(); + return (u32)hv_kernel_cntv_delta(); +} + +static void hv_kernel_cntv_write_ctl(u64 val) +{ + hv_kernel_cntv_init(); + PERCPU(kernel_cntv_ctl) = val & (CNTx_CTL_ENABLE | CNTx_CTL_IMASK); +} + +static void hv_kernel_cntv_write_tval(u64 val) +{ + hv_kernel_cntv_init(); + PERCPU(kernel_cntv_cval) = hv_kernel_cntv_now() + (s64)(s32)(u32)val; +} + +static bool hv_kernel_cntv_pending(void) +{ + if (!PERCPU(kernel_cntv_valid)) + return false; + + u64 ctl = hv_kernel_cntv_read_ctl(); + return (ctl & (CNTx_CTL_ISTATUS | CNTx_CTL_IMASK | CNTx_CTL_ENABLE)) == + (CNTx_CTL_ISTATUS | CNTx_CTL_ENABLE); +} + void hv_exit_guest(void) __attribute__((noreturn)); -static u64 stolen_time = 0; static u64 exc_entry_time; extern u64 hv_cpus_in_guest; @@ -57,6 +140,7 @@ static void _hv_exc_proxy(struct exc_info *ctx, uartproxy_boot_reason_t reason, hv_rendezvous(); u64 entry_time = mrs(CNTPCT_EL0); + u64 kernel_cntv_entry_time = mrs(SYS_IMP_APL_CNTVCT_ALIAS_EL0); ctx->elr_phys = hv_translate(ctx->elr, false, false, NULL); ctx->far_phys = hv_translate(ctx->far, false, false, NULL); @@ -77,8 +161,9 @@ static void _hv_exc_proxy(struct exc_info *ctx, uartproxy_boot_reason_t reason, case EXC_RET_HANDLED: hv_wdt_breadcrumb('p'); if (time_stealing) { - u64 lost = mrs(CNTPCT_EL0) - entry_time; - stolen_time += lost; + stolen_time += mrs(CNTPCT_EL0) - entry_time; + stolen_time_kernel_cntv += mrs(SYS_IMP_APL_CNTVCT_ALIAS_EL0) - + kernel_cntv_entry_time; } break; case EXC_EXIT_GUEST: @@ -142,8 +227,10 @@ void hv_exc_proxy(struct exc_info *ctx, uartproxy_boot_reason_t reason, u32 type void hv_set_time_stealing(bool enabled, bool reset) { time_stealing = enabled; - if (reset) + if (reset) { stolen_time = 0; + stolen_time_kernel_cntv = 0; + } hv_apply_time_stealing_offset(); } @@ -154,7 +241,14 @@ void hv_apply_time_stealing_offset(void) void hv_add_time(s64 time) { + u64 ticks = time < 0 ? (u64)-time : (u64)time; + u64 kernel_ticks = (u64)((__uint128_t)ticks * 24000000ULL / mrs(CNTFRQ_EL0)); + stolen_time -= (u64)time; + if (time < 0) + stolen_time_kernel_cntv += kernel_ticks; + else + stolen_time_kernel_cntv -= kernel_ticks; } static void hv_update_fiq(void) @@ -177,6 +271,7 @@ static void hv_update_fiq(void) } fiq_pending |= PERCPU(ipi_pending) || PERCPU(pmc_pending); + fiq_pending |= hv_kernel_cntv_pending(); sysop("isb"); @@ -239,6 +334,21 @@ static bool hv_handle_msr_unlocked(struct exc_info *ctx, u64 iss) SYSREG_MAP(SYS_CNTP_CTL_EL0, SYS_CNTP_CTL_EL02) SYSREG_MAP(SYS_CNTP_CVAL_EL0, SYS_CNTP_CVAL_EL02) SYSREG_MAP(SYS_CNTP_TVAL_EL0, SYS_CNTP_TVAL_EL02) + /* Apple kernel deadline timer that we shadow soft-modeled, CTL/TVAL + * are write-locked at EL2; the counters read through. */ + case SYSREG_ISS(SYS_IMP_APL_KERNEL_CNTV_CTL_EL0): + if (is_read) + regs[rt] = hv_kernel_cntv_read_ctl(); + else + hv_kernel_cntv_write_ctl(regs[rt]); + return true; + case SYSREG_ISS(SYS_IMP_APL_KERNEL_CNTV_TVAL_EL0): + if (is_read) + regs[rt] = hv_kernel_cntv_read_tval(); + else + hv_kernel_cntv_write_tval(regs[rt]); + return true; + SYSREG_PASS(SYS_IMP_APL_KERNEL_CNTVCT_EL0) /* Spammy stuff seen on t600x p-cores */ /* These are PMU/PMC registers */ SYSREG_PASS(sys_reg(3, 2, 15, 12, 0)); From 315f9f3fb0b6b3f08b10c55320918a281c4214a1 Mon Sep 17 00:00:00 2001 From: Cody Ho Date: Fri, 10 Jul 2026 02:51:40 -0700 Subject: [PATCH 27/33] hv: virtualize Apple kernel CNTKCTL Signed-off-by: Cody Ho --- src/cpu_regs.h | 1 + src/hv_exc.c | 29 +++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/src/cpu_regs.h b/src/cpu_regs.h index 0ba5b9073..980b01d69 100644 --- a/src/cpu_regs.h +++ b/src/cpu_regs.h @@ -24,6 +24,7 @@ #define SYS_IMP_APL_KERNEL_CNTV_CTL_EL0 sys_reg(3, 1, 15, 0, 5) #define SYS_IMP_APL_KERNEL_CNTV_TVAL_EL0 sys_reg(3, 1, 15, 15, 4) #define SYS_IMP_APL_KERNEL_CNTVCT_EL0 sys_reg(3, 4, 15, 11, 7) +#define SYS_IMP_APL_KERNEL_CNTKCTL_EL1 sys_reg(3, 4, 15, 9, 6) /* HID registers */ #define SYS_IMP_APL_HID0 sys_reg(3, 0, 15, 0, 0) diff --git a/src/hv_exc.c b/src/hv_exc.c index f2baf8324..fbceff208 100644 --- a/src/hv_exc.c +++ b/src/hv_exc.c @@ -29,6 +29,7 @@ struct hv_pcpu_data { u64 exc_entry_pmcr0_cnt; u64 kernel_cntv_cval; u64 kernel_cntv_ctl; + u64 kernel_cntkctl_el1; bool kernel_cntv_valid; } ALIGNED(64); @@ -115,6 +116,32 @@ static bool hv_kernel_cntv_pending(void) (CNTx_CTL_ISTATUS | CNTx_CTL_ENABLE); } +#define KERNEL_CNTKCTL_CNTHCTL_MASK \ + (CNTHCTL_EL0PCTEN | CNTHCTL_EL0VCTEN | CNTHCTL_EVNTI | CNTHCTL_EVNTDIR | CNTHCTL_EVNTEN) + +static bool hv_handle_kernel_cntkctl(struct exc_info *ctx, u64 rt, bool is_read) +{ + if (is_read) { + if (rt < 31) + ctx->regs[rt] = PERCPU(kernel_cntkctl_el1); + return true; + } + + u64 val = rt < 31 ? ctx->regs[rt] : 0; + PERCPU(kernel_cntkctl_el1) = val; + + /* + * XNU programs WFE timeout events and EL0 timebase access through this + * Apple alias. On guarded SoCs, direct EL2 writes to the Apple register can + * fault, so mirror only the architectural CNTHCTL_EL2 bits that correspond + * to the guest-visible Apple control state. + */ + msr(SYS_CNTHCTL_EL2, (mrs(SYS_CNTHCTL_EL2) & ~KERNEL_CNTKCTL_CNTHCTL_MASK) | + (val & KERNEL_CNTKCTL_CNTHCTL_MASK)); + sysop("isb"); + return true; +} + void hv_exit_guest(void) __attribute__((noreturn)); static u64 exc_entry_time; @@ -348,6 +375,8 @@ static bool hv_handle_msr_unlocked(struct exc_info *ctx, u64 iss) else hv_kernel_cntv_write_tval(regs[rt]); return true; + case SYSREG_ISS(SYS_IMP_APL_KERNEL_CNTKCTL_EL1): + return hv_handle_kernel_cntkctl(ctx, rt, is_read); SYSREG_PASS(SYS_IMP_APL_KERNEL_CNTVCT_EL0) /* Spammy stuff seen on t600x p-cores */ /* These are PMU/PMC registers */ From cc3c2be097743e026779eaaaf6ea408739250cb8 Mon Sep 17 00:00:00 2001 From: Cody Ho Date: Fri, 10 Jul 2026 02:57:25 -0700 Subject: [PATCH 28/33] hv: soft-cache guarded impdef round-trip sysregs XNU round-trips several Apple impdef registers through RMW and context save/restore paths, but m1n1 does not consume their state. On guarded SoCs those accesses trap to EL2, so keep per-CPU shadows to preserve guest read-after-write behavior. Signed-off-by: Cody Ho --- src/cpu_regs.h | 8 ++++++++ src/hv_exc.c | 22 ++++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/src/cpu_regs.h b/src/cpu_regs.h index 980b01d69..ad5f9f4f9 100644 --- a/src/cpu_regs.h +++ b/src/cpu_regs.h @@ -25,6 +25,7 @@ #define SYS_IMP_APL_KERNEL_CNTV_TVAL_EL0 sys_reg(3, 1, 15, 15, 4) #define SYS_IMP_APL_KERNEL_CNTVCT_EL0 sys_reg(3, 4, 15, 11, 7) #define SYS_IMP_APL_KERNEL_CNTKCTL_EL1 sys_reg(3, 4, 15, 9, 6) +#define SYS_IMP_APL_JCTL_EL0 sys_reg(3, 4, 15, 15, 6) /* HID registers */ #define SYS_IMP_APL_HID0 sys_reg(3, 0, 15, 0, 0) @@ -468,6 +469,13 @@ #define AGTCNTRDIR_DISABLE BIT(1) #define AGTCNTRDIR_EL0_TRAP_CTL BIT(0) +/* Impdef state XNU round-trips, locked on M4+ */ +#define SYS_IMP_APL_WATCHDOGDIAG0 sys_reg(3, 5, 15, 2, 6) +#define SYS_IMP_APL_S3_1_C15_C13_4 sys_reg(3, 1, 15, 13, 4) +#define SYS_IMP_APL_S3_6_C15_C0_4 sys_reg(3, 6, 15, 0, 4) +#define SYS_IMP_APL_S3_6_C15_C0_5 sys_reg(3, 6, 15, 0, 5) +#define SYS_IMP_APL_S3_4_C15_C12_0 sys_reg(3, 4, 15, 12, 0) + // Performance monitor registers #define SYS_IMP_APL_PMCR0 sys_reg(3, 1, 15, 0, 0) #define PMCR0_CNT_EN_MASK (MASK(8) | GENMASK(33, 32)) diff --git a/src/hv_exc.c b/src/hv_exc.c index fbceff208..876469bde 100644 --- a/src/hv_exc.c +++ b/src/hv_exc.c @@ -31,6 +31,16 @@ struct hv_pcpu_data { u64 kernel_cntv_ctl; u64 kernel_cntkctl_el1; bool kernel_cntv_valid; + /* Soft-cached guarded impdef registers (see SYSREG_SHADOW users below). */ + u64 agtcntrdir_el1; + u64 agtcntrdir_el12; + u64 siq_cfg_el1; + u64 jctl_el0; + u64 watchdogdiag0; + u64 impdef_c13_4; + u64 impdef_c15_c0_4; + u64 impdef_c15_c0_5; + u64 impdef_c12_0; } ALIGNED(64); struct hv_pcpu_data pcpu[MAX_CPUS]; @@ -400,6 +410,18 @@ static bool hv_handle_msr_unlocked(struct exc_info *ctx, u64 iss) SYSREG_MAP(SYS_IMP_APL_ELR_GL1, SYS_IMP_APL_ELR_GL12) SYSREG_MAP(SYS_IMP_APL_ESR_GL1, SYS_IMP_APL_ESR_GL12) SYSREG_MAP(SYS_IMP_APL_SPRR_PERM_EL1, SYS_IMP_APL_SPRR_PERM_EL12) + /* + * Impdef sysregs locked on M4+ that are required by XNU + */ + SYSREG_SHADOW(SYS_IMP_APL_AGTCNTRDIR_EL1, PERCPU(agtcntrdir_el1)) + SYSREG_SHADOW(SYS_IMP_APL_AGTCNTRDIR_EL12, PERCPU(agtcntrdir_el12)) + SYSREG_SHADOW(SYS_IMP_APL_SIQ_CFG_EL1, PERCPU(siq_cfg_el1)) + SYSREG_SHADOW(SYS_IMP_APL_JCTL_EL0, PERCPU(jctl_el0)) + SYSREG_SHADOW(SYS_IMP_APL_WATCHDOGDIAG0, PERCPU(watchdogdiag0)) + SYSREG_SHADOW(SYS_IMP_APL_S3_1_C15_C13_4, PERCPU(impdef_c13_4)) + SYSREG_SHADOW(SYS_IMP_APL_S3_6_C15_C0_4, PERCPU(impdef_c15_c0_4)) + SYSREG_SHADOW(SYS_IMP_APL_S3_6_C15_C0_5, PERCPU(impdef_c15_c0_5)) + SYSREG_SHADOW(SYS_IMP_APL_S3_4_C15_C12_0, PERCPU(impdef_c12_0)) SYSREG_MAP(SYS_IMP_APL_APCTL_EL1, SYS_IMP_APL_APCTL_EL12) SYSREG_MAP(SYS_IMP_APL_AMX_CTL_EL1, SYS_IMP_APL_AMX_CTL_EL12) /* FIXME:Might be wrong */ From 401793120cd743c4169bb75cd0915b22fe8716b1 Mon Sep 17 00:00:00 2001 From: Cody Ho Date: Fri, 10 Jul 2026 03:05:26 -0700 Subject: [PATCH 29/33] hv: virtualize guarded AMX sysregs XNU still probes and context-switches Apple AMX state on M4-class SoCs even though userspace uses SME there. The AMX control/state registers fault at EL2 on guarded systems, so preserve guest read-after-write behavior with per-CPU shadows while keeping the existing real-register path on unlocked SoCs. Return a valid AMXIDR value on guarded systems so XNU accepts the AMX version probe during boot. Signed-off-by: Cody Ho --- src/cpu_regs.h | 2 ++ src/hv_exc.c | 47 ++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/src/cpu_regs.h b/src/cpu_regs.h index ad5f9f4f9..449a3ea5c 100644 --- a/src/cpu_regs.h +++ b/src/cpu_regs.h @@ -14,6 +14,8 @@ #define SYS_IMP_APL_AMX_CTL_EL1 sys_reg(3, 4, 15, 1, 4) #define SYS_IMP_APL_AMX_CTL_EL2 sys_reg(3, 4, 15, 4, 7) #define SYS_IMP_APL_AMX_CTL_EL12 sys_reg(3, 4, 15, 4, 6) +/* AMX/AIDR2 version-ID probe; XNU boot-panics if it reads an unknown value. */ +#define SYS_IMP_APL_AMXIDR_EL1 sys_reg(3, 6, 15, 2, 7) #define AMX_CTL_EN BIT(63) #define AMX_CTL_EN_EL1 BIT(62) diff --git a/src/hv_exc.c b/src/hv_exc.c index 876469bde..6aba1de32 100644 --- a/src/hv_exc.c +++ b/src/hv_exc.c @@ -41,6 +41,9 @@ struct hv_pcpu_data { u64 impdef_c15_c0_4; u64 impdef_c15_c0_5; u64 impdef_c12_0; + u64 amx_ctx_el1; + u64 amx_ctl_el1; + u64 amx_state_t; } ALIGNED(64); struct hv_pcpu_data pcpu[MAX_CPUS]; @@ -423,9 +426,47 @@ static bool hv_handle_msr_unlocked(struct exc_info *ctx, u64 iss) SYSREG_SHADOW(SYS_IMP_APL_S3_6_C15_C0_5, PERCPU(impdef_c15_c0_5)) SYSREG_SHADOW(SYS_IMP_APL_S3_4_C15_C12_0, PERCPU(impdef_c12_0)) SYSREG_MAP(SYS_IMP_APL_APCTL_EL1, SYS_IMP_APL_APCTL_EL12) - SYSREG_MAP(SYS_IMP_APL_AMX_CTL_EL1, SYS_IMP_APL_AMX_CTL_EL12) - /* FIXME:Might be wrong */ - SYSREG_PASS(SYS_IMP_APL_AMX_STATE_T) + /* + * AMX. On M4 userspace uses SME, so AMX is inert, but XNU's per-CPU + * init and context switch still touch these regs, which fault at EL2 on + * guarded SoCs. Advertise a valid version (AMXIDR/AIDR2, else boot + * panics) and soft-cache the control/state on guarded SoCs so XNU's + * read-after-write is coherent; the real AMX unit is never used. Unlocked + * SoCs keep the original map/pass behavior. + */ + case SYSREG_ISS(SYS_IMP_APL_AMXIDR_EL1): + if (!is_read) + return false; + if (cpu_features->apple_sysregs_unlocked) + regs[rt] = _mrs(sr_tkn(SYS_IMP_APL_AMXIDR_EL1)); + else + regs[rt] = BIT(5); /* AMX v6 */ + return true; + SYSREG_SHADOW(SYS_IMP_APL_AMX_CTX_EL1, PERCPU(amx_ctx_el1)) + case SYSREG_ISS(SYS_IMP_APL_AMX_CTL_EL1): + if (cpu_features->apple_sysregs_unlocked) { + if (is_read) + regs[rt] = _mrs(sr_tkn(SYS_IMP_APL_AMX_CTL_EL12)); + else + _msr(sr_tkn(SYS_IMP_APL_AMX_CTL_EL12), regs[rt]); + } else if (is_read) { + regs[rt] = PERCPU(amx_ctl_el1); + } else { + PERCPU(amx_ctl_el1) = regs[rt]; + } + return true; + case SYSREG_ISS(SYS_IMP_APL_AMX_STATE_T): + if (cpu_features->apple_sysregs_unlocked) { + if (is_read) + regs[rt] = _mrs(sr_tkn(SYS_IMP_APL_AMX_STATE_T)); + else + _msr(sr_tkn(SYS_IMP_APL_AMX_STATE_T), regs[rt]); + } else if (is_read) { + regs[rt] = PERCPU(amx_state_t); + } else { + PERCPU(amx_state_t) = regs[rt]; + } + return true; /* pass through PMU handling */ SYSREG_PASS(SYS_IMP_APL_PMCR1) SYSREG_PASS(SYS_IMP_APL_PMCR2) From f4003b74891b8ae76d9ff1e9a68d4f1c1bcc06d2 Mon Sep 17 00:00:00 2001 From: Cody Ho Date: Thu, 9 Jul 2026 03:52:07 -0700 Subject: [PATCH 30/33] hv: shadow T8140 SPRR_UMPRR_EL1 Signed-off-by: Cody Ho --- src/cpu_regs.h | 1 + src/hv_exc.c | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/src/cpu_regs.h b/src/cpu_regs.h index 449a3ea5c..1adb16eab 100644 --- a/src/cpu_regs.h +++ b/src/cpu_regs.h @@ -730,6 +730,7 @@ #define SYS_IMP_APL_SPRR_PERM_EL1 sys_reg(3, 6, 15, 1, 6) #define SYS_IMP_APL_SPRR_PERM_EL02 sys_reg(3, 4, 15, 5, 2) #define SYS_IMP_APL_SPRR_PERM_EL12 sys_reg(3, 6, 15, 15, 7) +#define SYS_IMP_APL_SPRR_UMPRR_EL1 sys_reg(3, 6, 15, 3, 0) #define SYS_IMP_APL_TPIDR_GL1 sys_reg(3, 6, 15, 10, 1) #define SYS_IMP_APL_VBAR_GL1 sys_reg(3, 6, 15, 10, 2) diff --git a/src/hv_exc.c b/src/hv_exc.c index 6aba1de32..188167cce 100644 --- a/src/hv_exc.c +++ b/src/hv_exc.c @@ -48,6 +48,8 @@ struct hv_pcpu_data { struct hv_pcpu_data pcpu[MAX_CPUS]; +static u64 hv_sprr_umprr_el1; + /* * Track EL2/proxy time hidden from the guest. The architectural counter and * Apple CNTVCT alias run in different clock domains, so each needs its own @@ -413,6 +415,12 @@ static bool hv_handle_msr_unlocked(struct exc_info *ctx, u64 iss) SYSREG_MAP(SYS_IMP_APL_ELR_GL1, SYS_IMP_APL_ELR_GL12) SYSREG_MAP(SYS_IMP_APL_ESR_GL1, SYS_IMP_APL_ESR_GL12) SYSREG_MAP(SYS_IMP_APL_SPRR_PERM_EL1, SYS_IMP_APL_SPRR_PERM_EL12) + case SYSREG_ISS(SYS_IMP_APL_SPRR_UMPRR_EL1): + if (is_read) + regs[rt] = hv_sprr_umprr_el1 & 0xffffffff; + else + hv_sprr_umprr_el1 = regs[rt] & 0xffffffff; + return true; /* * Impdef sysregs locked on M4+ that are required by XNU */ From 234f91c93db7d19c2a4f0b04f70755d1bd9b7846 Mon Sep 17 00:00:00 2001 From: Cody Ho Date: Thu, 9 Jul 2026 04:33:41 -0700 Subject: [PATCH 31/33] hv: shadow guarded SPRR EL0 permissions SPRR registers are locked on M4+ SoCs. Seed and soft-cache SPRR_PERM_EL0 with the sane default values. Signed-off-by: Cody Ho --- src/hv_exc.c | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/src/hv_exc.c b/src/hv_exc.c index 188167cce..1e6ce218e 100644 --- a/src/hv_exc.c +++ b/src/hv_exc.c @@ -48,6 +48,12 @@ struct hv_pcpu_data { struct hv_pcpu_data pcpu[MAX_CPUS]; +/* + * SPRR permission-remap table, soft-cached on M4+ SoCs where the real register + * faults at EL2. Values taken from Sven's blog post which appear to work. + */ +static u64 hv_sprr_perm_el0 = 0x2010002030100000; +static u64 hv_sprr_perm_el1 = 0x2020a506f020f0e0; static u64 hv_sprr_umprr_el1; /* @@ -414,7 +420,29 @@ static bool hv_handle_msr_unlocked(struct exc_info *ctx, u64 iss) SYSREG_MAP(SYS_IMP_APL_ASPSR_GL1, SYS_IMP_APL_ASPSR_GL12) SYSREG_MAP(SYS_IMP_APL_ELR_GL1, SYS_IMP_APL_ELR_GL12) SYSREG_MAP(SYS_IMP_APL_ESR_GL1, SYS_IMP_APL_ESR_GL12) - SYSREG_MAP(SYS_IMP_APL_SPRR_PERM_EL1, SYS_IMP_APL_SPRR_PERM_EL12) + /* + * SPRR permission remapping. On unlocked SoCs, redirect to the EL12 + * alias as before; on M4+ SoCs the register faults at EL2, so + * soft-cache it + */ + case SYSREG_ISS(SYS_IMP_APL_SPRR_PERM_EL0): + if (is_read) + regs[rt] = hv_sprr_perm_el0; + else + hv_sprr_perm_el0 = regs[rt]; + return true; + case SYSREG_ISS(SYS_IMP_APL_SPRR_PERM_EL1): + if (cpu_features->apple_sysregs_unlocked) { + if (is_read) + regs[rt] = _mrs(sr_tkn(SYS_IMP_APL_SPRR_PERM_EL12)); + else + _msr(sr_tkn(SYS_IMP_APL_SPRR_PERM_EL12), regs[rt]); + } else if (is_read) { + regs[rt] = hv_sprr_perm_el1; + } else { + hv_sprr_perm_el1 = regs[rt]; + } + return true; case SYSREG_ISS(SYS_IMP_APL_SPRR_UMPRR_EL1): if (is_read) regs[rt] = hv_sprr_umprr_el1 & 0xffffffff; From 0556a0b65b0abce120332d0694aecdf791aa45b8 Mon Sep 17 00:00:00 2001 From: Cody Ho Date: Thu, 9 Jul 2026 04:29:48 -0700 Subject: [PATCH 32/33] hv: soft-model T8140 CLPC sampler registers XNU's AppleT8140CLPC (and presumably other SoCs) path accesses Apple-private sampler registers such as S3_1_C15_C0_3 which are locked on M4+ SoCs. Handle the previously proven CLPC sampler family in C: keep read-after-write coherent state for control registers and provide monotonic guest-visible counters for CLPC perf/accumulator reads. This avoids the Python/EL2 replay fallback without claiming to drive physical CLPC hardware. Signed-off-by: Cody Ho --- src/cpu_regs.h | 17 +++++++++ src/hv_exc.c | 100 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 117 insertions(+) diff --git a/src/cpu_regs.h b/src/cpu_regs.h index 1adb16eab..9169fb374 100644 --- a/src/cpu_regs.h +++ b/src/cpu_regs.h @@ -478,6 +478,23 @@ #define SYS_IMP_APL_S3_6_C15_C0_5 sys_reg(3, 6, 15, 0, 5) #define SYS_IMP_APL_S3_4_C15_C12_0 sys_reg(3, 4, 15, 12, 0) +/* T8140 CLPC sampler controls/counters. Live EL2 access faults in the SPTM path. */ +#define SYS_IMP_APL_CLPC_CORE_PERF0 sys_reg(3, 7, 15, 0, 3) +#define SYS_IMP_APL_CLPC_CORE_PERF1 sys_reg(3, 7, 15, 2, 3) +#define SYS_IMP_APL_CLPC_CORE_PERF2 sys_reg(3, 7, 15, 4, 3) +#define SYS_IMP_APL_CLPC_CORE_PERF_ENABLE0 sys_reg(3, 7, 15, 1, 3) +#define SYS_IMP_APL_CLPC_CORE_PERF_ENABLE1 sys_reg(3, 7, 15, 3, 3) +#define SYS_IMP_APL_CLPC_CORE_PERF_ENABLE2 sys_reg(3, 7, 15, 5, 3) +#define SYS_IMP_APL_CLPC_CORE_PERF_CTRL2 sys_reg(3, 7, 15, 5, 0) +#define SYS_IMP_APL_CLPC_CORE_ACC0 sys_reg(3, 1, 15, 8, 3) +#define SYS_IMP_APL_CLPC_CORE_ACC1 sys_reg(3, 1, 15, 9, 3) +#define SYS_IMP_APL_CLPC_CORE_ACC2 sys_reg(3, 1, 15, 10, 3) +#define SYS_IMP_APL_CLPC_CORE_ACC3 sys_reg(3, 1, 15, 11, 3) +#define SYS_IMP_APL_CLPC_CORE_ACC4 sys_reg(3, 1, 15, 12, 3) +#define SYS_IMP_APL_CLPC_MEM_STALL_CFG0 sys_reg(3, 1, 15, 0, 3) +#define SYS_IMP_APL_CLPC_MEM_STALL_CFG1 sys_reg(3, 1, 15, 1, 3) +#define SYS_IMP_APL_CLPC_MEM_STALL_CFG2 sys_reg(3, 1, 15, 2, 3) + // Performance monitor registers #define SYS_IMP_APL_PMCR0 sys_reg(3, 1, 15, 0, 0) #define PMCR0_CNT_EN_MASK (MASK(8) | GENMASK(33, 32)) diff --git a/src/hv_exc.c b/src/hv_exc.c index 1e6ce218e..b835cb513 100644 --- a/src/hv_exc.c +++ b/src/hv_exc.c @@ -48,6 +48,10 @@ struct hv_pcpu_data { struct hv_pcpu_data pcpu[MAX_CPUS]; +static u64 hv_clpc_core_perf_ctrl2[MAX_CPUS]; +static u64 hv_clpc_mem_stall_cfg[MAX_CPUS][3]; +static s64 hv_clpc_counter_bias[MAX_CPUS][16]; + /* * SPRR permission-remap table, soft-cached on M4+ SoCs where the real register * faults at EL2. Values taken from Sven's blog post which appear to work. @@ -361,6 +365,72 @@ static void hv_update_fiq(void) (store) = regs[rt]; \ return true; +static bool hv_handle_clpc_counter(struct exc_info *ctx, u64 rt, bool is_read, unsigned int counter) +{ + /* + * Apple's PMGR kext consumes these as perf/power deltas. Live EL2 access + * is unsafe in the current T8140/SPTM context, so provide coherent + * monotonic guest-visible counters. + */ + if (counter >= 16) + return false; + + u64 cpu = mrs(TPIDR_EL2); + u64 base = mrs(CNTPCT_EL0) + (counter * 0x1000000ULL); + + if (is_read) { + if (rt < 31) + ctx->regs[rt] = (u64)((s64)base + hv_clpc_counter_bias[cpu][counter]); + } else { + u64 val = rt < 31 ? ctx->regs[rt] : 0; + hv_clpc_counter_bias[cpu][counter] = (s64)val - (s64)base; + } + + return true; +} + +static bool hv_handle_clpc_mem_stall_config(struct exc_info *ctx, u64 rt, bool is_read, + unsigned int regno) +{ + /* + * Apple's PMGR kext programs these CoreMemStallSampler controls with + * read/modify/write. Keep readback coherent, but do not touch live EL2 + * hardware registers. + */ + if (regno >= 3) + return false; + + u64 cpu = mrs(TPIDR_EL2); + u64 *cache = &hv_clpc_mem_stall_cfg[cpu][regno]; + + if (is_read) { + if (rt < 31) + ctx->regs[rt] = *cache; + } else { + *cache = rt < 31 ? ctx->regs[rt] : 0; + } + + return true; +} + +static bool hv_handle_clpc_core_perf_ctrl2(struct exc_info *ctx, u64 rt, bool is_read) +{ + /* + * Another sysreg used by Apple's PMGR. XNU only needs coherent state here; + * live execution faults in this raw guest path. + */ + u64 cpu = mrs(TPIDR_EL2); + + if (is_read) { + if (rt < 31) + ctx->regs[rt] = hv_clpc_core_perf_ctrl2[cpu]; + } else { + hv_clpc_core_perf_ctrl2[cpu] = rt < 31 ? ctx->regs[rt] : 0; + } + + return true; +} + static bool hv_handle_msr_unlocked(struct exc_info *ctx, u64 iss) { u64 reg = iss & (ESR_ISS_MSR_OP0 | ESR_ISS_MSR_OP2 | ESR_ISS_MSR_OP1 | ESR_ISS_MSR_CRn | @@ -461,6 +531,36 @@ static bool hv_handle_msr_unlocked(struct exc_info *ctx, u64 iss) SYSREG_SHADOW(SYS_IMP_APL_S3_6_C15_C0_4, PERCPU(impdef_c15_c0_4)) SYSREG_SHADOW(SYS_IMP_APL_S3_6_C15_C0_5, PERCPU(impdef_c15_c0_5)) SYSREG_SHADOW(SYS_IMP_APL_S3_4_C15_C12_0, PERCPU(impdef_c12_0)) + case SYSREG_ISS(SYS_IMP_APL_CLPC_CORE_PERF0): + return hv_handle_clpc_counter(ctx, rt, is_read, 0); + case SYSREG_ISS(SYS_IMP_APL_CLPC_CORE_PERF1): + return hv_handle_clpc_counter(ctx, rt, is_read, 1); + case SYSREG_ISS(SYS_IMP_APL_CLPC_CORE_PERF2): + return hv_handle_clpc_counter(ctx, rt, is_read, 2); + case SYSREG_ISS(SYS_IMP_APL_CLPC_CORE_ACC0): + return hv_handle_clpc_counter(ctx, rt, is_read, 0); + case SYSREG_ISS(SYS_IMP_APL_CLPC_CORE_ACC1): + return hv_handle_clpc_counter(ctx, rt, is_read, 1); + case SYSREG_ISS(SYS_IMP_APL_CLPC_CORE_ACC2): + return hv_handle_clpc_counter(ctx, rt, is_read, 2); + case SYSREG_ISS(SYS_IMP_APL_CLPC_CORE_ACC3): + return hv_handle_clpc_counter(ctx, rt, is_read, 3); + case SYSREG_ISS(SYS_IMP_APL_CLPC_CORE_ACC4): + return hv_handle_clpc_counter(ctx, rt, is_read, 4); + case SYSREG_ISS(SYS_IMP_APL_CLPC_CORE_PERF_ENABLE0): + case SYSREG_ISS(SYS_IMP_APL_CLPC_CORE_PERF_ENABLE1): + case SYSREG_ISS(SYS_IMP_APL_CLPC_CORE_PERF_ENABLE2): + if (is_read) + regs[rt] = 0; + return true; + case SYSREG_ISS(SYS_IMP_APL_CLPC_CORE_PERF_CTRL2): + return hv_handle_clpc_core_perf_ctrl2(ctx, rt, is_read); + case SYSREG_ISS(SYS_IMP_APL_CLPC_MEM_STALL_CFG0): + return hv_handle_clpc_mem_stall_config(ctx, rt, is_read, 0); + case SYSREG_ISS(SYS_IMP_APL_CLPC_MEM_STALL_CFG1): + return hv_handle_clpc_mem_stall_config(ctx, rt, is_read, 1); + case SYSREG_ISS(SYS_IMP_APL_CLPC_MEM_STALL_CFG2): + return hv_handle_clpc_mem_stall_config(ctx, rt, is_read, 2); SYSREG_MAP(SYS_IMP_APL_APCTL_EL1, SYS_IMP_APL_APCTL_EL12) /* * AMX. On M4 userspace uses SME, so AMX is inert, but XNU's per-CPU From ce1c1574757e19acc44fdd28fa49c7a500a23210 Mon Sep 17 00:00:00 2001 From: Cody Ho Date: Thu, 9 Jul 2026 05:07:36 -0700 Subject: [PATCH 33/33] hv: skip guarded ACTLR_EL12 on secondaries ACTLR_EL2 is locked on M4+ SoCs so skip the write. Signed-off-by: Cody Ho --- src/hv.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/hv.c b/src/hv.c index 89f7ea464..4135a7b40 100644 --- a/src/hv.c +++ b/src/hv.c @@ -295,10 +295,12 @@ static void hv_init_secondary(struct hv_secondary_info_t *info) msr(SYS_IMP_APL_APSTS_EL12, info->apsts); } msr(ACTLR_EL2, info->actlr_el2); - if (cpu_features->actlr_el2) - msr(SYS_ACTLR_EL12, info->actlr_el1); - else - msr(SYS_IMP_APL_ACTLR_EL12, info->actlr_el1); + if (cpu_features->apple_sysregs_unlocked) { + if (cpu_features->actlr_el2) + msr(SYS_ACTLR_EL12, info->actlr_el1); + else + msr(SYS_IMP_APL_ACTLR_EL12, info->actlr_el1); + } msr(CNTHCTL_EL2, info->cnthctl); if (cpu_features->apple_sysregs_unlocked) { msr(SYS_IMP_APL_SPRR_CONFIG_EL1, info->sprr_config);