Hypervisor Support for M4+ SoCs

Makefile

@@ -141,7 +141,7 @@ OBJECTS := \
 	firmware.o \
 	gxf.o gxf_asm.o \
 	heapblock.o \
-	hv.o hv_vm.o hv_exc.o hv_vuart.o hv_wdt.o hv_asm.o hv_aic.o hv_virtio.o \
+	hv.o hv_vm.o hv_exc.o hv_vuart.o hv_wdt.o hv_asm.o hv_aic.o hv_t8140.o hv_virtio.o \
 	i2c.o \
 	iodev.o \
 	iova.o \

proxyclient/m1n1/hv/__init__.py

@@ -109,6 +109,10 @@ def __init__(self, iface, proxy, utils):
         self.hook_exceptions = False
         self.started_cpus = {}
         self.started = False
+        self.start_entry = None
+        self.start_args = None
+        self.secondary_start_entry = None
+        self.secondary_start_args = None
         self.ctx = None
         self.hvcall_handlers = {}
         self.switching_context = False
@@ -714,7 +718,10 @@ def handle_msr(self, ctx, iss=None):
                 sys.stdout.flush()
                 if enc in xlate:
                     value = self.p.hv_translate(value, True, False)
-                self.u.msr(enc2, value, call=self.p.gl2_call)
+                call=None
+                if self.u.cpu_features.apple_sysregs_unlocked:
+                    call=self.p.gl2_call
+                self.u.msr(enc2, value, call=call)
                 self.log(f"Pass: msr {name}, x{iss.Rt} = {value:x} (OK) ({sysreg_name(enc2)})")
 
         ctx.elr += 4
@@ -1429,15 +1436,16 @@ def init(self):
         self.u.msr(MDCR_EL2, mdcr.value)
         self.u.msr(MDSCR_EL1, MDSCR(MDE=1).value)
 
-        # Enable AMX
-        amx_ctl = AMX_CONFIG(self.u.mrs(AMX_CONFIG_EL1))
-        amx_ctl.EN_EL1 = 1
-        self.u.msr(AMX_CONFIG_EL1, amx_ctl.value)
+        if self.u.cpu_features.apple_sysregs_unlocked:
+            # Enable AMX
+            amx_ctl = AMX_CONFIG(self.u.mrs(AMX_CONFIG_EL1))
+            amx_ctl.EN_EL1 = 1
+            self.u.msr(AMX_CONFIG_EL1, amx_ctl.value)
 
-        # Set guest AP keys
-        self.u.msr(VMKEYLO_EL2, 0x4E7672476F6E6147)
-        self.u.msr(VMKEYHI_EL2, 0x697665596F755570)
-        self.u.msr(APSTS_EL12, 1)
+            # Set guest AP keys
+            self.u.msr(VMKEYLO_EL2, 0x4E7672476F6E6147)
+            self.u.msr(VMKEYHI_EL2, 0x697665596F755570)
+            self.u.msr(APSTS_EL12, 1)
 
         self.map_vuart()
 
@@ -1475,10 +1483,12 @@ def rh(base, off, width):
             self.log(f"PMGR R {base:x}+{off:x}:{width} = 0x{data:x} -> 0x{ret:x}")
             return ret
 
-        atc = f"ATC{self.iodev - IODEV.USB0}_USB"
-        atc_aon = f"ATC{self.iodev - IODEV.USB0}_USB_AON"
-
-        hook_devs = ["UART0", atc, atc_aon]
+        # When the proxy is on the debugusb link there is no ATC PHY to protect
+        # and the index would go negative (ie, ATC-3_USB)
+        hook_devs = ["UART0"]
+        if self.iodev >= IODEV.USB0:
+            idx = self.iodev - IODEV.USB0
+            hook_devs += [f"ATC{idx}_USB", f"ATC{idx}_USB_AON"]
 
         pmgr = self.adt["/arm-io/pmgr"]
         dev_by_name = {dev.name: dev for dev in pmgr.devices}
@@ -1487,13 +1497,11 @@ def rh(base, off, width):
         pmgr_hooks = []
 
         def hook_pmgr_dev(dev):
-            ps = pmgr.ps_regs[dev.psreg]
-            if dev.psidx or dev.psreg:
-                addr = pmgr.get_reg(ps.reg)[0] + ps.offset + dev.psidx * 8
-                pmgr_hooks.append(addr)
-                for idx in self.adt.pmgr_dev_get_parents(dev):
-                    if idx in dev_by_id:
-                        hook_pmgr_dev(dev_by_id[idx])
+            addr = self.adt.pmgr_dev_get_addr(dev)
+            pmgr_hooks.append(addr)
+            for idx in self.adt.pmgr_dev_get_parents(dev):
+                if idx in dev_by_id:
+                    hook_pmgr_dev(dev_by_id[idx])
 
         for name in hook_devs:
             dev = dev_by_name[name]
@@ -1562,11 +1570,11 @@ def cpustart_wh(base, off, data, width):
             chip_id = self.u.adt["/chosen"].chip_id
             if chip_id in (0x8103, 0x6000, 0x6001, 0x6002):
                 cpu_start = 0x54000 + die * 0x20_0000_0000
-            elif chip_id in (0x8112, 0x8122, 0x6030):
+            elif chip_id in (0x8112, 0x8122, 0x8132, 0x8140, 0x6030):
                 cpu_start = 0x34000 + die * 0x20_0000_0000
             elif chip_id in (0x6020, 0x6021, 0x6022):
                 cpu_start = 0x28000 + die * 0x20_0000_0000
-            elif chip_id in (0x6031,):
+            elif chip_id in (0x6031, 0x6034, 0x6040):
                 cpu_start = 0x88000 + die * 0x20_0000_0000
             else:
                 self.log("CPUSTART unknown for this SoC!")
@@ -1576,6 +1584,19 @@ def cpustart_wh(base, off, data, width):
             self.map_hook(pmgr0_start + cpu_start, 0x20, write=cpustart_wh)
             self.add_tracer(zone, "CPU_START", TraceMode.RESERVED)
 
+        if not self.u.cpu_features.apple_sysregs_unlocked:
+
+            def rvbar_rh(base, off, width):
+                ret = self.entry & ~0xfff | 1
+                self.log(f"RVBAR R {base:x}+{off:x}:{width} -> 0x{ret:x}")
+                return ret
+
+            for cpu in self.adt["cpus"]:
+                addr, _ = cpu.cpu_impl_reg
+                zone = irange(addr, 4)
+                self.map_hook(addr, 4, read=rvbar_rh)
+                self.add_tracer(zone, "RVBAR", TraceMode.RESERVED)
+
     def start_secondary(self, die, cluster, cpu):
         self.log(f"Starting guest secondary {die}:{cluster}:{cpu}")
 
@@ -1586,13 +1607,24 @@ def start_secondary(self, die, cluster, cpu):
             self.log("CPU not found!")
             return
 
-        entry = self.p.read64(node.cpu_impl_reg[0]) & 0xfffffffffff
+        secondary_entry = self.secondary_start_entry
+        secondary_args = self.secondary_start_args
+
+        if secondary_entry is not None:
+            entry = secondary_entry
+        elif self.u.cpu_features.apple_sysregs_unlocked:
+            entry = self.p.read64(node.cpu_impl_reg[0]) & 0xfffffffffff
+        else:
+            entry = self.entry & ~0xfff
         index = node.cpu_id
         self.log(f" CPU #{index}: RVBAR = {entry:#x}")
 
         self.sysreg[index] = {}
         self.started_cpus[index] = (die, cluster, cpu)
-        self.p.hv_start_secondary(index, entry)
+        if secondary_args is not None:
+            self.p.hv_start_secondary(index, entry, *secondary_args)
+        else:
+            self.p.hv_start_secondary(index, entry)
 
     def setup_adt(self):
         self.adt["product"].product_name += " on m1n1 hypervisor"
@@ -1779,14 +1811,15 @@ def remove_oslog(node):
         elif self.tba.revision == 3:
             self.iface.writemem(guest_base + self.bootargs_off, BootArgs_r3.build(self.tba))
 
-        print("Setting secondary CPU RVBARs...")
-        rvbar = self.entry & ~0xfff
-        for cpu in self.adt["cpus"]:
-            if cpu.state == "running":
-                continue
-            addr, size = cpu.cpu_impl_reg
-            print(f"  {cpu.name}: [0x{addr:x}] = 0x{rvbar:x}")
-            self.p.write64(addr, rvbar)
+        if self.u.cpu_features.apple_sysregs_unlocked:
+            print("Setting secondary CPU RVBARs...")
+            rvbar = self.entry & ~0xfff
+            for cpu in self.adt["cpus"]:
+                if cpu.state == "running":
+                    continue
+                addr, size = cpu.cpu_impl_reg
+                print(f"  {cpu.name}: [0x{addr:x}] = 0x{rvbar:x}")
+                self.p.write64(addr, rvbar)
 
     def _load_macho_symbols(self):
         self.symbol_dict = self.macho.symbols
@@ -1882,10 +1915,84 @@ def load_hook_m3(data, segname, size, fileoff, dest):
             print("Done.")
             return a.tobytes()
 
-        #image = macho.prepare_image(load_hook)
+        def load_hook_m4(data, segname, size, fileoff, dest):
+            if segname not in ("__TEXT_EXEC", "__TEXT_BOOT_EXEC"):
+                return data
+
+            nop = 0xd503201f
+            print(f"Patching segment {segname}...")
+
+            a = array.array("I", data)
+            seg_vmaddr = macho.vmin + dest
+
+            # genter -> HVC to allow us to catch SPTM entry calls
+            p = 0
+            while (p := data.find(b"\x20\x00", p)) != -1:
+                if (p & 3) != 2:
+                    p += 1
+                    continue
+                opcode = a[p // 4]
+                if 0x00201420 <= opcode <= 0x00201424:  # genter #imm (imm 0..4)
+                    a[p // 4] = self.hvc(0)
+                p += 4
+
+            # sysreg neutralization: NOP writes to guarded impdef regs and
+            # SME control register
+            nop_msr = [
+                0xd51cf100,   # msr KERNKEYLO_EL1   (s3_4_c15_c1_0)
+                0xd51cf120,   # msr KERNKEYHI_EL1   (s3_4_c15_c1_1)
+                0xd51cfdc0,   # msr s3_4_c15_c13_6  (Apple PAC key)
+                0xd51cfde0,   # msr s3_4_c15_c13_7  (Apple PAC key)
+                0xd51c12a0,   # msr SMPRIMAP_EL2    (s3_4_c1_c2_5)
+            ]
+            p = 0
+            while (p := data.find(b"\x1c\xd5", p)) != -1:
+                if (p & 3) != 2:
+                    p += 1
+                    continue
+                opcode = a[p // 4] & ~0x1f
+                if opcode in nop_msr:
+                    if opcode == 0xd51c12a0:
+                        print(f"  0x{seg_vmaddr + (p & ~3):x}: SMPRIMAP_EL2 -> noop")
+                    a[p // 4] = nop
+                p += 4
+
+            # commpage: force XNU to publish VM-safe userspace feature policy
+            # bytes. The Apple timebase path and HW-TPRO/SPRR path use
+            # userspace-inaccessible Apple registers under this EL1 guest.
+            #
+            # hard coded kernelcache addresses is a hack we should find a way
+            # to avoid
+            off = 0xfffffe000b5a3330 - seg_vmaddr
+            if 0 <= off < size:
+                if a[off // 4] != 0x52800068:
+                    raise RuntimeError("_COMM_PAGE_USER_TIMEBASE patch target mismatch")
+                a[off // 4] = 0x52800008
+                print(f"  0xfffffe000b5a3330: _COMM_PAGE_USER_TIMEBASE=0")
+
+            off = 0xfffffe000b5a3388 - seg_vmaddr
+            if 0 <= off < size:
+                if a[off // 4] != 0x52800028:
+                    raise RuntimeError("_COMM_PAGE_CONT_HWCLOCK patch target mismatch")
+                a[off // 4] = 0x52800008
+                print(f"  0xfffffe000b5a3388: _COMM_PAGE_CONT_HWCLOCK=0")
+
+            off = 0xfffffe000b5a358c - seg_vmaddr
+            if 0 <= off < size:
+                if a[off // 4] != 0x39043128:
+                    raise RuntimeError("_COMM_PAGE_HW_TPRO patch target mismatch")
+                a[off // 4] = 0x3904313f
+                print(f"  0xfffffe000b5a358c: _COMM_PAGE_HW_TPRO=0")
+
+            print("Done.")
+            return a.tobytes()
+
         chip_id = self.u.adt["/chosen"].chip_id
         if chip_id in (0x8122, 0x6030, 0x6031, 0x6032, 0x6034):
             image = macho.prepare_image(load_hook_m3)
+        elif chip_id in (0x8132, 0x8140, 0x6040, 0x6041):
+            # M4 / A18 Pro class: T8132, T8140, T6040 (M4 Pro), T6041 (M4 Max)
+            image = macho.prepare_image(load_hook_m4)
         else:
             image = macho.prepare_image()
         self.load_raw(image, entryoffset=(macho.entry - macho.vmin), use_xnu_symbols=self.xnu_mode and symfile is not None, vmin=macho.vmin)
@@ -1968,13 +2075,19 @@ def start(self):
         print("Shutting down framebuffer...")
         self.p.fb_shutdown(True)
 
-        print("Enabling SPRR...")
-        self.u.msr(SPRR_CONFIG_EL1, 1)
+        if self.u.cpu_features.apple_sysregs_unlocked:
+            print("Enabling SPRR...")
+            self.u.msr(SPRR_CONFIG_EL1, 1)
+
+            print("Enabling GXF...")
+            self.u.msr(GXF_CONFIG_EL1, 1)
 
-        print("Enabling GXF...")
-        self.u.msr(GXF_CONFIG_EL1, 1)
+        entry = self.start_entry if self.start_entry is not None else self.entry
+        args = self.start_args
+        if args is None:
+            args = (self.guest_base + self.bootargs_off,)
 
-        print(f"Jumping to entrypoint at 0x{self.entry:x}")
+        print(f"Jumping to entrypoint at 0x{entry:x}")
 
         self.iface.dev.timeout = None
         self.default_sigint = signal.signal(signal.SIGINT, self._handle_sigint)
@@ -1991,6 +2104,6 @@ def start(self):
                 break
         self.started_cpus[cpu_node.cpu_id] = (getattr(cpu_node, "die_id", 0), cpu_node.cluster_id, cpu_node.cpu_id)
         self.sysreg[cpu_node.cpu_id] = {}
-        self.p.hv_start(self.entry, self.guest_base + self.bootargs_off)
+        self.p.hv_start(entry, *args)
 
 from .. import trace

proxyclient/m1n1/proxy.py

@@ -507,7 +507,7 @@ class GUARD(IntFlag):
     "amx" / bool_,
     "actlr_el2" / bool_,
     "counter_redirect" / bool_,
-    "padding" / Bytes(1),
+    "sme_enabled" / bool_,
 )
 
 # Uses UartInterface.proxyreq() to send requests to M1N1 and process
@@ -1189,8 +1189,8 @@ def display_is_external(self):
 
     def dapf_init_all(self):
         return self.request(self.P_DAPF_INIT_ALL)
-    def dapf_init(self, path):
-        return self.request(self.P_DAPF_INIT, path)
+    def dapf_init(self, path, index=1):
+        return self.request(self.P_DAPF_INIT, path, index)
 
     def cpufreq_init(self):
         return self.request(self.P_CPUFREQ_INIT)