From 9f95af44eaa8c2bc1f35480123c1a2c6958f55f9 Mon Sep 17 00:00:00 2001 From: verbiiyo Date: Thu, 28 May 2026 09:26:01 +0900 Subject: [PATCH 01/29] add metadata parsing --- .../minekov/client/XosViewportOverlay.java | 31 ++++++++++----- .../minekov/client/XosViewportRuntime.java | 39 +++++++++++++++++-- src/xos/examples/inventory.py | 21 ++++++---- 3 files changed, 70 insertions(+), 21 deletions(-) diff --git a/src/main/java/com/verbii/minekov/client/XosViewportOverlay.java b/src/main/java/com/verbii/minekov/client/XosViewportOverlay.java index b097072..3dc9a23 100644 --- a/src/main/java/com/verbii/minekov/client/XosViewportOverlay.java +++ b/src/main/java/com/verbii/minekov/client/XosViewportOverlay.java @@ -509,6 +509,13 @@ private static boolean panelContains(double mx, double my, Minecraft mc) { /** Horizontal inset from the pill’s right edge for the online (green) / offline (gray) status dot. */ private static final int MINIMIZED_STATUS_DOT_INSET = 10; + /** Private-use codepoints used to preserve modifier-aware editor keys over Java host char events. */ + private static final int XOS_KEY_SHIFT_TAB = 0xE100; + private static final int XOS_KEY_SHIFT_LEFT = 0xE101; + private static final int XOS_KEY_SHIFT_RIGHT = 0xE102; + private static final int XOS_KEY_SHIFT_UP = 0xE103; + private static final int XOS_KEY_SHIFT_DOWN = 0xE104; + private static final int XOS_KEY_TOGGLE_COMMENT = 0xE105; private static boolean shouldRouteKeyboardToXos(Minecraft mc) { if (!(mc.screen instanceof ChatScreen)) { @@ -561,16 +568,17 @@ private static boolean isModifierOrToggleKey(int key) { * {@link ScreenEvent.CharacterTyped.Pre}. Returns -1 for keys that should only come through character * events. */ - private static int navigationKeyToCodePoint(int key) { + private static int navigationKeyToCodePoint(int key, int modifiers) { + boolean shiftDown = (modifiers & GLFW.GLFW_MOD_SHIFT) != 0; return switch (key) { case GLFW.GLFW_KEY_BACKSPACE -> '\b'; case GLFW.GLFW_KEY_DELETE -> 0x7F; case GLFW.GLFW_KEY_ENTER, GLFW.GLFW_KEY_KP_ENTER -> '\n'; - case GLFW.GLFW_KEY_TAB -> '\t'; - case GLFW.GLFW_KEY_LEFT -> 0x2190; - case GLFW.GLFW_KEY_RIGHT -> 0x2192; - case GLFW.GLFW_KEY_UP -> 0x2191; - case GLFW.GLFW_KEY_DOWN -> 0x2193; + case GLFW.GLFW_KEY_TAB -> shiftDown ? XOS_KEY_SHIFT_TAB : '\t'; + case GLFW.GLFW_KEY_LEFT -> shiftDown ? XOS_KEY_SHIFT_LEFT : 0x2190; + case GLFW.GLFW_KEY_RIGHT -> shiftDown ? XOS_KEY_SHIFT_RIGHT : 0x2192; + case GLFW.GLFW_KEY_UP -> shiftDown ? XOS_KEY_SHIFT_UP : 0x2191; + case GLFW.GLFW_KEY_DOWN -> shiftDown ? XOS_KEY_SHIFT_DOWN : 0x2193; default -> -1; }; } @@ -692,8 +700,13 @@ public static void onScreenKeyPressed(ScreenEvent.KeyPressed.Pre event) { } int key = event.getKeyCode(); int modifiers = event.getModifiers(); - boolean ctrlDown = (modifiers & GLFW.GLFW_MOD_CONTROL) != 0; - if (ctrlDown) { + boolean commandDown = (modifiers & (GLFW.GLFW_MOD_CONTROL | GLFW.GLFW_MOD_SUPER)) != 0; + if (commandDown && (key == GLFW.GLFW_KEY_SLASH || key == GLFW.GLFW_KEY_KP_DIVIDE)) { + XosViewportRuntime.sendKeyCharToEngine(XOS_KEY_TOGGLE_COMMENT); + event.setCanceled(true); + return; + } + if (commandDown) { int actionCode = ctrlShortcutToActionCode(key, modifiers); if (actionCode != -1) { XosNative.onShortcut(actionCode); @@ -713,7 +726,7 @@ public static void onScreenKeyPressed(ScreenEvent.KeyPressed.Pre event) { event.setCanceled(true); return; } - int nav = navigationKeyToCodePoint(key); + int nav = navigationKeyToCodePoint(key, modifiers); if (nav != -1) { XosViewportRuntime.sendKeyCharToEngine(nav); event.setCanceled(true); diff --git a/src/main/java/com/verbii/minekov/client/XosViewportRuntime.java b/src/main/java/com/verbii/minekov/client/XosViewportRuntime.java index 02997fe..bd3b3ea 100644 --- a/src/main/java/com/verbii/minekov/client/XosViewportRuntime.java +++ b/src/main/java/com/verbii/minekov/client/XosViewportRuntime.java @@ -2,13 +2,16 @@ import ai.xlate.xos.XosNative; +import com.google.gson.JsonElement; import com.mojang.blaze3d.platform.NativeImage; +import com.mojang.serialization.JsonOps; import com.verbii.minekov.Minekov; import com.verbii.minekov.entities.RLOperator; import net.minecraft.client.Minecraft; import net.minecraft.client.gui.GuiGraphics; import net.minecraft.client.renderer.texture.DynamicTexture; import net.minecraft.core.registries.BuiltInRegistries; +import net.minecraft.nbt.NbtOps; import net.minecraft.network.chat.Component; import net.minecraft.resources.ResourceLocation; import net.minecraft.world.entity.Entity; @@ -521,7 +524,15 @@ private static String encodeItemStack(ItemStack stack) { if (stack.isDamageableItem()) { durability = Integer.toString(Math.max(0, stack.getMaxDamage() - stack.getDamageValue())); } - String meta = stack.getTag() != null ? stack.getTag().toString() : "{}"; + String meta = "{}"; + if (stack.getTag() != null) { + try { + JsonElement asJson = NbtOps.INSTANCE.convertTo(JsonOps.INSTANCE, stack.getTag()); + meta = asJson.toString(); + } catch (Throwable ignored) { + meta = stack.getTag().toString(); + } + } return sanitizeItemWireValue(type) + ITEM_FIELD_SEPARATOR + count @@ -1093,9 +1104,29 @@ def _parse_item(raw): durability = int(durability_text) except Exception: durability = None - meta = parts[3] if parts[3] else "{}" + meta_raw = parts[3] if parts[3] else "{}" + try: + meta_obj = xos.json.loads(meta_raw) + except Exception: + meta_obj = {} + meta = _normalize_meta(meta_obj) return Item(item_type=item_type, count=count, durability=durability, meta=meta) +def _normalize_meta(value): + if isinstance(value, dict): + out = {} + for key, sub in value.items(): + out[str(key)] = _normalize_meta(sub) + return out + if isinstance(value, list): + out = {} + for i, sub in enumerate(value): + out[str(i)] = _normalize_meta(sub) + return out + if value is None: + return "null" + return str(value) + def _parse_item_list(raw): if raw is None: return [] @@ -1132,11 +1163,11 @@ def look(self, pitch, yaw): self._call("look", f"{float(pitch)},{float(yaw)}") class Item: - def __init__(self, item_type="minecraft:air", count=0, durability=None, meta="{}"): + def __init__(self, item_type="minecraft:air", count=0, durability=None, meta=None): self.type = str(item_type) self.count = int(count) self.durability = durability - self.meta = str(meta) if meta is not None else "{}" + self.meta = meta if isinstance(meta, dict) else {} def __repr__(self): return f"mc.Item(type={{{self.type}}}, count={self.count}, meta={self.meta})" diff --git a/src/xos/examples/inventory.py b/src/xos/examples/inventory.py index a05a5f1..7235493 100644 --- a/src/xos/examples/inventory.py +++ b/src/xos/examples/inventory.py @@ -1,12 +1,17 @@ import mc item = mc.player.inventory.hand -mc.chat(str(item)) -print(item) +#mc.chat(str(item)) +#print(item) + +hand = mc.player.inventory.hand +print(hand.type) +print(type(hand.meta)) +#print(hand.meta["GunID"]) +#print("hand:", hand) +#print("offhand:", mc.player.inventory.offhand) +#print("hotbar:", mc.player.inventory.hotbar) +#print("armor:", mc.player.inventory.armor) +#print("contents:", mc.player.inventory.contents) +#print("enderchest:", mc.player.inventory.enderchest) -print("hand:", mc.player.inventory.hand) -print("offhand:", mc.player.inventory.offhand) -print("hotbar:", mc.player.inventory.hotbar) -print("armor:", mc.player.inventory.armor) -print("contents:", mc.player.inventory.contents) -print("enderchest:", mc.player.inventory.enderchest) From 33c162d0c805846f5374817f8069077567e1d8eb Mon Sep 17 00:00:00 2001 From: verbiiyo Date: Mon, 1 Jun 2026 10:42:31 +0900 Subject: [PATCH 02/29] move towards me actually kinda works! --- .../minekov/client/XosViewportRuntime.java | 122 ++++++------------ .../verbii/minekov/pymodule/McPyModule.java | 69 ++++++++++ .../resources/xos/examples/walk_towards_me.py | 32 ++--- src/xos/examples/walk_towards_me.py | 32 ++--- 4 files changed, 141 insertions(+), 114 deletions(-) create mode 100644 src/main/java/com/verbii/minekov/pymodule/McPyModule.java diff --git a/src/main/java/com/verbii/minekov/client/XosViewportRuntime.java b/src/main/java/com/verbii/minekov/client/XosViewportRuntime.java index bd3b3ea..5fa1699 100644 --- a/src/main/java/com/verbii/minekov/client/XosViewportRuntime.java +++ b/src/main/java/com/verbii/minekov/client/XosViewportRuntime.java @@ -7,6 +7,7 @@ import com.mojang.serialization.JsonOps; import com.verbii.minekov.Minekov; import com.verbii.minekov.entities.RLOperator; +import com.verbii.minekov.pymodule.McPyModule; import net.minecraft.client.Minecraft; import net.minecraft.client.gui.GuiGraphics; import net.minecraft.client.renderer.texture.DynamicTexture; @@ -39,6 +40,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Comparator; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; @@ -140,7 +142,6 @@ public final class XosViewportRuntime { private static volatile int maxPumpsPerSecond; private static long lastPumpNanos; - private static boolean hostBindingsRegistered; /** * Rotation lock requested from mc Python bindings. Values are [yaw, pitch] and are re-applied each * frame so Minecraft AI/network updates do not drift agents off target. @@ -722,26 +723,7 @@ private static void setEntityVelocityWithServerMirror(Minecraft mc, Entity entit } private static void applyMovementImpulse(Entity entity, double forward, double strafe) { - if (entity instanceof RLOperator rl) { - double speed = movementImpulseFor(rl); - if (forward > 0.0) { - rl.moveTowards(180.0f, (float) speed); - return; - } - if (forward < 0.0) { - rl.moveTowards(0.0f, (float) speed); - return; - } - if (strafe < 0.0) { - rl.moveTowards(-90.0f, (float) speed); - return; - } - if (strafe > 0.0) { - rl.moveTowards(90.0f, (float) speed); - return; - } - } - + // Keep WASD strictly look-relative for every controllable entity (player + agents). double speed = movementImpulseFor(entity); double yawRad = Math.toRadians(entity.getYRot()); double fx = -Math.sin(yawRad); @@ -750,8 +732,10 @@ private static void applyMovementImpulse(Entity entity, double forward, double s double lz = fx; Vec3 cur = entity.getDeltaMovement(); - double vx = cur.x + (fx * forward + lx * strafe) * speed; - double vz = cur.z + (fz * forward + lz * strafe) * speed; + // Treat scripted WASD as desired movement for this tick (no momentum accumulation), + // so heading updates from mc.agents.rotations immediately affect travel direction. + double vx = (fx * forward + lx * strafe) * speed; + double vz = (fz * forward + lz * strafe) * speed; entity.setDeltaMovement(vx, cur.y, vz); entity.hurtMarked = true; } @@ -819,6 +803,18 @@ private static void applyAgentRotation(RLOperator agent, float yaw, float pitch) agent.setYBodyRot(yaw); } + private static void applyAgentRotationWithServerMirror( + Minecraft mc, RLOperator agent, float yaw, float pitch) { + if (agent == null) { + return; + } + applyAgentRotation(agent, yaw, pitch); + Entity mirrorEntity = resolveServerMirrorEntity(mc, agent); + if (mirrorEntity instanceof RLOperator mirror && mirror != agent) { + applyAgentRotation(mirror, yaw, pitch); + } + } + private static void applyForcedAgentRotations(Minecraft mc) { if (forcedAgentRotations.isEmpty()) { return; @@ -828,7 +824,7 @@ private static void applyForcedAgentRotations(Minecraft mc) { if (rot == null || rot.length < 2) { continue; } - applyAgentRotation(agent, rot[0], rot[1]); + applyAgentRotationWithServerMirror(mc, agent, rot[0], rot[1]); } } @@ -852,6 +848,9 @@ private static List collectAgents(Minecraft mc) { out.add(operator); } } + // Keep index-based Python APIs deterministic across calls: + // mc.agents.positions / rotations / move / look all rely on the same ordering. + out.sort(Comparator.comparing(RLOperator::getStringUUID)); return out; } @@ -1580,7 +1579,7 @@ private static String invokeHostBinding(Minecraft mc, String moduleName, String return; } float[] yp = parsePair(ypRaw, agent.getYRot(), agent.getXRot()); - applyAgentRotation(agent, yp[0], yp[1]); + applyAgentRotationWithServerMirror(mc, agent, yp[0], yp[1]); forcedAgentRotations.put(agent.getStringUUID(), new float[] {yp[0], yp[1]}); }); yield null; @@ -1598,8 +1597,9 @@ private static String invokeHostBinding(Minecraft mc, String moduleName, String return; } float yaw = parsePair(yawRaw + ",0", agent.getYRot(), agent.getXRot())[0]; - applyAgentRotation(agent, yaw, agent.getXRot()); - forcedAgentRotations.put(agent.getStringUUID(), new float[] {yaw, agent.getXRot()}); + float pitch = agent.getXRot(); + applyAgentRotationWithServerMirror(mc, agent, yaw, pitch); + forcedAgentRotations.put(agent.getStringUUID(), new float[] {yaw, pitch}); }); yield null; } @@ -1616,8 +1616,9 @@ private static String invokeHostBinding(Minecraft mc, String moduleName, String return; } float pitch = parsePair("0," + pitchRaw, agent.getYRot(), agent.getXRot())[1]; - applyAgentRotation(agent, agent.getYRot(), pitch); - forcedAgentRotations.put(agent.getStringUUID(), new float[] {agent.getYRot(), pitch}); + float yaw = agent.getYRot(); + applyAgentRotationWithServerMirror(mc, agent, yaw, pitch); + forcedAgentRotations.put(agent.getStringUUID(), new float[] {yaw, pitch}); }); yield null; } @@ -1765,7 +1766,7 @@ private static String invokeHostBinding(Minecraft mc, String moduleName, String RLOperator agent = agents.get(i); float yaw = (float) rows.get(i)[0]; float pitch = (float) rows.get(i)[1]; - applyAgentRotation(agent, yaw, pitch); + applyAgentRotationWithServerMirror(mc, agent, yaw, pitch); forcedAgentRotations.put(agent.getStringUUID(), new float[] {yaw, pitch}); } }); @@ -1779,8 +1780,9 @@ private static String invokeHostBinding(Minecraft mc, String moduleName, String for (int i = 0; i < n; i++) { RLOperator agent = agents.get(i); float yaw = vals.get(i); - applyAgentRotation(agent, yaw, agent.getXRot()); - forcedAgentRotations.put(agent.getStringUUID(), new float[] {yaw, agent.getXRot()}); + float pitch = agent.getXRot(); + applyAgentRotationWithServerMirror(mc, agent, yaw, pitch); + forcedAgentRotations.put(agent.getStringUUID(), new float[] {yaw, pitch}); } }); yield null; @@ -1793,8 +1795,9 @@ private static String invokeHostBinding(Minecraft mc, String moduleName, String for (int i = 0; i < n; i++) { RLOperator agent = agents.get(i); float pitch = vals.get(i); - applyAgentRotation(agent, agent.getYRot(), pitch); - forcedAgentRotations.put(agent.getStringUUID(), new float[] {agent.getYRot(), pitch}); + float yaw = agent.getYRot(); + applyAgentRotationWithServerMirror(mc, agent, yaw, pitch); + forcedAgentRotations.put(agent.getStringUUID(), new float[] {yaw, pitch}); } }); yield null; @@ -1810,58 +1813,9 @@ private static String invokeHostBinding(Minecraft mc, String moduleName, String } private static void configureHostBindings(Minecraft mc) { - if (hostBindingsRegistered) { - return; - } - XosNative.setHostBindingCallback( + McPyModule.configureHostBindings( (moduleName, functionName, arg0) -> invokeHostBinding(mc, moduleName, functionName, arg0)); - XosNative.clearHostPythonModules(); - XosNative.registerHostPythonModule( - "mc", - new String[] { - "chat", - "player_position", - "player_set_position", - "player_rotation", - "player_set_rotation", - "player_set_yaw", - "player_set_pitch", - "player_velocity", - "player_set_velocity", - "player_looking_velocity", - "player_set_looking_velocity", - "player_inventory_armor", - "player_inventory_contents", - "player_inventory_hotbar", - "player_inventory_offhand", - "player_inventory_hand", - "player_inventory_enderchest", - "agent_ids", - "agent_position", - "agent_set_position", - "agent_rotation", - "agent_set_rotation", - "agent_set_yaw", - "agent_set_pitch", - "agent_velocity", - "agent_set_velocity", - "agent_looking_velocity", - "agent_set_looking_velocity", - "entity_action", - "agents_look", - "agents_move", - "agents_positions", - "agents_rotations", - "agents_yaws", - "agents_pitches", - "agents_set_positions", - "agents_set_rotations", - "agents_set_yaws", - "agents_set_pitches", - "__bootstrap__" - }); - hostBindingsRegistered = true; } private static boolean configureCoderScriptsDirectory(Minecraft mc) { diff --git a/src/main/java/com/verbii/minekov/pymodule/McPyModule.java b/src/main/java/com/verbii/minekov/pymodule/McPyModule.java new file mode 100644 index 0000000..28bc150 --- /dev/null +++ b/src/main/java/com/verbii/minekov/pymodule/McPyModule.java @@ -0,0 +1,69 @@ +package com.verbii.minekov.pymodule; + +import ai.xlate.xos.XosNative; + +public final class McPyModule { + private static boolean hostBindingsRegistered; + + private static final String[] EXPORTED_FUNCTIONS = { + "chat", + "player_position", + "player_set_position", + "player_rotation", + "player_set_rotation", + "player_set_yaw", + "player_set_pitch", + "player_velocity", + "player_set_velocity", + "player_looking_velocity", + "player_set_looking_velocity", + "player_inventory_armor", + "player_inventory_contents", + "player_inventory_hotbar", + "player_inventory_offhand", + "player_inventory_hand", + "player_inventory_enderchest", + "agent_ids", + "agent_position", + "agent_set_position", + "agent_rotation", + "agent_set_rotation", + "agent_set_yaw", + "agent_set_pitch", + "agent_velocity", + "agent_set_velocity", + "agent_looking_velocity", + "agent_set_looking_velocity", + "entity_action", + "agents_look", + "agents_move", + "agents_positions", + "agents_rotations", + "agents_yaws", + "agents_pitches", + "agents_set_positions", + "agents_set_rotations", + "agents_set_yaws", + "agents_set_pitches", + "__bootstrap__" + }; + + private McPyModule() {} + + @FunctionalInterface + public interface HostBindingInvoker { + String invoke(String moduleName, String functionName, String arg0); + } + + public static void configureHostBindings(HostBindingInvoker invoker) { + if (hostBindingsRegistered) { + return; + } + XosNative.setHostBindingCallback( + (moduleName, functionName, arg0) -> + invoker.invoke(moduleName, functionName, arg0)); + XosNative.clearHostPythonModules(); + XosNative.registerHostPythonModule("mc", EXPORTED_FUNCTIONS); + hostBindingsRegistered = true; + } +} diff --git a/src/main/resources/xos/examples/walk_towards_me.py b/src/main/resources/xos/examples/walk_towards_me.py index 01ef582..3089b7e 100644 --- a/src/main/resources/xos/examples/walk_towards_me.py +++ b/src/main/resources/xos/examples/walk_towards_me.py @@ -1,31 +1,33 @@ import mc import xos + + + while True: agent_positions = mc.agents.positions num_agents = int(agent_positions.shape[0]) if len(agent_positions.shape) > 0 else 0 - if num_agents <= 0: - xos.sleep(0.1) - continue - player_position = mc.player.position - player_positions = xos.tensor( - [player_position[0], player_position[1], player_position[2]], (1, 3) - ).repeat(num_agents, axis=0) + # Batch player position to (N,3) tensor, index-aligned to mc.agents + p = mc.player.position + player_positions = ( + xos.tensor([p[0], p[1], p[2]], (1, 3)) + .repeat(num_agents, axis=0) + ) delta = player_positions - agent_positions dx = delta[:, 0] dy = delta[:, 1] dz = delta[:, 2] - target_yaws = xos.math.degrees(xos.math.atan2(dz, dx)) - 90.0 + # Vectorized look-at in degrees (Minecraft yaw/pitch convention) + yaws = xos.math.degrees(xos.math.atan2(dz, dx)) - 90.0 horizontal = xos.math.sqrt(dx * dx + dz * dz) - target_pitches = -xos.math.degrees(xos.math.atan2(dy, horizontal)) - target_rotations = xos.stack([target_pitches, target_yaws], axis=1) + pitches = -xos.math.degrees(xos.math.atan2(dy, horizontal)) - current_rotations = mc.agents.rotations - look_deltas = target_rotations - current_rotations - mc.agents.look(look_deltas) - mc.agents.move((True, False, False, False)) + rotations_tensor = xos.stack([yaws, pitches], axis=1) + mc.agents.rotations = rotations_tensor - xos.sleep(0.1) + print("updated", num_agents, "agent rotations to look at player") + mc.agents.move((True, False, False, False)) + xos.sleep(0.1) \ No newline at end of file diff --git a/src/xos/examples/walk_towards_me.py b/src/xos/examples/walk_towards_me.py index 01ef582..3089b7e 100644 --- a/src/xos/examples/walk_towards_me.py +++ b/src/xos/examples/walk_towards_me.py @@ -1,31 +1,33 @@ import mc import xos + + + while True: agent_positions = mc.agents.positions num_agents = int(agent_positions.shape[0]) if len(agent_positions.shape) > 0 else 0 - if num_agents <= 0: - xos.sleep(0.1) - continue - player_position = mc.player.position - player_positions = xos.tensor( - [player_position[0], player_position[1], player_position[2]], (1, 3) - ).repeat(num_agents, axis=0) + # Batch player position to (N,3) tensor, index-aligned to mc.agents + p = mc.player.position + player_positions = ( + xos.tensor([p[0], p[1], p[2]], (1, 3)) + .repeat(num_agents, axis=0) + ) delta = player_positions - agent_positions dx = delta[:, 0] dy = delta[:, 1] dz = delta[:, 2] - target_yaws = xos.math.degrees(xos.math.atan2(dz, dx)) - 90.0 + # Vectorized look-at in degrees (Minecraft yaw/pitch convention) + yaws = xos.math.degrees(xos.math.atan2(dz, dx)) - 90.0 horizontal = xos.math.sqrt(dx * dx + dz * dz) - target_pitches = -xos.math.degrees(xos.math.atan2(dy, horizontal)) - target_rotations = xos.stack([target_pitches, target_yaws], axis=1) + pitches = -xos.math.degrees(xos.math.atan2(dy, horizontal)) - current_rotations = mc.agents.rotations - look_deltas = target_rotations - current_rotations - mc.agents.look(look_deltas) - mc.agents.move((True, False, False, False)) + rotations_tensor = xos.stack([yaws, pitches], axis=1) + mc.agents.rotations = rotations_tensor - xos.sleep(0.1) + print("updated", num_agents, "agent rotations to look at player") + mc.agents.move((True, False, False, False)) + xos.sleep(0.1) \ No newline at end of file From 5526ca68e2ae4e5bfe53cf9c1dd7e88c23969d52 Mon Sep 17 00:00:00 2001 From: verbiiyo Date: Mon, 1 Jun 2026 10:51:26 +0900 Subject: [PATCH 03/29] rl operators can collide with each other --- .../java/com/verbii/minekov/entities/AIOperator.java | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/verbii/minekov/entities/AIOperator.java b/src/main/java/com/verbii/minekov/entities/AIOperator.java index 29799c6..cc7e60c 100644 --- a/src/main/java/com/verbii/minekov/entities/AIOperator.java +++ b/src/main/java/com/verbii/minekov/entities/AIOperator.java @@ -79,25 +79,22 @@ public static AttributeSupplier.Builder createAttributes() { @Override public boolean canCollideWith(Entity entity) { - // Use the centralized interaction logic - // return TrainingIsolationHandler.shouldEntitiesInteract(this, entity); - return false; + return super.canCollideWith(entity); } @Override public void push(Entity entity) { - // if (!TrainingIsolationHandler.shouldEntitiesInteract(this, entity)) return; - // super.push(entity); + super.push(entity); } @Override public boolean canBeCollidedWith() { - return false; + return true; } @Override public boolean isPushable() { - return false; + return true; } @Override From c1bfcd8b7943e00318c020385a8a2e95db459bac Mon Sep 17 00:00:00 2001 From: verbiiyo Date: Mon, 1 Jun 2026 11:19:06 +0900 Subject: [PATCH 04/29] agent controller *almost* dispalys the frame from the agent's perspective. --- .../minekov/client/XosViewportRuntime.java | 136 ++++++++++++++++++ .../verbii/minekov/pymodule/McPyModule.java | 1 + .../xos/examples/agent_controller.py | 68 ++++++++- src/xos/examples/agent_controller.py | 70 ++++++++- 4 files changed, 269 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/verbii/minekov/client/XosViewportRuntime.java b/src/main/java/com/verbii/minekov/client/XosViewportRuntime.java index 5fa1699..a785309 100644 --- a/src/main/java/com/verbii/minekov/client/XosViewportRuntime.java +++ b/src/main/java/com/verbii/minekov/client/XosViewportRuntime.java @@ -11,6 +11,7 @@ import net.minecraft.client.Minecraft; import net.minecraft.client.gui.GuiGraphics; import net.minecraft.client.renderer.texture.DynamicTexture; +import net.minecraft.core.BlockPos; import net.minecraft.core.registries.BuiltInRegistries; import net.minecraft.nbt.NbtOps; import net.minecraft.network.chat.Component; @@ -74,6 +75,9 @@ public final class XosViewportRuntime { private static final int VIEWPORT_ALPHA_HOVER = Math.round(255 * 0.8f); private static final String ITEM_FIELD_SEPARATOR = "\u001f"; private static final String ITEM_ROW_SEPARATOR = "\u001e"; + private static final float AGENT_VIEW_FOV_DEG = 70.0f; + private static final double AGENT_VIEW_MAX_DISTANCE = 24.0; + private static final double AGENT_VIEW_STEP = 1.0; private static final List STARTER_SCRIPT_NAMES = List.of( "balls_many.py", @@ -610,6 +614,93 @@ private static String encodePlayerInventoryEnderChest(Minecraft mc) { return out.toString(); } + private static int clampViewSize(int value) { + return Math.max(16, Math.min(128, value)); + } + + private static int rgbFromBlockState(net.minecraft.world.level.block.state.BlockState state) { + ResourceLocation key = BuiltInRegistries.BLOCK.getKey(state.getBlock()); + int hash = key != null ? key.toString().hashCode() : 0x77AA77; + int r = 64 + (Math.abs(hash) & 0x7F); + int g = 64 + ((Math.abs(hash >> 8)) & 0x7F); + int b = 64 + ((Math.abs(hash >> 16)) & 0x7F); + return (r << 16) | (g << 8) | b; + } + + private static String renderAgentViewRgbaHex(Minecraft mc, RLOperator agent, int reqW, int reqH) { + int width = clampViewSize(reqW); + int height = clampViewSize(reqH); + if (mc == null || mc.level == null || agent == null) { + return ""; + } + + Vec3 origin = new Vec3(agent.getX(), agent.getEyeY(), agent.getZ()); + float yaw = agent.getYRot(); + float pitch = agent.getXRot(); + Vec3 forward = forwardFromRotation(yaw, pitch); + Vec3 worldUp = new Vec3(0.0, 1.0, 0.0); + Vec3 right = worldUp.cross(forward); + if (right.lengthSqr() < 1.0e-8) { + right = rightFromYaw(yaw); + } else { + right = right.normalize(); + } + Vec3 up = forward.cross(right).normalize(); + + double tanHalfFov = Math.tan(Math.toRadians(AGENT_VIEW_FOV_DEG * 0.5)); + double aspect = (double) width / (double) height; + StringBuilder hex = new StringBuilder(width * height * 8); + + for (int y = 0; y < height; y++) { + double ny = 1.0 - (2.0 * (y + 0.5) / (double) height); + for (int x = 0; x < width; x++) { + double nx = (2.0 * (x + 0.5) / (double) width) - 1.0; + Vec3 dir = forward + .add(right.scale(nx * tanHalfFov * aspect)) + .add(up.scale(ny * tanHalfFov)) + .normalize(); + + int r = 110; + int g = 150; + int b = 210; + boolean hit = false; + double distance = 0.0; + + for (double t = AGENT_VIEW_STEP; t <= AGENT_VIEW_MAX_DISTANCE; t += AGENT_VIEW_STEP) { + Vec3 sample = origin.add(dir.scale(t)); + BlockPos bp = BlockPos.containing(sample.x, sample.y, sample.z); + var state = mc.level.getBlockState(bp); + if (!state.isAir()) { + int rgb = rgbFromBlockState(state); + distance = t; + double shade = Math.max(0.22, 1.0 - (t / AGENT_VIEW_MAX_DISTANCE)); + r = (int) Math.max(0, Math.min(255, ((rgb >> 16) & 0xFF) * shade)); + g = (int) Math.max(0, Math.min(255, ((rgb >> 8) & 0xFF) * shade)); + b = (int) Math.max(0, Math.min(255, (rgb & 0xFF) * shade)); + hit = true; + break; + } + } + + if (!hit) { + double tSky = Math.max(0.0, Math.min(1.0, (dir.y + 1.0) * 0.5)); + r = (int) (70 + 60 * tSky); + g = (int) (100 + 90 * tSky); + b = (int) (150 + 80 * tSky); + } else { + // Tiny depth cue so nearby geometry pops in the mini-view. + int fog = (int) Math.max(0, Math.min(40, (AGENT_VIEW_MAX_DISTANCE - distance) * 0.8)); + r = Math.min(255, r + fog); + g = Math.min(255, g + fog); + b = Math.min(255, b + fog); + } + + hex.append(String.format(Locale.ROOT, "%02x%02x%02x%02x", r, g, b, 255)); + } + } + return hex.toString(); + } + private static double[] parseTriple(String raw, double fallbackX, double fallbackY, double fallbackZ) { if (raw == null || raw.isBlank()) { return new double[] {fallbackX, fallbackY, fallbackZ}; @@ -1134,6 +1225,21 @@ def _parse_item_list(raw): return [] return [_parse_item(part) for part in text.split("\\x1e")] +def _decode_rgba_hex_to_tensor(raw_hex, w, h): + if raw_hex is None: + return xos.zeros((h, w, 4), dtype="uint8") + text = str(raw_hex).strip() + need = int(w) * int(h) * 4 * 2 + if len(text) < need: + return xos.zeros((h, w, 4), dtype="uint8") + vals = [] + try: + for i in range(0, need, 2): + vals.append(int(text[i:i + 2], 16)) + except Exception: + return xos.zeros((h, w, 4), dtype="uint8") + return xos.tensor(vals, (h, w, 4), dtype="uint8") + class Actions: def __init__(self, entity_kind, entity_id=""): self._kind = str(entity_kind) @@ -1331,6 +1437,12 @@ def looking_velocity(self, value): def actions(self): return self._actions + def visual(self, h=128, w=128): + hh = int(h) + ww = int(w) + raw = __module__._host_call("agent_view_rgba", f"{self._id};{ww};{hh}") + return _decode_rgba_hex_to_tensor(raw, ww, hh) + class Agents: def _list(self): raw = __module__._host_call("agent_ids", "") @@ -1679,6 +1791,30 @@ private static String invokeHostBinding(Minecraft mc, String moduleName, String }); yield null; } + case "agent_view_rgba" -> { + AtomicReference viewRef = new AtomicReference<>(""); + runOnClientThreadSync(mc, () -> { + String[] parts = arg.split(";", 3); + if (parts.length < 3) { + return; + } + String id = parts[0].trim(); + int w; + int h; + try { + w = Integer.parseInt(parts[1].trim()); + h = Integer.parseInt(parts[2].trim()); + } catch (Exception ignored) { + return; + } + RLOperator agent = findAgentById(mc, id); + if (agent == null) { + return; + } + viewRef.set(renderAgentViewRgbaHex(mc, agent, w, h)); + }); + yield viewRef.get(); + } case "entity_action" -> { runOnClientThreadSync(mc, () -> { String[] parts = arg.split(";", 4); diff --git a/src/main/java/com/verbii/minekov/pymodule/McPyModule.java b/src/main/java/com/verbii/minekov/pymodule/McPyModule.java index 28bc150..c0e7057 100644 --- a/src/main/java/com/verbii/minekov/pymodule/McPyModule.java +++ b/src/main/java/com/verbii/minekov/pymodule/McPyModule.java @@ -34,6 +34,7 @@ public final class McPyModule { "agent_set_velocity", "agent_looking_velocity", "agent_set_looking_velocity", + "agent_view_rgba", "entity_action", "agents_look", "agents_move", diff --git a/src/main/resources/xos/examples/agent_controller.py b/src/main/resources/xos/examples/agent_controller.py index 3a76a2f..0859d68 100644 --- a/src/main/resources/xos/examples/agent_controller.py +++ b/src/main/resources/xos/examples/agent_controller.py @@ -1,6 +1,68 @@ import mc +import xos -print(mc.player.position) -for agent in mc.agents: - mc.chat(agent.position) +class AgentController(xos.Application): + def __init__(self): + super().__init__(max_fps=15) + self.key_active_until = {} + self.jump_cooldown_until = 0.0 + self.view_w = 128 + self.view_h = 128 + + def _now(self): + return xos.time.perf_counter() + + def _pressed(self, key): + return self._now() < self.key_active_until.get(key, 0.0) + + def _mark_pressed(self, key, hold_s=0.35): + self.key_active_until[key] = self._now() + hold_s + + def tick(self): + agents = list(mc.agents) + if len(agents) == 0: + self.frame.clear((10, 10, 10, 255)) + return + + agent = agents[0] + now = self._now() + + if self._pressed("w"): + agent.actions.w() + if self._pressed("a"): + agent.actions.a() + if self._pressed("s"): + agent.actions.s() + if self._pressed("d"): + agent.actions.d() + if self._pressed("space") and now >= self.jump_cooldown_until: + agent.actions.jump() + self.jump_cooldown_until = now + 0.25 + + view = agent.visual(h=self.view_h, w=self.view_w) + try: + self.frame.tensor[:] = view + except Exception as e: + print("agent_controller render error:", e) + self.frame.clear((0, 0, 0, 255)) + + def on_events(self): + ev = getattr(self, "_xos_event", None) + if not isinstance(ev, dict): + return + if ev.get("kind") != "key_char": + return + + ch = str(ev.get("char", "")) + if not ch: + return + c = ch.lower() + if c in ("w", "a", "s", "d"): + self._mark_pressed(c) + elif ch == " ": + self._mark_pressed("space", hold_s=0.2) + + +if __name__ == "__main__": + AgentController().run() diff --git a/src/xos/examples/agent_controller.py b/src/xos/examples/agent_controller.py index 3a76a2f..4417d70 100644 --- a/src/xos/examples/agent_controller.py +++ b/src/xos/examples/agent_controller.py @@ -1,6 +1,70 @@ import mc +import xos -print(mc.player.position) -for agent in mc.agents: - mc.chat(agent.position) +class AgentController(xos.Application): + def __init__(self): + super().__init__(max_fps=15) + self.key_active_until = {} + self.jump_cooldown_until = 0.0 + self.view_w = 128 + self.view_h = 128 + + def _now(self): + return xos.time.perf_counter() + + def _pressed(self, key): + return self._now() < self.key_active_until.get(key, 0.0) + + def _mark_pressed(self, key, hold_s=0.35): + self.key_active_until[key] = self._now() + hold_s + + def tick(self): + agents = list(mc.agents) + if len(agents) == 0: + self.frame.clear((10, 10, 10, 255)) + return + + agent = agents[0] + now = self._now() + + # Movement controls (single controllable agent: index 0). + if self._pressed("w"): + agent.actions.w() + if self._pressed("a"): + agent.actions.a() + if self._pressed("s"): + agent.actions.s() + if self._pressed("d"): + agent.actions.d() + if self._pressed("space") and now >= self.jump_cooldown_until: + agent.actions.jump() + self.jump_cooldown_until = now + 0.25 + + # Fixed-size viewport from the agent perspective. + view = agent.visual(h=self.view_h, w=self.view_w) + try: + self.frame.tensor[:] = view + except Exception as e: + print("agent_controller render error:", e) + self.frame.clear((0, 0, 0, 255)) + + def on_events(self): + ev = getattr(self, "_xos_event", None) + if not isinstance(ev, dict): + return + if ev.get("kind") != "key_char": + return + + ch = str(ev.get("char", "")) + if not ch: + return + c = ch.lower() + if c in ("w", "a", "s", "d"): + self._mark_pressed(c) + elif ch == " ": + self._mark_pressed("space", hold_s=0.2) + + +if __name__ == "__main__": + AgentController().run() From fa1cab7d028a18632aae0e19639dc275535ba1c7 Mon Sep 17 00:00:00 2001 From: verbiiyo Date: Mon, 1 Jun 2026 11:25:30 +0900 Subject: [PATCH 05/29] it kinda displays a frame slightly --- src/main/resources/xos/examples/agent_controller.py | 3 ++- src/xos/examples/agent_controller.py | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/resources/xos/examples/agent_controller.py b/src/main/resources/xos/examples/agent_controller.py index 0859d68..b96ab2c 100644 --- a/src/main/resources/xos/examples/agent_controller.py +++ b/src/main/resources/xos/examples/agent_controller.py @@ -42,7 +42,8 @@ def tick(self): view = agent.visual(h=self.view_h, w=self.view_w) try: - self.frame.tensor[:] = view + self.frame.clear((0, 0, 0, 255)) + xos.rasterizer._frame_blit_aspect_fit_norm_rect(view, 0.0, 0.0, 1.0, 1.0) except Exception as e: print("agent_controller render error:", e) self.frame.clear((0, 0, 0, 255)) diff --git a/src/xos/examples/agent_controller.py b/src/xos/examples/agent_controller.py index 4417d70..6c853d0 100644 --- a/src/xos/examples/agent_controller.py +++ b/src/xos/examples/agent_controller.py @@ -44,7 +44,9 @@ def tick(self): # Fixed-size viewport from the agent perspective. view = agent.visual(h=self.view_h, w=self.view_w) try: - self.frame.tensor[:] = view + # Fill the live viewport with letterboxed aspect-fit scaling of the 128x128 feed. + self.frame.clear((0, 0, 0, 255)) + xos.rasterizer._frame_blit_aspect_fit_norm_rect(view, 0.0, 0.0, 1.0, 1.0) except Exception as e: print("agent_controller render error:", e) self.frame.clear((0, 0, 0, 255)) From 69ad281b486b27daaafb8606c93e7b1751d56fd5 Mon Sep 17 00:00:00 2001 From: verbiiyo Date: Mon, 1 Jun 2026 13:43:31 +0900 Subject: [PATCH 06/29] controller.py copied in --- src/main/resources/xos/examples/controller.py | 98 +++++++++++++++++++ src/xos/examples/controller.py | 98 +++++++++++++++++++ 2 files changed, 196 insertions(+) create mode 100644 src/main/resources/xos/examples/controller.py create mode 100644 src/xos/examples/controller.py diff --git a/src/main/resources/xos/examples/controller.py b/src/main/resources/xos/examples/controller.py new file mode 100644 index 0000000..7ba0447 --- /dev/null +++ b/src/main/resources/xos/examples/controller.py @@ -0,0 +1,98 @@ +import xos + +BG_COLOR = (8, 10, 16, 255) +BALL_COLOR = (80, 220, 255, 255) +NOSE_COLOR = (255, 255, 255, 255) + +BALL_RADIUS_NORM = 0.024 +NOSE_RADIUS_NORM = 0.008 +NOSE_DISTANCE_NORM = BALL_RADIUS_NORM * 1.6 + +# Units: normalized-screen-units. +THRUST = 2.2 +DRAG_PER_SEC = 2.8 +MAX_SPEED = 0.95 +MAX_DT = 1.0 / 30.0 + + +def _wrap01(value): + return value % 1.0 + + +def _normalized_or_zero(x, y): + mag = xos.math.sqrt(x * x + y * y) + if mag <= 1e-8: + return 0.0, 0.0 + return x / mag, y / mag + + +def _limit_speed(vx, vy, limit): + speed = xos.math.sqrt(vx * vx + vy * vy) + if speed <= limit or speed <= 1e-8: + return vx, vy, speed + scale = limit / speed + return vx * scale, vy * scale, limit + + +class ControllerApp(xos.Application): + def __init__(self): + super().__init__(max_fps=120) + self.x = 0.5 + self.y = 0.5 + self.vx = 0.0 + self.vy = 0.0 + + def _input_axis(self): + keys = self.state.keyboard.keys + ax = 0.0 + ay = 0.0 + if keys.W.is_pressed(): + ay -= 1.0 + if keys.A.is_pressed(): + ax -= 1.0 + if keys.S.is_pressed(): + ay += 1.0 + if keys.D.is_pressed(): + ax += 1.0 + return _normalized_or_zero(ax, ay) + + def tick(self): + dt = min(MAX_DT, max(0.0, self.dt)) + ax, ay = self._input_axis() + + self.vx += ax * THRUST * dt + self.vy += ay * THRUST * dt + + drag = max(0.0, 1.0 - DRAG_PER_SEC * dt) + self.vx *= drag + self.vy *= drag + self.vx, self.vy, speed = _limit_speed(self.vx, self.vy, MAX_SPEED) + + self.x = _wrap01(self.x + self.vx * dt) + self.y = _wrap01(self.y + self.vy * dt) + + self.frame.clear(BG_COLOR) + w, h = self.get_width(), self.get_height() + px = self.x * w + py = self.y * h + scale_px = max(w, h) + + xos.rasterizer.circles(self.frame, [(px, py)], [BALL_RADIUS_NORM * scale_px], BALL_COLOR) + + if speed > 0.02: + nx, ny = self.vx / speed, self.vy / speed + else: + nx, ny = 0.0, -1.0 + + nose_x = _wrap01(self.x + nx * NOSE_DISTANCE_NORM) * w + nose_y = _wrap01(self.y + ny * NOSE_DISTANCE_NORM) * h + xos.rasterizer.circles( + self.frame, + [(nose_x, nose_y)], + [NOSE_RADIUS_NORM * scale_px], + NOSE_COLOR, + ) + + +if __name__ == "__main__": + ControllerApp().run() diff --git a/src/xos/examples/controller.py b/src/xos/examples/controller.py new file mode 100644 index 0000000..7ba0447 --- /dev/null +++ b/src/xos/examples/controller.py @@ -0,0 +1,98 @@ +import xos + +BG_COLOR = (8, 10, 16, 255) +BALL_COLOR = (80, 220, 255, 255) +NOSE_COLOR = (255, 255, 255, 255) + +BALL_RADIUS_NORM = 0.024 +NOSE_RADIUS_NORM = 0.008 +NOSE_DISTANCE_NORM = BALL_RADIUS_NORM * 1.6 + +# Units: normalized-screen-units. +THRUST = 2.2 +DRAG_PER_SEC = 2.8 +MAX_SPEED = 0.95 +MAX_DT = 1.0 / 30.0 + + +def _wrap01(value): + return value % 1.0 + + +def _normalized_or_zero(x, y): + mag = xos.math.sqrt(x * x + y * y) + if mag <= 1e-8: + return 0.0, 0.0 + return x / mag, y / mag + + +def _limit_speed(vx, vy, limit): + speed = xos.math.sqrt(vx * vx + vy * vy) + if speed <= limit or speed <= 1e-8: + return vx, vy, speed + scale = limit / speed + return vx * scale, vy * scale, limit + + +class ControllerApp(xos.Application): + def __init__(self): + super().__init__(max_fps=120) + self.x = 0.5 + self.y = 0.5 + self.vx = 0.0 + self.vy = 0.0 + + def _input_axis(self): + keys = self.state.keyboard.keys + ax = 0.0 + ay = 0.0 + if keys.W.is_pressed(): + ay -= 1.0 + if keys.A.is_pressed(): + ax -= 1.0 + if keys.S.is_pressed(): + ay += 1.0 + if keys.D.is_pressed(): + ax += 1.0 + return _normalized_or_zero(ax, ay) + + def tick(self): + dt = min(MAX_DT, max(0.0, self.dt)) + ax, ay = self._input_axis() + + self.vx += ax * THRUST * dt + self.vy += ay * THRUST * dt + + drag = max(0.0, 1.0 - DRAG_PER_SEC * dt) + self.vx *= drag + self.vy *= drag + self.vx, self.vy, speed = _limit_speed(self.vx, self.vy, MAX_SPEED) + + self.x = _wrap01(self.x + self.vx * dt) + self.y = _wrap01(self.y + self.vy * dt) + + self.frame.clear(BG_COLOR) + w, h = self.get_width(), self.get_height() + px = self.x * w + py = self.y * h + scale_px = max(w, h) + + xos.rasterizer.circles(self.frame, [(px, py)], [BALL_RADIUS_NORM * scale_px], BALL_COLOR) + + if speed > 0.02: + nx, ny = self.vx / speed, self.vy / speed + else: + nx, ny = 0.0, -1.0 + + nose_x = _wrap01(self.x + nx * NOSE_DISTANCE_NORM) * w + nose_y = _wrap01(self.y + ny * NOSE_DISTANCE_NORM) * h + xos.rasterizer.circles( + self.frame, + [(nose_x, nose_y)], + [NOSE_RADIUS_NORM * scale_px], + NOSE_COLOR, + ) + + +if __name__ == "__main__": + ControllerApp().run() From 65f985e9acc71df0840f10e046499e53e7be38bb Mon Sep 17 00:00:00 2001 From: verbiiyo Date: Mon, 1 Jun 2026 13:57:34 +0900 Subject: [PATCH 07/29] fix controller.py import into the game (doesnt listen to WASD however) --- .../minekov/client/XosViewportOverlay.java | 125 ++++++++++++++++++ .../minekov/client/XosViewportRuntime.java | 29 ++++ 2 files changed, 154 insertions(+) diff --git a/src/main/java/com/verbii/minekov/client/XosViewportOverlay.java b/src/main/java/com/verbii/minekov/client/XosViewportOverlay.java index 3dc9a23..fb0ff1a 100644 --- a/src/main/java/com/verbii/minekov/client/XosViewportOverlay.java +++ b/src/main/java/com/verbii/minekov/client/XosViewportOverlay.java @@ -597,6 +597,107 @@ private static int ctrlShortcutToActionCode(int key, int modifiers) { }; } + /** + * GLFW key -> xos binding token used by `self.state.keyboard.keys..is_pressed()`. + * Returns null for unknown keys. + */ + private static String keyCodeToBindingName(int key, int scanCode) { + String named = GLFW.glfwGetKeyName(key, scanCode); + if (named != null && named.length() == 1) { + char ch = Character.toUpperCase(named.charAt(0)); + if (Character.isLetterOrDigit(ch)) { + return String.valueOf(ch); + } + } + return switch (key) { + case GLFW.GLFW_KEY_SPACE -> "SPACE"; + case GLFW.GLFW_KEY_APOSTROPHE -> "APOSTROPHE"; + case GLFW.GLFW_KEY_COMMA -> "COMMA"; + case GLFW.GLFW_KEY_MINUS -> "MINUS"; + case GLFW.GLFW_KEY_PERIOD -> "PERIOD"; + case GLFW.GLFW_KEY_SLASH -> "SLASH"; + case GLFW.GLFW_KEY_SEMICOLON -> "SEMICOLON"; + case GLFW.GLFW_KEY_EQUAL -> "EQUAL"; + case GLFW.GLFW_KEY_LEFT_BRACKET -> "LEFT_BRACKET"; + case GLFW.GLFW_KEY_BACKSLASH -> "BACKSLASH"; + case GLFW.GLFW_KEY_RIGHT_BRACKET -> "RIGHT_BRACKET"; + case GLFW.GLFW_KEY_GRAVE_ACCENT -> "BACKQUOTE"; + case GLFW.GLFW_KEY_WORLD_1 -> "WORLD_1"; + case GLFW.GLFW_KEY_WORLD_2 -> "WORLD_2"; + case GLFW.GLFW_KEY_ESCAPE -> "ESCAPE"; + case GLFW.GLFW_KEY_ENTER -> "ENTER"; + case GLFW.GLFW_KEY_TAB -> "TAB"; + case GLFW.GLFW_KEY_BACKSPACE -> "BACKSPACE"; + case GLFW.GLFW_KEY_INSERT -> "INSERT"; + case GLFW.GLFW_KEY_DELETE -> "DELETE"; + case GLFW.GLFW_KEY_RIGHT -> "ARROW_RIGHT"; + case GLFW.GLFW_KEY_LEFT -> "ARROW_LEFT"; + case GLFW.GLFW_KEY_DOWN -> "ARROW_DOWN"; + case GLFW.GLFW_KEY_UP -> "ARROW_UP"; + case GLFW.GLFW_KEY_PAGE_UP -> "PAGE_UP"; + case GLFW.GLFW_KEY_PAGE_DOWN -> "PAGE_DOWN"; + case GLFW.GLFW_KEY_HOME -> "HOME"; + case GLFW.GLFW_KEY_END -> "END"; + case GLFW.GLFW_KEY_CAPS_LOCK -> "CAPS_LOCK"; + case GLFW.GLFW_KEY_SCROLL_LOCK -> "SCROLL_LOCK"; + case GLFW.GLFW_KEY_NUM_LOCK -> "NUM_LOCK"; + case GLFW.GLFW_KEY_PRINT_SCREEN -> "PRINT_SCREEN"; + case GLFW.GLFW_KEY_PAUSE -> "PAUSE"; + case GLFW.GLFW_KEY_F1 -> "F1"; + case GLFW.GLFW_KEY_F2 -> "F2"; + case GLFW.GLFW_KEY_F3 -> "F3"; + case GLFW.GLFW_KEY_F4 -> "F4"; + case GLFW.GLFW_KEY_F5 -> "F5"; + case GLFW.GLFW_KEY_F6 -> "F6"; + case GLFW.GLFW_KEY_F7 -> "F7"; + case GLFW.GLFW_KEY_F8 -> "F8"; + case GLFW.GLFW_KEY_F9 -> "F9"; + case GLFW.GLFW_KEY_F10 -> "F10"; + case GLFW.GLFW_KEY_F11 -> "F11"; + case GLFW.GLFW_KEY_F12 -> "F12"; + case GLFW.GLFW_KEY_F13 -> "F13"; + case GLFW.GLFW_KEY_F14 -> "F14"; + case GLFW.GLFW_KEY_F15 -> "F15"; + case GLFW.GLFW_KEY_F16 -> "F16"; + case GLFW.GLFW_KEY_F17 -> "F17"; + case GLFW.GLFW_KEY_F18 -> "F18"; + case GLFW.GLFW_KEY_F19 -> "F19"; + case GLFW.GLFW_KEY_F20 -> "F20"; + case GLFW.GLFW_KEY_F21 -> "F21"; + case GLFW.GLFW_KEY_F22 -> "F22"; + case GLFW.GLFW_KEY_F23 -> "F23"; + case GLFW.GLFW_KEY_F24 -> "F24"; + case GLFW.GLFW_KEY_F25 -> "F25"; + case GLFW.GLFW_KEY_KP_0 -> "KP_0"; + case GLFW.GLFW_KEY_KP_1 -> "KP_1"; + case GLFW.GLFW_KEY_KP_2 -> "KP_2"; + case GLFW.GLFW_KEY_KP_3 -> "KP_3"; + case GLFW.GLFW_KEY_KP_4 -> "KP_4"; + case GLFW.GLFW_KEY_KP_5 -> "KP_5"; + case GLFW.GLFW_KEY_KP_6 -> "KP_6"; + case GLFW.GLFW_KEY_KP_7 -> "KP_7"; + case GLFW.GLFW_KEY_KP_8 -> "KP_8"; + case GLFW.GLFW_KEY_KP_9 -> "KP_9"; + case GLFW.GLFW_KEY_KP_DECIMAL -> "KP_DECIMAL"; + case GLFW.GLFW_KEY_KP_DIVIDE -> "KP_DIVIDE"; + case GLFW.GLFW_KEY_KP_MULTIPLY -> "KP_MULTIPLY"; + case GLFW.GLFW_KEY_KP_SUBTRACT -> "KP_SUBTRACT"; + case GLFW.GLFW_KEY_KP_ADD -> "KP_ADD"; + case GLFW.GLFW_KEY_KP_ENTER -> "KP_ENTER"; + case GLFW.GLFW_KEY_KP_EQUAL -> "KP_EQUAL"; + case GLFW.GLFW_KEY_LEFT_SHIFT -> "LEFT_SHIFT"; + case GLFW.GLFW_KEY_LEFT_CONTROL -> "LEFT_CONTROL"; + case GLFW.GLFW_KEY_LEFT_ALT -> "LEFT_ALT"; + case GLFW.GLFW_KEY_LEFT_SUPER -> "LEFT_SUPER"; + case GLFW.GLFW_KEY_RIGHT_SHIFT -> "RIGHT_SHIFT"; + case GLFW.GLFW_KEY_RIGHT_CONTROL -> "RIGHT_CONTROL"; + case GLFW.GLFW_KEY_RIGHT_ALT -> "RIGHT_ALT"; + case GLFW.GLFW_KEY_RIGHT_SUPER -> "RIGHT_SUPER"; + case GLFW.GLFW_KEY_MENU -> "MENU"; + default -> null; + }; + } + private static double scaledMouseX(Minecraft mc, int sw) { return mc.mouseHandler.xpos() * (double) sw / (double) mc.getWindow().getScreenWidth(); } @@ -699,6 +800,10 @@ public static void onScreenKeyPressed(ScreenEvent.KeyPressed.Pre event) { return; } int key = event.getKeyCode(); + String bindingName = keyCodeToBindingName(key, event.getScanCode()); + if (bindingName != null) { + XosViewportRuntime.sendKeyDownToEngine(bindingName); + } int modifiers = event.getModifiers(); boolean commandDown = (modifiers & (GLFW.GLFW_MOD_CONTROL | GLFW.GLFW_MOD_SUPER)) != 0; if (commandDown && (key == GLFW.GLFW_KEY_SLASH || key == GLFW.GLFW_KEY_KP_DIVIDE)) { @@ -735,6 +840,26 @@ public static void onScreenKeyPressed(ScreenEvent.KeyPressed.Pre event) { event.setCanceled(true); } + @SubscribeEvent + public static void onScreenKeyReleased(ScreenEvent.KeyReleased.Pre event) { + if (!(event.getScreen() instanceof ChatScreen)) { + return; + } + Minecraft mc = Minecraft.getInstance(); + if (!shouldRouteKeyboardToXos(mc)) { + return; + } + int key = event.getKeyCode(); + if (key == GLFW.GLFW_KEY_ESCAPE) { + return; + } + String bindingName = keyCodeToBindingName(key, event.getScanCode()); + if (bindingName != null) { + XosViewportRuntime.sendKeyUpToEngine(bindingName); + } + event.setCanceled(true); + } + @SubscribeEvent public static void onScreenCharTyped(ScreenEvent.CharacterTyped.Pre event) { if (!(event.getScreen() instanceof ChatScreen)) { diff --git a/src/main/java/com/verbii/minekov/client/XosViewportRuntime.java b/src/main/java/com/verbii/minekov/client/XosViewportRuntime.java index a785309..5c12b89 100644 --- a/src/main/java/com/verbii/minekov/client/XosViewportRuntime.java +++ b/src/main/java/com/verbii/minekov/client/XosViewportRuntime.java @@ -82,6 +82,7 @@ public final class XosViewportRuntime { List.of( "balls_many.py", "demo_mod.py", + "controller.py", "inventory.py", "agent_controller.py", "look_at_me.py", @@ -368,6 +369,34 @@ public static void sendKeyCharToEngine(int codepoint) { XosNative.onKeyChar(codepoint); } + /** + * Forward low-latency key pressed state into xos (`self.state.keyboard.keys..is_pressed()`). + */ + public static void sendKeyDownToEngine(String bindingName) { + if (!runSession || bindingName == null || bindingName.isEmpty()) { + return; + } + tryLoadLibrary(); + if (!libraryOk || !engineRunning) { + return; + } + XosNative.onKeyDown(bindingName); + } + + /** + * Clear low-latency key pressed state in xos. + */ + public static void sendKeyUpToEngine(String bindingName) { + if (!runSession || bindingName == null || bindingName.isEmpty()) { + return; + } + tryLoadLibrary(); + if (!libraryOk || !engineRunning) { + return; + } + XosNative.onKeyUp(bindingName); + } + /** F3 → global FPS overlay toggle (matches desktop xos; not a text character). */ public static void sendF3ToEngine() { if (!runSession) { From 6df4bfb64d1e01f4dc7301bf381e099ec6e51920 Mon Sep 17 00:00:00 2001 From: verbiiyo Date: Mon, 1 Jun 2026 14:03:14 +0900 Subject: [PATCH 08/29] key listener works! alongside xos changes! --- .../minekov/client/XosViewportOverlay.java | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/src/main/java/com/verbii/minekov/client/XosViewportOverlay.java b/src/main/java/com/verbii/minekov/client/XosViewportOverlay.java index fb0ff1a..08ccd76 100644 --- a/src/main/java/com/verbii/minekov/client/XosViewportOverlay.java +++ b/src/main/java/com/verbii/minekov/client/XosViewportOverlay.java @@ -610,6 +610,42 @@ private static String keyCodeToBindingName(int key, int scanCode) { } } return switch (key) { + case GLFW.GLFW_KEY_0 -> "0"; + case GLFW.GLFW_KEY_1 -> "1"; + case GLFW.GLFW_KEY_2 -> "2"; + case GLFW.GLFW_KEY_3 -> "3"; + case GLFW.GLFW_KEY_4 -> "4"; + case GLFW.GLFW_KEY_5 -> "5"; + case GLFW.GLFW_KEY_6 -> "6"; + case GLFW.GLFW_KEY_7 -> "7"; + case GLFW.GLFW_KEY_8 -> "8"; + case GLFW.GLFW_KEY_9 -> "9"; + case GLFW.GLFW_KEY_A -> "A"; + case GLFW.GLFW_KEY_B -> "B"; + case GLFW.GLFW_KEY_C -> "C"; + case GLFW.GLFW_KEY_D -> "D"; + case GLFW.GLFW_KEY_E -> "E"; + case GLFW.GLFW_KEY_F -> "F"; + case GLFW.GLFW_KEY_G -> "G"; + case GLFW.GLFW_KEY_H -> "H"; + case GLFW.GLFW_KEY_I -> "I"; + case GLFW.GLFW_KEY_J -> "J"; + case GLFW.GLFW_KEY_K -> "K"; + case GLFW.GLFW_KEY_L -> "L"; + case GLFW.GLFW_KEY_M -> "M"; + case GLFW.GLFW_KEY_N -> "N"; + case GLFW.GLFW_KEY_O -> "O"; + case GLFW.GLFW_KEY_P -> "P"; + case GLFW.GLFW_KEY_Q -> "Q"; + case GLFW.GLFW_KEY_R -> "R"; + case GLFW.GLFW_KEY_S -> "S"; + case GLFW.GLFW_KEY_T -> "T"; + case GLFW.GLFW_KEY_U -> "U"; + case GLFW.GLFW_KEY_V -> "V"; + case GLFW.GLFW_KEY_W -> "W"; + case GLFW.GLFW_KEY_X -> "X"; + case GLFW.GLFW_KEY_Y -> "Y"; + case GLFW.GLFW_KEY_Z -> "Z"; case GLFW.GLFW_KEY_SPACE -> "SPACE"; case GLFW.GLFW_KEY_APOSTROPHE -> "APOSTROPHE"; case GLFW.GLFW_KEY_COMMA -> "COMMA"; From e174dcef54017d373d57c7fbb9ab6d20bafe84c4 Mon Sep 17 00:00:00 2001 From: verbiiyo Date: Mon, 1 Jun 2026 14:05:37 +0900 Subject: [PATCH 09/29] agent controller wasd starting to work! --- .../xos/examples/agent_controller.py | 34 ++++--------------- src/xos/examples/agent_controller.py | 34 ++++--------------- 2 files changed, 12 insertions(+), 56 deletions(-) diff --git a/src/main/resources/xos/examples/agent_controller.py b/src/main/resources/xos/examples/agent_controller.py index b96ab2c..067593a 100644 --- a/src/main/resources/xos/examples/agent_controller.py +++ b/src/main/resources/xos/examples/agent_controller.py @@ -5,7 +5,6 @@ class AgentController(xos.Application): def __init__(self): super().__init__(max_fps=15) - self.key_active_until = {} self.jump_cooldown_until = 0.0 self.view_w = 128 self.view_h = 128 @@ -13,12 +12,6 @@ def __init__(self): def _now(self): return xos.time.perf_counter() - def _pressed(self, key): - return self._now() < self.key_active_until.get(key, 0.0) - - def _mark_pressed(self, key, hold_s=0.35): - self.key_active_until[key] = self._now() + hold_s - def tick(self): agents = list(mc.agents) if len(agents) == 0: @@ -27,16 +20,17 @@ def tick(self): agent = agents[0] now = self._now() + keys = self.state.keyboard.keys - if self._pressed("w"): + if keys.W.is_pressed(): agent.actions.w() - if self._pressed("a"): + if keys.A.is_pressed(): agent.actions.a() - if self._pressed("s"): + if keys.S.is_pressed(): agent.actions.s() - if self._pressed("d"): + if keys.D.is_pressed(): agent.actions.d() - if self._pressed("space") and now >= self.jump_cooldown_until: + if keys.SPACE.is_pressed() and now >= self.jump_cooldown_until: agent.actions.jump() self.jump_cooldown_until = now + 0.25 @@ -48,22 +42,6 @@ def tick(self): print("agent_controller render error:", e) self.frame.clear((0, 0, 0, 255)) - def on_events(self): - ev = getattr(self, "_xos_event", None) - if not isinstance(ev, dict): - return - if ev.get("kind") != "key_char": - return - - ch = str(ev.get("char", "")) - if not ch: - return - c = ch.lower() - if c in ("w", "a", "s", "d"): - self._mark_pressed(c) - elif ch == " ": - self._mark_pressed("space", hold_s=0.2) - if __name__ == "__main__": AgentController().run() diff --git a/src/xos/examples/agent_controller.py b/src/xos/examples/agent_controller.py index 6c853d0..356b6db 100644 --- a/src/xos/examples/agent_controller.py +++ b/src/xos/examples/agent_controller.py @@ -5,7 +5,6 @@ class AgentController(xos.Application): def __init__(self): super().__init__(max_fps=15) - self.key_active_until = {} self.jump_cooldown_until = 0.0 self.view_w = 128 self.view_h = 128 @@ -13,12 +12,6 @@ def __init__(self): def _now(self): return xos.time.perf_counter() - def _pressed(self, key): - return self._now() < self.key_active_until.get(key, 0.0) - - def _mark_pressed(self, key, hold_s=0.35): - self.key_active_until[key] = self._now() + hold_s - def tick(self): agents = list(mc.agents) if len(agents) == 0: @@ -27,17 +20,18 @@ def tick(self): agent = agents[0] now = self._now() + keys = self.state.keyboard.keys # Movement controls (single controllable agent: index 0). - if self._pressed("w"): + if keys.W.is_pressed(): agent.actions.w() - if self._pressed("a"): + if keys.A.is_pressed(): agent.actions.a() - if self._pressed("s"): + if keys.S.is_pressed(): agent.actions.s() - if self._pressed("d"): + if keys.D.is_pressed(): agent.actions.d() - if self._pressed("space") and now >= self.jump_cooldown_until: + if keys.SPACE.is_pressed() and now >= self.jump_cooldown_until: agent.actions.jump() self.jump_cooldown_until = now + 0.25 @@ -51,22 +45,6 @@ def tick(self): print("agent_controller render error:", e) self.frame.clear((0, 0, 0, 255)) - def on_events(self): - ev = getattr(self, "_xos_event", None) - if not isinstance(ev, dict): - return - if ev.get("kind") != "key_char": - return - - ch = str(ev.get("char", "")) - if not ch: - return - c = ch.lower() - if c in ("w", "a", "s", "d"): - self._mark_pressed(c) - elif ch == " ": - self._mark_pressed("space", hold_s=0.2) - if __name__ == "__main__": AgentController().run() From b30d00d92dd20a4f518ae86d27d55e6f991bb257 Mon Sep 17 00:00:00 2001 From: verbiiyo Date: Mon, 1 Jun 2026 14:42:36 +0900 Subject: [PATCH 10/29] entities are orbs? lmao --- .../minekov/client/XosViewportRuntime.java | 145 ++++++++++++++++-- 1 file changed, 128 insertions(+), 17 deletions(-) diff --git a/src/main/java/com/verbii/minekov/client/XosViewportRuntime.java b/src/main/java/com/verbii/minekov/client/XosViewportRuntime.java index 5c12b89..a4ed06c 100644 --- a/src/main/java/com/verbii/minekov/client/XosViewportRuntime.java +++ b/src/main/java/com/verbii/minekov/client/XosViewportRuntime.java @@ -21,7 +21,9 @@ import net.minecraft.world.entity.ai.attributes.Attributes; import net.minecraft.world.entity.player.Inventory; import net.minecraft.world.item.ItemStack; +import net.minecraft.world.level.ClipContext; import net.minecraft.world.level.storage.LevelResource; +import net.minecraft.world.phys.HitResult; import net.minecraft.world.phys.Vec3; import net.minecraftforge.api.distmarker.Dist; import net.minecraftforge.api.distmarker.OnlyIn; @@ -77,7 +79,7 @@ public final class XosViewportRuntime { private static final String ITEM_ROW_SEPARATOR = "\u001e"; private static final float AGENT_VIEW_FOV_DEG = 70.0f; private static final double AGENT_VIEW_MAX_DISTANCE = 24.0; - private static final double AGENT_VIEW_STEP = 1.0; + private static final char[] HEX = "0123456789abcdef".toCharArray(); private static final List STARTER_SCRIPT_NAMES = List.of( "balls_many.py", @@ -644,10 +646,18 @@ private static String encodePlayerInventoryEnderChest(Minecraft mc) { } private static int clampViewSize(int value) { - return Math.max(16, Math.min(128, value)); + return Math.max(16, Math.min(512, value)); } - private static int rgbFromBlockState(net.minecraft.world.level.block.state.BlockState state) { + private static int rgbFromBlockState( + Minecraft mc, BlockPos pos, net.minecraft.world.level.block.state.BlockState state) { + try { + var map = state.getMapColor(mc.level, pos); + if (map != null && map.col != 0) { + return map.col & 0x00FFFFFF; + } + } catch (Throwable ignored) { + } ResourceLocation key = BuiltInRegistries.BLOCK.getKey(state.getBlock()); int hash = key != null ? key.toString().hashCode() : 0x77AA77; int r = 64 + (Math.abs(hash) & 0x7F); @@ -656,6 +666,28 @@ private static int rgbFromBlockState(net.minecraft.world.level.block.state.Block return (r << 16) | (g << 8) | b; } + private static int rgbFromEntity(Entity entity) { + int hash = entity.getType().toString().hashCode(); + int r = 96 + (Math.abs(hash) & 0x5F); + int g = 96 + ((Math.abs(hash >> 8)) & 0x5F); + int b = 96 + ((Math.abs(hash >> 16)) & 0x5F); + if (entity instanceof LivingEntity) { + r = Math.min(255, r + 24); + } + return (r << 16) | (g << 8) | b; + } + + private static String encodeRgbaHex(byte[] rgba) { + char[] out = new char[rgba.length * 2]; + int j = 0; + for (byte b : rgba) { + int v = b & 0xFF; + out[j++] = HEX[v >>> 4]; + out[j++] = HEX[v & 0x0F]; + } + return new String(out); + } + private static String renderAgentViewRgbaHex(Minecraft mc, RLOperator agent, int reqW, int reqH) { int width = clampViewSize(reqW); int height = clampViewSize(reqH); @@ -678,7 +710,12 @@ private static String renderAgentViewRgbaHex(Minecraft mc, RLOperator agent, int double tanHalfFov = Math.tan(Math.toRadians(AGENT_VIEW_FOV_DEG * 0.5)); double aspect = (double) width / (double) height; - StringBuilder hex = new StringBuilder(width * height * 8); + int pixelCount = width * height; + byte[] rgba = new byte[pixelCount * 4]; + double[] depth = new double[pixelCount]; + for (int i = 0; i < pixelCount; i++) { + depth[i] = Double.POSITIVE_INFINITY; + } for (int y = 0; y < height; y++) { double ny = 1.0 - (2.0 * (y + 0.5) / (double) height); @@ -694,20 +731,32 @@ private static String renderAgentViewRgbaHex(Minecraft mc, RLOperator agent, int int b = 210; boolean hit = false; double distance = 0.0; - - for (double t = AGENT_VIEW_STEP; t <= AGENT_VIEW_MAX_DISTANCE; t += AGENT_VIEW_STEP) { - Vec3 sample = origin.add(dir.scale(t)); - BlockPos bp = BlockPos.containing(sample.x, sample.y, sample.z); + Vec3 rayEnd = origin.add(dir.scale(AGENT_VIEW_MAX_DISTANCE)); + var hitRes = mc.level.clip(new ClipContext( + origin, + rayEnd, + ClipContext.Block.OUTLINE, + ClipContext.Fluid.NONE, + agent)); + if (hitRes.getType() != HitResult.Type.MISS) { + BlockPos bp = hitRes.getBlockPos(); var state = mc.level.getBlockState(bp); if (!state.isAir()) { - int rgb = rgbFromBlockState(state); - distance = t; - double shade = Math.max(0.22, 1.0 - (t / AGENT_VIEW_MAX_DISTANCE)); - r = (int) Math.max(0, Math.min(255, ((rgb >> 16) & 0xFF) * shade)); - g = (int) Math.max(0, Math.min(255, ((rgb >> 8) & 0xFF) * shade)); - b = (int) Math.max(0, Math.min(255, (rgb & 0xFF) * shade)); + int rgb = rgbFromBlockState(mc, bp, state); + distance = hitRes.getLocation().distanceTo(origin); + double shade = Math.max(0.28, 1.0 - (distance / AGENT_VIEW_MAX_DISTANCE)); + // Face-aware brighten/darken helps depth cues in tiny viewports. + double faceShade = switch (hitRes.getDirection()) { + case UP -> 1.08; + case DOWN -> 0.72; + case NORTH, SOUTH -> 0.94; + default -> 1.00; + }; + double totalShade = Math.max(0.20, Math.min(1.15, shade * faceShade)); + r = (int) Math.max(0, Math.min(255, ((rgb >> 16) & 0xFF) * totalShade)); + g = (int) Math.max(0, Math.min(255, ((rgb >> 8) & 0xFF) * totalShade)); + b = (int) Math.max(0, Math.min(255, (rgb & 0xFF) * totalShade)); hit = true; - break; } } @@ -723,11 +772,73 @@ private static String renderAgentViewRgbaHex(Minecraft mc, RLOperator agent, int g = Math.min(255, g + fog); b = Math.min(255, b + fog); } + int p = y * width + x; + int i = p * 4; + rgba[i] = (byte) r; + rgba[i + 1] = (byte) g; + rgba[i + 2] = (byte) b; + rgba[i + 3] = (byte) 255; + depth[p] = hit ? distance : Double.POSITIVE_INFINITY; + } + } + + // Overlay nearby entities as simple depth-tested billboards so they are visible in the agent feed. + var renderables = mc.level.entitiesForRendering(); + for (Entity e : renderables) { + if (e == null || e == agent || e.isRemoved()) { + continue; + } + Vec3 rel = new Vec3(e.getX() - origin.x, e.getEyeY() - origin.y, e.getZ() - origin.z); + double camX = rel.dot(right); + double camY = rel.dot(up); + double camZ = rel.dot(forward); + if (camZ <= 0.01 || camZ > AGENT_VIEW_MAX_DISTANCE) { + continue; + } - hex.append(String.format(Locale.ROOT, "%02x%02x%02x%02x", r, g, b, 255)); + double ndcX = camX / (camZ * tanHalfFov * aspect); + double ndcY = camY / (camZ * tanHalfFov); + if (Math.abs(ndcX) > 1.5 || Math.abs(ndcY) > 1.5) { + continue; + } + + int cx = (int) Math.round((ndcX + 1.0) * 0.5 * (width - 1)); + int cy = (int) Math.round((1.0 - (ndcY + 1.0) * 0.5) * (height - 1)); + double radiusPx = Math.max( + 1.0, + (Math.max(0.25, e.getBbWidth()) / (camZ * tanHalfFov * aspect)) * (width * 0.28)); + int rad = (int) Math.min(Math.max(1, Math.round(radiusPx)), Math.max(width, height)); + int color = rgbFromEntity(e); + int er = (color >> 16) & 0xFF; + int eg = (color >> 8) & 0xFF; + int eb = color & 0xFF; + int x1 = Math.max(0, cx - rad); + int x2 = Math.min(width - 1, cx + rad); + int y1 = Math.max(0, cy - rad); + int y2 = Math.min(height - 1, cy + rad); + int rr = rad * rad; + for (int py = y1; py <= y2; py++) { + int dy = py - cy; + for (int px = x1; px <= x2; px++) { + int dx = px - cx; + if ((dx * dx + dy * dy) > rr) { + continue; + } + int p = py * width + px; + if (camZ >= depth[p]) { + continue; + } + int i = p * 4; + rgba[i] = (byte) er; + rgba[i + 1] = (byte) eg; + rgba[i + 2] = (byte) eb; + rgba[i + 3] = (byte) 255; + depth[p] = camZ; + } } } - return hex.toString(); + + return encodeRgbaHex(rgba); } private static double[] parseTriple(String raw, double fallbackX, double fallbackY, double fallbackZ) { From 4c790a70df1174dfb847d0c8c1eaa9b66b8798a3 Mon Sep 17 00:00:00 2001 From: verbiiyo Date: Mon, 1 Jun 2026 15:01:25 +0900 Subject: [PATCH 11/29] it kinda looks better/crispier but its mirrored --- .../minekov/client/XosViewportRuntime.java | 160 ++++++++++++++++++ 1 file changed, 160 insertions(+) diff --git a/src/main/java/com/verbii/minekov/client/XosViewportRuntime.java b/src/main/java/com/verbii/minekov/client/XosViewportRuntime.java index a4ed06c..e9e5a9b 100644 --- a/src/main/java/com/verbii/minekov/client/XosViewportRuntime.java +++ b/src/main/java/com/verbii/minekov/client/XosViewportRuntime.java @@ -2,8 +2,12 @@ import ai.xlate.xos.XosNative; +import com.mojang.blaze3d.pipeline.RenderTarget; +import com.mojang.blaze3d.pipeline.TextureTarget; import com.google.gson.JsonElement; import com.mojang.blaze3d.platform.NativeImage; +import com.mojang.blaze3d.systems.RenderSystem; +import com.mojang.blaze3d.vertex.PoseStack; import com.mojang.serialization.JsonOps; import com.verbii.minekov.Minekov; import com.verbii.minekov.entities.RLOperator; @@ -32,6 +36,7 @@ import org.slf4j.Logger; import java.lang.reflect.Field; +import java.lang.reflect.Method; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.IntBuffer; @@ -49,6 +54,7 @@ import java.util.concurrent.atomic.AtomicReference; import org.lwjgl.system.MemoryUtil; +import org.lwjgl.opengl.GL11C; /** * Runs the xos engine (ball app via JNI) when {@link #setRunSession(boolean) run session} is on; @@ -80,6 +86,11 @@ public final class XosViewportRuntime { private static final float AGENT_VIEW_FOV_DEG = 70.0f; private static final double AGENT_VIEW_MAX_DISTANCE = 24.0; private static final char[] HEX = "0123456789abcdef".toCharArray(); + // Keep disabled until we have a fully isolated offscreen pass that cannot touch the live HUD/chat viewport. + private static final boolean ENABLE_REAL_AGENT_VIEW_RENDER = false; + private static TextureTarget agentViewRenderTarget; + private static Method gameRendererRenderLevelMethod; + private static Field minecraftMainRenderTargetField; private static final List STARTER_SCRIPT_NAMES = List.of( "balls_many.py", @@ -694,6 +705,14 @@ private static String renderAgentViewRgbaHex(Minecraft mc, RLOperator agent, int if (mc == null || mc.level == null || agent == null) { return ""; } + if (ENABLE_REAL_AGENT_VIEW_RENDER) { + if (ENABLE_REAL_AGENT_VIEW_RENDER) { + byte[] rendered = tryRenderAgentViewWithMinecraftRenderer(mc, agent, width, height); + if (rendered != null) { + return encodeRgbaHex(rendered); + } + } + } Vec3 origin = new Vec3(agent.getX(), agent.getEyeY(), agent.getZ()); float yaw = agent.getYRot(); @@ -841,6 +860,147 @@ private static String renderAgentViewRgbaHex(Minecraft mc, RLOperator agent, int return encodeRgbaHex(rgba); } + private static byte[] tryRenderAgentViewWithMinecraftRenderer( + Minecraft mc, RLOperator agent, int width, int height) { + // Keep this on the render thread only; host binding runs on the client thread already. + if (!RenderSystem.isOnRenderThread()) { + return null; + } + try { + if (agentViewRenderTarget == null + || agentViewRenderTarget.width != width + || agentViewRenderTarget.height != height) { + if (agentViewRenderTarget != null) { + agentViewRenderTarget.destroyBuffers(); + } + agentViewRenderTarget = new TextureTarget(width, height, true, Minecraft.ON_OSX); + } + } catch (Throwable t) { + return null; + } + + RenderTarget previousTarget = mc.getMainRenderTarget(); + Entity previousCamera = mc.getCameraEntity(); + boolean swappedMainTarget = setMinecraftMainRenderTarget(mc, previousTarget, agentViewRenderTarget); + if (!swappedMainTarget) { + // If we cannot isolate render context, fall back to the safe custom renderer. + return null; + } + try { + mc.setCameraEntity(agent); + agentViewRenderTarget.bindWrite(true); + RenderSystem.viewport(0, 0, width, height); + agentViewRenderTarget.setClearColor(0f, 0f, 0f, 1f); + agentViewRenderTarget.clear(Minecraft.ON_OSX); + + if (!invokeGameRendererLevelPass(mc)) { + return null; + } + + ByteBuffer buf = MemoryUtil.memAlloc(width * height * 4); + try { + GL11C.glReadPixels(0, 0, width, height, GL11C.GL_RGBA, GL11C.GL_UNSIGNED_BYTE, buf); + byte[] out = new byte[width * height * 4]; + int rowBytes = width * 4; + for (int y = 0; y < height; y++) { + int src = (height - 1 - y) * rowBytes; + int dst = y * rowBytes; + buf.position(src); + buf.get(out, dst, rowBytes); + } + return out; + } finally { + MemoryUtil.memFree(buf); + } + } catch (Throwable t) { + return null; + } finally { + try { + mc.setCameraEntity(previousCamera); + } catch (Throwable ignored) { + } + try { + if (swappedMainTarget) { + setMinecraftMainRenderTarget(mc, agentViewRenderTarget, previousTarget); + } + previousTarget.bindWrite(true); + RenderSystem.viewport(0, 0, previousTarget.width, previousTarget.height); + } catch (Throwable ignored) { + } + } + } + + private static boolean invokeGameRendererLevelPass(Minecraft mc) { + try { + if (gameRendererRenderLevelMethod == null) { + for (Method m : mc.gameRenderer.getClass().getDeclaredMethods()) { + if ("renderLevel".equals(m.getName())) { + m.setAccessible(true); + gameRendererRenderLevelMethod = m; + break; + } + } + } + if (gameRendererRenderLevelMethod == null) { + return false; + } + Class[] p = gameRendererRenderLevelMethod.getParameterTypes(); + Object[] args = new Object[p.length]; + for (int i = 0; i < p.length; i++) { + Class t = p[i]; + if (t == float.class || t == Float.class) { + args[i] = mc.getFrameTime(); + } else if (t == double.class || t == Double.class) { + args[i] = (double) mc.getFrameTime(); + } else if (t == long.class || t == Long.class) { + args[i] = System.nanoTime(); + } else if (t == int.class || t == Integer.class) { + args[i] = 0; + } else if (t == boolean.class || t == Boolean.class) { + // Avoid leaking selection/outline/debug overlays into the offscreen pass. + args[i] = Boolean.FALSE; + } else if (PoseStack.class.isAssignableFrom(t)) { + args[i] = new PoseStack(); + } else { + args[i] = null; + } + } + gameRendererRenderLevelMethod.invoke(mc.gameRenderer, args); + return true; + } catch (Throwable t) { + return false; + } + } + + private static boolean setMinecraftMainRenderTarget( + Minecraft mc, RenderTarget expectedCurrent, RenderTarget target) { + try { + if (minecraftMainRenderTargetField == null) { + for (Field f : Minecraft.class.getDeclaredFields()) { + if (RenderTarget.class.isAssignableFrom(f.getType())) { + f.setAccessible(true); + Object cur = f.get(mc); + if (cur == expectedCurrent) { + minecraftMainRenderTargetField = f; + break; + } + } + } + } + if (minecraftMainRenderTargetField == null) { + return false; + } + Object cur = minecraftMainRenderTargetField.get(mc); + if (cur != expectedCurrent) { + return false; + } + minecraftMainRenderTargetField.set(mc, target); + return true; + } catch (Throwable t) { + return false; + } + } + private static double[] parseTriple(String raw, double fallbackX, double fallbackY, double fallbackZ) { if (raw == null || raw.isBlank()) { return new double[] {fallbackX, fallbackY, fallbackZ}; From a843b499968666311f99246db61be82d4df890e9 Mon Sep 17 00:00:00 2001 From: verbiiyo Date: Mon, 1 Jun 2026 16:03:56 +0900 Subject: [PATCH 12/29] add the mouse show/hide and lock/unlock --- .../minekov/client/XosViewportRuntime.java | 75 +++++++++++++++++++ src/main/resources/xos/examples/controller.py | 10 ++- 2 files changed, 84 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/verbii/minekov/client/XosViewportRuntime.java b/src/main/java/com/verbii/minekov/client/XosViewportRuntime.java index e9e5a9b..878b3ac 100644 --- a/src/main/java/com/verbii/minekov/client/XosViewportRuntime.java +++ b/src/main/java/com/verbii/minekov/client/XosViewportRuntime.java @@ -55,6 +55,7 @@ import org.lwjgl.system.MemoryUtil; import org.lwjgl.opengl.GL11C; +import org.lwjgl.glfw.GLFW; /** * Runs the xos engine (ball app via JNI) when {@link #setRunSession(boolean) run session} is on; @@ -160,6 +161,16 @@ public final class XosViewportRuntime { private static volatile int maxPumpsPerSecond; private static long lastPumpNanos; + private static int nativeCursorMode = GLFW.GLFW_CURSOR_NORMAL; + private static boolean nativeMouseLocked; + private static boolean nativeMouseHidden; + private static boolean lockAnchorInitialized; + private static double lockAnchorWindowX; + private static double lockAnchorWindowY; + private static float lockSyntheticLocalX; + private static float lockSyntheticLocalY; + private static final double[] CURSOR_X = new double[1]; + private static final double[] CURSOR_Y = new double[1]; /** * Rotation lock requested from mc Python bindings. Values are [yaw, pitch] and are re-applied each * frame so Minecraft AI/network updates do not drift agents off target. @@ -201,10 +212,46 @@ public static boolean isEngineReady() { public static void setRunSession(boolean active) { if (!active && runSession) { stopActiveExecution(); + releaseNativeMouseCapture(Minecraft.getInstance()); } runSession = active; } + private static void releaseNativeMouseCapture(Minecraft mc) { + if (mc == null || mc.getWindow() == null) { + nativeCursorMode = GLFW.GLFW_CURSOR_NORMAL; + nativeMouseLocked = false; + nativeMouseHidden = false; + lockAnchorInitialized = false; + return; + } + long win = mc.getWindow().getWindow(); + GLFW.glfwSetInputMode(win, GLFW.GLFW_CURSOR, GLFW.GLFW_CURSOR_NORMAL); + nativeCursorMode = GLFW.GLFW_CURSOR_NORMAL; + nativeMouseLocked = false; + nativeMouseHidden = false; + lockAnchorInitialized = false; + } + + private static void applyNativeMouseCaptureState(Minecraft mc, boolean locked, boolean hidden) { + if (mc == null || mc.getWindow() == null) { + return; + } + long win = mc.getWindow().getWindow(); + int mode = locked + ? GLFW.GLFW_CURSOR_DISABLED + : (hidden ? GLFW.GLFW_CURSOR_HIDDEN : GLFW.GLFW_CURSOR_NORMAL); + if (mode != nativeCursorMode) { + GLFW.glfwSetInputMode(win, GLFW.GLFW_CURSOR, mode); + nativeCursorMode = mode; + } + if (!locked) { + lockAnchorInitialized = false; + } + nativeMouseLocked = locked; + nativeMouseHidden = hidden; + } + /** Stops currently running Coder execution(s) inside xos, if engine is initialized. */ public static void stopActiveExecution() { if (!libraryOk || !engineRunning) { @@ -305,6 +352,9 @@ public static void pumpFrame(Minecraft mc, int guiBodyW, int guiBodyH) { applyForcedAgentRotations(mc); XosNative.setMinecraftViewportAlpha(panelHovered ? VIEWPORT_ALPHA_HOVER : VIEWPORT_ALPHA_IDLE); XosNative.tick(); + nativeMouseLocked = XosNative.isMouseLocked(); + nativeMouseHidden = XosNative.isMouseCursorHidden(); + applyNativeMouseCaptureState(mc, nativeMouseLocked, nativeMouseHidden || nativeMouseLocked); ByteBuffer buf = cachedPackedFrameBuffer; if (buf == null || nativeImage == null) { @@ -343,6 +393,29 @@ public static void syncPointer( if (!libraryOk || !engineRunning || !runSession || bodyW < 1 || bodyH < 1) { return; } + if (nativeMouseLocked && mc != null && mc.getWindow() != null) { + float s = (float) effectiveGuiScale(mc); + long win = mc.getWindow().getWindow(); + if (!lockAnchorInitialized) { + GLFW.glfwGetCursorPos(win, CURSOR_X, CURSOR_Y); + lockAnchorWindowX = CURSOR_X[0]; + lockAnchorWindowY = CURSOR_Y[0]; + lockSyntheticLocalX = (float) (mouseX - bodyLeft) * s; + lockSyntheticLocalY = (float) (mouseY - bodyTop) * s; + lockAnchorInitialized = true; + } else { + GLFW.glfwGetCursorPos(win, CURSOR_X, CURSOR_Y); + double dx = CURSOR_X[0] - lockAnchorWindowX; + double dy = CURSOR_Y[0] - lockAnchorWindowY; + if (Math.abs(dx) > 0.001 || Math.abs(dy) > 0.001) { + lockSyntheticLocalX += (float) dx * s; + lockSyntheticLocalY += (float) dy * s; + XosNative.onMouseMove(lockSyntheticLocalX, lockSyntheticLocalY); + GLFW.glfwSetCursorPos(win, lockAnchorWindowX, lockAnchorWindowY); + } + } + return; + } if (mouseX >= bodyLeft && mouseX < bodyLeft + bodyW && mouseY >= bodyTop @@ -449,6 +522,7 @@ public static void disposeEngine() { stopActiveExecution(); runSession = false; if (!libraryOk || !engineRunning) { + releaseNativeMouseCapture(Minecraft.getInstance()); return; } Minecraft mc = Minecraft.getInstance(); @@ -468,6 +542,7 @@ public static void disposeEngine() { panelHovered = false; prewarmAttempted = false; lastPumpNanos = 0L; + releaseNativeMouseCapture(mc); } private static Path resolvePlayerCoderScriptsDirectory(Minecraft mc) { diff --git a/src/main/resources/xos/examples/controller.py b/src/main/resources/xos/examples/controller.py index 7ba0447..e530c88 100644 --- a/src/main/resources/xos/examples/controller.py +++ b/src/main/resources/xos/examples/controller.py @@ -44,6 +44,7 @@ def __init__(self): def _input_axis(self): keys = self.state.keyboard.keys + mouse = self.state.mouse ax = 0.0 ay = 0.0 if keys.W.is_pressed(): @@ -54,15 +55,23 @@ def _input_axis(self): ay += 1.0 if keys.D.is_pressed(): ax += 1.0 + if mouse.LEFT_CLICK.is_pressed(): + mouse.lock(); mouse.hide() + ax += mouse.dx * 0.02 + ay += mouse.dy * 0.02 + else: + mouse.unlock(); mouse.show() return _normalized_or_zero(ax, ay) def tick(self): dt = min(MAX_DT, max(0.0, self.dt)) ax, ay = self._input_axis() + # Apply thrust from current directional intent. self.vx += ax * THRUST * dt self.vy += ay * THRUST * dt + # Linear drag: easier to reason about than exponential damping. drag = max(0.0, 1.0 - DRAG_PER_SEC * dt) self.vx *= drag self.vy *= drag @@ -93,6 +102,5 @@ def tick(self): NOSE_COLOR, ) - if __name__ == "__main__": ControllerApp().run() From f5153af76745df53e3c2295f24cd1bc5bcdbfa27 Mon Sep 17 00:00:00 2001 From: verbiiyo Date: Mon, 1 Jun 2026 16:22:39 +0900 Subject: [PATCH 13/29] patched mouse lock and hide in-game --- .../verbii/minekov/client/XosViewportOverlay.java | 8 ++++++-- .../verbii/minekov/client/XosViewportRuntime.java | 15 ++++++++++----- src/xos/examples/controller.py | 10 +++++++++- 3 files changed, 25 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/verbii/minekov/client/XosViewportOverlay.java b/src/main/java/com/verbii/minekov/client/XosViewportOverlay.java index 08ccd76..9055728 100644 --- a/src/main/java/com/verbii/minekov/client/XosViewportOverlay.java +++ b/src/main/java/com/verbii/minekov/client/XosViewportOverlay.java @@ -1005,7 +1005,9 @@ public static void onScreenRenderPost(ScreenEvent.Render.Post event) { RenderSystem.disableBlend(); - updateResizeCursor(mc, sw, sh, mx, my); + if (!XosViewportRuntime.isMouseCaptureActive()) { + updateResizeCursor(mc, sw, sh, mx, my); + } return; } @@ -1073,7 +1075,9 @@ public static void onScreenRenderPost(ScreenEvent.Render.Post event) { RenderSystem.disableBlend(); - updateResizeCursor(mc, sw, sh, mx, my); + if (!XosViewportRuntime.isMouseCaptureActive()) { + updateResizeCursor(mc, sw, sh, mx, my); + } } @SubscribeEvent diff --git a/src/main/java/com/verbii/minekov/client/XosViewportRuntime.java b/src/main/java/com/verbii/minekov/client/XosViewportRuntime.java index 878b3ac..4f7ed99 100644 --- a/src/main/java/com/verbii/minekov/client/XosViewportRuntime.java +++ b/src/main/java/com/verbii/minekov/client/XosViewportRuntime.java @@ -204,6 +204,11 @@ public static boolean isEngineReady() { return libraryOk && engineRunning; } + /** True when xos requests host-level cursor capture/hidden behavior. */ + public static boolean isMouseCaptureActive() { + return nativeMouseLocked || nativeMouseHidden; + } + /** * Starts or pauses the xos simulation. Pausing does not tear down the native * engine (library stays loaded, {@link XosNative#init} state kept) so Run resumes instantly. @@ -238,9 +243,7 @@ private static void applyNativeMouseCaptureState(Minecraft mc, boolean locked, b return; } long win = mc.getWindow().getWindow(); - int mode = locked - ? GLFW.GLFW_CURSOR_DISABLED - : (hidden ? GLFW.GLFW_CURSOR_HIDDEN : GLFW.GLFW_CURSOR_NORMAL); + int mode = (locked || hidden) ? GLFW.GLFW_CURSOR_HIDDEN : GLFW.GLFW_CURSOR_NORMAL; if (mode != nativeCursorMode) { GLFW.glfwSetInputMode(win, GLFW.GLFW_CURSOR, mode); nativeCursorMode = mode; @@ -342,6 +345,9 @@ public static void pumpFrame(Minecraft mc, int guiBodyW, int guiBodyH) { long minNs = 1_000_000_000L / (long) cap; long now = System.nanoTime(); if (lastPumpNanos != 0L && now - lastPumpNanos < minNs) { + nativeMouseLocked = XosNative.isMouseLocked(); + nativeMouseHidden = XosNative.isMouseCursorHidden(); + applyNativeMouseCaptureState(mc, nativeMouseLocked, nativeMouseHidden || nativeMouseLocked); return; } lastPumpNanos = now; @@ -396,15 +402,14 @@ public static void syncPointer( if (nativeMouseLocked && mc != null && mc.getWindow() != null) { float s = (float) effectiveGuiScale(mc); long win = mc.getWindow().getWindow(); + GLFW.glfwGetCursorPos(win, CURSOR_X, CURSOR_Y); if (!lockAnchorInitialized) { - GLFW.glfwGetCursorPos(win, CURSOR_X, CURSOR_Y); lockAnchorWindowX = CURSOR_X[0]; lockAnchorWindowY = CURSOR_Y[0]; lockSyntheticLocalX = (float) (mouseX - bodyLeft) * s; lockSyntheticLocalY = (float) (mouseY - bodyTop) * s; lockAnchorInitialized = true; } else { - GLFW.glfwGetCursorPos(win, CURSOR_X, CURSOR_Y); double dx = CURSOR_X[0] - lockAnchorWindowX; double dy = CURSOR_Y[0] - lockAnchorWindowY; if (Math.abs(dx) > 0.001 || Math.abs(dy) > 0.001) { diff --git a/src/xos/examples/controller.py b/src/xos/examples/controller.py index 7ba0447..e530c88 100644 --- a/src/xos/examples/controller.py +++ b/src/xos/examples/controller.py @@ -44,6 +44,7 @@ def __init__(self): def _input_axis(self): keys = self.state.keyboard.keys + mouse = self.state.mouse ax = 0.0 ay = 0.0 if keys.W.is_pressed(): @@ -54,15 +55,23 @@ def _input_axis(self): ay += 1.0 if keys.D.is_pressed(): ax += 1.0 + if mouse.LEFT_CLICK.is_pressed(): + mouse.lock(); mouse.hide() + ax += mouse.dx * 0.02 + ay += mouse.dy * 0.02 + else: + mouse.unlock(); mouse.show() return _normalized_or_zero(ax, ay) def tick(self): dt = min(MAX_DT, max(0.0, self.dt)) ax, ay = self._input_axis() + # Apply thrust from current directional intent. self.vx += ax * THRUST * dt self.vy += ay * THRUST * dt + # Linear drag: easier to reason about than exponential damping. drag = max(0.0, 1.0 - DRAG_PER_SEC * dt) self.vx *= drag self.vy *= drag @@ -93,6 +102,5 @@ def tick(self): NOSE_COLOR, ) - if __name__ == "__main__": ControllerApp().run() From b318e4c00c34326a52a5d953d23dc9f499ce4e7e Mon Sep 17 00:00:00 2001 From: verbiiyo Date: Mon, 1 Jun 2026 16:28:11 +0900 Subject: [PATCH 14/29] start moving like controller script but for the agent vision! --- src/main/resources/xos/examples/agent_controller.py | 8 ++++++++ src/xos/examples/agent_controller.py | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/src/main/resources/xos/examples/agent_controller.py b/src/main/resources/xos/examples/agent_controller.py index 067593a..9dbe7f4 100644 --- a/src/main/resources/xos/examples/agent_controller.py +++ b/src/main/resources/xos/examples/agent_controller.py @@ -1,6 +1,8 @@ import mc import xos +LOOK_SENS = 0.08 + class AgentController(xos.Application): def __init__(self): @@ -21,6 +23,7 @@ def tick(self): agent = agents[0] now = self._now() keys = self.state.keyboard.keys + mouse = self.state.mouse if keys.W.is_pressed(): agent.actions.w() @@ -33,6 +36,11 @@ def tick(self): if keys.SPACE.is_pressed() and now >= self.jump_cooldown_until: agent.actions.jump() self.jump_cooldown_until = now + 0.25 + if mouse.LEFT_CLICK.is_pressed(): + mouse.lock(); mouse.hide() + agent.actions.look(-mouse.dy * LOOK_SENS, mouse.dx * LOOK_SENS) + else: + mouse.unlock(); mouse.show() view = agent.visual(h=self.view_h, w=self.view_w) try: diff --git a/src/xos/examples/agent_controller.py b/src/xos/examples/agent_controller.py index 356b6db..d8c179a 100644 --- a/src/xos/examples/agent_controller.py +++ b/src/xos/examples/agent_controller.py @@ -1,6 +1,8 @@ import mc import xos +LOOK_SENS = 0.08 + class AgentController(xos.Application): def __init__(self): @@ -21,6 +23,7 @@ def tick(self): agent = agents[0] now = self._now() keys = self.state.keyboard.keys + mouse = self.state.mouse # Movement controls (single controllable agent: index 0). if keys.W.is_pressed(): @@ -34,6 +37,11 @@ def tick(self): if keys.SPACE.is_pressed() and now >= self.jump_cooldown_until: agent.actions.jump() self.jump_cooldown_until = now + 0.25 + if mouse.LEFT_CLICK.is_pressed(): + mouse.lock(); mouse.hide() + agent.actions.look(-mouse.dy * LOOK_SENS, mouse.dx * LOOK_SENS) + else: + mouse.unlock(); mouse.show() # Fixed-size viewport from the agent perspective. view = agent.visual(h=self.view_h, w=self.view_w) From 49afd72a4e29bfe3c703a9e2bcfcea948bada9a0 Mon Sep 17 00:00:00 2001 From: verbiiyo Date: Mon, 1 Jun 2026 16:36:36 +0900 Subject: [PATCH 15/29] much better --- .../com/verbii/minekov/client/XosViewportRuntime.java | 10 ++++++++++ src/main/resources/xos/examples/agent_controller.py | 2 +- src/xos/examples/agent_controller.py | 2 +- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/verbii/minekov/client/XosViewportRuntime.java b/src/main/java/com/verbii/minekov/client/XosViewportRuntime.java index 4f7ed99..d876ce8 100644 --- a/src/main/java/com/verbii/minekov/client/XosViewportRuntime.java +++ b/src/main/java/com/verbii/minekov/client/XosViewportRuntime.java @@ -1618,6 +1618,16 @@ def _decode_rgba_hex_to_tensor(raw_hex, w, h): vals.append(int(text[i:i + 2], 16)) except Exception: return xos.zeros((h, w, 4), dtype="uint8") + ww = int(w) + hh = int(h) + row_stride = ww * 4 + flipped = [] + for y in range(hh): + row = vals[y * row_stride:(y + 1) * row_stride] + for x in range(ww - 1, -1, -1): + i = x * 4 + flipped.extend(row[i:i + 4]) + vals = flipped return xos.tensor(vals, (h, w, 4), dtype="uint8") class Actions: diff --git a/src/main/resources/xos/examples/agent_controller.py b/src/main/resources/xos/examples/agent_controller.py index 9dbe7f4..43ba50b 100644 --- a/src/main/resources/xos/examples/agent_controller.py +++ b/src/main/resources/xos/examples/agent_controller.py @@ -38,7 +38,7 @@ def tick(self): self.jump_cooldown_until = now + 0.25 if mouse.LEFT_CLICK.is_pressed(): mouse.lock(); mouse.hide() - agent.actions.look(-mouse.dy * LOOK_SENS, mouse.dx * LOOK_SENS) + agent.actions.look(mouse.dy * LOOK_SENS, mouse.dx * LOOK_SENS) else: mouse.unlock(); mouse.show() diff --git a/src/xos/examples/agent_controller.py b/src/xos/examples/agent_controller.py index d8c179a..31095af 100644 --- a/src/xos/examples/agent_controller.py +++ b/src/xos/examples/agent_controller.py @@ -39,7 +39,7 @@ def tick(self): self.jump_cooldown_until = now + 0.25 if mouse.LEFT_CLICK.is_pressed(): mouse.lock(); mouse.hide() - agent.actions.look(-mouse.dy * LOOK_SENS, mouse.dx * LOOK_SENS) + agent.actions.look(mouse.dy * LOOK_SENS, mouse.dx * LOOK_SENS) else: mouse.unlock(); mouse.show() From ae6b076d8c2ad49279d1df2275d6bf4f677188c0 Mon Sep 17 00:00:00 2001 From: verbiiyo Date: Mon, 1 Jun 2026 16:39:42 +0900 Subject: [PATCH 16/29] even more slightly better --- src/main/resources/xos/examples/agent_controller.py | 10 +++++++++- src/xos/examples/agent_controller.py | 10 +++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/src/main/resources/xos/examples/agent_controller.py b/src/main/resources/xos/examples/agent_controller.py index 43ba50b..7abb203 100644 --- a/src/main/resources/xos/examples/agent_controller.py +++ b/src/main/resources/xos/examples/agent_controller.py @@ -2,6 +2,8 @@ import xos LOOK_SENS = 0.08 +PITCH_MIN = -89.0 +PITCH_MAX = 89.0 class AgentController(xos.Application): @@ -10,6 +12,8 @@ def __init__(self): self.jump_cooldown_until = 0.0 self.view_w = 128 self.view_h = 128 + self.look_yaw = None + self.look_pitch = None def _now(self): return xos.time.perf_counter() @@ -24,6 +28,8 @@ def tick(self): now = self._now() keys = self.state.keyboard.keys mouse = self.state.mouse + if self.look_yaw is None or self.look_pitch is None: + self.look_yaw, self.look_pitch = map(float, agent.rotation) if keys.W.is_pressed(): agent.actions.w() @@ -38,9 +44,11 @@ def tick(self): self.jump_cooldown_until = now + 0.25 if mouse.LEFT_CLICK.is_pressed(): mouse.lock(); mouse.hide() - agent.actions.look(mouse.dy * LOOK_SENS, mouse.dx * LOOK_SENS) + self.look_yaw += mouse.dx * LOOK_SENS + self.look_pitch = max(PITCH_MIN, min(PITCH_MAX, self.look_pitch + mouse.dy * LOOK_SENS)) else: mouse.unlock(); mouse.show() + agent.rotation = (self.look_yaw, self.look_pitch) view = agent.visual(h=self.view_h, w=self.view_w) try: diff --git a/src/xos/examples/agent_controller.py b/src/xos/examples/agent_controller.py index 31095af..9d43216 100644 --- a/src/xos/examples/agent_controller.py +++ b/src/xos/examples/agent_controller.py @@ -2,6 +2,8 @@ import xos LOOK_SENS = 0.08 +PITCH_MIN = -89.0 +PITCH_MAX = 89.0 class AgentController(xos.Application): @@ -10,6 +12,8 @@ def __init__(self): self.jump_cooldown_until = 0.0 self.view_w = 128 self.view_h = 128 + self.look_yaw = None + self.look_pitch = None def _now(self): return xos.time.perf_counter() @@ -24,6 +28,8 @@ def tick(self): now = self._now() keys = self.state.keyboard.keys mouse = self.state.mouse + if self.look_yaw is None or self.look_pitch is None: + self.look_yaw, self.look_pitch = map(float, agent.rotation) # Movement controls (single controllable agent: index 0). if keys.W.is_pressed(): @@ -39,9 +45,11 @@ def tick(self): self.jump_cooldown_until = now + 0.25 if mouse.LEFT_CLICK.is_pressed(): mouse.lock(); mouse.hide() - agent.actions.look(mouse.dy * LOOK_SENS, mouse.dx * LOOK_SENS) + self.look_yaw += mouse.dx * LOOK_SENS + self.look_pitch = max(PITCH_MIN, min(PITCH_MAX, self.look_pitch + mouse.dy * LOOK_SENS)) else: mouse.unlock(); mouse.show() + agent.rotation = (self.look_yaw, self.look_pitch) # Fixed-size viewport from the agent perspective. view = agent.visual(h=self.view_h, w=self.view_w) From 662b3734c68445e513c72ad577e185b9b6332370 Mon Sep 17 00:00:00 2001 From: verbiiyo Date: Mon, 1 Jun 2026 16:40:31 +0900 Subject: [PATCH 17/29] remove all look momentum (feels awesome) --- src/main/resources/xos/examples/agent_controller.py | 13 +++++++++++-- src/xos/examples/agent_controller.py | 13 +++++++++++-- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/src/main/resources/xos/examples/agent_controller.py b/src/main/resources/xos/examples/agent_controller.py index 7abb203..a381ed6 100644 --- a/src/main/resources/xos/examples/agent_controller.py +++ b/src/main/resources/xos/examples/agent_controller.py @@ -14,6 +14,8 @@ def __init__(self): self.view_h = 128 self.look_yaw = None self.look_pitch = None + self.last_look_x = None + self.last_look_y = None def _now(self): return xos.time.perf_counter() @@ -44,10 +46,17 @@ def tick(self): self.jump_cooldown_until = now + 0.25 if mouse.LEFT_CLICK.is_pressed(): mouse.lock(); mouse.hide() - self.look_yaw += mouse.dx * LOOK_SENS - self.look_pitch = max(PITCH_MIN, min(PITCH_MAX, self.look_pitch + mouse.dy * LOOK_SENS)) + if self.last_look_x is None or self.last_look_y is None: + self.last_look_x, self.last_look_y = float(mouse.x), float(mouse.y) + mdx = float(mouse.x) - self.last_look_x + mdy = float(mouse.y) - self.last_look_y + self.last_look_x, self.last_look_y = float(mouse.x), float(mouse.y) + self.look_yaw += mdx * LOOK_SENS + self.look_pitch = max(PITCH_MIN, min(PITCH_MAX, self.look_pitch + mdy * LOOK_SENS)) else: mouse.unlock(); mouse.show() + self.last_look_x = None + self.last_look_y = None agent.rotation = (self.look_yaw, self.look_pitch) view = agent.visual(h=self.view_h, w=self.view_w) diff --git a/src/xos/examples/agent_controller.py b/src/xos/examples/agent_controller.py index 9d43216..245f9e7 100644 --- a/src/xos/examples/agent_controller.py +++ b/src/xos/examples/agent_controller.py @@ -14,6 +14,8 @@ def __init__(self): self.view_h = 128 self.look_yaw = None self.look_pitch = None + self.last_look_x = None + self.last_look_y = None def _now(self): return xos.time.perf_counter() @@ -45,10 +47,17 @@ def tick(self): self.jump_cooldown_until = now + 0.25 if mouse.LEFT_CLICK.is_pressed(): mouse.lock(); mouse.hide() - self.look_yaw += mouse.dx * LOOK_SENS - self.look_pitch = max(PITCH_MIN, min(PITCH_MAX, self.look_pitch + mouse.dy * LOOK_SENS)) + if self.last_look_x is None or self.last_look_y is None: + self.last_look_x, self.last_look_y = float(mouse.x), float(mouse.y) + mdx = float(mouse.x) - self.last_look_x + mdy = float(mouse.y) - self.last_look_y + self.last_look_x, self.last_look_y = float(mouse.x), float(mouse.y) + self.look_yaw += mdx * LOOK_SENS + self.look_pitch = max(PITCH_MIN, min(PITCH_MAX, self.look_pitch + mdy * LOOK_SENS)) else: mouse.unlock(); mouse.show() + self.last_look_x = None + self.last_look_y = None agent.rotation = (self.look_yaw, self.look_pitch) # Fixed-size viewport from the agent perspective. From d2ba7a86788f4b8894785e74ff7a57748095d007 Mon Sep 17 00:00:00 2001 From: verbiiyo Date: Mon, 1 Jun 2026 17:04:50 +0900 Subject: [PATCH 18/29] reractor to Vision.java --- .../com/verbii/minekov/client/Vision.java | 228 +++++++++++ .../minekov/client/XosViewportRuntime.java | 360 +----------------- 2 files changed, 229 insertions(+), 359 deletions(-) create mode 100644 src/main/java/com/verbii/minekov/client/Vision.java diff --git a/src/main/java/com/verbii/minekov/client/Vision.java b/src/main/java/com/verbii/minekov/client/Vision.java new file mode 100644 index 0000000..400766b --- /dev/null +++ b/src/main/java/com/verbii/minekov/client/Vision.java @@ -0,0 +1,228 @@ +package com.verbii.minekov.client; + +import com.verbii.minekov.entities.RLOperator; +import net.minecraft.client.Minecraft; +import net.minecraft.core.BlockPos; +import net.minecraft.core.registries.BuiltInRegistries; +import net.minecraft.resources.ResourceLocation; +import net.minecraft.world.entity.Entity; +import net.minecraft.world.entity.LivingEntity; +import net.minecraft.world.level.ClipContext; +import net.minecraft.world.phys.HitResult; +import net.minecraft.world.phys.Vec3; + +/** Agent vision renderer used by Python `agent.visual(...)` host binding. */ +final class Vision { + private static final float AGENT_VIEW_FOV_DEG = 70.0f; + private static final double AGENT_VIEW_MAX_DISTANCE = 24.0; + private static final char[] HEX = "0123456789abcdef".toCharArray(); + + private Vision() {} + + static String renderAgentViewRgbaHex(Minecraft mc, RLOperator agent, int reqW, int reqH) { + int width = clampViewSize(reqW); + int height = clampViewSize(reqH); + if (mc == null || mc.level == null || agent == null) { + return ""; + } + + Vec3 origin = new Vec3(agent.getX(), agent.getEyeY(), agent.getZ()); + float yaw = agent.getYRot(); + float pitch = agent.getXRot(); + Vec3 forward = forwardFromRotation(yaw, pitch); + Vec3 worldUp = new Vec3(0.0, 1.0, 0.0); + Vec3 right = worldUp.cross(forward); + if (right.lengthSqr() < 1.0e-8) { + right = rightFromYaw(yaw); + } else { + right = right.normalize(); + } + Vec3 up = forward.cross(right).normalize(); + + double tanHalfFov = Math.tan(Math.toRadians(AGENT_VIEW_FOV_DEG * 0.5)); + double aspect = (double) width / (double) height; + int pixelCount = width * height; + byte[] rgba = new byte[pixelCount * 4]; + double[] depth = new double[pixelCount]; + for (int i = 0; i < pixelCount; i++) { + depth[i] = Double.POSITIVE_INFINITY; + } + + for (int y = 0; y < height; y++) { + double ny = 1.0 - (2.0 * (y + 0.5) / (double) height); + for (int x = 0; x < width; x++) { + double nx = (2.0 * (x + 0.5) / (double) width) - 1.0; + Vec3 dir = forward + .add(right.scale(nx * tanHalfFov * aspect)) + .add(up.scale(ny * tanHalfFov)) + .normalize(); + + int r = 110; + int g = 150; + int b = 210; + boolean hit = false; + double distance = 0.0; + Vec3 rayEnd = origin.add(dir.scale(AGENT_VIEW_MAX_DISTANCE)); + var hitRes = mc.level.clip(new ClipContext( + origin, + rayEnd, + ClipContext.Block.OUTLINE, + ClipContext.Fluid.NONE, + agent)); + if (hitRes.getType() != HitResult.Type.MISS) { + BlockPos bp = hitRes.getBlockPos(); + var state = mc.level.getBlockState(bp); + if (!state.isAir()) { + int rgb = rgbFromBlockState(mc, bp, state); + distance = hitRes.getLocation().distanceTo(origin); + double shade = Math.max(0.28, 1.0 - (distance / AGENT_VIEW_MAX_DISTANCE)); + double faceShade = switch (hitRes.getDirection()) { + case UP -> 1.08; + case DOWN -> 0.72; + case NORTH, SOUTH -> 0.94; + default -> 1.00; + }; + double totalShade = Math.max(0.20, Math.min(1.15, shade * faceShade)); + r = (int) Math.max(0, Math.min(255, ((rgb >> 16) & 0xFF) * totalShade)); + g = (int) Math.max(0, Math.min(255, ((rgb >> 8) & 0xFF) * totalShade)); + b = (int) Math.max(0, Math.min(255, (rgb & 0xFF) * totalShade)); + hit = true; + } + } + + if (!hit) { + double tSky = Math.max(0.0, Math.min(1.0, (dir.y + 1.0) * 0.5)); + r = (int) (70 + 60 * tSky); + g = (int) (100 + 90 * tSky); + b = (int) (150 + 80 * tSky); + } else { + int fog = (int) Math.max(0, Math.min(40, (AGENT_VIEW_MAX_DISTANCE - distance) * 0.8)); + r = Math.min(255, r + fog); + g = Math.min(255, g + fog); + b = Math.min(255, b + fog); + } + int p = y * width + x; + int i = p * 4; + rgba[i] = (byte) r; + rgba[i + 1] = (byte) g; + rgba[i + 2] = (byte) b; + rgba[i + 3] = (byte) 255; + depth[p] = hit ? distance : Double.POSITIVE_INFINITY; + } + } + + var renderables = mc.level.entitiesForRendering(); + for (Entity e : renderables) { + if (e == null || e == agent || e.isRemoved()) { + continue; + } + Vec3 rel = new Vec3(e.getX() - origin.x, e.getEyeY() - origin.y, e.getZ() - origin.z); + double camX = rel.dot(right); + double camY = rel.dot(up); + double camZ = rel.dot(forward); + if (camZ <= 0.01 || camZ > AGENT_VIEW_MAX_DISTANCE) { + continue; + } + + double ndcX = camX / (camZ * tanHalfFov * aspect); + double ndcY = camY / (camZ * tanHalfFov); + if (Math.abs(ndcX) > 1.5 || Math.abs(ndcY) > 1.5) { + continue; + } + + int cx = (int) Math.round((ndcX + 1.0) * 0.5 * (width - 1)); + int cy = (int) Math.round((1.0 - (ndcY + 1.0) * 0.5) * (height - 1)); + double radiusPx = Math.max( + 1.0, + (Math.max(0.25, e.getBbWidth()) / (camZ * tanHalfFov * aspect)) * (width * 0.28)); + int rad = (int) Math.min(Math.max(1, Math.round(radiusPx)), Math.max(width, height)); + int color = rgbFromEntity(e); + int er = (color >> 16) & 0xFF; + int eg = (color >> 8) & 0xFF; + int eb = color & 0xFF; + int x1 = Math.max(0, cx - rad); + int x2 = Math.min(width - 1, cx + rad); + int y1 = Math.max(0, cy - rad); + int y2 = Math.min(height - 1, cy + rad); + int rr = rad * rad; + for (int py = y1; py <= y2; py++) { + int dy = py - cy; + for (int px = x1; px <= x2; px++) { + int dx = px - cx; + if ((dx * dx + dy * dy) > rr) { + continue; + } + int p = py * width + px; + if (camZ >= depth[p]) { + continue; + } + int i = p * 4; + rgba[i] = (byte) er; + rgba[i + 1] = (byte) eg; + rgba[i + 2] = (byte) eb; + rgba[i + 3] = (byte) 255; + depth[p] = camZ; + } + } + } + + return encodeRgbaHex(rgba); + } + + private static int clampViewSize(int value) { + return Math.max(16, Math.min(512, value)); + } + + private static int rgbFromBlockState( + Minecraft mc, BlockPos pos, net.minecraft.world.level.block.state.BlockState state) { + try { + var map = state.getMapColor(mc.level, pos); + if (map != null && map.col != 0) { + return map.col & 0x00FFFFFF; + } + } catch (Throwable ignored) { + } + ResourceLocation key = BuiltInRegistries.BLOCK.getKey(state.getBlock()); + int hash = key != null ? key.toString().hashCode() : 0x77AA77; + int r = 64 + (Math.abs(hash) & 0x7F); + int g = 64 + ((Math.abs(hash >> 8)) & 0x7F); + int b = 64 + ((Math.abs(hash >> 16)) & 0x7F); + return (r << 16) | (g << 8) | b; + } + + private static int rgbFromEntity(Entity entity) { + int hash = entity.getType().toString().hashCode(); + int r = 96 + (Math.abs(hash) & 0x5F); + int g = 96 + ((Math.abs(hash >> 8)) & 0x5F); + int b = 96 + ((Math.abs(hash >> 16)) & 0x5F); + if (entity instanceof LivingEntity) { + r = Math.min(255, r + 24); + } + return (r << 16) | (g << 8) | b; + } + + private static String encodeRgbaHex(byte[] rgba) { + char[] out = new char[rgba.length * 2]; + int j = 0; + for (byte b : rgba) { + int v = b & 0xFF; + out[j++] = HEX[v >>> 4]; + out[j++] = HEX[v & 0x0F]; + } + return new String(out); + } + + private static Vec3 forwardFromRotation(float yawDeg, float pitchDeg) { + double yaw = Math.toRadians(yawDeg); + double pitch = Math.toRadians(pitchDeg); + double x = -Math.sin(yaw) * Math.cos(pitch); + double y = -Math.sin(pitch); + double z = Math.cos(yaw) * Math.cos(pitch); + return new Vec3(x, y, z).normalize(); + } + + private static Vec3 rightFromYaw(float yawDeg) { + double yaw = Math.toRadians(yawDeg); + return new Vec3(-Math.cos(yaw), 0.0, -Math.sin(yaw)).normalize(); + } +} diff --git a/src/main/java/com/verbii/minekov/client/XosViewportRuntime.java b/src/main/java/com/verbii/minekov/client/XosViewportRuntime.java index d876ce8..1dc29ff 100644 --- a/src/main/java/com/verbii/minekov/client/XosViewportRuntime.java +++ b/src/main/java/com/verbii/minekov/client/XosViewportRuntime.java @@ -2,12 +2,8 @@ import ai.xlate.xos.XosNative; -import com.mojang.blaze3d.pipeline.RenderTarget; -import com.mojang.blaze3d.pipeline.TextureTarget; import com.google.gson.JsonElement; import com.mojang.blaze3d.platform.NativeImage; -import com.mojang.blaze3d.systems.RenderSystem; -import com.mojang.blaze3d.vertex.PoseStack; import com.mojang.serialization.JsonOps; import com.verbii.minekov.Minekov; import com.verbii.minekov.entities.RLOperator; @@ -15,7 +11,6 @@ import net.minecraft.client.Minecraft; import net.minecraft.client.gui.GuiGraphics; import net.minecraft.client.renderer.texture.DynamicTexture; -import net.minecraft.core.BlockPos; import net.minecraft.core.registries.BuiltInRegistries; import net.minecraft.nbt.NbtOps; import net.minecraft.network.chat.Component; @@ -25,9 +20,7 @@ import net.minecraft.world.entity.ai.attributes.Attributes; import net.minecraft.world.entity.player.Inventory; import net.minecraft.world.item.ItemStack; -import net.minecraft.world.level.ClipContext; import net.minecraft.world.level.storage.LevelResource; -import net.minecraft.world.phys.HitResult; import net.minecraft.world.phys.Vec3; import net.minecraftforge.api.distmarker.Dist; import net.minecraftforge.api.distmarker.OnlyIn; @@ -36,7 +29,6 @@ import org.slf4j.Logger; import java.lang.reflect.Field; -import java.lang.reflect.Method; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.IntBuffer; @@ -54,7 +46,6 @@ import java.util.concurrent.atomic.AtomicReference; import org.lwjgl.system.MemoryUtil; -import org.lwjgl.opengl.GL11C; import org.lwjgl.glfw.GLFW; /** @@ -84,14 +75,6 @@ public final class XosViewportRuntime { private static final int VIEWPORT_ALPHA_HOVER = Math.round(255 * 0.8f); private static final String ITEM_FIELD_SEPARATOR = "\u001f"; private static final String ITEM_ROW_SEPARATOR = "\u001e"; - private static final float AGENT_VIEW_FOV_DEG = 70.0f; - private static final double AGENT_VIEW_MAX_DISTANCE = 24.0; - private static final char[] HEX = "0123456789abcdef".toCharArray(); - // Keep disabled until we have a fully isolated offscreen pass that cannot touch the live HUD/chat viewport. - private static final boolean ENABLE_REAL_AGENT_VIEW_RENDER = false; - private static TextureTarget agentViewRenderTarget; - private static Method gameRendererRenderLevelMethod; - private static Field minecraftMainRenderTargetField; private static final List STARTER_SCRIPT_NAMES = List.of( "balls_many.py", @@ -736,349 +719,8 @@ private static String encodePlayerInventoryEnderChest(Minecraft mc) { return out.toString(); } - private static int clampViewSize(int value) { - return Math.max(16, Math.min(512, value)); - } - - private static int rgbFromBlockState( - Minecraft mc, BlockPos pos, net.minecraft.world.level.block.state.BlockState state) { - try { - var map = state.getMapColor(mc.level, pos); - if (map != null && map.col != 0) { - return map.col & 0x00FFFFFF; - } - } catch (Throwable ignored) { - } - ResourceLocation key = BuiltInRegistries.BLOCK.getKey(state.getBlock()); - int hash = key != null ? key.toString().hashCode() : 0x77AA77; - int r = 64 + (Math.abs(hash) & 0x7F); - int g = 64 + ((Math.abs(hash >> 8)) & 0x7F); - int b = 64 + ((Math.abs(hash >> 16)) & 0x7F); - return (r << 16) | (g << 8) | b; - } - - private static int rgbFromEntity(Entity entity) { - int hash = entity.getType().toString().hashCode(); - int r = 96 + (Math.abs(hash) & 0x5F); - int g = 96 + ((Math.abs(hash >> 8)) & 0x5F); - int b = 96 + ((Math.abs(hash >> 16)) & 0x5F); - if (entity instanceof LivingEntity) { - r = Math.min(255, r + 24); - } - return (r << 16) | (g << 8) | b; - } - - private static String encodeRgbaHex(byte[] rgba) { - char[] out = new char[rgba.length * 2]; - int j = 0; - for (byte b : rgba) { - int v = b & 0xFF; - out[j++] = HEX[v >>> 4]; - out[j++] = HEX[v & 0x0F]; - } - return new String(out); - } - private static String renderAgentViewRgbaHex(Minecraft mc, RLOperator agent, int reqW, int reqH) { - int width = clampViewSize(reqW); - int height = clampViewSize(reqH); - if (mc == null || mc.level == null || agent == null) { - return ""; - } - if (ENABLE_REAL_AGENT_VIEW_RENDER) { - if (ENABLE_REAL_AGENT_VIEW_RENDER) { - byte[] rendered = tryRenderAgentViewWithMinecraftRenderer(mc, agent, width, height); - if (rendered != null) { - return encodeRgbaHex(rendered); - } - } - } - - Vec3 origin = new Vec3(agent.getX(), agent.getEyeY(), agent.getZ()); - float yaw = agent.getYRot(); - float pitch = agent.getXRot(); - Vec3 forward = forwardFromRotation(yaw, pitch); - Vec3 worldUp = new Vec3(0.0, 1.0, 0.0); - Vec3 right = worldUp.cross(forward); - if (right.lengthSqr() < 1.0e-8) { - right = rightFromYaw(yaw); - } else { - right = right.normalize(); - } - Vec3 up = forward.cross(right).normalize(); - - double tanHalfFov = Math.tan(Math.toRadians(AGENT_VIEW_FOV_DEG * 0.5)); - double aspect = (double) width / (double) height; - int pixelCount = width * height; - byte[] rgba = new byte[pixelCount * 4]; - double[] depth = new double[pixelCount]; - for (int i = 0; i < pixelCount; i++) { - depth[i] = Double.POSITIVE_INFINITY; - } - - for (int y = 0; y < height; y++) { - double ny = 1.0 - (2.0 * (y + 0.5) / (double) height); - for (int x = 0; x < width; x++) { - double nx = (2.0 * (x + 0.5) / (double) width) - 1.0; - Vec3 dir = forward - .add(right.scale(nx * tanHalfFov * aspect)) - .add(up.scale(ny * tanHalfFov)) - .normalize(); - - int r = 110; - int g = 150; - int b = 210; - boolean hit = false; - double distance = 0.0; - Vec3 rayEnd = origin.add(dir.scale(AGENT_VIEW_MAX_DISTANCE)); - var hitRes = mc.level.clip(new ClipContext( - origin, - rayEnd, - ClipContext.Block.OUTLINE, - ClipContext.Fluid.NONE, - agent)); - if (hitRes.getType() != HitResult.Type.MISS) { - BlockPos bp = hitRes.getBlockPos(); - var state = mc.level.getBlockState(bp); - if (!state.isAir()) { - int rgb = rgbFromBlockState(mc, bp, state); - distance = hitRes.getLocation().distanceTo(origin); - double shade = Math.max(0.28, 1.0 - (distance / AGENT_VIEW_MAX_DISTANCE)); - // Face-aware brighten/darken helps depth cues in tiny viewports. - double faceShade = switch (hitRes.getDirection()) { - case UP -> 1.08; - case DOWN -> 0.72; - case NORTH, SOUTH -> 0.94; - default -> 1.00; - }; - double totalShade = Math.max(0.20, Math.min(1.15, shade * faceShade)); - r = (int) Math.max(0, Math.min(255, ((rgb >> 16) & 0xFF) * totalShade)); - g = (int) Math.max(0, Math.min(255, ((rgb >> 8) & 0xFF) * totalShade)); - b = (int) Math.max(0, Math.min(255, (rgb & 0xFF) * totalShade)); - hit = true; - } - } - - if (!hit) { - double tSky = Math.max(0.0, Math.min(1.0, (dir.y + 1.0) * 0.5)); - r = (int) (70 + 60 * tSky); - g = (int) (100 + 90 * tSky); - b = (int) (150 + 80 * tSky); - } else { - // Tiny depth cue so nearby geometry pops in the mini-view. - int fog = (int) Math.max(0, Math.min(40, (AGENT_VIEW_MAX_DISTANCE - distance) * 0.8)); - r = Math.min(255, r + fog); - g = Math.min(255, g + fog); - b = Math.min(255, b + fog); - } - int p = y * width + x; - int i = p * 4; - rgba[i] = (byte) r; - rgba[i + 1] = (byte) g; - rgba[i + 2] = (byte) b; - rgba[i + 3] = (byte) 255; - depth[p] = hit ? distance : Double.POSITIVE_INFINITY; - } - } - - // Overlay nearby entities as simple depth-tested billboards so they are visible in the agent feed. - var renderables = mc.level.entitiesForRendering(); - for (Entity e : renderables) { - if (e == null || e == agent || e.isRemoved()) { - continue; - } - Vec3 rel = new Vec3(e.getX() - origin.x, e.getEyeY() - origin.y, e.getZ() - origin.z); - double camX = rel.dot(right); - double camY = rel.dot(up); - double camZ = rel.dot(forward); - if (camZ <= 0.01 || camZ > AGENT_VIEW_MAX_DISTANCE) { - continue; - } - - double ndcX = camX / (camZ * tanHalfFov * aspect); - double ndcY = camY / (camZ * tanHalfFov); - if (Math.abs(ndcX) > 1.5 || Math.abs(ndcY) > 1.5) { - continue; - } - - int cx = (int) Math.round((ndcX + 1.0) * 0.5 * (width - 1)); - int cy = (int) Math.round((1.0 - (ndcY + 1.0) * 0.5) * (height - 1)); - double radiusPx = Math.max( - 1.0, - (Math.max(0.25, e.getBbWidth()) / (camZ * tanHalfFov * aspect)) * (width * 0.28)); - int rad = (int) Math.min(Math.max(1, Math.round(radiusPx)), Math.max(width, height)); - int color = rgbFromEntity(e); - int er = (color >> 16) & 0xFF; - int eg = (color >> 8) & 0xFF; - int eb = color & 0xFF; - int x1 = Math.max(0, cx - rad); - int x2 = Math.min(width - 1, cx + rad); - int y1 = Math.max(0, cy - rad); - int y2 = Math.min(height - 1, cy + rad); - int rr = rad * rad; - for (int py = y1; py <= y2; py++) { - int dy = py - cy; - for (int px = x1; px <= x2; px++) { - int dx = px - cx; - if ((dx * dx + dy * dy) > rr) { - continue; - } - int p = py * width + px; - if (camZ >= depth[p]) { - continue; - } - int i = p * 4; - rgba[i] = (byte) er; - rgba[i + 1] = (byte) eg; - rgba[i + 2] = (byte) eb; - rgba[i + 3] = (byte) 255; - depth[p] = camZ; - } - } - } - - return encodeRgbaHex(rgba); - } - - private static byte[] tryRenderAgentViewWithMinecraftRenderer( - Minecraft mc, RLOperator agent, int width, int height) { - // Keep this on the render thread only; host binding runs on the client thread already. - if (!RenderSystem.isOnRenderThread()) { - return null; - } - try { - if (agentViewRenderTarget == null - || agentViewRenderTarget.width != width - || agentViewRenderTarget.height != height) { - if (agentViewRenderTarget != null) { - agentViewRenderTarget.destroyBuffers(); - } - agentViewRenderTarget = new TextureTarget(width, height, true, Minecraft.ON_OSX); - } - } catch (Throwable t) { - return null; - } - - RenderTarget previousTarget = mc.getMainRenderTarget(); - Entity previousCamera = mc.getCameraEntity(); - boolean swappedMainTarget = setMinecraftMainRenderTarget(mc, previousTarget, agentViewRenderTarget); - if (!swappedMainTarget) { - // If we cannot isolate render context, fall back to the safe custom renderer. - return null; - } - try { - mc.setCameraEntity(agent); - agentViewRenderTarget.bindWrite(true); - RenderSystem.viewport(0, 0, width, height); - agentViewRenderTarget.setClearColor(0f, 0f, 0f, 1f); - agentViewRenderTarget.clear(Minecraft.ON_OSX); - - if (!invokeGameRendererLevelPass(mc)) { - return null; - } - - ByteBuffer buf = MemoryUtil.memAlloc(width * height * 4); - try { - GL11C.glReadPixels(0, 0, width, height, GL11C.GL_RGBA, GL11C.GL_UNSIGNED_BYTE, buf); - byte[] out = new byte[width * height * 4]; - int rowBytes = width * 4; - for (int y = 0; y < height; y++) { - int src = (height - 1 - y) * rowBytes; - int dst = y * rowBytes; - buf.position(src); - buf.get(out, dst, rowBytes); - } - return out; - } finally { - MemoryUtil.memFree(buf); - } - } catch (Throwable t) { - return null; - } finally { - try { - mc.setCameraEntity(previousCamera); - } catch (Throwable ignored) { - } - try { - if (swappedMainTarget) { - setMinecraftMainRenderTarget(mc, agentViewRenderTarget, previousTarget); - } - previousTarget.bindWrite(true); - RenderSystem.viewport(0, 0, previousTarget.width, previousTarget.height); - } catch (Throwable ignored) { - } - } - } - - private static boolean invokeGameRendererLevelPass(Minecraft mc) { - try { - if (gameRendererRenderLevelMethod == null) { - for (Method m : mc.gameRenderer.getClass().getDeclaredMethods()) { - if ("renderLevel".equals(m.getName())) { - m.setAccessible(true); - gameRendererRenderLevelMethod = m; - break; - } - } - } - if (gameRendererRenderLevelMethod == null) { - return false; - } - Class[] p = gameRendererRenderLevelMethod.getParameterTypes(); - Object[] args = new Object[p.length]; - for (int i = 0; i < p.length; i++) { - Class t = p[i]; - if (t == float.class || t == Float.class) { - args[i] = mc.getFrameTime(); - } else if (t == double.class || t == Double.class) { - args[i] = (double) mc.getFrameTime(); - } else if (t == long.class || t == Long.class) { - args[i] = System.nanoTime(); - } else if (t == int.class || t == Integer.class) { - args[i] = 0; - } else if (t == boolean.class || t == Boolean.class) { - // Avoid leaking selection/outline/debug overlays into the offscreen pass. - args[i] = Boolean.FALSE; - } else if (PoseStack.class.isAssignableFrom(t)) { - args[i] = new PoseStack(); - } else { - args[i] = null; - } - } - gameRendererRenderLevelMethod.invoke(mc.gameRenderer, args); - return true; - } catch (Throwable t) { - return false; - } - } - - private static boolean setMinecraftMainRenderTarget( - Minecraft mc, RenderTarget expectedCurrent, RenderTarget target) { - try { - if (minecraftMainRenderTargetField == null) { - for (Field f : Minecraft.class.getDeclaredFields()) { - if (RenderTarget.class.isAssignableFrom(f.getType())) { - f.setAccessible(true); - Object cur = f.get(mc); - if (cur == expectedCurrent) { - minecraftMainRenderTargetField = f; - break; - } - } - } - } - if (minecraftMainRenderTargetField == null) { - return false; - } - Object cur = minecraftMainRenderTargetField.get(mc); - if (cur != expectedCurrent) { - return false; - } - minecraftMainRenderTargetField.set(mc, target); - return true; - } catch (Throwable t) { - return false; - } + return Vision.renderAgentViewRgbaHex(mc, agent, reqW, reqH); } private static double[] parseTriple(String raw, double fallbackX, double fallbackY, double fallbackZ) { From d6a630bcab71b4630f7aba655885cc49e82280d3 Mon Sep 17 00:00:00 2001 From: verbiiyo Date: Mon, 1 Jun 2026 17:48:01 +0900 Subject: [PATCH 19/29] use real render pipeline? contained!!! but its not displaying 100% --- .../com/verbii/minekov/client/Vision.java | 440 ++++++++++-------- .../minekov/client/XosViewportOverlay.java | 13 +- .../minekov/client/XosViewportRuntime.java | 14 + 3 files changed, 283 insertions(+), 184 deletions(-) diff --git a/src/main/java/com/verbii/minekov/client/Vision.java b/src/main/java/com/verbii/minekov/client/Vision.java index 400766b..f1c5192 100644 --- a/src/main/java/com/verbii/minekov/client/Vision.java +++ b/src/main/java/com/verbii/minekov/client/Vision.java @@ -1,228 +1,302 @@ package com.verbii.minekov.client; import com.verbii.minekov.entities.RLOperator; +import com.mojang.blaze3d.pipeline.RenderTarget; +import com.mojang.blaze3d.pipeline.TextureTarget; +import com.mojang.blaze3d.systems.RenderSystem; +import com.mojang.blaze3d.vertex.PoseStack; import net.minecraft.client.Minecraft; -import net.minecraft.core.BlockPos; -import net.minecraft.core.registries.BuiltInRegistries; -import net.minecraft.resources.ResourceLocation; import net.minecraft.world.entity.Entity; -import net.minecraft.world.entity.LivingEntity; -import net.minecraft.world.level.ClipContext; -import net.minecraft.world.phys.HitResult; -import net.minecraft.world.phys.Vec3; +import org.lwjgl.opengl.GL11C; +import org.lwjgl.system.MemoryUtil; + +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.Map; /** Agent vision renderer used by Python `agent.visual(...)` host binding. */ final class Vision { - private static final float AGENT_VIEW_FOV_DEG = 70.0f; - private static final double AGENT_VIEW_MAX_DISTANCE = 24.0; + private static final int MIN_VIEW_SIZE = 16; + private static final int MAX_VIEW_SIZE = 1024; private static final char[] HEX = "0123456789abcdef".toCharArray(); + private static final VisionManager MANAGER = new VisionManager(); private Vision() {} static String renderAgentViewRgbaHex(Minecraft mc, RLOperator agent, int reqW, int reqH) { - int width = clampViewSize(reqW); - int height = clampViewSize(reqH); - if (mc == null || mc.level == null || agent == null) { - return ""; - } + return MANAGER.renderAgentViewRgbaHex(mc, agent, reqW, reqH); + } + + static void flushDeferredCaptures(Minecraft mc) { + MANAGER.flushDeferredCaptures(mc); + } - Vec3 origin = new Vec3(agent.getX(), agent.getEyeY(), agent.getZ()); - float yaw = agent.getYRot(); - float pitch = agent.getXRot(); - Vec3 forward = forwardFromRotation(yaw, pitch); - Vec3 worldUp = new Vec3(0.0, 1.0, 0.0); - Vec3 right = worldUp.cross(forward); - if (right.lengthSqr() < 1.0e-8) { - right = rightFromYaw(yaw); - } else { - right = right.normalize(); + private static String encodeRgbaHex(byte[] rgba) { + char[] out = new char[rgba.length * 2]; + int j = 0; + for (byte b : rgba) { + int v = b & 0xFF; + out[j++] = HEX[v >>> 4]; + out[j++] = HEX[v & 0x0F]; } - Vec3 up = forward.cross(right).normalize(); - - double tanHalfFov = Math.tan(Math.toRadians(AGENT_VIEW_FOV_DEG * 0.5)); - double aspect = (double) width / (double) height; - int pixelCount = width * height; - byte[] rgba = new byte[pixelCount * 4]; - double[] depth = new double[pixelCount]; - for (int i = 0; i < pixelCount; i++) { - depth[i] = Double.POSITIVE_INFINITY; + return new String(out); + } + + private static final class VisionManager { + private final Map cameras = new HashMap<>(); + private Method gameRendererRenderLevelMethod; + private Field minecraftMainRenderTargetField; + + String renderAgentViewRgbaHex(Minecraft mc, RLOperator agent, int reqW, int reqH) { + if (mc == null || mc.level == null || agent == null) { + return ""; + } + int width = clampViewSize(reqW); + int height = clampViewSize(reqH); + CameraContext camera = cameraFor(agent.getStringUUID()); + byte[] rgba = renderCameraRgba(mc, camera, agent, width, height); + if (rgba == null) { + rgba = camera.lastRgba; + } + return rgba == null ? "" : encodeRgbaHex(rgba); } - for (int y = 0; y < height; y++) { - double ny = 1.0 - (2.0 * (y + 0.5) / (double) height); - for (int x = 0; x < width; x++) { - double nx = (2.0 * (x + 0.5) / (double) width) - 1.0; - Vec3 dir = forward - .add(right.scale(nx * tanHalfFov * aspect)) - .add(up.scale(ny * tanHalfFov)) - .normalize(); - - int r = 110; - int g = 150; - int b = 210; - boolean hit = false; - double distance = 0.0; - Vec3 rayEnd = origin.add(dir.scale(AGENT_VIEW_MAX_DISTANCE)); - var hitRes = mc.level.clip(new ClipContext( - origin, - rayEnd, - ClipContext.Block.OUTLINE, - ClipContext.Fluid.NONE, - agent)); - if (hitRes.getType() != HitResult.Type.MISS) { - BlockPos bp = hitRes.getBlockPos(); - var state = mc.level.getBlockState(bp); - if (!state.isAir()) { - int rgb = rgbFromBlockState(mc, bp, state); - distance = hitRes.getLocation().distanceTo(origin); - double shade = Math.max(0.28, 1.0 - (distance / AGENT_VIEW_MAX_DISTANCE)); - double faceShade = switch (hitRes.getDirection()) { - case UP -> 1.08; - case DOWN -> 0.72; - case NORTH, SOUTH -> 0.94; - default -> 1.00; - }; - double totalShade = Math.max(0.20, Math.min(1.15, shade * faceShade)); - r = (int) Math.max(0, Math.min(255, ((rgb >> 16) & 0xFF) * totalShade)); - g = (int) Math.max(0, Math.min(255, ((rgb >> 8) & 0xFF) * totalShade)); - b = (int) Math.max(0, Math.min(255, (rgb & 0xFF) * totalShade)); - hit = true; + void flushDeferredCaptures(Minecraft mc) { + if (mc == null || !RenderSystem.isOnRenderThread()) { + return; + } + synchronized (this) { + for (CameraContext camera : cameras.values()) { + RLOperator entity; + int w; + int h; + synchronized (camera) { + if (!camera.captureQueued || camera.pendingEntity == null) { + continue; + } + entity = camera.pendingEntity; + w = camera.pendingWidth; + h = camera.pendingHeight; + camera.captureQueued = false; + } + if (entity != null && !entity.isRemoved()) { + renderCameraRgbaOnRenderThread(mc, camera, entity, w, h); } } + } + } - if (!hit) { - double tSky = Math.max(0.0, Math.min(1.0, (dir.y + 1.0) * 0.5)); - r = (int) (70 + 60 * tSky); - g = (int) (100 + 90 * tSky); - b = (int) (150 + 80 * tSky); - } else { - int fog = (int) Math.max(0, Math.min(40, (AGENT_VIEW_MAX_DISTANCE - distance) * 0.8)); - r = Math.min(255, r + fog); - g = Math.min(255, g + fog); - b = Math.min(255, b + fog); - } - int p = y * width + x; - int i = p * 4; - rgba[i] = (byte) r; - rgba[i + 1] = (byte) g; - rgba[i + 2] = (byte) b; - rgba[i + 3] = (byte) 255; - depth[p] = hit ? distance : Double.POSITIVE_INFINITY; + private synchronized CameraContext cameraFor(String cameraId) { + return cameras.computeIfAbsent(cameraId, CameraContext::new); + } + + private static int clampViewSize(int value) { + return Math.max(MIN_VIEW_SIZE, Math.min(MAX_VIEW_SIZE, value)); + } + + private byte[] renderCameraRgba( + Minecraft mc, + CameraContext camera, + RLOperator cameraEntity, + int width, + int height) { + if (XosViewportRuntime.isOverlayRenderPassActive()) { + queueCapture(camera, cameraEntity, width, height); + return camera.lastRgba; } + if (!RenderSystem.isOnRenderThread()) { + queueCapture(camera, cameraEntity, width, height); + return camera.lastRgba; + } + return renderCameraRgbaOnRenderThread(mc, camera, cameraEntity, width, height); } - var renderables = mc.level.entitiesForRendering(); - for (Entity e : renderables) { - if (e == null || e == agent || e.isRemoved()) { - continue; + private void queueCapture( + CameraContext camera, RLOperator cameraEntity, int width, int height) { + synchronized (camera) { + camera.pendingEntity = cameraEntity; + camera.pendingWidth = width; + camera.pendingHeight = height; + camera.captureQueued = true; } - Vec3 rel = new Vec3(e.getX() - origin.x, e.getEyeY() - origin.y, e.getZ() - origin.z); - double camX = rel.dot(right); - double camY = rel.dot(up); - double camZ = rel.dot(forward); - if (camZ <= 0.01 || camZ > AGENT_VIEW_MAX_DISTANCE) { - continue; + } + + private byte[] renderCameraRgbaOnRenderThread( + Minecraft mc, + CameraContext camera, + RLOperator cameraEntity, + int width, + int height) { + try { + ensureTarget(camera, width, height); + } catch (Throwable ignored) { + return null; } - double ndcX = camX / (camZ * tanHalfFov * aspect); - double ndcY = camY / (camZ * tanHalfFov); - if (Math.abs(ndcX) > 1.5 || Math.abs(ndcY) > 1.5) { - continue; + RenderTarget previousTarget = mc.getMainRenderTarget(); + int previousWidth = previousTarget.width; + int previousHeight = previousTarget.height; + Entity previousCamera = mc.getCameraEntity(); + if (!setMinecraftMainRenderTarget(mc, camera.target)) { + return null; } + try { + mc.setCameraEntity(cameraEntity); + camera.target.bindWrite(true); + RenderSystem.viewport(0, 0, width, height); + camera.target.setClearColor(0f, 0f, 0f, 1f); + camera.target.clear(Minecraft.ON_OSX); - int cx = (int) Math.round((ndcX + 1.0) * 0.5 * (width - 1)); - int cy = (int) Math.round((1.0 - (ndcY + 1.0) * 0.5) * (height - 1)); - double radiusPx = Math.max( - 1.0, - (Math.max(0.25, e.getBbWidth()) / (camZ * tanHalfFov * aspect)) * (width * 0.28)); - int rad = (int) Math.min(Math.max(1, Math.round(radiusPx)), Math.max(width, height)); - int color = rgbFromEntity(e); - int er = (color >> 16) & 0xFF; - int eg = (color >> 8) & 0xFF; - int eb = color & 0xFF; - int x1 = Math.max(0, cx - rad); - int x2 = Math.min(width - 1, cx + rad); - int y1 = Math.max(0, cy - rad); - int y2 = Math.min(height - 1, cy + rad); - int rr = rad * rad; - for (int py = y1; py <= y2; py++) { - int dy = py - cy; - for (int px = x1; px <= x2; px++) { - int dx = px - cx; - if ((dx * dx + dy * dy) > rr) { - continue; - } - int p = py * width + px; - if (camZ >= depth[p]) { - continue; + if (!invokeGameRendererLevelPass(mc)) { + return null; + } + + ByteBuffer buf = MemoryUtil.memAlloc(width * height * 4); + try { + GL11C.glReadPixels(0, 0, width, height, GL11C.GL_RGBA, GL11C.GL_UNSIGNED_BYTE, buf); + byte[] out = new byte[width * height * 4]; + int rowBytes = width * 4; + for (int y = 0; y < height; y++) { + int src = (height - 1 - y) * rowBytes; + int dst = y * rowBytes; + buf.position(src); + buf.get(out, dst, rowBytes); } - int i = p * 4; - rgba[i] = (byte) er; - rgba[i + 1] = (byte) eg; - rgba[i + 2] = (byte) eb; - rgba[i + 3] = (byte) 255; - depth[p] = camZ; + camera.lastRgba = out; + return out; + } finally { + MemoryUtil.memFree(buf); + } + } catch (Throwable ignored) { + return null; + } finally { + try { + mc.setCameraEntity(previousCamera); + } catch (Throwable ignored) { + } + try { + setMinecraftMainRenderTarget(mc, previousTarget); + previousTarget.bindWrite(true); + RenderSystem.viewport(0, 0, previousWidth, previousHeight); + // Defensive reset after offscreen world pass so GUI overlays remain visible. + RenderSystem.disableDepthTest(); + RenderSystem.depthMask(true); + RenderSystem.disableCull(); + RenderSystem.disableScissor(); + RenderSystem.setShaderColor(1.0F, 1.0F, 1.0F, 1.0F); + } catch (Throwable ignored) { } } } - return encodeRgbaHex(rgba); - } - - private static int clampViewSize(int value) { - return Math.max(16, Math.min(512, value)); - } + private void ensureTarget(CameraContext camera, int width, int height) { + if (camera.target == null || camera.width != width || camera.height != height) { + if (camera.target != null) { + camera.target.destroyBuffers(); + } + camera.target = new TextureTarget(width, height, true, Minecraft.ON_OSX); + camera.width = width; + camera.height = height; + } + } - private static int rgbFromBlockState( - Minecraft mc, BlockPos pos, net.minecraft.world.level.block.state.BlockState state) { - try { - var map = state.getMapColor(mc.level, pos); - if (map != null && map.col != 0) { - return map.col & 0x00FFFFFF; + private boolean invokeGameRendererLevelPass(Minecraft mc) { + try { + if (gameRendererRenderLevelMethod == null) { + for (Method m : mc.gameRenderer.getClass().getDeclaredMethods()) { + if ("renderLevel".equals(m.getName())) { + m.setAccessible(true); + gameRendererRenderLevelMethod = m; + break; + } + } + } + if (gameRendererRenderLevelMethod == null) { + return false; + } + Class[] p = gameRendererRenderLevelMethod.getParameterTypes(); + Object[] args = new Object[p.length]; + for (int i = 0; i < p.length; i++) { + Class t = p[i]; + if (t == float.class || t == Float.class) { + args[i] = mc.getFrameTime(); + } else if (t == double.class || t == Double.class) { + args[i] = (double) mc.getFrameTime(); + } else if (t == long.class || t == Long.class) { + args[i] = System.nanoTime(); + } else if (t == int.class || t == Integer.class) { + args[i] = 0; + } else if (t == boolean.class || t == Boolean.class) { + args[i] = Boolean.FALSE; + } else if (PoseStack.class.isAssignableFrom(t)) { + args[i] = new PoseStack(); + } else { + args[i] = null; + } + } + gameRendererRenderLevelMethod.invoke(mc.gameRenderer, args); + return true; + } catch (Throwable ignored) { + return false; } - } catch (Throwable ignored) { } - ResourceLocation key = BuiltInRegistries.BLOCK.getKey(state.getBlock()); - int hash = key != null ? key.toString().hashCode() : 0x77AA77; - int r = 64 + (Math.abs(hash) & 0x7F); - int g = 64 + ((Math.abs(hash >> 8)) & 0x7F); - int b = 64 + ((Math.abs(hash >> 16)) & 0x7F); - return (r << 16) | (g << 8) | b; - } - private static int rgbFromEntity(Entity entity) { - int hash = entity.getType().toString().hashCode(); - int r = 96 + (Math.abs(hash) & 0x5F); - int g = 96 + ((Math.abs(hash >> 8)) & 0x5F); - int b = 96 + ((Math.abs(hash >> 16)) & 0x5F); - if (entity instanceof LivingEntity) { - r = Math.min(255, r + 24); + private boolean setMinecraftMainRenderTarget(Minecraft mc, RenderTarget target) { + try { + if (minecraftMainRenderTargetField == null) { + minecraftMainRenderTargetField = resolveMainRenderTargetField(mc); + } + if (minecraftMainRenderTargetField == null) { + return false; + } + minecraftMainRenderTargetField.set(mc, target); + return true; + } catch (Throwable ignored) { + return false; + } } - return (r << 16) | (g << 8) | b; - } - private static String encodeRgbaHex(byte[] rgba) { - char[] out = new char[rgba.length * 2]; - int j = 0; - for (byte b : rgba) { - int v = b & 0xFF; - out[j++] = HEX[v >>> 4]; - out[j++] = HEX[v & 0x0F]; + private Field resolveMainRenderTargetField(Minecraft mc) { + try { + for (Field f : Minecraft.class.getDeclaredFields()) { + if (f.getType() == RenderTarget.class && "mainRenderTarget".equals(f.getName())) { + f.setAccessible(true); + return f; + } + } + RenderTarget current = mc.getMainRenderTarget(); + for (Field f : Minecraft.class.getDeclaredFields()) { + if (!RenderTarget.class.isAssignableFrom(f.getType())) { + continue; + } + f.setAccessible(true); + Object value = f.get(mc); + if (value == current) { + return f; + } + } + } catch (Throwable ignored) { + } + return null; } - return new String(out); - } - private static Vec3 forwardFromRotation(float yawDeg, float pitchDeg) { - double yaw = Math.toRadians(yawDeg); - double pitch = Math.toRadians(pitchDeg); - double x = -Math.sin(yaw) * Math.cos(pitch); - double y = -Math.sin(pitch); - double z = Math.cos(yaw) * Math.cos(pitch); - return new Vec3(x, y, z).normalize(); } - private static Vec3 rightFromYaw(float yawDeg) { - double yaw = Math.toRadians(yawDeg); - return new Vec3(-Math.cos(yaw), 0.0, -Math.sin(yaw)).normalize(); + private static final class CameraContext { + private final String cameraId; + private TextureTarget target; + private int width; + private int height; + private byte[] lastRgba; + private boolean captureQueued; + private RLOperator pendingEntity; + private int pendingWidth; + private int pendingHeight; + + private CameraContext(String cameraId) { + this.cameraId = cameraId; + } } } diff --git a/src/main/java/com/verbii/minekov/client/XosViewportOverlay.java b/src/main/java/com/verbii/minekov/client/XosViewportOverlay.java index 9055728..0d353c2 100644 --- a/src/main/java/com/verbii/minekov/client/XosViewportOverlay.java +++ b/src/main/java/com/verbii/minekov/client/XosViewportOverlay.java @@ -951,7 +951,12 @@ public static void onScreenRenderPost(ScreenEvent.Render.Post event) { boolean hover = panelContains(mx, my, mc); XosViewportRuntime.setPanelHovered(hover); - XosViewportRuntime.pumpFrame(mc, panelW, contentH); + XosViewportRuntime.setOverlayRenderPassActive(true); + try { + XosViewportRuntime.pumpFrame(mc, panelW, contentH); + } finally { + XosViewportRuntime.setOverlayRenderPassActive(false); + } float drawA = hover ? ALPHA_HOVER : ALPHA_IDLE; int neon = neonArgb(drawA); int neonBorder = neonBorderArgb(drawA); @@ -963,6 +968,11 @@ public static void onScreenRenderPost(ScreenEvent.Render.Post event) { // Reset tint so blit() uses framebuffer α as-uploaded (60% idle / 80% hover), not × leftover shader. RenderSystem.setShaderColor(1.0F, 1.0F, 1.0F, 1.0F); g.setColor(1.0F, 1.0F, 1.0F, 1.0F); + // Vision's offscreen world pass can leave GL state dirty (depth/scissor/cull). Force GUI-safe state. + RenderSystem.disableDepthTest(); + RenderSystem.depthMask(true); + RenderSystem.disableCull(); + RenderSystem.disableScissor(); if (minimized) { int bx = MINIMIZED_BTN_MARGIN; @@ -1259,6 +1269,7 @@ public static void onRenderTickBackgroundPump(TickEvent.RenderTickEvent event) { return; } Minecraft mc = Minecraft.getInstance(); + XosViewportRuntime.flushDeferredVisionCaptures(mc); if (mc.screen instanceof ChatScreen) { return; } diff --git a/src/main/java/com/verbii/minekov/client/XosViewportRuntime.java b/src/main/java/com/verbii/minekov/client/XosViewportRuntime.java index 1dc29ff..2c6ab86 100644 --- a/src/main/java/com/verbii/minekov/client/XosViewportRuntime.java +++ b/src/main/java/com/verbii/minekov/client/XosViewportRuntime.java @@ -128,6 +128,8 @@ public final class XosViewportRuntime { /** Set each frame before {@link #pumpFrame} (chat) or background pump (not hovered). */ private static boolean panelHovered; + /** True only while chat overlay draw is executing. Used to avoid re-entrant offscreen world renders. */ + private static volatile boolean overlayRenderPassActive; /** * Caps how often {@link #pumpFrame} runs the native {@link XosNative#tick} + texture upload. @@ -182,6 +184,14 @@ public static boolean isRunSession() { return runSession; } + static void setOverlayRenderPassActive(boolean active) { + overlayRenderPassActive = active; + } + + static boolean isOverlayRenderPassActive() { + return overlayRenderPassActive; + } + /** Native library loaded and {@link XosNative#init} completed (e.g. after {@link #prewarmEngine}). */ public static boolean isEngineReady() { return libraryOk && engineRunning; @@ -723,6 +733,10 @@ private static String renderAgentViewRgbaHex(Minecraft mc, RLOperator agent, int return Vision.renderAgentViewRgbaHex(mc, agent, reqW, reqH); } + static void flushDeferredVisionCaptures(Minecraft mc) { + Vision.flushDeferredCaptures(mc); + } + private static double[] parseTriple(String raw, double fallbackX, double fallbackY, double fallbackZ) { if (raw == null || raw.isBlank()) { return new double[] {fallbackX, fallbackY, fallbackZ}; From e07bc5f1b419be7abbf5cc6e524a8ea755aeff04 Mon Sep 17 00:00:00 2001 From: verbiiyo Date: Mon, 1 Jun 2026 20:06:53 +0900 Subject: [PATCH 20/29] low poly renders! --- .../com/verbii/minekov/client/Vision.java | 60 ++++++++++++++----- 1 file changed, 46 insertions(+), 14 deletions(-) diff --git a/src/main/java/com/verbii/minekov/client/Vision.java b/src/main/java/com/verbii/minekov/client/Vision.java index f1c5192..dbd9f17 100644 --- a/src/main/java/com/verbii/minekov/client/Vision.java +++ b/src/main/java/com/verbii/minekov/client/Vision.java @@ -129,23 +129,28 @@ private byte[] renderCameraRgbaOnRenderThread( RLOperator cameraEntity, int width, int height) { - try { - ensureTarget(camera, width, height); - } catch (Throwable ignored) { - return null; - } - RenderTarget previousTarget = mc.getMainRenderTarget(); int previousWidth = previousTarget.width; int previousHeight = previousTarget.height; Entity previousCamera = mc.getCameraEntity(); + int renderWidth = Math.max(1, previousWidth); + int renderHeight = Math.max(1, previousHeight); + try { + ensureTarget(camera, renderWidth, renderHeight); + } catch (Throwable ignored) { + return null; + } if (!setMinecraftMainRenderTarget(mc, camera.target)) { return null; } try { mc.setCameraEntity(cameraEntity); + // Ensure the world pass is not clipped by chat/UI scissor state. + RenderSystem.disableScissor(); + RenderSystem.enableDepthTest(); + RenderSystem.depthMask(true); camera.target.bindWrite(true); - RenderSystem.viewport(0, 0, width, height); + RenderSystem.viewport(0, 0, renderWidth, renderHeight); camera.target.setClearColor(0f, 0f, 0f, 1f); camera.target.clear(Minecraft.ON_OSX); @@ -153,17 +158,26 @@ private byte[] renderCameraRgbaOnRenderThread( return null; } - ByteBuffer buf = MemoryUtil.memAlloc(width * height * 4); + ByteBuffer buf = MemoryUtil.memAlloc(renderWidth * renderHeight * 4); try { - GL11C.glReadPixels(0, 0, width, height, GL11C.GL_RGBA, GL11C.GL_UNSIGNED_BYTE, buf); - byte[] out = new byte[width * height * 4]; - int rowBytes = width * 4; - for (int y = 0; y < height; y++) { - int src = (height - 1 - y) * rowBytes; + // renderLevel can rebind other targets; force readback from this camera target. + camera.target.bindWrite(true); + RenderSystem.viewport(0, 0, renderWidth, renderHeight); + GL11C.glPixelStorei(GL11C.GL_PACK_ALIGNMENT, 1); + GL11C.glReadPixels( + 0, 0, renderWidth, renderHeight, GL11C.GL_RGBA, GL11C.GL_UNSIGNED_BYTE, buf); + byte[] full = new byte[renderWidth * renderHeight * 4]; + int rowBytes = renderWidth * 4; + for (int y = 0; y < renderHeight; y++) { + int src = (renderHeight - 1 - y) * rowBytes; int dst = y * rowBytes; buf.position(src); - buf.get(out, dst, rowBytes); + buf.get(full, dst, rowBytes); } + byte[] out = + (renderWidth == width && renderHeight == height) + ? full + : downscaleNearest(full, renderWidth, renderHeight, width, height); camera.lastRgba = out; return out; } finally { @@ -191,6 +205,24 @@ private byte[] renderCameraRgbaOnRenderThread( } } + private byte[] downscaleNearest( + byte[] src, int srcW, int srcH, int dstW, int dstH) { + byte[] dst = new byte[Math.max(1, dstW * dstH * 4)]; + for (int y = 0; y < dstH; y++) { + int sy = Math.min(srcH - 1, (y * srcH) / Math.max(1, dstH)); + for (int x = 0; x < dstW; x++) { + int sx = Math.min(srcW - 1, (x * srcW) / Math.max(1, dstW)); + int si = (sy * srcW + sx) * 4; + int di = (y * dstW + x) * 4; + dst[di] = src[si]; + dst[di + 1] = src[si + 1]; + dst[di + 2] = src[si + 2]; + dst[di + 3] = src[si + 3]; + } + } + return dst; + } + private void ensureTarget(CameraContext camera, int width, int height) { if (camera.target == null || camera.width != width || camera.height != height) { if (camera.target != null) { From a7a106030859c88b3131431c7fafbe742bbb81e9 Mon Sep 17 00:00:00 2001 From: verbiiyo Date: Wed, 3 Jun 2026 12:08:10 +0900 Subject: [PATCH 21/29] add some notes --- MinecraftRenderingNotes.md | 42 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 MinecraftRenderingNotes.md diff --git a/MinecraftRenderingNotes.md b/MinecraftRenderingNotes.md new file mode 100644 index 0000000..d8f37f8 --- /dev/null +++ b/MinecraftRenderingNotes.md @@ -0,0 +1,42 @@ +# Minecraft Rendering Notes + +This document tracks what we currently understand about the offscreen agent camera path in `Vision.java`, what is working, and what is still unresolved. + +## Current Status (2026-06-03) + +- We switched the offscreen pass to call the real game renderer entry point: + - `mc.gameRenderer.render(mc.getFrameTime(), System.nanoTime(), true)` +- We also set the camera entity before rendering and restore it afterward: + - `mc.setCameraEntity(cameraEntity)` + - `mainCamera.setup(...)` +- We suppress first-person hand rendering during capture with `RenderHandEvent` while a capture is active. + +## Observable Result From Latest Change + +- The agent view can now render the player's `xos` viewport inside the `xos` viewport (infinite mirror effect). +- This confirms the capture path is now running through an actual frame render path, not only a synthetic/raycast path. +- The mirrored content appears at low resolution because capture readback currently scales from the offscreen target to requested output size. + +## What Works + +- **Entity rendering:** entities are visible in the agent output. +- **First-person hand/item handling:** we can control hand rendering in capture via `RenderHandEvent` cancellation during active capture. +- **Offscreen capture loop:** capture + readback + RGBA hex encoding are stable enough to produce consistent frames. + +## What Is Not Understood Yet + +- **World block rendering is still incorrect/incomplete** in the agent output. +- We currently observe sky rendering reliably, but in-world block terrain is not rendering as expected. +- We do not yet have a confirmed explanation for why terrain/chunk world geometry is missing while other frame elements still appear. + +## Important Historical Context + +- A previous commit (`d2ba7a86788f4b8894785e74ff7a57748095d007`) rendered blocks using a custom raycast + color approximation path. +- That old path could show block-like structure, but it was not real textured world rendering. +- The current objective is real renderer parity (world blocks + entities + expected first-person behavior), not approximate visualization. + +## Next Investigation Direction + +- Keep using the real renderer path. +- Focus debugging on why terrain passes are absent in the offscreen target while other passes still appear. +- Treat this as a render-target / pipeline-state issue until proven otherwise. From 3839da8482181b77f38f7b5fa5a1d86fda067c8a Mon Sep 17 00:00:00 2001 From: verbiiyo Date: Wed, 3 Jun 2026 12:14:18 +0900 Subject: [PATCH 22/29] update notes --- MinecraftRenderingNotes.md | 146 ++++++++++++++++++++++++++++++------- 1 file changed, 118 insertions(+), 28 deletions(-) diff --git a/MinecraftRenderingNotes.md b/MinecraftRenderingNotes.md index d8f37f8..0a24d2b 100644 --- a/MinecraftRenderingNotes.md +++ b/MinecraftRenderingNotes.md @@ -1,42 +1,132 @@ # Minecraft Rendering Notes -This document tracks what we currently understand about the offscreen agent camera path in `Vision.java`, what is working, and what is still unresolved. +This document separates the vision stack into two independently understandable systems: -## Current Status (2026-06-03) +1. **Entity Rendering** (what is now accurate in the current setup) +2. **World Rendering** (what `4c790a70...` / `b30d00d9...` were doing, and why entities became orbs there) -- We switched the offscreen pass to call the real game renderer entry point: - - `mc.gameRenderer.render(mc.getFrameTime(), System.nanoTime(), true)` -- We also set the camera entity before rendering and restore it afterward: - - `mc.setCameraEntity(cameraEntity)` - - `mainCamera.setup(...)` -- We suppress first-person hand rendering during capture with `RenderHandEvent` while a capture is active. +The goal is to keep both paths clear so they can be recombined intentionally. -## Observable Result From Latest Change +--- -- The agent view can now render the player's `xos` viewport inside the `xos` viewport (infinite mirror effect). -- This confirms the capture path is now running through an actual frame render path, not only a synthetic/raycast path. -- The mirrored content appears at low resolution because capture readback currently scales from the offscreen target to requested output size. +## 1) Entity Rendering -## What Works +### Current state (today): accurate entity rendering path -- **Entity rendering:** entities are visible in the agent output. -- **First-person hand/item handling:** we can control hand rendering in capture via `RenderHandEvent` cancellation during active capture. -- **Offscreen capture loop:** capture + readback + RGBA hex encoding are stable enough to produce consistent frames. +The current `Vision.java` pipeline is the first path that is designed around real renderer internals instead of synthetic overlays: -## What Is Not Understood Yet +- Per-camera context (`CameraContext`) keyed by agent UUID. +- Offscreen `TextureTarget` lifecycle management. +- Render-thread scheduling (`queueCapture` + `flushDeferredCaptures`) so world pass work stays on the correct thread. +- Capture using actual game render invocation (`renderLevel`) and readback. -- **World block rendering is still incorrect/incomplete** in the agent output. -- We currently observe sky rendering reliably, but in-world block terrain is not rendering as expected. -- We do not yet have a confirmed explanation for why terrain/chunk world geometry is missing while other frame elements still appear. +Tiny internals example: -## Important Historical Context +```java +camera.target = new TextureTarget(width, height, true, Minecraft.ON_OSX); +mc.setCameraEntity(cameraEntity); +gameRendererRenderLevelMethod.invoke(mc.gameRenderer, args); +GL11C.glReadPixels(..., GL11C.GL_RGBA, GL11C.GL_UNSIGNED_BYTE, buf); +``` -- A previous commit (`d2ba7a86788f4b8894785e74ff7a57748095d007`) rendered blocks using a custom raycast + color approximation path. -- That old path could show block-like structure, but it was not real textured world rendering. -- The current objective is real renderer parity (world blocks + entities + expected first-person behavior), not approximate visualization. +Why this section is considered "working now": -## Next Investigation Direction +- It is camera-entity based (agent POV), not just projected world math. +- It uses renderer passes rather than hand-authored entity glyphs. +- It keeps per-camera state and deferred capture semantics needed for correctness. -- Keep using the real renderer path. -- Focus debugging on why terrain passes are absent in the offscreen target while other passes still appear. -- Treat this as a render-target / pipeline-state issue until proven otherwise. +### Why entities were orbs in `b30d00d9...` (and still in fallback behavior) + +`b30d00d9...` introduced explicit **entity overlay as projected circles** over raycasted world color. +That is why entity visibility improved but fidelity became "orb-like". + +Tiny orb logic example: + +```java +int color = rgbFromEntity(e); +for (int py = y1; py <= y2; py++) { + for (int px = x1; px <= x2; px++) { + if ((dx * dx + dy * dy) <= rr) { /* paint disk */ } + } +} +``` + +So in that commit lineage: + +- Entities were **represented**, but not mesh-rendered. +- Entity depth was approximated against ray depth. +- Result: stable and readable, but never true model render. + +--- + +## 2) World Rendering + +### What `b30d00d9...` did for world rendering + +This commit improved world perception by replacing coarse stepping with proper block hit queries (`ClipContext`) and better block color lookup (`MapColor` fallback to hash). + +Tiny world hit example: + +```java +var hit = mc.level.clip(new ClipContext(origin, rayEnd, OUTLINE, NONE, agent)); +BlockPos bp = hit.getBlockPos(); +int rgb = rgbFromBlockState(mc, bp, state); +``` + +What this gives: + +- World **state structure** is closer to reality (actual block intersections). +- Lighting cue approximation (distance + face shade). + +What it does **not** give: + +- Real block textures/material pipeline. +- Full parity with Minecraft frame rendering. + +So this path is "world-state true-ish", not "framebuffer true". + +### What `4c790a70...` added + +This commit introduced an experimental offscreen real-render attempt: + +- `TextureTarget` offscreen buffer +- Reflective `renderLevel` invocation +- temporary camera swap and readback + +Tiny real-pass attempt example: + +```java +agentViewRenderTarget = new TextureTarget(w, h, true, Minecraft.ON_OSX); +setMinecraftMainRenderTarget(mc, previous, agentViewRenderTarget); +invokeGameRendererLevelPass(mc); +``` + +Important internal note: + +- The commit includes `ENABLE_REAL_AGENT_VIEW_RENDER = false`. +- Therefore fallback behavior from `b30d00d9...` remained the dominant observed mode unless manually toggled. +- This is why "world looked improved in state" while entities still appeared as orbs in that line of behavior. + +### Why world and entities diverged in those commits + +In `b30d00d9...` + `4c790a70...` practical behavior, the pipeline mixed two concepts: + +1. **World via ray/block sampling** (real block-state lookup, non-textured shading) +2. **Entities via synthetic disk overlay** (explicit orb rendering) + +That combination explains the observed result exactly: + +- Blocks/world topology looked meaningful. +- Entities were visible but stylized as circles. +- Not yet full camera parity with native Minecraft scene rendering. + +--- + +## Practical interpretation for integration work + +- **Section 1 (Entity Rendering)** now refers to the camera/render-pass architecture in `Vision.java` (accurate target direction). +- **Section 2 (World Rendering)** documents what the two historical commits proved: + - `b30d00d9...`: strong world-state approximation, synthetic entities. + - `4c790a70...`: first real offscreen render plumbing attempt, not yet stable as a unified final path. + +This separation is intentional and useful: both systems can be validated independently before merging into one robust camera pipeline. From 5020512c394d8c4f4b27b93d6f6659cfc16606a0 Mon Sep 17 00:00:00 2001 From: verbiiyo Date: Wed, 3 Jun 2026 12:24:24 +0900 Subject: [PATCH 23/29] both versions of the renderer in the same codebase --- .../client/{Vision.java => EntityVision.java} | 8 +- .../verbii/minekov/client/WorldVision.java | 392 ++++++++++++++++++ .../minekov/client/XosViewportRuntime.java | 8 +- 3 files changed, 400 insertions(+), 8 deletions(-) rename src/main/java/com/verbii/minekov/client/{Vision.java => EntityVision.java} (97%) create mode 100644 src/main/java/com/verbii/minekov/client/WorldVision.java diff --git a/src/main/java/com/verbii/minekov/client/Vision.java b/src/main/java/com/verbii/minekov/client/EntityVision.java similarity index 97% rename from src/main/java/com/verbii/minekov/client/Vision.java rename to src/main/java/com/verbii/minekov/client/EntityVision.java index dbd9f17..f45bad6 100644 --- a/src/main/java/com/verbii/minekov/client/Vision.java +++ b/src/main/java/com/verbii/minekov/client/EntityVision.java @@ -17,13 +17,13 @@ import java.util.Map; /** Agent vision renderer used by Python `agent.visual(...)` host binding. */ -final class Vision { +final class EntityVision { private static final int MIN_VIEW_SIZE = 16; private static final int MAX_VIEW_SIZE = 1024; private static final char[] HEX = "0123456789abcdef".toCharArray(); private static final VisionManager MANAGER = new VisionManager(); - private Vision() {} + private EntityVision() {} static String renderAgentViewRgbaHex(Minecraft mc, RLOperator agent, int reqW, int reqH) { return MANAGER.renderAgentViewRgbaHex(mc, agent, reqW, reqH); @@ -145,7 +145,6 @@ private byte[] renderCameraRgbaOnRenderThread( } try { mc.setCameraEntity(cameraEntity); - // Ensure the world pass is not clipped by chat/UI scissor state. RenderSystem.disableScissor(); RenderSystem.enableDepthTest(); RenderSystem.depthMask(true); @@ -160,7 +159,6 @@ private byte[] renderCameraRgbaOnRenderThread( ByteBuffer buf = MemoryUtil.memAlloc(renderWidth * renderHeight * 4); try { - // renderLevel can rebind other targets; force readback from this camera target. camera.target.bindWrite(true); RenderSystem.viewport(0, 0, renderWidth, renderHeight); GL11C.glPixelStorei(GL11C.GL_PACK_ALIGNMENT, 1); @@ -194,7 +192,6 @@ private byte[] renderCameraRgbaOnRenderThread( setMinecraftMainRenderTarget(mc, previousTarget); previousTarget.bindWrite(true); RenderSystem.viewport(0, 0, previousWidth, previousHeight); - // Defensive reset after offscreen world pass so GUI overlays remain visible. RenderSystem.disableDepthTest(); RenderSystem.depthMask(true); RenderSystem.disableCull(); @@ -313,7 +310,6 @@ private Field resolveMainRenderTargetField(Minecraft mc) { } return null; } - } private static final class CameraContext { diff --git a/src/main/java/com/verbii/minekov/client/WorldVision.java b/src/main/java/com/verbii/minekov/client/WorldVision.java new file mode 100644 index 0000000..c5cab22 --- /dev/null +++ b/src/main/java/com/verbii/minekov/client/WorldVision.java @@ -0,0 +1,392 @@ +package com.verbii.minekov.client; + +import com.verbii.minekov.entities.RLOperator; +import com.mojang.blaze3d.pipeline.RenderTarget; +import com.mojang.blaze3d.pipeline.TextureTarget; +import com.mojang.blaze3d.systems.RenderSystem; +import com.mojang.blaze3d.vertex.PoseStack; +import net.minecraft.client.Minecraft; +import net.minecraft.core.BlockPos; +import net.minecraft.core.registries.BuiltInRegistries; +import net.minecraft.resources.ResourceLocation; +import net.minecraft.world.entity.Entity; +import net.minecraft.world.entity.LivingEntity; +import net.minecraft.world.level.ClipContext; +import net.minecraft.world.phys.HitResult; +import net.minecraft.world.phys.Vec3; +import org.lwjgl.opengl.GL11C; +import org.lwjgl.system.MemoryUtil; + +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.nio.ByteBuffer; + +/** + * World-state renderer path from the 4c790a70 / b30d00d9 lineage. + * - Blocks: ray/clip sampled + shaded colors. + * - Entities: projected depth-tested orb overlay. + */ +final class WorldVision { + private static final float AGENT_VIEW_FOV_DEG = 70.0f; + private static final double AGENT_VIEW_MAX_DISTANCE = 24.0; + private static final char[] HEX = "0123456789abcdef".toCharArray(); + + // Keep false unless explicitly experimenting with old real-render attempt. + private static final boolean ENABLE_REAL_AGENT_VIEW_RENDER = false; + private static TextureTarget agentViewRenderTarget; + private static Method gameRendererRenderLevelMethod; + private static Field minecraftMainRenderTargetField; + + private WorldVision() {} + + static String renderAgentViewRgbaHex(Minecraft mc, RLOperator agent, int reqW, int reqH) { + int width = clampViewSize(reqW); + int height = clampViewSize(reqH); + if (mc == null || mc.level == null || agent == null) { + return ""; + } + if (ENABLE_REAL_AGENT_VIEW_RENDER) { + byte[] rendered = tryRenderAgentViewWithMinecraftRenderer(mc, agent, width, height); + if (rendered != null) { + return encodeRgbaHex(rendered); + } + } + + Vec3 origin = new Vec3(agent.getX(), agent.getEyeY(), agent.getZ()); + float yaw = agent.getYRot(); + float pitch = agent.getXRot(); + Vec3 forward = forwardFromRotation(yaw, pitch); + Vec3 worldUp = new Vec3(0.0, 1.0, 0.0); + Vec3 right = worldUp.cross(forward); + if (right.lengthSqr() < 1.0e-8) { + right = rightFromYaw(yaw); + } else { + right = right.normalize(); + } + Vec3 up = forward.cross(right).normalize(); + + double tanHalfFov = Math.tan(Math.toRadians(AGENT_VIEW_FOV_DEG * 0.5)); + double aspect = (double) width / (double) height; + int pixelCount = width * height; + byte[] rgba = new byte[pixelCount * 4]; + double[] depth = new double[pixelCount]; + for (int i = 0; i < pixelCount; i++) { + depth[i] = Double.POSITIVE_INFINITY; + } + + for (int y = 0; y < height; y++) { + double ny = 1.0 - (2.0 * (y + 0.5) / (double) height); + for (int x = 0; x < width; x++) { + double nx = (2.0 * (x + 0.5) / (double) width) - 1.0; + Vec3 dir = forward + .add(right.scale(nx * tanHalfFov * aspect)) + .add(up.scale(ny * tanHalfFov)) + .normalize(); + + int r = 110; + int g = 150; + int b = 210; + boolean hit = false; + double distance = 0.0; + Vec3 rayEnd = origin.add(dir.scale(AGENT_VIEW_MAX_DISTANCE)); + var hitRes = mc.level.clip(new ClipContext( + origin, + rayEnd, + ClipContext.Block.OUTLINE, + ClipContext.Fluid.NONE, + agent)); + if (hitRes.getType() != HitResult.Type.MISS) { + BlockPos bp = hitRes.getBlockPos(); + var state = mc.level.getBlockState(bp); + if (!state.isAir()) { + int rgb = rgbFromBlockState(mc, bp, state); + distance = hitRes.getLocation().distanceTo(origin); + double shade = Math.max(0.28, 1.0 - (distance / AGENT_VIEW_MAX_DISTANCE)); + double faceShade = switch (hitRes.getDirection()) { + case UP -> 1.08; + case DOWN -> 0.72; + case NORTH, SOUTH -> 0.94; + default -> 1.00; + }; + double totalShade = Math.max(0.20, Math.min(1.15, shade * faceShade)); + r = (int) Math.max(0, Math.min(255, ((rgb >> 16) & 0xFF) * totalShade)); + g = (int) Math.max(0, Math.min(255, ((rgb >> 8) & 0xFF) * totalShade)); + b = (int) Math.max(0, Math.min(255, (rgb & 0xFF) * totalShade)); + hit = true; + } + } + + if (!hit) { + double tSky = Math.max(0.0, Math.min(1.0, (dir.y + 1.0) * 0.5)); + r = (int) (70 + 60 * tSky); + g = (int) (100 + 90 * tSky); + b = (int) (150 + 80 * tSky); + } else { + int fog = (int) Math.max(0, Math.min(40, (AGENT_VIEW_MAX_DISTANCE - distance) * 0.8)); + r = Math.min(255, r + fog); + g = Math.min(255, g + fog); + b = Math.min(255, b + fog); + } + int p = y * width + x; + int i = p * 4; + rgba[i] = (byte) r; + rgba[i + 1] = (byte) g; + rgba[i + 2] = (byte) b; + rgba[i + 3] = (byte) 255; + depth[p] = hit ? distance : Double.POSITIVE_INFINITY; + } + } + + var renderables = mc.level.entitiesForRendering(); + for (Entity e : renderables) { + if (e == null || e == agent || e.isRemoved()) { + continue; + } + Vec3 rel = new Vec3(e.getX() - origin.x, e.getEyeY() - origin.y, e.getZ() - origin.z); + double camX = rel.dot(right); + double camY = rel.dot(up); + double camZ = rel.dot(forward); + if (camZ <= 0.01 || camZ > AGENT_VIEW_MAX_DISTANCE) { + continue; + } + + double ndcX = camX / (camZ * tanHalfFov * aspect); + double ndcY = camY / (camZ * tanHalfFov); + if (Math.abs(ndcX) > 1.5 || Math.abs(ndcY) > 1.5) { + continue; + } + + int cx = (int) Math.round((ndcX + 1.0) * 0.5 * (width - 1)); + int cy = (int) Math.round((1.0 - (ndcY + 1.0) * 0.5) * (height - 1)); + double radiusPx = Math.max( + 1.0, + (Math.max(0.25, e.getBbWidth()) / (camZ * tanHalfFov * aspect)) * (width * 0.28)); + int rad = (int) Math.min(Math.max(1, Math.round(radiusPx)), Math.max(width, height)); + int color = rgbFromEntity(e); + int er = (color >> 16) & 0xFF; + int eg = (color >> 8) & 0xFF; + int eb = color & 0xFF; + int x1 = Math.max(0, cx - rad); + int x2 = Math.min(width - 1, cx + rad); + int y1 = Math.max(0, cy - rad); + int y2 = Math.min(height - 1, cy + rad); + int rr = rad * rad; + for (int py = y1; py <= y2; py++) { + int dy = py - cy; + for (int px = x1; px <= x2; px++) { + int dx = px - cx; + if ((dx * dx + dy * dy) > rr) { + continue; + } + int p = py * width + px; + if (camZ >= depth[p]) { + continue; + } + int i = p * 4; + rgba[i] = (byte) er; + rgba[i + 1] = (byte) eg; + rgba[i + 2] = (byte) eb; + rgba[i + 3] = (byte) 255; + depth[p] = camZ; + } + } + } + return encodeRgbaHex(rgba); + } + + static void flushDeferredCaptures(Minecraft mc) { + // No-op in world-state renderer path (kept for easy call-site swapping). + } + + private static int clampViewSize(int value) { + return Math.max(16, Math.min(512, value)); + } + + private static int rgbFromBlockState( + Minecraft mc, BlockPos pos, net.minecraft.world.level.block.state.BlockState state) { + try { + var map = state.getMapColor(mc.level, pos); + if (map != null && map.col != 0) { + return map.col & 0x00FFFFFF; + } + } catch (Throwable ignored) { + } + ResourceLocation key = BuiltInRegistries.BLOCK.getKey(state.getBlock()); + int hash = key != null ? key.toString().hashCode() : 0x77AA77; + int r = 64 + (Math.abs(hash) & 0x7F); + int g = 64 + ((Math.abs(hash >> 8)) & 0x7F); + int b = 64 + ((Math.abs(hash >> 16)) & 0x7F); + return (r << 16) | (g << 8) | b; + } + + private static int rgbFromEntity(Entity entity) { + int hash = entity.getType().toString().hashCode(); + int r = 96 + (Math.abs(hash) & 0x5F); + int g = 96 + ((Math.abs(hash >> 8)) & 0x5F); + int b = 96 + ((Math.abs(hash >> 16)) & 0x5F); + if (entity instanceof LivingEntity) { + r = Math.min(255, r + 24); + } + return (r << 16) | (g << 8) | b; + } + + private static String encodeRgbaHex(byte[] rgba) { + char[] out = new char[rgba.length * 2]; + int j = 0; + for (byte b : rgba) { + int v = b & 0xFF; + out[j++] = HEX[v >>> 4]; + out[j++] = HEX[v & 0x0F]; + } + return new String(out); + } + + private static Vec3 forwardFromRotation(float yawDeg, float pitchDeg) { + double yaw = Math.toRadians(yawDeg); + double pitch = Math.toRadians(pitchDeg); + double x = -Math.sin(yaw) * Math.cos(pitch); + double y = -Math.sin(pitch); + double z = Math.cos(yaw) * Math.cos(pitch); + return new Vec3(x, y, z).normalize(); + } + + private static Vec3 rightFromYaw(float yawDeg) { + double yaw = Math.toRadians(yawDeg); + return new Vec3(-Math.cos(yaw), 0.0, -Math.sin(yaw)).normalize(); + } + + private static byte[] tryRenderAgentViewWithMinecraftRenderer( + Minecraft mc, RLOperator agent, int width, int height) { + if (!RenderSystem.isOnRenderThread()) { + return null; + } + try { + if (agentViewRenderTarget == null + || agentViewRenderTarget.width != width + || agentViewRenderTarget.height != height) { + if (agentViewRenderTarget != null) { + agentViewRenderTarget.destroyBuffers(); + } + agentViewRenderTarget = new TextureTarget(width, height, true, Minecraft.ON_OSX); + } + } catch (Throwable t) { + return null; + } + + RenderTarget previousTarget = mc.getMainRenderTarget(); + Entity previousCamera = mc.getCameraEntity(); + if (!setMinecraftMainRenderTarget(mc, previousTarget, agentViewRenderTarget)) { + return null; + } + try { + mc.setCameraEntity(agent); + agentViewRenderTarget.bindWrite(true); + RenderSystem.viewport(0, 0, width, height); + agentViewRenderTarget.setClearColor(0f, 0f, 0f, 1f); + agentViewRenderTarget.clear(Minecraft.ON_OSX); + + if (!invokeGameRendererLevelPass(mc)) { + return null; + } + + ByteBuffer buf = MemoryUtil.memAlloc(width * height * 4); + try { + GL11C.glReadPixels(0, 0, width, height, GL11C.GL_RGBA, GL11C.GL_UNSIGNED_BYTE, buf); + byte[] out = new byte[width * height * 4]; + int rowBytes = width * 4; + for (int y = 0; y < height; y++) { + int src = (height - 1 - y) * rowBytes; + int dst = y * rowBytes; + buf.position(src); + buf.get(out, dst, rowBytes); + } + return out; + } finally { + MemoryUtil.memFree(buf); + } + } catch (Throwable t) { + return null; + } finally { + try { + mc.setCameraEntity(previousCamera); + } catch (Throwable ignored) { + } + try { + setMinecraftMainRenderTarget(mc, agentViewRenderTarget, previousTarget); + previousTarget.bindWrite(true); + RenderSystem.viewport(0, 0, previousTarget.width, previousTarget.height); + } catch (Throwable ignored) { + } + } + } + + private static boolean invokeGameRendererLevelPass(Minecraft mc) { + try { + if (gameRendererRenderLevelMethod == null) { + for (Method m : mc.gameRenderer.getClass().getDeclaredMethods()) { + if ("renderLevel".equals(m.getName())) { + m.setAccessible(true); + gameRendererRenderLevelMethod = m; + break; + } + } + } + if (gameRendererRenderLevelMethod == null) { + return false; + } + Class[] p = gameRendererRenderLevelMethod.getParameterTypes(); + Object[] args = new Object[p.length]; + for (int i = 0; i < p.length; i++) { + Class t = p[i]; + if (t == float.class || t == Float.class) { + args[i] = mc.getFrameTime(); + } else if (t == double.class || t == Double.class) { + args[i] = (double) mc.getFrameTime(); + } else if (t == long.class || t == Long.class) { + args[i] = System.nanoTime(); + } else if (t == int.class || t == Integer.class) { + args[i] = 0; + } else if (t == boolean.class || t == Boolean.class) { + args[i] = Boolean.FALSE; + } else if (PoseStack.class.isAssignableFrom(t)) { + args[i] = new PoseStack(); + } else { + args[i] = null; + } + } + gameRendererRenderLevelMethod.invoke(mc.gameRenderer, args); + return true; + } catch (Throwable t) { + return false; + } + } + + private static boolean setMinecraftMainRenderTarget( + Minecraft mc, RenderTarget expectedCurrent, RenderTarget target) { + try { + if (minecraftMainRenderTargetField == null) { + for (Field f : Minecraft.class.getDeclaredFields()) { + if (RenderTarget.class.isAssignableFrom(f.getType())) { + f.setAccessible(true); + Object cur = f.get(mc); + if (cur == expectedCurrent) { + minecraftMainRenderTargetField = f; + break; + } + } + } + } + if (minecraftMainRenderTargetField == null) { + return false; + } + Object cur = minecraftMainRenderTargetField.get(mc); + if (cur != expectedCurrent) { + return false; + } + minecraftMainRenderTargetField.set(mc, target); + return true; + } catch (Throwable t) { + return false; + } + } +} diff --git a/src/main/java/com/verbii/minekov/client/XosViewportRuntime.java b/src/main/java/com/verbii/minekov/client/XosViewportRuntime.java index 2c6ab86..15ada8e 100644 --- a/src/main/java/com/verbii/minekov/client/XosViewportRuntime.java +++ b/src/main/java/com/verbii/minekov/client/XosViewportRuntime.java @@ -730,11 +730,15 @@ private static String encodePlayerInventoryEnderChest(Minecraft mc) { } private static String renderAgentViewRgbaHex(Minecraft mc, RLOperator agent, int reqW, int reqH) { - return Vision.renderAgentViewRgbaHex(mc, agent, reqW, reqH); + // Toggle renderer by swapping these calls. + return WorldVision.renderAgentViewRgbaHex(mc, agent, reqW, reqH); + // return EntityVision.renderAgentViewRgbaHex(mc, agent, reqW, reqH); } static void flushDeferredVisionCaptures(Minecraft mc) { - Vision.flushDeferredCaptures(mc); + // Keep paired with the selected renderer above. + WorldVision.flushDeferredCaptures(mc); + // EntityVision.flushDeferredCaptures(mc); } private static double[] parseTriple(String raw, double fallbackX, double fallbackY, double fallbackZ) { From 5d0ede55a7a37fe55faf607dbd03ac7ea5982919 Mon Sep 17 00:00:00 2001 From: verbiiyo Date: Wed, 3 Jun 2026 12:32:51 +0900 Subject: [PATCH 24/29] updates to notes! --- MinecraftRenderingNotes.md | 118 +++++++++++++------------------------ 1 file changed, 42 insertions(+), 76 deletions(-) diff --git a/MinecraftRenderingNotes.md b/MinecraftRenderingNotes.md index 0a24d2b..4328b24 100644 --- a/MinecraftRenderingNotes.md +++ b/MinecraftRenderingNotes.md @@ -1,19 +1,22 @@ # Minecraft Rendering Notes -This document separates the vision stack into two independently understandable systems: +This document tracks the current split vision stack and the historical commit lineage. -1. **Entity Rendering** (what is now accurate in the current setup) -2. **World Rendering** (what `4c790a70...` / `b30d00d9...` were doing, and why entities became orbs there) +Current branch snapshot: -The goal is to keep both paths clear so they can be recombined intentionally. +- **Current commit**: `5020512c394d8c4f4b27b93d6f6659cfc16606a0` +- **Both renderers now exist in the same branch/commit**: + - `EntityVision.java` + - `WorldVision.java` +- Runtime selection is currently in `XosViewportRuntime.renderAgentViewRgbaHex(...)` and `flushDeferredVisionCaptures(...)` with an easy comment swap. --- ## 1) Entity Rendering -### Current state (today): accurate entity rendering path +### What `EntityVision` does now -The current `Vision.java` pipeline is the first path that is designed around real renderer internals instead of synthetic overlays: +`EntityVision` is the camera-driven render-pass path intended for accurate POV rendering: - Per-camera context (`CameraContext`) keyed by agent UUID. - Offscreen `TextureTarget` lifecycle management. @@ -29,41 +32,24 @@ gameRendererRenderLevelMethod.invoke(mc.gameRenderer, args); GL11C.glReadPixels(..., GL11C.GL_RGBA, GL11C.GL_UNSIGNED_BYTE, buf); ``` -Why this section is considered "working now": +Why `EntityVision` is the "accurate entity" direction: - It is camera-entity based (agent POV), not just projected world math. -- It uses renderer passes rather than hand-authored entity glyphs. +- It uses renderer passes rather than hand-authored entity shapes. - It keeps per-camera state and deferred capture semantics needed for correctness. -### Why entities were orbs in `b30d00d9...` (and still in fallback behavior) - -`b30d00d9...` introduced explicit **entity overlay as projected circles** over raycasted world color. -That is why entity visibility improved but fidelity became "orb-like". - -Tiny orb logic example: - -```java -int color = rgbFromEntity(e); -for (int py = y1; py <= y2; py++) { - for (int px = x1; px <= x2; px++) { - if ((dx * dx + dy * dy) <= rr) { /* paint disk */ } - } -} -``` - -So in that commit lineage: - -- Entities were **represented**, but not mesh-rendered. -- Entity depth was approximated against ray depth. -- Result: stable and readable, but never true model render. - --- ## 2) World Rendering -### What `b30d00d9...` did for world rendering +### What `WorldVision` does now (ported from `4c790a70...` lineage) + +`WorldVision` keeps the block-state ray/clip approach from the `b30d00d9...` and `4c790a70...` line: -This commit improved world perception by replacing coarse stepping with proper block hit queries (`ClipContext`) and better block color lookup (`MapColor` fallback to hash). +- Per-pixel world ray casting using `ClipContext`. +- Block color from `MapColor` with registry-hash fallback. +- Distance and face-based shading for depth/readability. +- Entity overlay rendered as projected disks (orbs) with depth checks. Tiny world hit example: @@ -73,60 +59,40 @@ BlockPos bp = hit.getBlockPos(); int rgb = rgbFromBlockState(mc, bp, state); ``` -What this gives: - -- World **state structure** is closer to reality (actual block intersections). -- Lighting cue approximation (distance + face shade). - -What it does **not** give: - -- Real block textures/material pipeline. -- Full parity with Minecraft frame rendering. - -So this path is "world-state true-ish", not "framebuffer true". - -### What `4c790a70...` added - -This commit introduced an experimental offscreen real-render attempt: - -- `TextureTarget` offscreen buffer -- Reflective `renderLevel` invocation -- temporary camera swap and readback - -Tiny real-pass attempt example: +Tiny orb overlay example: ```java -agentViewRenderTarget = new TextureTarget(w, h, true, Minecraft.ON_OSX); -setMinecraftMainRenderTarget(mc, previous, agentViewRenderTarget); -invokeGameRendererLevelPass(mc); +int color = rgbFromEntity(e); +if ((dx * dx + dy * dy) <= rr && camZ < depth[p]) { + // paint projected entity disk +} ``` -Important internal note: +What `WorldVision` gives: -- The commit includes `ENABLE_REAL_AGENT_VIEW_RENDER = false`. -- Therefore fallback behavior from `b30d00d9...` remained the dominant observed mode unless manually toggled. -- This is why "world looked improved in state" while entities still appeared as orbs in that line of behavior. +- World **state structure** that is close to live block layout. +- Stable, readable rendering and predictable performance shape. +- Explicit fallback behavior that is easy to reason about. -### Why world and entities diverged in those commits +What `WorldVision` does **not** provide: -In `b30d00d9...` + `4c790a70...` practical behavior, the pipeline mixed two concepts: +- Mesh/textured entity rendering. +- Full native-frame parity with Minecraft's standard on-screen camera pass. -1. **World via ray/block sampling** (real block-state lookup, non-textured shading) -2. **Entities via synthetic disk overlay** (explicit orb rendering) - -That combination explains the observed result exactly: +--- -- Blocks/world topology looked meaningful. -- Entities were visible but stylized as circles. -- Not yet full camera parity with native Minecraft scene rendering. +## EntityVision vs WorldVision ---- +Both renderers now exist side by side in one branch so each can be stabilized independently. -## Practical interpretation for integration work +- **`EntityVision`** + - Camera/render-pass architecture (`renderLevel` + offscreen target + readback). + - Best path toward true POV fidelity (including proper entity rendering). + - More sensitive to render-thread/target-state correctness. -- **Section 1 (Entity Rendering)** now refers to the camera/render-pass architecture in `Vision.java` (accurate target direction). -- **Section 2 (World Rendering)** documents what the two historical commits proved: - - `b30d00d9...`: strong world-state approximation, synthetic entities. - - `4c790a70...`: first real offscreen render plumbing attempt, not yet stable as a unified final path. +- **`WorldVision`** + - Deterministic ray/clip world sampler with shaded block colors. + - Entity visibility via synthetic projected orbs. + - Easier to reason about for world-state output and debugging. -This separation is intentional and useful: both systems can be validated independently before merging into one robust camera pipeline. +This split is intentional: verify each path independently, then merge strengths into one robust final renderer. From 36149057025eddcfcf52863ee35e46aa95bfc50a Mon Sep 17 00:00:00 2001 From: verbiiyo Date: Wed, 3 Jun 2026 13:25:56 +0900 Subject: [PATCH 25/29] split into a third vision class and start adding TexturedWorldVision, right now its not doing great but the 3 file split will be a helpful "refactor" --- ...WorldVision.java => BasicWorldVision.java} | 6 +- .../minekov/client/TexturedWorldVision.java | 454 ++++++++++++++++++ .../minekov/client/XosViewportRuntime.java | 6 +- 3 files changed, 461 insertions(+), 5 deletions(-) rename src/main/java/com/verbii/minekov/client/{WorldVision.java => BasicWorldVision.java} (99%) create mode 100644 src/main/java/com/verbii/minekov/client/TexturedWorldVision.java diff --git a/src/main/java/com/verbii/minekov/client/WorldVision.java b/src/main/java/com/verbii/minekov/client/BasicWorldVision.java similarity index 99% rename from src/main/java/com/verbii/minekov/client/WorldVision.java rename to src/main/java/com/verbii/minekov/client/BasicWorldVision.java index c5cab22..7e1a2b2 100644 --- a/src/main/java/com/verbii/minekov/client/WorldVision.java +++ b/src/main/java/com/verbii/minekov/client/BasicWorldVision.java @@ -22,11 +22,11 @@ import java.nio.ByteBuffer; /** - * World-state renderer path from the 4c790a70 / b30d00d9 lineage. + * Basic world-state renderer path from the 4c790a70 / b30d00d9 lineage. * - Blocks: ray/clip sampled + shaded colors. * - Entities: projected depth-tested orb overlay. */ -final class WorldVision { +final class BasicWorldVision { private static final float AGENT_VIEW_FOV_DEG = 70.0f; private static final double AGENT_VIEW_MAX_DISTANCE = 24.0; private static final char[] HEX = "0123456789abcdef".toCharArray(); @@ -37,7 +37,7 @@ final class WorldVision { private static Method gameRendererRenderLevelMethod; private static Field minecraftMainRenderTargetField; - private WorldVision() {} + private BasicWorldVision() {} static String renderAgentViewRgbaHex(Minecraft mc, RLOperator agent, int reqW, int reqH) { int width = clampViewSize(reqW); diff --git a/src/main/java/com/verbii/minekov/client/TexturedWorldVision.java b/src/main/java/com/verbii/minekov/client/TexturedWorldVision.java new file mode 100644 index 0000000..744a1e7 --- /dev/null +++ b/src/main/java/com/verbii/minekov/client/TexturedWorldVision.java @@ -0,0 +1,454 @@ +package com.verbii.minekov.client; + +import com.verbii.minekov.entities.RLOperator; +import com.mojang.blaze3d.pipeline.RenderTarget; +import com.mojang.blaze3d.pipeline.TextureTarget; +import com.mojang.blaze3d.systems.RenderSystem; +import com.mojang.blaze3d.vertex.PoseStack; +import net.minecraft.client.Minecraft; +import net.minecraft.core.BlockPos; +import net.minecraft.core.Direction; +import net.minecraft.core.registries.BuiltInRegistries; +import net.minecraft.resources.ResourceLocation; +import net.minecraft.world.entity.Entity; +import net.minecraft.world.entity.LivingEntity; +import net.minecraft.world.level.ClipContext; +import net.minecraft.world.level.block.state.BlockState; +import net.minecraft.world.phys.HitResult; +import net.minecraft.world.phys.Vec3; +import org.lwjgl.opengl.GL11C; +import org.lwjgl.system.MemoryUtil; + +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.nio.ByteBuffer; + +/** + * Textured world-state renderer. + * Starts from BasicWorldVision and adds deterministic texture-like variation per hit face. + */ +final class TexturedWorldVision { + private static final float AGENT_VIEW_FOV_DEG = 70.0f; + private static final double AGENT_VIEW_MAX_DISTANCE = 24.0; + private static final char[] HEX = "0123456789abcdef".toCharArray(); + + // Keep false unless explicitly experimenting with old real-render attempt. + private static final boolean ENABLE_REAL_AGENT_VIEW_RENDER = false; + private static TextureTarget agentViewRenderTarget; + private static Method gameRendererRenderLevelMethod; + private static Field minecraftMainRenderTargetField; + + private TexturedWorldVision() {} + + static String renderAgentViewRgbaHex(Minecraft mc, RLOperator agent, int reqW, int reqH) { + int width = clampViewSize(reqW); + int height = clampViewSize(reqH); + if (mc == null || mc.level == null || agent == null) { + return ""; + } + if (ENABLE_REAL_AGENT_VIEW_RENDER) { + byte[] rendered = tryRenderAgentViewWithMinecraftRenderer(mc, agent, width, height); + if (rendered != null) { + return encodeRgbaHex(rendered); + } + } + + Vec3 origin = new Vec3(agent.getX(), agent.getEyeY(), agent.getZ()); + float yaw = agent.getYRot(); + float pitch = agent.getXRot(); + Vec3 forward = forwardFromRotation(yaw, pitch); + Vec3 worldUp = new Vec3(0.0, 1.0, 0.0); + Vec3 right = worldUp.cross(forward); + if (right.lengthSqr() < 1.0e-8) { + right = rightFromYaw(yaw); + } else { + right = right.normalize(); + } + Vec3 up = forward.cross(right).normalize(); + + double tanHalfFov = Math.tan(Math.toRadians(AGENT_VIEW_FOV_DEG * 0.5)); + double aspect = (double) width / (double) height; + int pixelCount = width * height; + byte[] rgba = new byte[pixelCount * 4]; + double[] depth = new double[pixelCount]; + for (int i = 0; i < pixelCount; i++) { + depth[i] = Double.POSITIVE_INFINITY; + } + + for (int y = 0; y < height; y++) { + double ny = 1.0 - (2.0 * (y + 0.5) / (double) height); + for (int x = 0; x < width; x++) { + double nx = (2.0 * (x + 0.5) / (double) width) - 1.0; + Vec3 dir = forward + .add(right.scale(nx * tanHalfFov * aspect)) + .add(up.scale(ny * tanHalfFov)) + .normalize(); + + int r = 110; + int g = 150; + int b = 210; + boolean hit = false; + double distance = 0.0; + Vec3 rayEnd = origin.add(dir.scale(AGENT_VIEW_MAX_DISTANCE)); + var hitRes = mc.level.clip(new ClipContext( + origin, + rayEnd, + ClipContext.Block.OUTLINE, + ClipContext.Fluid.NONE, + agent)); + if (hitRes.getType() != HitResult.Type.MISS) { + BlockPos bp = hitRes.getBlockPos(); + var state = mc.level.getBlockState(bp); + if (!state.isAir()) { + int rgb = rgbFromBlockState(mc, bp, state); + rgb = applySyntheticTexture(rgb, state, hitRes.getLocation(), hitRes.getDirection(), bp); + distance = hitRes.getLocation().distanceTo(origin); + double shade = Math.max(0.28, 1.0 - (distance / AGENT_VIEW_MAX_DISTANCE)); + double faceShade = switch (hitRes.getDirection()) { + case UP -> 1.08; + case DOWN -> 0.72; + case NORTH, SOUTH -> 0.94; + default -> 1.00; + }; + double totalShade = Math.max(0.20, Math.min(1.15, shade * faceShade)); + r = (int) Math.max(0, Math.min(255, ((rgb >> 16) & 0xFF) * totalShade)); + g = (int) Math.max(0, Math.min(255, ((rgb >> 8) & 0xFF) * totalShade)); + b = (int) Math.max(0, Math.min(255, (rgb & 0xFF) * totalShade)); + hit = true; + } + } + + if (!hit) { + double tSky = Math.max(0.0, Math.min(1.0, (dir.y + 1.0) * 0.5)); + r = (int) (70 + 60 * tSky); + g = (int) (100 + 90 * tSky); + b = (int) (150 + 80 * tSky); + } else { + int fog = (int) Math.max(0, Math.min(40, (AGENT_VIEW_MAX_DISTANCE - distance) * 0.8)); + r = Math.min(255, r + fog); + g = Math.min(255, g + fog); + b = Math.min(255, b + fog); + } + int p = y * width + x; + int i = p * 4; + rgba[i] = (byte) r; + rgba[i + 1] = (byte) g; + rgba[i + 2] = (byte) b; + rgba[i + 3] = (byte) 255; + depth[p] = hit ? distance : Double.POSITIVE_INFINITY; + } + } + + var renderables = mc.level.entitiesForRendering(); + for (Entity e : renderables) { + if (e == null || e == agent || e.isRemoved()) { + continue; + } + Vec3 rel = new Vec3(e.getX() - origin.x, e.getEyeY() - origin.y, e.getZ() - origin.z); + double camX = rel.dot(right); + double camY = rel.dot(up); + double camZ = rel.dot(forward); + if (camZ <= 0.01 || camZ > AGENT_VIEW_MAX_DISTANCE) { + continue; + } + + double ndcX = camX / (camZ * tanHalfFov * aspect); + double ndcY = camY / (camZ * tanHalfFov); + if (Math.abs(ndcX) > 1.5 || Math.abs(ndcY) > 1.5) { + continue; + } + + int cx = (int) Math.round((ndcX + 1.0) * 0.5 * (width - 1)); + int cy = (int) Math.round((1.0 - (ndcY + 1.0) * 0.5) * (height - 1)); + double radiusPx = Math.max( + 1.0, + (Math.max(0.25, e.getBbWidth()) / (camZ * tanHalfFov * aspect)) * (width * 0.28)); + int rad = (int) Math.min(Math.max(1, Math.round(radiusPx)), Math.max(width, height)); + int color = rgbFromEntity(e); + int er = (color >> 16) & 0xFF; + int eg = (color >> 8) & 0xFF; + int eb = color & 0xFF; + int x1 = Math.max(0, cx - rad); + int x2 = Math.min(width - 1, cx + rad); + int y1 = Math.max(0, cy - rad); + int y2 = Math.min(height - 1, cy + rad); + int rr = rad * rad; + for (int py = y1; py <= y2; py++) { + int dy = py - cy; + for (int px = x1; px <= x2; px++) { + int dx = px - cx; + if ((dx * dx + dy * dy) > rr) { + continue; + } + int p = py * width + px; + if (camZ >= depth[p]) { + continue; + } + int i = p * 4; + rgba[i] = (byte) er; + rgba[i + 1] = (byte) eg; + rgba[i + 2] = (byte) eb; + rgba[i + 3] = (byte) 255; + depth[p] = camZ; + } + } + } + return encodeRgbaHex(rgba); + } + + static void flushDeferredCaptures(Minecraft mc) { + // No-op in world-state renderer path (kept for easy call-site swapping). + } + + private static int clampViewSize(int value) { + return Math.max(16, Math.min(512, value)); + } + + private static int rgbFromBlockState( + Minecraft mc, BlockPos pos, BlockState state) { + try { + var map = state.getMapColor(mc.level, pos); + if (map != null && map.col != 0) { + return map.col & 0x00FFFFFF; + } + } catch (Throwable ignored) { + } + ResourceLocation key = BuiltInRegistries.BLOCK.getKey(state.getBlock()); + int hash = key != null ? key.toString().hashCode() : 0x77AA77; + int r = 64 + (Math.abs(hash) & 0x7F); + int g = 64 + ((Math.abs(hash >> 8)) & 0x7F); + int b = 64 + ((Math.abs(hash >> 16)) & 0x7F); + return (r << 16) | (g << 8) | b; + } + + private static int applySyntheticTexture( + int baseRgb, BlockState state, Vec3 hit, Direction face, BlockPos blockPos) { + double[] uv = faceUv(hit, face); + int checkerX = (int) Math.floor(uv[0] * 16.0); + int checkerY = (int) Math.floor(uv[1] * 16.0); + int blockSeed = (blockPos.getX() * 734287 + blockPos.getY() * 912931 + blockPos.getZ() * 193939); + int noise = (checkerX * 11939) ^ (checkerY * 27653) ^ blockSeed ^ state.getBlock().hashCode(); + + double checker = (((checkerX + checkerY) & 1) == 0) ? 1.06 : 0.92; + double grain = (((noise & 0xFF) / 255.0) - 0.5) * 0.14; + + double tintR = 1.0; + double tintG = 1.0; + double tintB = 1.0; + ResourceLocation key = BuiltInRegistries.BLOCK.getKey(state.getBlock()); + String name = key == null ? "" : key.toString(); + if (name.contains("grass") || name.contains("leaves")) { + tintR = 0.92; + tintG = 1.08; + tintB = 0.92; + } else if (name.contains("water")) { + tintR = 0.82; + tintG = 0.95; + tintB = 1.12; + } else if (name.contains("sand")) { + tintR = 1.07; + tintG = 1.03; + tintB = 0.92; + } else if (name.contains("stone") || name.contains("deepslate")) { + tintR = 0.96; + tintG = 0.98; + tintB = 1.02; + } + + double mult = Math.max(0.72, Math.min(1.26, checker + grain)); + int r = clampColor((int) (((baseRgb >> 16) & 0xFF) * mult * tintR)); + int g = clampColor((int) (((baseRgb >> 8) & 0xFF) * mult * tintG)); + int b = clampColor((int) ((baseRgb & 0xFF) * mult * tintB)); + return (r << 16) | (g << 8) | b; + } + + private static double[] faceUv(Vec3 hit, Direction face) { + double fx = frac(hit.x); + double fy = frac(hit.y); + double fz = frac(hit.z); + return switch (face) { + case UP, DOWN -> new double[] {fx, fz}; + case NORTH, SOUTH -> new double[] {fx, fy}; + case EAST, WEST -> new double[] {fz, fy}; + }; + } + + private static double frac(double value) { + return value - Math.floor(value); + } + + private static int clampColor(int value) { + return Math.max(0, Math.min(255, value)); + } + + private static int rgbFromEntity(Entity entity) { + int hash = entity.getType().toString().hashCode(); + int r = 96 + (Math.abs(hash) & 0x5F); + int g = 96 + ((Math.abs(hash >> 8)) & 0x5F); + int b = 96 + ((Math.abs(hash >> 16)) & 0x5F); + if (entity instanceof LivingEntity) { + r = Math.min(255, r + 24); + } + return (r << 16) | (g << 8) | b; + } + + private static String encodeRgbaHex(byte[] rgba) { + char[] out = new char[rgba.length * 2]; + int j = 0; + for (byte b : rgba) { + int v = b & 0xFF; + out[j++] = HEX[v >>> 4]; + out[j++] = HEX[v & 0x0F]; + } + return new String(out); + } + + private static Vec3 forwardFromRotation(float yawDeg, float pitchDeg) { + double yaw = Math.toRadians(yawDeg); + double pitch = Math.toRadians(pitchDeg); + double x = -Math.sin(yaw) * Math.cos(pitch); + double y = -Math.sin(pitch); + double z = Math.cos(yaw) * Math.cos(pitch); + return new Vec3(x, y, z).normalize(); + } + + private static Vec3 rightFromYaw(float yawDeg) { + double yaw = Math.toRadians(yawDeg); + return new Vec3(-Math.cos(yaw), 0.0, -Math.sin(yaw)).normalize(); + } + + private static byte[] tryRenderAgentViewWithMinecraftRenderer( + Minecraft mc, RLOperator agent, int width, int height) { + if (!RenderSystem.isOnRenderThread()) { + return null; + } + try { + if (agentViewRenderTarget == null + || agentViewRenderTarget.width != width + || agentViewRenderTarget.height != height) { + if (agentViewRenderTarget != null) { + agentViewRenderTarget.destroyBuffers(); + } + agentViewRenderTarget = new TextureTarget(width, height, true, Minecraft.ON_OSX); + } + } catch (Throwable t) { + return null; + } + + RenderTarget previousTarget = mc.getMainRenderTarget(); + Entity previousCamera = mc.getCameraEntity(); + if (!setMinecraftMainRenderTarget(mc, previousTarget, agentViewRenderTarget)) { + return null; + } + try { + mc.setCameraEntity(agent); + agentViewRenderTarget.bindWrite(true); + RenderSystem.viewport(0, 0, width, height); + agentViewRenderTarget.setClearColor(0f, 0f, 0f, 1f); + agentViewRenderTarget.clear(Minecraft.ON_OSX); + + if (!invokeGameRendererLevelPass(mc)) { + return null; + } + + ByteBuffer buf = MemoryUtil.memAlloc(width * height * 4); + try { + GL11C.glReadPixels(0, 0, width, height, GL11C.GL_RGBA, GL11C.GL_UNSIGNED_BYTE, buf); + byte[] out = new byte[width * height * 4]; + int rowBytes = width * 4; + for (int y = 0; y < height; y++) { + int src = (height - 1 - y) * rowBytes; + int dst = y * rowBytes; + buf.position(src); + buf.get(out, dst, rowBytes); + } + return out; + } finally { + MemoryUtil.memFree(buf); + } + } catch (Throwable t) { + return null; + } finally { + try { + mc.setCameraEntity(previousCamera); + } catch (Throwable ignored) { + } + try { + setMinecraftMainRenderTarget(mc, agentViewRenderTarget, previousTarget); + previousTarget.bindWrite(true); + RenderSystem.viewport(0, 0, previousTarget.width, previousTarget.height); + } catch (Throwable ignored) { + } + } + } + + private static boolean invokeGameRendererLevelPass(Minecraft mc) { + try { + if (gameRendererRenderLevelMethod == null) { + for (Method m : mc.gameRenderer.getClass().getDeclaredMethods()) { + if ("renderLevel".equals(m.getName())) { + m.setAccessible(true); + gameRendererRenderLevelMethod = m; + break; + } + } + } + if (gameRendererRenderLevelMethod == null) { + return false; + } + Class[] p = gameRendererRenderLevelMethod.getParameterTypes(); + Object[] args = new Object[p.length]; + for (int i = 0; i < p.length; i++) { + Class t = p[i]; + if (t == float.class || t == Float.class) { + args[i] = mc.getFrameTime(); + } else if (t == double.class || t == Double.class) { + args[i] = (double) mc.getFrameTime(); + } else if (t == long.class || t == Long.class) { + args[i] = System.nanoTime(); + } else if (t == int.class || t == Integer.class) { + args[i] = 0; + } else if (t == boolean.class || t == Boolean.class) { + args[i] = Boolean.FALSE; + } else if (PoseStack.class.isAssignableFrom(t)) { + args[i] = new PoseStack(); + } else { + args[i] = null; + } + } + gameRendererRenderLevelMethod.invoke(mc.gameRenderer, args); + return true; + } catch (Throwable t) { + return false; + } + } + + private static boolean setMinecraftMainRenderTarget( + Minecraft mc, RenderTarget expectedCurrent, RenderTarget target) { + try { + if (minecraftMainRenderTargetField == null) { + for (Field f : Minecraft.class.getDeclaredFields()) { + if (RenderTarget.class.isAssignableFrom(f.getType())) { + f.setAccessible(true); + Object cur = f.get(mc); + if (cur == expectedCurrent) { + minecraftMainRenderTargetField = f; + break; + } + } + } + } + if (minecraftMainRenderTargetField == null) { + return false; + } + Object cur = minecraftMainRenderTargetField.get(mc); + if (cur != expectedCurrent) { + return false; + } + minecraftMainRenderTargetField.set(mc, target); + return true; + } catch (Throwable t) { + return false; + } + } +} diff --git a/src/main/java/com/verbii/minekov/client/XosViewportRuntime.java b/src/main/java/com/verbii/minekov/client/XosViewportRuntime.java index 15ada8e..9b656a5 100644 --- a/src/main/java/com/verbii/minekov/client/XosViewportRuntime.java +++ b/src/main/java/com/verbii/minekov/client/XosViewportRuntime.java @@ -731,13 +731,15 @@ private static String encodePlayerInventoryEnderChest(Minecraft mc) { private static String renderAgentViewRgbaHex(Minecraft mc, RLOperator agent, int reqW, int reqH) { // Toggle renderer by swapping these calls. - return WorldVision.renderAgentViewRgbaHex(mc, agent, reqW, reqH); + return TexturedWorldVision.renderAgentViewRgbaHex(mc, agent, reqW, reqH); + // return BasicWorldVision.renderAgentViewRgbaHex(mc, agent, reqW, reqH); // return EntityVision.renderAgentViewRgbaHex(mc, agent, reqW, reqH); } static void flushDeferredVisionCaptures(Minecraft mc) { // Keep paired with the selected renderer above. - WorldVision.flushDeferredCaptures(mc); + TexturedWorldVision.flushDeferredCaptures(mc); + // BasicWorldVision.flushDeferredCaptures(mc); // EntityVision.flushDeferredCaptures(mc); } From 68162e40e3b7f8d64efba02495728892be000c66 Mon Sep 17 00:00:00 2001 From: verbiiyo Date: Wed, 3 Jun 2026 14:26:20 +0900 Subject: [PATCH 26/29] liquid dream stitching (not great but its a start of merging entities with the world blocks) --- .../minekov/client/TexturedWorldVision.java | 468 ++---------------- 1 file changed, 54 insertions(+), 414 deletions(-) diff --git a/src/main/java/com/verbii/minekov/client/TexturedWorldVision.java b/src/main/java/com/verbii/minekov/client/TexturedWorldVision.java index 744a1e7..33b2eeb 100644 --- a/src/main/java/com/verbii/minekov/client/TexturedWorldVision.java +++ b/src/main/java/com/verbii/minekov/client/TexturedWorldVision.java @@ -1,295 +1,84 @@ package com.verbii.minekov.client; import com.verbii.minekov.entities.RLOperator; -import com.mojang.blaze3d.pipeline.RenderTarget; -import com.mojang.blaze3d.pipeline.TextureTarget; -import com.mojang.blaze3d.systems.RenderSystem; -import com.mojang.blaze3d.vertex.PoseStack; import net.minecraft.client.Minecraft; -import net.minecraft.core.BlockPos; -import net.minecraft.core.Direction; -import net.minecraft.core.registries.BuiltInRegistries; -import net.minecraft.resources.ResourceLocation; -import net.minecraft.world.entity.Entity; -import net.minecraft.world.entity.LivingEntity; -import net.minecraft.world.level.ClipContext; -import net.minecraft.world.level.block.state.BlockState; -import net.minecraft.world.phys.HitResult; -import net.minecraft.world.phys.Vec3; -import org.lwjgl.opengl.GL11C; -import org.lwjgl.system.MemoryUtil; - -import java.lang.reflect.Field; -import java.lang.reflect.Method; -import java.nio.ByteBuffer; /** - * Textured world-state renderer. - * Starts from BasicWorldVision and adds deterministic texture-like variation per hit face. + * Unified vision path that composes world-geometry sampling with render-pass captures. + *

+ * This keeps BasicWorldVision's stable block structure output, while allowing EntityVision's + * camera-rendered frame to provide accurate entity pixels when available. */ final class TexturedWorldVision { - private static final float AGENT_VIEW_FOV_DEG = 70.0f; - private static final double AGENT_VIEW_MAX_DISTANCE = 24.0; + private static final int PIXEL_STRIDE = 4; private static final char[] HEX = "0123456789abcdef".toCharArray(); - // Keep false unless explicitly experimenting with old real-render attempt. - private static final boolean ENABLE_REAL_AGENT_VIEW_RENDER = false; - private static TextureTarget agentViewRenderTarget; - private static Method gameRendererRenderLevelMethod; - private static Field minecraftMainRenderTargetField; - private TexturedWorldVision() {} static String renderAgentViewRgbaHex(Minecraft mc, RLOperator agent, int reqW, int reqH) { - int width = clampViewSize(reqW); - int height = clampViewSize(reqH); - if (mc == null || mc.level == null || agent == null) { - return ""; - } - if (ENABLE_REAL_AGENT_VIEW_RENDER) { - byte[] rendered = tryRenderAgentViewWithMinecraftRenderer(mc, agent, width, height); - if (rendered != null) { - return encodeRgbaHex(rendered); - } - } + String worldHex = BasicWorldVision.renderAgentViewRgbaHex(mc, agent, reqW, reqH); + String entityHex = EntityVision.renderAgentViewRgbaHex(mc, agent, reqW, reqH); + return mergeVisionHex(worldHex, entityHex); + } - Vec3 origin = new Vec3(agent.getX(), agent.getEyeY(), agent.getZ()); - float yaw = agent.getYRot(); - float pitch = agent.getXRot(); - Vec3 forward = forwardFromRotation(yaw, pitch); - Vec3 worldUp = new Vec3(0.0, 1.0, 0.0); - Vec3 right = worldUp.cross(forward); - if (right.lengthSqr() < 1.0e-8) { - right = rightFromYaw(yaw); - } else { - right = right.normalize(); - } - Vec3 up = forward.cross(right).normalize(); + static void flushDeferredCaptures(Minecraft mc) { + // Basic path does not queue work; entity path does. + EntityVision.flushDeferredCaptures(mc); + } - double tanHalfFov = Math.tan(Math.toRadians(AGENT_VIEW_FOV_DEG * 0.5)); - double aspect = (double) width / (double) height; - int pixelCount = width * height; - byte[] rgba = new byte[pixelCount * 4]; - double[] depth = new double[pixelCount]; - for (int i = 0; i < pixelCount; i++) { - depth[i] = Double.POSITIVE_INFINITY; + private static String mergeVisionHex(String worldHex, String entityHex) { + if (entityHex == null || entityHex.isEmpty()) { + return worldHex == null ? "" : worldHex; } - - for (int y = 0; y < height; y++) { - double ny = 1.0 - (2.0 * (y + 0.5) / (double) height); - for (int x = 0; x < width; x++) { - double nx = (2.0 * (x + 0.5) / (double) width) - 1.0; - Vec3 dir = forward - .add(right.scale(nx * tanHalfFov * aspect)) - .add(up.scale(ny * tanHalfFov)) - .normalize(); - - int r = 110; - int g = 150; - int b = 210; - boolean hit = false; - double distance = 0.0; - Vec3 rayEnd = origin.add(dir.scale(AGENT_VIEW_MAX_DISTANCE)); - var hitRes = mc.level.clip(new ClipContext( - origin, - rayEnd, - ClipContext.Block.OUTLINE, - ClipContext.Fluid.NONE, - agent)); - if (hitRes.getType() != HitResult.Type.MISS) { - BlockPos bp = hitRes.getBlockPos(); - var state = mc.level.getBlockState(bp); - if (!state.isAir()) { - int rgb = rgbFromBlockState(mc, bp, state); - rgb = applySyntheticTexture(rgb, state, hitRes.getLocation(), hitRes.getDirection(), bp); - distance = hitRes.getLocation().distanceTo(origin); - double shade = Math.max(0.28, 1.0 - (distance / AGENT_VIEW_MAX_DISTANCE)); - double faceShade = switch (hitRes.getDirection()) { - case UP -> 1.08; - case DOWN -> 0.72; - case NORTH, SOUTH -> 0.94; - default -> 1.00; - }; - double totalShade = Math.max(0.20, Math.min(1.15, shade * faceShade)); - r = (int) Math.max(0, Math.min(255, ((rgb >> 16) & 0xFF) * totalShade)); - g = (int) Math.max(0, Math.min(255, ((rgb >> 8) & 0xFF) * totalShade)); - b = (int) Math.max(0, Math.min(255, (rgb & 0xFF) * totalShade)); - hit = true; - } - } - - if (!hit) { - double tSky = Math.max(0.0, Math.min(1.0, (dir.y + 1.0) * 0.5)); - r = (int) (70 + 60 * tSky); - g = (int) (100 + 90 * tSky); - b = (int) (150 + 80 * tSky); - } else { - int fog = (int) Math.max(0, Math.min(40, (AGENT_VIEW_MAX_DISTANCE - distance) * 0.8)); - r = Math.min(255, r + fog); - g = Math.min(255, g + fog); - b = Math.min(255, b + fog); - } - int p = y * width + x; - int i = p * 4; - rgba[i] = (byte) r; - rgba[i + 1] = (byte) g; - rgba[i + 2] = (byte) b; - rgba[i + 3] = (byte) 255; - depth[p] = hit ? distance : Double.POSITIVE_INFINITY; - } + if (worldHex == null || worldHex.isEmpty()) { + return entityHex; } - var renderables = mc.level.entitiesForRendering(); - for (Entity e : renderables) { - if (e == null || e == agent || e.isRemoved()) { - continue; - } - Vec3 rel = new Vec3(e.getX() - origin.x, e.getEyeY() - origin.y, e.getZ() - origin.z); - double camX = rel.dot(right); - double camY = rel.dot(up); - double camZ = rel.dot(forward); - if (camZ <= 0.01 || camZ > AGENT_VIEW_MAX_DISTANCE) { - continue; - } - - double ndcX = camX / (camZ * tanHalfFov * aspect); - double ndcY = camY / (camZ * tanHalfFov); - if (Math.abs(ndcX) > 1.5 || Math.abs(ndcY) > 1.5) { - continue; - } - - int cx = (int) Math.round((ndcX + 1.0) * 0.5 * (width - 1)); - int cy = (int) Math.round((1.0 - (ndcY + 1.0) * 0.5) * (height - 1)); - double radiusPx = Math.max( - 1.0, - (Math.max(0.25, e.getBbWidth()) / (camZ * tanHalfFov * aspect)) * (width * 0.28)); - int rad = (int) Math.min(Math.max(1, Math.round(radiusPx)), Math.max(width, height)); - int color = rgbFromEntity(e); - int er = (color >> 16) & 0xFF; - int eg = (color >> 8) & 0xFF; - int eb = color & 0xFF; - int x1 = Math.max(0, cx - rad); - int x2 = Math.min(width - 1, cx + rad); - int y1 = Math.max(0, cy - rad); - int y2 = Math.min(height - 1, cy + rad); - int rr = rad * rad; - for (int py = y1; py <= y2; py++) { - int dy = py - cy; - for (int px = x1; px <= x2; px++) { - int dx = px - cx; - if ((dx * dx + dy * dy) > rr) { - continue; - } - int p = py * width + px; - if (camZ >= depth[p]) { - continue; - } - int i = p * 4; - rgba[i] = (byte) er; - rgba[i + 1] = (byte) eg; - rgba[i + 2] = (byte) eb; - rgba[i + 3] = (byte) 255; - depth[p] = camZ; - } - } + byte[] world = decodeRgbaHex(worldHex); + byte[] entity = decodeRgbaHex(entityHex); + if (world == null || entity == null || world.length != entity.length) { + return entityHex; } - return encodeRgbaHex(rgba); - } - static void flushDeferredCaptures(Minecraft mc) { - // No-op in world-state renderer path (kept for easy call-site swapping). - } - - private static int clampViewSize(int value) { - return Math.max(16, Math.min(512, value)); - } + byte[] out = new byte[world.length]; + for (int i = 0; i < world.length; i += PIXEL_STRIDE) { + int er = entity[i] & 0xFF; + int eg = entity[i + 1] & 0xFF; + int eb = entity[i + 2] & 0xFF; + int ea = entity[i + 3] & 0xFF; - private static int rgbFromBlockState( - Minecraft mc, BlockPos pos, BlockState state) { - try { - var map = state.getMapColor(mc.level, pos); - if (map != null && map.col != 0) { - return map.col & 0x00FFFFFF; + // EntityVision target clear color is black; keep world sample for clear pixels. + boolean entityPixelPresent = ea > 0 && (er > 6 || eg > 6 || eb > 6); + if (entityPixelPresent) { + out[i] = entity[i]; + out[i + 1] = entity[i + 1]; + out[i + 2] = entity[i + 2]; + out[i + 3] = entity[i + 3]; + } else { + out[i] = world[i]; + out[i + 1] = world[i + 1]; + out[i + 2] = world[i + 2]; + out[i + 3] = world[i + 3]; } - } catch (Throwable ignored) { } - ResourceLocation key = BuiltInRegistries.BLOCK.getKey(state.getBlock()); - int hash = key != null ? key.toString().hashCode() : 0x77AA77; - int r = 64 + (Math.abs(hash) & 0x7F); - int g = 64 + ((Math.abs(hash >> 8)) & 0x7F); - int b = 64 + ((Math.abs(hash >> 16)) & 0x7F); - return (r << 16) | (g << 8) | b; + return encodeRgbaHex(out); } - private static int applySyntheticTexture( - int baseRgb, BlockState state, Vec3 hit, Direction face, BlockPos blockPos) { - double[] uv = faceUv(hit, face); - int checkerX = (int) Math.floor(uv[0] * 16.0); - int checkerY = (int) Math.floor(uv[1] * 16.0); - int blockSeed = (blockPos.getX() * 734287 + blockPos.getY() * 912931 + blockPos.getZ() * 193939); - int noise = (checkerX * 11939) ^ (checkerY * 27653) ^ blockSeed ^ state.getBlock().hashCode(); - - double checker = (((checkerX + checkerY) & 1) == 0) ? 1.06 : 0.92; - double grain = (((noise & 0xFF) / 255.0) - 0.5) * 0.14; - - double tintR = 1.0; - double tintG = 1.0; - double tintB = 1.0; - ResourceLocation key = BuiltInRegistries.BLOCK.getKey(state.getBlock()); - String name = key == null ? "" : key.toString(); - if (name.contains("grass") || name.contains("leaves")) { - tintR = 0.92; - tintG = 1.08; - tintB = 0.92; - } else if (name.contains("water")) { - tintR = 0.82; - tintG = 0.95; - tintB = 1.12; - } else if (name.contains("sand")) { - tintR = 1.07; - tintG = 1.03; - tintB = 0.92; - } else if (name.contains("stone") || name.contains("deepslate")) { - tintR = 0.96; - tintG = 0.98; - tintB = 1.02; + private static byte[] decodeRgbaHex(String hex) { + int n = hex.length(); + if ((n & 1) != 0) { + return null; } - - double mult = Math.max(0.72, Math.min(1.26, checker + grain)); - int r = clampColor((int) (((baseRgb >> 16) & 0xFF) * mult * tintR)); - int g = clampColor((int) (((baseRgb >> 8) & 0xFF) * mult * tintG)); - int b = clampColor((int) ((baseRgb & 0xFF) * mult * tintB)); - return (r << 16) | (g << 8) | b; - } - - private static double[] faceUv(Vec3 hit, Direction face) { - double fx = frac(hit.x); - double fy = frac(hit.y); - double fz = frac(hit.z); - return switch (face) { - case UP, DOWN -> new double[] {fx, fz}; - case NORTH, SOUTH -> new double[] {fx, fy}; - case EAST, WEST -> new double[] {fz, fy}; - }; - } - - private static double frac(double value) { - return value - Math.floor(value); - } - - private static int clampColor(int value) { - return Math.max(0, Math.min(255, value)); - } - - private static int rgbFromEntity(Entity entity) { - int hash = entity.getType().toString().hashCode(); - int r = 96 + (Math.abs(hash) & 0x5F); - int g = 96 + ((Math.abs(hash >> 8)) & 0x5F); - int b = 96 + ((Math.abs(hash >> 16)) & 0x5F); - if (entity instanceof LivingEntity) { - r = Math.min(255, r + 24); + byte[] out = new byte[n / 2]; + for (int i = 0; i < n; i += 2) { + int hi = Character.digit(hex.charAt(i), 16); + int lo = Character.digit(hex.charAt(i + 1), 16); + if (hi < 0 || lo < 0) { + return null; + } + out[i / 2] = (byte) ((hi << 4) | lo); } - return (r << 16) | (g << 8) | b; + return out; } private static String encodeRgbaHex(byte[] rgba) { @@ -302,153 +91,4 @@ private static String encodeRgbaHex(byte[] rgba) { } return new String(out); } - - private static Vec3 forwardFromRotation(float yawDeg, float pitchDeg) { - double yaw = Math.toRadians(yawDeg); - double pitch = Math.toRadians(pitchDeg); - double x = -Math.sin(yaw) * Math.cos(pitch); - double y = -Math.sin(pitch); - double z = Math.cos(yaw) * Math.cos(pitch); - return new Vec3(x, y, z).normalize(); - } - - private static Vec3 rightFromYaw(float yawDeg) { - double yaw = Math.toRadians(yawDeg); - return new Vec3(-Math.cos(yaw), 0.0, -Math.sin(yaw)).normalize(); - } - - private static byte[] tryRenderAgentViewWithMinecraftRenderer( - Minecraft mc, RLOperator agent, int width, int height) { - if (!RenderSystem.isOnRenderThread()) { - return null; - } - try { - if (agentViewRenderTarget == null - || agentViewRenderTarget.width != width - || agentViewRenderTarget.height != height) { - if (agentViewRenderTarget != null) { - agentViewRenderTarget.destroyBuffers(); - } - agentViewRenderTarget = new TextureTarget(width, height, true, Minecraft.ON_OSX); - } - } catch (Throwable t) { - return null; - } - - RenderTarget previousTarget = mc.getMainRenderTarget(); - Entity previousCamera = mc.getCameraEntity(); - if (!setMinecraftMainRenderTarget(mc, previousTarget, agentViewRenderTarget)) { - return null; - } - try { - mc.setCameraEntity(agent); - agentViewRenderTarget.bindWrite(true); - RenderSystem.viewport(0, 0, width, height); - agentViewRenderTarget.setClearColor(0f, 0f, 0f, 1f); - agentViewRenderTarget.clear(Minecraft.ON_OSX); - - if (!invokeGameRendererLevelPass(mc)) { - return null; - } - - ByteBuffer buf = MemoryUtil.memAlloc(width * height * 4); - try { - GL11C.glReadPixels(0, 0, width, height, GL11C.GL_RGBA, GL11C.GL_UNSIGNED_BYTE, buf); - byte[] out = new byte[width * height * 4]; - int rowBytes = width * 4; - for (int y = 0; y < height; y++) { - int src = (height - 1 - y) * rowBytes; - int dst = y * rowBytes; - buf.position(src); - buf.get(out, dst, rowBytes); - } - return out; - } finally { - MemoryUtil.memFree(buf); - } - } catch (Throwable t) { - return null; - } finally { - try { - mc.setCameraEntity(previousCamera); - } catch (Throwable ignored) { - } - try { - setMinecraftMainRenderTarget(mc, agentViewRenderTarget, previousTarget); - previousTarget.bindWrite(true); - RenderSystem.viewport(0, 0, previousTarget.width, previousTarget.height); - } catch (Throwable ignored) { - } - } - } - - private static boolean invokeGameRendererLevelPass(Minecraft mc) { - try { - if (gameRendererRenderLevelMethod == null) { - for (Method m : mc.gameRenderer.getClass().getDeclaredMethods()) { - if ("renderLevel".equals(m.getName())) { - m.setAccessible(true); - gameRendererRenderLevelMethod = m; - break; - } - } - } - if (gameRendererRenderLevelMethod == null) { - return false; - } - Class[] p = gameRendererRenderLevelMethod.getParameterTypes(); - Object[] args = new Object[p.length]; - for (int i = 0; i < p.length; i++) { - Class t = p[i]; - if (t == float.class || t == Float.class) { - args[i] = mc.getFrameTime(); - } else if (t == double.class || t == Double.class) { - args[i] = (double) mc.getFrameTime(); - } else if (t == long.class || t == Long.class) { - args[i] = System.nanoTime(); - } else if (t == int.class || t == Integer.class) { - args[i] = 0; - } else if (t == boolean.class || t == Boolean.class) { - args[i] = Boolean.FALSE; - } else if (PoseStack.class.isAssignableFrom(t)) { - args[i] = new PoseStack(); - } else { - args[i] = null; - } - } - gameRendererRenderLevelMethod.invoke(mc.gameRenderer, args); - return true; - } catch (Throwable t) { - return false; - } - } - - private static boolean setMinecraftMainRenderTarget( - Minecraft mc, RenderTarget expectedCurrent, RenderTarget target) { - try { - if (minecraftMainRenderTargetField == null) { - for (Field f : Minecraft.class.getDeclaredFields()) { - if (RenderTarget.class.isAssignableFrom(f.getType())) { - f.setAccessible(true); - Object cur = f.get(mc); - if (cur == expectedCurrent) { - minecraftMainRenderTargetField = f; - break; - } - } - } - } - if (minecraftMainRenderTargetField == null) { - return false; - } - Object cur = minecraftMainRenderTargetField.get(mc); - if (cur != expectedCurrent) { - return false; - } - minecraftMainRenderTargetField.set(mc, target); - return true; - } catch (Throwable t) { - return false; - } - } } From 2041b6935996e12543d8211d49de50055a56e1b4 Mon Sep 17 00:00:00 2001 From: verbiiyo Date: Wed, 3 Jun 2026 15:05:32 +0900 Subject: [PATCH 27/29] just use basic world vision fo rnow. --- .../com/verbii/minekov/client/XosViewportRuntime.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/verbii/minekov/client/XosViewportRuntime.java b/src/main/java/com/verbii/minekov/client/XosViewportRuntime.java index 9b656a5..3fa265f 100644 --- a/src/main/java/com/verbii/minekov/client/XosViewportRuntime.java +++ b/src/main/java/com/verbii/minekov/client/XosViewportRuntime.java @@ -731,15 +731,15 @@ private static String encodePlayerInventoryEnderChest(Minecraft mc) { private static String renderAgentViewRgbaHex(Minecraft mc, RLOperator agent, int reqW, int reqH) { // Toggle renderer by swapping these calls. - return TexturedWorldVision.renderAgentViewRgbaHex(mc, agent, reqW, reqH); - // return BasicWorldVision.renderAgentViewRgbaHex(mc, agent, reqW, reqH); + // return TexturedWorldVision.renderAgentViewRgbaHex(mc, agent, reqW, reqH); + return BasicWorldVision.renderAgentViewRgbaHex(mc, agent, reqW, reqH); // return EntityVision.renderAgentViewRgbaHex(mc, agent, reqW, reqH); } static void flushDeferredVisionCaptures(Minecraft mc) { // Keep paired with the selected renderer above. - TexturedWorldVision.flushDeferredCaptures(mc); - // BasicWorldVision.flushDeferredCaptures(mc); + // TexturedWorldVision.flushDeferredCaptures(mc); + BasicWorldVision.flushDeferredCaptures(mc); // EntityVision.flushDeferredCaptures(mc); } From ed91050cf992fcbfff632261eb377cde1fcc41b7 Mon Sep 17 00:00:00 2001 From: verbiiyo Date: Wed, 3 Jun 2026 15:12:01 +0900 Subject: [PATCH 28/29] esc out --- .../minekov/client/XosViewportOverlay.java | 3 +++ .../resources/xos/examples/agent_controller.py | 17 ++++++++++++++++- src/xos/examples/agent_controller.py | 17 ++++++++++++++++- 3 files changed, 35 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/verbii/minekov/client/XosViewportOverlay.java b/src/main/java/com/verbii/minekov/client/XosViewportOverlay.java index 0d353c2..f56af92 100644 --- a/src/main/java/com/verbii/minekov/client/XosViewportOverlay.java +++ b/src/main/java/com/verbii/minekov/client/XosViewportOverlay.java @@ -856,6 +856,7 @@ public static void onScreenKeyPressed(ScreenEvent.KeyPressed.Pre event) { } } if (key == GLFW.GLFW_KEY_ESCAPE) { + event.setCanceled(true); return; } if (key == GLFW.GLFW_KEY_F3) { @@ -887,6 +888,8 @@ public static void onScreenKeyReleased(ScreenEvent.KeyReleased.Pre event) { } int key = event.getKeyCode(); if (key == GLFW.GLFW_KEY_ESCAPE) { + XosViewportRuntime.sendKeyUpToEngine("ESCAPE"); + event.setCanceled(true); return; } String bindingName = keyCodeToBindingName(key, event.getScanCode()); diff --git a/src/main/resources/xos/examples/agent_controller.py b/src/main/resources/xos/examples/agent_controller.py index a381ed6..df2eb7a 100644 --- a/src/main/resources/xos/examples/agent_controller.py +++ b/src/main/resources/xos/examples/agent_controller.py @@ -14,6 +14,9 @@ def __init__(self): self.view_h = 128 self.look_yaw = None self.look_pitch = None + self.look_capture_active = False + self.prev_left_click = False + self.prev_escape = False self.last_look_x = None self.last_look_y = None @@ -44,7 +47,17 @@ def tick(self): if keys.SPACE.is_pressed() and now >= self.jump_cooldown_until: agent.actions.jump() self.jump_cooldown_until = now + 0.25 - if mouse.LEFT_CLICK.is_pressed(): + + left_click_down = mouse.LEFT_CLICK.is_pressed() + escape_down = keys.ESCAPE.is_pressed() + if left_click_down and not self.prev_left_click: + self.look_capture_active = True + self.last_look_x = float(mouse.x) + self.last_look_y = float(mouse.y) + if escape_down and not self.prev_escape: + self.look_capture_active = False + + if self.look_capture_active: mouse.lock(); mouse.hide() if self.last_look_x is None or self.last_look_y is None: self.last_look_x, self.last_look_y = float(mouse.x), float(mouse.y) @@ -57,6 +70,8 @@ def tick(self): mouse.unlock(); mouse.show() self.last_look_x = None self.last_look_y = None + self.prev_left_click = left_click_down + self.prev_escape = escape_down agent.rotation = (self.look_yaw, self.look_pitch) view = agent.visual(h=self.view_h, w=self.view_w) diff --git a/src/xos/examples/agent_controller.py b/src/xos/examples/agent_controller.py index 245f9e7..48a6f43 100644 --- a/src/xos/examples/agent_controller.py +++ b/src/xos/examples/agent_controller.py @@ -14,6 +14,9 @@ def __init__(self): self.view_h = 128 self.look_yaw = None self.look_pitch = None + self.look_capture_active = False + self.prev_left_click = False + self.prev_escape = False self.last_look_x = None self.last_look_y = None @@ -45,7 +48,17 @@ def tick(self): if keys.SPACE.is_pressed() and now >= self.jump_cooldown_until: agent.actions.jump() self.jump_cooldown_until = now + 0.25 - if mouse.LEFT_CLICK.is_pressed(): + + left_click_down = mouse.LEFT_CLICK.is_pressed() + escape_down = keys.ESCAPE.is_pressed() + if left_click_down and not self.prev_left_click: + self.look_capture_active = True + self.last_look_x = float(mouse.x) + self.last_look_y = float(mouse.y) + if escape_down and not self.prev_escape: + self.look_capture_active = False + + if self.look_capture_active: mouse.lock(); mouse.hide() if self.last_look_x is None or self.last_look_y is None: self.last_look_x, self.last_look_y = float(mouse.x), float(mouse.y) @@ -58,6 +71,8 @@ def tick(self): mouse.unlock(); mouse.show() self.last_look_x = None self.last_look_y = None + self.prev_left_click = left_click_down + self.prev_escape = escape_down agent.rotation = (self.look_yaw, self.look_pitch) # Fixed-size viewport from the agent perspective. From 471607ef47e557b3d9f162fa73d6433c54373f71 Mon Sep 17 00:00:00 2001 From: verbiiyo Date: Wed, 3 Jun 2026 15:37:58 +0900 Subject: [PATCH 29/29] shoot from viewport --- .../minekov/client/XosViewportRuntime.java | 38 ++++++++++++++++++- .../xos/examples/agent_controller.py | 22 ++++++++++- src/xos/examples/agent_controller.py | 22 ++++++++++- 3 files changed, 79 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/verbii/minekov/client/XosViewportRuntime.java b/src/main/java/com/verbii/minekov/client/XosViewportRuntime.java index 3fa265f..ae44497 100644 --- a/src/main/java/com/verbii/minekov/client/XosViewportRuntime.java +++ b/src/main/java/com/verbii/minekov/client/XosViewportRuntime.java @@ -19,6 +19,7 @@ import net.minecraft.world.entity.LivingEntity; import net.minecraft.world.entity.ai.attributes.Attributes; import net.minecraft.world.entity.player.Inventory; +import net.minecraft.world.InteractionHand; import net.minecraft.world.item.ItemStack; import net.minecraft.world.level.storage.LevelResource; import net.minecraft.world.phys.Vec3; @@ -886,8 +887,15 @@ private static void applyLookDelta(Entity entity, float pitchDelta, float yawDel private static void applyEntityActionWithServerMirror( Minecraft mc, Entity entity, String action, String payload) { - applyEntityAction(entity, action, payload); Entity mirror = resolveServerMirrorEntity(mc, entity); + boolean preferServerMirror = + "left_click".equals(action) || "right_click".equals(action); + if (preferServerMirror && mirror != null && mirror != entity) { + applyEntityAction(mirror, action, payload); + return; + } + + applyEntityAction(entity, action, payload); if (mirror != null && mirror != entity) { applyEntityAction(mirror, action, payload); } @@ -908,6 +916,8 @@ private static void applyEntityAction(Entity entity, String action, String paylo entity.hurtMarked = true; } } + case "left_click" -> applyLeftClickAction(entity); + case "right_click" -> applyRightClickAction(entity); case "look" -> { float[] delta = parsePair(payload, 0.0f, 0.0f); // API order is look(pitch, yaw) @@ -917,6 +927,26 @@ private static void applyEntityAction(Entity entity, String action, String paylo } } + private static void applyLeftClickAction(Entity entity) { + if (entity instanceof RLOperator rl) { + // Try gun-fire path first (works for TACZ loadouts). + rl.shootForward(); + } + if (entity instanceof LivingEntity living) { + living.swing(InteractionHand.MAIN_HAND, true); + } + } + + private static void applyRightClickAction(Entity entity) { + if (entity instanceof RLOperator rl) { + rl.aim(true); + return; + } + if (entity instanceof LivingEntity living) { + living.swing(InteractionHand.OFF_HAND, true); + } + } + private static RLOperator findAgentById(Minecraft mc, String id) { if (mc == null || mc.level == null || id == null || id.isBlank()) { return null; @@ -1316,6 +1346,12 @@ def d(self): def jump(self): self._call("jump", "") + def left_click(self): + self._call("left_click", "") + + def right_click(self): + self._call("right_click", "") + def look(self, pitch, yaw): self._call("look", f"{float(pitch)},{float(yaw)}") diff --git a/src/main/resources/xos/examples/agent_controller.py b/src/main/resources/xos/examples/agent_controller.py index df2eb7a..b173863 100644 --- a/src/main/resources/xos/examples/agent_controller.py +++ b/src/main/resources/xos/examples/agent_controller.py @@ -4,18 +4,22 @@ LOOK_SENS = 0.08 PITCH_MIN = -89.0 PITCH_MAX = 89.0 +CLICK_REPEAT_SEC = 0.12 class AgentController(xos.Application): def __init__(self): - super().__init__(max_fps=15) + super().__init__(max_fps=60) self.jump_cooldown_until = 0.0 + self.left_click_cooldown_until = 0.0 + self.right_click_cooldown_until = 0.0 self.view_w = 128 self.view_h = 128 self.look_yaw = None self.look_pitch = None self.look_capture_active = False self.prev_left_click = False + self.prev_right_click = False self.prev_escape = False self.last_look_x = None self.last_look_y = None @@ -49,9 +53,13 @@ def tick(self): self.jump_cooldown_until = now + 0.25 left_click_down = mouse.LEFT_CLICK.is_pressed() + right_click_btn = getattr(mouse, "RIGHT_CLICK", None) + right_click_down = right_click_btn.is_pressed() if right_click_btn is not None else False escape_down = keys.ESCAPE.is_pressed() + entered_look_this_tick = False if left_click_down and not self.prev_left_click: self.look_capture_active = True + entered_look_this_tick = True self.last_look_x = float(mouse.x) self.last_look_y = float(mouse.y) if escape_down and not self.prev_escape: @@ -66,11 +74,23 @@ def tick(self): self.last_look_x, self.last_look_y = float(mouse.x), float(mouse.y) self.look_yaw += mdx * LOOK_SENS self.look_pitch = max(PITCH_MIN, min(PITCH_MAX, self.look_pitch + mdy * LOOK_SENS)) + # Apply latest aim first so click actions use current look direction. + agent.rotation = (self.look_yaw, self.look_pitch) + if not entered_look_this_tick: + left_click_edge = left_click_down and not self.prev_left_click + right_click_edge = right_click_down and not self.prev_right_click + if left_click_down and (left_click_edge or now >= self.left_click_cooldown_until): + agent.actions.left_click() + self.left_click_cooldown_until = now + CLICK_REPEAT_SEC + if right_click_down and (right_click_edge or now >= self.right_click_cooldown_until): + agent.actions.right_click() + self.right_click_cooldown_until = now + CLICK_REPEAT_SEC else: mouse.unlock(); mouse.show() self.last_look_x = None self.last_look_y = None self.prev_left_click = left_click_down + self.prev_right_click = right_click_down self.prev_escape = escape_down agent.rotation = (self.look_yaw, self.look_pitch) diff --git a/src/xos/examples/agent_controller.py b/src/xos/examples/agent_controller.py index 48a6f43..291d659 100644 --- a/src/xos/examples/agent_controller.py +++ b/src/xos/examples/agent_controller.py @@ -4,18 +4,22 @@ LOOK_SENS = 0.08 PITCH_MIN = -89.0 PITCH_MAX = 89.0 +CLICK_REPEAT_SEC = 0.12 class AgentController(xos.Application): def __init__(self): - super().__init__(max_fps=15) + super().__init__(max_fps=60) self.jump_cooldown_until = 0.0 + self.left_click_cooldown_until = 0.0 + self.right_click_cooldown_until = 0.0 self.view_w = 128 self.view_h = 128 self.look_yaw = None self.look_pitch = None self.look_capture_active = False self.prev_left_click = False + self.prev_right_click = False self.prev_escape = False self.last_look_x = None self.last_look_y = None @@ -50,9 +54,13 @@ def tick(self): self.jump_cooldown_until = now + 0.25 left_click_down = mouse.LEFT_CLICK.is_pressed() + right_click_btn = getattr(mouse, "RIGHT_CLICK", None) + right_click_down = right_click_btn.is_pressed() if right_click_btn is not None else False escape_down = keys.ESCAPE.is_pressed() + entered_look_this_tick = False if left_click_down and not self.prev_left_click: self.look_capture_active = True + entered_look_this_tick = True self.last_look_x = float(mouse.x) self.last_look_y = float(mouse.y) if escape_down and not self.prev_escape: @@ -67,11 +75,23 @@ def tick(self): self.last_look_x, self.last_look_y = float(mouse.x), float(mouse.y) self.look_yaw += mdx * LOOK_SENS self.look_pitch = max(PITCH_MIN, min(PITCH_MAX, self.look_pitch + mdy * LOOK_SENS)) + # Apply latest aim first so click actions use current look direction. + agent.rotation = (self.look_yaw, self.look_pitch) + if not entered_look_this_tick: + left_click_edge = left_click_down and not self.prev_left_click + right_click_edge = right_click_down and not self.prev_right_click + if left_click_down and (left_click_edge or now >= self.left_click_cooldown_until): + agent.actions.left_click() + self.left_click_cooldown_until = now + CLICK_REPEAT_SEC + if right_click_down and (right_click_edge or now >= self.right_click_cooldown_until): + agent.actions.right_click() + self.right_click_cooldown_until = now + CLICK_REPEAT_SEC else: mouse.unlock(); mouse.show() self.last_look_x = None self.last_look_y = None self.prev_left_click = left_click_down + self.prev_right_click = right_click_down self.prev_escape = escape_down agent.rotation = (self.look_yaw, self.look_pitch)