diff --git a/MinecraftRenderingNotes.md b/MinecraftRenderingNotes.md new file mode 100644 index 0000000..4328b24 --- /dev/null +++ b/MinecraftRenderingNotes.md @@ -0,0 +1,98 @@ +# Minecraft Rendering Notes + +This document tracks the current split vision stack and the historical commit lineage. + +Current branch snapshot: + +- **Current commit**: `5020512c394d8c4f4b27b93d6f6659cfc16606a0` +- **Both renderers now exist in the same branch/commit**: + - `EntityVision.java` + - `WorldVision.java` +- Runtime selection is currently in `XosViewportRuntime.renderAgentViewRgbaHex(...)` and `flushDeferredVisionCaptures(...)` with an easy comment swap. + +--- + +## 1) Entity Rendering + +### What `EntityVision` does now + +`EntityVision` is the camera-driven render-pass path intended for accurate POV rendering: + +- Per-camera context (`CameraContext`) keyed by agent UUID. +- Offscreen `TextureTarget` lifecycle management. +- Render-thread scheduling (`queueCapture` + `flushDeferredCaptures`) so world pass work stays on the correct thread. +- Capture using actual game render invocation (`renderLevel`) and readback. + +Tiny internals example: + +```java +camera.target = new TextureTarget(width, height, true, Minecraft.ON_OSX); +mc.setCameraEntity(cameraEntity); +gameRendererRenderLevelMethod.invoke(mc.gameRenderer, args); +GL11C.glReadPixels(..., GL11C.GL_RGBA, GL11C.GL_UNSIGNED_BYTE, buf); +``` + +Why `EntityVision` is the "accurate entity" direction: + +- It is camera-entity based (agent POV), not just projected world math. +- It uses renderer passes rather than hand-authored entity shapes. +- It keeps per-camera state and deferred capture semantics needed for correctness. + +--- + +## 2) World Rendering + +### What `WorldVision` does now (ported from `4c790a70...` lineage) + +`WorldVision` keeps the block-state ray/clip approach from the `b30d00d9...` and `4c790a70...` line: + +- Per-pixel world ray casting using `ClipContext`. +- Block color from `MapColor` with registry-hash fallback. +- Distance and face-based shading for depth/readability. +- Entity overlay rendered as projected disks (orbs) with depth checks. + +Tiny world hit example: + +```java +var hit = mc.level.clip(new ClipContext(origin, rayEnd, OUTLINE, NONE, agent)); +BlockPos bp = hit.getBlockPos(); +int rgb = rgbFromBlockState(mc, bp, state); +``` + +Tiny orb overlay example: + +```java +int color = rgbFromEntity(e); +if ((dx * dx + dy * dy) <= rr && camZ < depth[p]) { + // paint projected entity disk +} +``` + +What `WorldVision` gives: + +- World **state structure** that is close to live block layout. +- Stable, readable rendering and predictable performance shape. +- Explicit fallback behavior that is easy to reason about. + +What `WorldVision` does **not** provide: + +- Mesh/textured entity rendering. +- Full native-frame parity with Minecraft's standard on-screen camera pass. + +--- + +## EntityVision vs WorldVision + +Both renderers now exist side by side in one branch so each can be stabilized independently. + +- **`EntityVision`** + - Camera/render-pass architecture (`renderLevel` + offscreen target + readback). + - Best path toward true POV fidelity (including proper entity rendering). + - More sensitive to render-thread/target-state correctness. + +- **`WorldVision`** + - Deterministic ray/clip world sampler with shaded block colors. + - Entity visibility via synthetic projected orbs. + - Easier to reason about for world-state output and debugging. + +This split is intentional: verify each path independently, then merge strengths into one robust final renderer. diff --git a/src/main/java/com/verbii/minekov/client/BasicWorldVision.java b/src/main/java/com/verbii/minekov/client/BasicWorldVision.java new file mode 100644 index 0000000..7e1a2b2 --- /dev/null +++ b/src/main/java/com/verbii/minekov/client/BasicWorldVision.java @@ -0,0 +1,392 @@ +package com.verbii.minekov.client; + +import com.verbii.minekov.entities.RLOperator; +import com.mojang.blaze3d.pipeline.RenderTarget; +import com.mojang.blaze3d.pipeline.TextureTarget; +import com.mojang.blaze3d.systems.RenderSystem; +import com.mojang.blaze3d.vertex.PoseStack; +import net.minecraft.client.Minecraft; +import net.minecraft.core.BlockPos; +import net.minecraft.core.registries.BuiltInRegistries; +import net.minecraft.resources.ResourceLocation; +import net.minecraft.world.entity.Entity; +import net.minecraft.world.entity.LivingEntity; +import net.minecraft.world.level.ClipContext; +import net.minecraft.world.phys.HitResult; +import net.minecraft.world.phys.Vec3; +import org.lwjgl.opengl.GL11C; +import org.lwjgl.system.MemoryUtil; + +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.nio.ByteBuffer; + +/** + * Basic world-state renderer path from the 4c790a70 / b30d00d9 lineage. + * - Blocks: ray/clip sampled + shaded colors. + * - Entities: projected depth-tested orb overlay. + */ +final class BasicWorldVision { + private static final float AGENT_VIEW_FOV_DEG = 70.0f; + private static final double AGENT_VIEW_MAX_DISTANCE = 24.0; + private static final char[] HEX = "0123456789abcdef".toCharArray(); + + // Keep false unless explicitly experimenting with old real-render attempt. + private static final boolean ENABLE_REAL_AGENT_VIEW_RENDER = false; + private static TextureTarget agentViewRenderTarget; + private static Method gameRendererRenderLevelMethod; + private static Field minecraftMainRenderTargetField; + + private BasicWorldVision() {} + + static String renderAgentViewRgbaHex(Minecraft mc, RLOperator agent, int reqW, int reqH) { + int width = clampViewSize(reqW); + int height = clampViewSize(reqH); + if (mc == null || mc.level == null || agent == null) { + return ""; + } + if (ENABLE_REAL_AGENT_VIEW_RENDER) { + byte[] rendered = tryRenderAgentViewWithMinecraftRenderer(mc, agent, width, height); + if (rendered != null) { + return encodeRgbaHex(rendered); + } + } + + Vec3 origin = new Vec3(agent.getX(), agent.getEyeY(), agent.getZ()); + float yaw = agent.getYRot(); + float pitch = agent.getXRot(); + Vec3 forward = forwardFromRotation(yaw, pitch); + Vec3 worldUp = new Vec3(0.0, 1.0, 0.0); + Vec3 right = worldUp.cross(forward); + if (right.lengthSqr() < 1.0e-8) { + right = rightFromYaw(yaw); + } else { + right = right.normalize(); + } + Vec3 up = forward.cross(right).normalize(); + + double tanHalfFov = Math.tan(Math.toRadians(AGENT_VIEW_FOV_DEG * 0.5)); + double aspect = (double) width / (double) height; + int pixelCount = width * height; + byte[] rgba = new byte[pixelCount * 4]; + double[] depth = new double[pixelCount]; + for (int i = 0; i < pixelCount; i++) { + depth[i] = Double.POSITIVE_INFINITY; + } + + for (int y = 0; y < height; y++) { + double ny = 1.0 - (2.0 * (y + 0.5) / (double) height); + for (int x = 0; x < width; x++) { + double nx = (2.0 * (x + 0.5) / (double) width) - 1.0; + Vec3 dir = forward + .add(right.scale(nx * tanHalfFov * aspect)) + .add(up.scale(ny * tanHalfFov)) + .normalize(); + + int r = 110; + int g = 150; + int b = 210; + boolean hit = false; + double distance = 0.0; + Vec3 rayEnd = origin.add(dir.scale(AGENT_VIEW_MAX_DISTANCE)); + var hitRes = mc.level.clip(new ClipContext( + origin, + rayEnd, + ClipContext.Block.OUTLINE, + ClipContext.Fluid.NONE, + agent)); + if (hitRes.getType() != HitResult.Type.MISS) { + BlockPos bp = hitRes.getBlockPos(); + var state = mc.level.getBlockState(bp); + if (!state.isAir()) { + int rgb = rgbFromBlockState(mc, bp, state); + distance = hitRes.getLocation().distanceTo(origin); + double shade = Math.max(0.28, 1.0 - (distance / AGENT_VIEW_MAX_DISTANCE)); + double faceShade = switch (hitRes.getDirection()) { + case UP -> 1.08; + case DOWN -> 0.72; + case NORTH, SOUTH -> 0.94; + default -> 1.00; + }; + double totalShade = Math.max(0.20, Math.min(1.15, shade * faceShade)); + r = (int) Math.max(0, Math.min(255, ((rgb >> 16) & 0xFF) * totalShade)); + g = (int) Math.max(0, Math.min(255, ((rgb >> 8) & 0xFF) * totalShade)); + b = (int) Math.max(0, Math.min(255, (rgb & 0xFF) * totalShade)); + hit = true; + } + } + + if (!hit) { + double tSky = Math.max(0.0, Math.min(1.0, (dir.y + 1.0) * 0.5)); + r = (int) (70 + 60 * tSky); + g = (int) (100 + 90 * tSky); + b = (int) (150 + 80 * tSky); + } else { + int fog = (int) Math.max(0, Math.min(40, (AGENT_VIEW_MAX_DISTANCE - distance) * 0.8)); + r = Math.min(255, r + fog); + g = Math.min(255, g + fog); + b = Math.min(255, b + fog); + } + int p = y * width + x; + int i = p * 4; + rgba[i] = (byte) r; + rgba[i + 1] = (byte) g; + rgba[i + 2] = (byte) b; + rgba[i + 3] = (byte) 255; + depth[p] = hit ? distance : Double.POSITIVE_INFINITY; + } + } + + var renderables = mc.level.entitiesForRendering(); + for (Entity e : renderables) { + if (e == null || e == agent || e.isRemoved()) { + continue; + } + Vec3 rel = new Vec3(e.getX() - origin.x, e.getEyeY() - origin.y, e.getZ() - origin.z); + double camX = rel.dot(right); + double camY = rel.dot(up); + double camZ = rel.dot(forward); + if (camZ <= 0.01 || camZ > AGENT_VIEW_MAX_DISTANCE) { + continue; + } + + double ndcX = camX / (camZ * tanHalfFov * aspect); + double ndcY = camY / (camZ * tanHalfFov); + if (Math.abs(ndcX) > 1.5 || Math.abs(ndcY) > 1.5) { + continue; + } + + int cx = (int) Math.round((ndcX + 1.0) * 0.5 * (width - 1)); + int cy = (int) Math.round((1.0 - (ndcY + 1.0) * 0.5) * (height - 1)); + double radiusPx = Math.max( + 1.0, + (Math.max(0.25, e.getBbWidth()) / (camZ * tanHalfFov * aspect)) * (width * 0.28)); + int rad = (int) Math.min(Math.max(1, Math.round(radiusPx)), Math.max(width, height)); + int color = rgbFromEntity(e); + int er = (color >> 16) & 0xFF; + int eg = (color >> 8) & 0xFF; + int eb = color & 0xFF; + int x1 = Math.max(0, cx - rad); + int x2 = Math.min(width - 1, cx + rad); + int y1 = Math.max(0, cy - rad); + int y2 = Math.min(height - 1, cy + rad); + int rr = rad * rad; + for (int py = y1; py <= y2; py++) { + int dy = py - cy; + for (int px = x1; px <= x2; px++) { + int dx = px - cx; + if ((dx * dx + dy * dy) > rr) { + continue; + } + int p = py * width + px; + if (camZ >= depth[p]) { + continue; + } + int i = p * 4; + rgba[i] = (byte) er; + rgba[i + 1] = (byte) eg; + rgba[i + 2] = (byte) eb; + rgba[i + 3] = (byte) 255; + depth[p] = camZ; + } + } + } + return encodeRgbaHex(rgba); + } + + static void flushDeferredCaptures(Minecraft mc) { + // No-op in world-state renderer path (kept for easy call-site swapping). + } + + private static int clampViewSize(int value) { + return Math.max(16, Math.min(512, value)); + } + + private static int rgbFromBlockState( + Minecraft mc, BlockPos pos, net.minecraft.world.level.block.state.BlockState state) { + try { + var map = state.getMapColor(mc.level, pos); + if (map != null && map.col != 0) { + return map.col & 0x00FFFFFF; + } + } catch (Throwable ignored) { + } + ResourceLocation key = BuiltInRegistries.BLOCK.getKey(state.getBlock()); + int hash = key != null ? key.toString().hashCode() : 0x77AA77; + int r = 64 + (Math.abs(hash) & 0x7F); + int g = 64 + ((Math.abs(hash >> 8)) & 0x7F); + int b = 64 + ((Math.abs(hash >> 16)) & 0x7F); + return (r << 16) | (g << 8) | b; + } + + private static int rgbFromEntity(Entity entity) { + int hash = entity.getType().toString().hashCode(); + int r = 96 + (Math.abs(hash) & 0x5F); + int g = 96 + ((Math.abs(hash >> 8)) & 0x5F); + int b = 96 + ((Math.abs(hash >> 16)) & 0x5F); + if (entity instanceof LivingEntity) { + r = Math.min(255, r + 24); + } + return (r << 16) | (g << 8) | b; + } + + private static String encodeRgbaHex(byte[] rgba) { + char[] out = new char[rgba.length * 2]; + int j = 0; + for (byte b : rgba) { + int v = b & 0xFF; + out[j++] = HEX[v >>> 4]; + out[j++] = HEX[v & 0x0F]; + } + return new String(out); + } + + private static Vec3 forwardFromRotation(float yawDeg, float pitchDeg) { + double yaw = Math.toRadians(yawDeg); + double pitch = Math.toRadians(pitchDeg); + double x = -Math.sin(yaw) * Math.cos(pitch); + double y = -Math.sin(pitch); + double z = Math.cos(yaw) * Math.cos(pitch); + return new Vec3(x, y, z).normalize(); + } + + private static Vec3 rightFromYaw(float yawDeg) { + double yaw = Math.toRadians(yawDeg); + return new Vec3(-Math.cos(yaw), 0.0, -Math.sin(yaw)).normalize(); + } + + private static byte[] tryRenderAgentViewWithMinecraftRenderer( + Minecraft mc, RLOperator agent, int width, int height) { + if (!RenderSystem.isOnRenderThread()) { + return null; + } + try { + if (agentViewRenderTarget == null + || agentViewRenderTarget.width != width + || agentViewRenderTarget.height != height) { + if (agentViewRenderTarget != null) { + agentViewRenderTarget.destroyBuffers(); + } + agentViewRenderTarget = new TextureTarget(width, height, true, Minecraft.ON_OSX); + } + } catch (Throwable t) { + return null; + } + + RenderTarget previousTarget = mc.getMainRenderTarget(); + Entity previousCamera = mc.getCameraEntity(); + if (!setMinecraftMainRenderTarget(mc, previousTarget, agentViewRenderTarget)) { + return null; + } + try { + mc.setCameraEntity(agent); + agentViewRenderTarget.bindWrite(true); + RenderSystem.viewport(0, 0, width, height); + agentViewRenderTarget.setClearColor(0f, 0f, 0f, 1f); + agentViewRenderTarget.clear(Minecraft.ON_OSX); + + if (!invokeGameRendererLevelPass(mc)) { + return null; + } + + ByteBuffer buf = MemoryUtil.memAlloc(width * height * 4); + try { + GL11C.glReadPixels(0, 0, width, height, GL11C.GL_RGBA, GL11C.GL_UNSIGNED_BYTE, buf); + byte[] out = new byte[width * height * 4]; + int rowBytes = width * 4; + for (int y = 0; y < height; y++) { + int src = (height - 1 - y) * rowBytes; + int dst = y * rowBytes; + buf.position(src); + buf.get(out, dst, rowBytes); + } + return out; + } finally { + MemoryUtil.memFree(buf); + } + } catch (Throwable t) { + return null; + } finally { + try { + mc.setCameraEntity(previousCamera); + } catch (Throwable ignored) { + } + try { + setMinecraftMainRenderTarget(mc, agentViewRenderTarget, previousTarget); + previousTarget.bindWrite(true); + RenderSystem.viewport(0, 0, previousTarget.width, previousTarget.height); + } catch (Throwable ignored) { + } + } + } + + private static boolean invokeGameRendererLevelPass(Minecraft mc) { + try { + if (gameRendererRenderLevelMethod == null) { + for (Method m : mc.gameRenderer.getClass().getDeclaredMethods()) { + if ("renderLevel".equals(m.getName())) { + m.setAccessible(true); + gameRendererRenderLevelMethod = m; + break; + } + } + } + if (gameRendererRenderLevelMethod == null) { + return false; + } + Class[] p = gameRendererRenderLevelMethod.getParameterTypes(); + Object[] args = new Object[p.length]; + for (int i = 0; i < p.length; i++) { + Class t = p[i]; + if (t == float.class || t == Float.class) { + args[i] = mc.getFrameTime(); + } else if (t == double.class || t == Double.class) { + args[i] = (double) mc.getFrameTime(); + } else if (t == long.class || t == Long.class) { + args[i] = System.nanoTime(); + } else if (t == int.class || t == Integer.class) { + args[i] = 0; + } else if (t == boolean.class || t == Boolean.class) { + args[i] = Boolean.FALSE; + } else if (PoseStack.class.isAssignableFrom(t)) { + args[i] = new PoseStack(); + } else { + args[i] = null; + } + } + gameRendererRenderLevelMethod.invoke(mc.gameRenderer, args); + return true; + } catch (Throwable t) { + return false; + } + } + + private static boolean setMinecraftMainRenderTarget( + Minecraft mc, RenderTarget expectedCurrent, RenderTarget target) { + try { + if (minecraftMainRenderTargetField == null) { + for (Field f : Minecraft.class.getDeclaredFields()) { + if (RenderTarget.class.isAssignableFrom(f.getType())) { + f.setAccessible(true); + Object cur = f.get(mc); + if (cur == expectedCurrent) { + minecraftMainRenderTargetField = f; + break; + } + } + } + } + if (minecraftMainRenderTargetField == null) { + return false; + } + Object cur = minecraftMainRenderTargetField.get(mc); + if (cur != expectedCurrent) { + return false; + } + minecraftMainRenderTargetField.set(mc, target); + return true; + } catch (Throwable t) { + return false; + } + } +} diff --git a/src/main/java/com/verbii/minekov/client/EntityVision.java b/src/main/java/com/verbii/minekov/client/EntityVision.java new file mode 100644 index 0000000..f45bad6 --- /dev/null +++ b/src/main/java/com/verbii/minekov/client/EntityVision.java @@ -0,0 +1,330 @@ +package com.verbii.minekov.client; + +import com.verbii.minekov.entities.RLOperator; +import com.mojang.blaze3d.pipeline.RenderTarget; +import com.mojang.blaze3d.pipeline.TextureTarget; +import com.mojang.blaze3d.systems.RenderSystem; +import com.mojang.blaze3d.vertex.PoseStack; +import net.minecraft.client.Minecraft; +import net.minecraft.world.entity.Entity; +import org.lwjgl.opengl.GL11C; +import org.lwjgl.system.MemoryUtil; + +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.Map; + +/** Agent vision renderer used by Python `agent.visual(...)` host binding. */ +final class EntityVision { + private static final int MIN_VIEW_SIZE = 16; + private static final int MAX_VIEW_SIZE = 1024; + private static final char[] HEX = "0123456789abcdef".toCharArray(); + private static final VisionManager MANAGER = new VisionManager(); + + private EntityVision() {} + + static String renderAgentViewRgbaHex(Minecraft mc, RLOperator agent, int reqW, int reqH) { + return MANAGER.renderAgentViewRgbaHex(mc, agent, reqW, reqH); + } + + static void flushDeferredCaptures(Minecraft mc) { + MANAGER.flushDeferredCaptures(mc); + } + + private static String encodeRgbaHex(byte[] rgba) { + char[] out = new char[rgba.length * 2]; + int j = 0; + for (byte b : rgba) { + int v = b & 0xFF; + out[j++] = HEX[v >>> 4]; + out[j++] = HEX[v & 0x0F]; + } + return new String(out); + } + + private static final class VisionManager { + private final Map cameras = new HashMap<>(); + private Method gameRendererRenderLevelMethod; + private Field minecraftMainRenderTargetField; + + String renderAgentViewRgbaHex(Minecraft mc, RLOperator agent, int reqW, int reqH) { + if (mc == null || mc.level == null || agent == null) { + return ""; + } + int width = clampViewSize(reqW); + int height = clampViewSize(reqH); + CameraContext camera = cameraFor(agent.getStringUUID()); + byte[] rgba = renderCameraRgba(mc, camera, agent, width, height); + if (rgba == null) { + rgba = camera.lastRgba; + } + return rgba == null ? "" : encodeRgbaHex(rgba); + } + + void flushDeferredCaptures(Minecraft mc) { + if (mc == null || !RenderSystem.isOnRenderThread()) { + return; + } + synchronized (this) { + for (CameraContext camera : cameras.values()) { + RLOperator entity; + int w; + int h; + synchronized (camera) { + if (!camera.captureQueued || camera.pendingEntity == null) { + continue; + } + entity = camera.pendingEntity; + w = camera.pendingWidth; + h = camera.pendingHeight; + camera.captureQueued = false; + } + if (entity != null && !entity.isRemoved()) { + renderCameraRgbaOnRenderThread(mc, camera, entity, w, h); + } + } + } + } + + private synchronized CameraContext cameraFor(String cameraId) { + return cameras.computeIfAbsent(cameraId, CameraContext::new); + } + + private static int clampViewSize(int value) { + return Math.max(MIN_VIEW_SIZE, Math.min(MAX_VIEW_SIZE, value)); + } + + private byte[] renderCameraRgba( + Minecraft mc, + CameraContext camera, + RLOperator cameraEntity, + int width, + int height) { + if (XosViewportRuntime.isOverlayRenderPassActive()) { + queueCapture(camera, cameraEntity, width, height); + return camera.lastRgba; + } + if (!RenderSystem.isOnRenderThread()) { + queueCapture(camera, cameraEntity, width, height); + return camera.lastRgba; + } + return renderCameraRgbaOnRenderThread(mc, camera, cameraEntity, width, height); + } + + private void queueCapture( + CameraContext camera, RLOperator cameraEntity, int width, int height) { + synchronized (camera) { + camera.pendingEntity = cameraEntity; + camera.pendingWidth = width; + camera.pendingHeight = height; + camera.captureQueued = true; + } + } + + private byte[] renderCameraRgbaOnRenderThread( + Minecraft mc, + CameraContext camera, + RLOperator cameraEntity, + int width, + int height) { + RenderTarget previousTarget = mc.getMainRenderTarget(); + int previousWidth = previousTarget.width; + int previousHeight = previousTarget.height; + Entity previousCamera = mc.getCameraEntity(); + int renderWidth = Math.max(1, previousWidth); + int renderHeight = Math.max(1, previousHeight); + try { + ensureTarget(camera, renderWidth, renderHeight); + } catch (Throwable ignored) { + return null; + } + if (!setMinecraftMainRenderTarget(mc, camera.target)) { + return null; + } + try { + mc.setCameraEntity(cameraEntity); + RenderSystem.disableScissor(); + RenderSystem.enableDepthTest(); + RenderSystem.depthMask(true); + camera.target.bindWrite(true); + RenderSystem.viewport(0, 0, renderWidth, renderHeight); + camera.target.setClearColor(0f, 0f, 0f, 1f); + camera.target.clear(Minecraft.ON_OSX); + + if (!invokeGameRendererLevelPass(mc)) { + return null; + } + + ByteBuffer buf = MemoryUtil.memAlloc(renderWidth * renderHeight * 4); + try { + camera.target.bindWrite(true); + RenderSystem.viewport(0, 0, renderWidth, renderHeight); + GL11C.glPixelStorei(GL11C.GL_PACK_ALIGNMENT, 1); + GL11C.glReadPixels( + 0, 0, renderWidth, renderHeight, GL11C.GL_RGBA, GL11C.GL_UNSIGNED_BYTE, buf); + byte[] full = new byte[renderWidth * renderHeight * 4]; + int rowBytes = renderWidth * 4; + for (int y = 0; y < renderHeight; y++) { + int src = (renderHeight - 1 - y) * rowBytes; + int dst = y * rowBytes; + buf.position(src); + buf.get(full, dst, rowBytes); + } + byte[] out = + (renderWidth == width && renderHeight == height) + ? full + : downscaleNearest(full, renderWidth, renderHeight, width, height); + camera.lastRgba = out; + return out; + } finally { + MemoryUtil.memFree(buf); + } + } catch (Throwable ignored) { + return null; + } finally { + try { + mc.setCameraEntity(previousCamera); + } catch (Throwable ignored) { + } + try { + setMinecraftMainRenderTarget(mc, previousTarget); + previousTarget.bindWrite(true); + RenderSystem.viewport(0, 0, previousWidth, previousHeight); + RenderSystem.disableDepthTest(); + RenderSystem.depthMask(true); + RenderSystem.disableCull(); + RenderSystem.disableScissor(); + RenderSystem.setShaderColor(1.0F, 1.0F, 1.0F, 1.0F); + } catch (Throwable ignored) { + } + } + } + + private byte[] downscaleNearest( + byte[] src, int srcW, int srcH, int dstW, int dstH) { + byte[] dst = new byte[Math.max(1, dstW * dstH * 4)]; + for (int y = 0; y < dstH; y++) { + int sy = Math.min(srcH - 1, (y * srcH) / Math.max(1, dstH)); + for (int x = 0; x < dstW; x++) { + int sx = Math.min(srcW - 1, (x * srcW) / Math.max(1, dstW)); + int si = (sy * srcW + sx) * 4; + int di = (y * dstW + x) * 4; + dst[di] = src[si]; + dst[di + 1] = src[si + 1]; + dst[di + 2] = src[si + 2]; + dst[di + 3] = src[si + 3]; + } + } + return dst; + } + + private void ensureTarget(CameraContext camera, int width, int height) { + if (camera.target == null || camera.width != width || camera.height != height) { + if (camera.target != null) { + camera.target.destroyBuffers(); + } + camera.target = new TextureTarget(width, height, true, Minecraft.ON_OSX); + camera.width = width; + camera.height = height; + } + } + + private boolean invokeGameRendererLevelPass(Minecraft mc) { + try { + if (gameRendererRenderLevelMethod == null) { + for (Method m : mc.gameRenderer.getClass().getDeclaredMethods()) { + if ("renderLevel".equals(m.getName())) { + m.setAccessible(true); + gameRendererRenderLevelMethod = m; + break; + } + } + } + if (gameRendererRenderLevelMethod == null) { + return false; + } + Class[] p = gameRendererRenderLevelMethod.getParameterTypes(); + Object[] args = new Object[p.length]; + for (int i = 0; i < p.length; i++) { + Class t = p[i]; + if (t == float.class || t == Float.class) { + args[i] = mc.getFrameTime(); + } else if (t == double.class || t == Double.class) { + args[i] = (double) mc.getFrameTime(); + } else if (t == long.class || t == Long.class) { + args[i] = System.nanoTime(); + } else if (t == int.class || t == Integer.class) { + args[i] = 0; + } else if (t == boolean.class || t == Boolean.class) { + args[i] = Boolean.FALSE; + } else if (PoseStack.class.isAssignableFrom(t)) { + args[i] = new PoseStack(); + } else { + args[i] = null; + } + } + gameRendererRenderLevelMethod.invoke(mc.gameRenderer, args); + return true; + } catch (Throwable ignored) { + return false; + } + } + + private boolean setMinecraftMainRenderTarget(Minecraft mc, RenderTarget target) { + try { + if (minecraftMainRenderTargetField == null) { + minecraftMainRenderTargetField = resolveMainRenderTargetField(mc); + } + if (minecraftMainRenderTargetField == null) { + return false; + } + minecraftMainRenderTargetField.set(mc, target); + return true; + } catch (Throwable ignored) { + return false; + } + } + + private Field resolveMainRenderTargetField(Minecraft mc) { + try { + for (Field f : Minecraft.class.getDeclaredFields()) { + if (f.getType() == RenderTarget.class && "mainRenderTarget".equals(f.getName())) { + f.setAccessible(true); + return f; + } + } + RenderTarget current = mc.getMainRenderTarget(); + for (Field f : Minecraft.class.getDeclaredFields()) { + if (!RenderTarget.class.isAssignableFrom(f.getType())) { + continue; + } + f.setAccessible(true); + Object value = f.get(mc); + if (value == current) { + return f; + } + } + } catch (Throwable ignored) { + } + return null; + } + } + + private static final class CameraContext { + private final String cameraId; + private TextureTarget target; + private int width; + private int height; + private byte[] lastRgba; + private boolean captureQueued; + private RLOperator pendingEntity; + private int pendingWidth; + private int pendingHeight; + + private CameraContext(String cameraId) { + this.cameraId = cameraId; + } + } +} diff --git a/src/main/java/com/verbii/minekov/client/TexturedWorldVision.java b/src/main/java/com/verbii/minekov/client/TexturedWorldVision.java new file mode 100644 index 0000000..33b2eeb --- /dev/null +++ b/src/main/java/com/verbii/minekov/client/TexturedWorldVision.java @@ -0,0 +1,94 @@ +package com.verbii.minekov.client; + +import com.verbii.minekov.entities.RLOperator; +import net.minecraft.client.Minecraft; + +/** + * Unified vision path that composes world-geometry sampling with render-pass captures. + *

+ * This keeps BasicWorldVision's stable block structure output, while allowing EntityVision's + * camera-rendered frame to provide accurate entity pixels when available. + */ +final class TexturedWorldVision { + private static final int PIXEL_STRIDE = 4; + private static final char[] HEX = "0123456789abcdef".toCharArray(); + + private TexturedWorldVision() {} + + static String renderAgentViewRgbaHex(Minecraft mc, RLOperator agent, int reqW, int reqH) { + String worldHex = BasicWorldVision.renderAgentViewRgbaHex(mc, agent, reqW, reqH); + String entityHex = EntityVision.renderAgentViewRgbaHex(mc, agent, reqW, reqH); + return mergeVisionHex(worldHex, entityHex); + } + + static void flushDeferredCaptures(Minecraft mc) { + // Basic path does not queue work; entity path does. + EntityVision.flushDeferredCaptures(mc); + } + + private static String mergeVisionHex(String worldHex, String entityHex) { + if (entityHex == null || entityHex.isEmpty()) { + return worldHex == null ? "" : worldHex; + } + if (worldHex == null || worldHex.isEmpty()) { + return entityHex; + } + + byte[] world = decodeRgbaHex(worldHex); + byte[] entity = decodeRgbaHex(entityHex); + if (world == null || entity == null || world.length != entity.length) { + return entityHex; + } + + byte[] out = new byte[world.length]; + for (int i = 0; i < world.length; i += PIXEL_STRIDE) { + int er = entity[i] & 0xFF; + int eg = entity[i + 1] & 0xFF; + int eb = entity[i + 2] & 0xFF; + int ea = entity[i + 3] & 0xFF; + + // EntityVision target clear color is black; keep world sample for clear pixels. + boolean entityPixelPresent = ea > 0 && (er > 6 || eg > 6 || eb > 6); + if (entityPixelPresent) { + out[i] = entity[i]; + out[i + 1] = entity[i + 1]; + out[i + 2] = entity[i + 2]; + out[i + 3] = entity[i + 3]; + } else { + out[i] = world[i]; + out[i + 1] = world[i + 1]; + out[i + 2] = world[i + 2]; + out[i + 3] = world[i + 3]; + } + } + return encodeRgbaHex(out); + } + + private static byte[] decodeRgbaHex(String hex) { + int n = hex.length(); + if ((n & 1) != 0) { + return null; + } + byte[] out = new byte[n / 2]; + for (int i = 0; i < n; i += 2) { + int hi = Character.digit(hex.charAt(i), 16); + int lo = Character.digit(hex.charAt(i + 1), 16); + if (hi < 0 || lo < 0) { + return null; + } + out[i / 2] = (byte) ((hi << 4) | lo); + } + return out; + } + + private static String encodeRgbaHex(byte[] rgba) { + char[] out = new char[rgba.length * 2]; + int j = 0; + for (byte b : rgba) { + int v = b & 0xFF; + out[j++] = HEX[v >>> 4]; + out[j++] = HEX[v & 0x0F]; + } + return new String(out); + } +} diff --git a/src/main/java/com/verbii/minekov/client/XosViewportOverlay.java b/src/main/java/com/verbii/minekov/client/XosViewportOverlay.java index b097072..f56af92 100644 --- a/src/main/java/com/verbii/minekov/client/XosViewportOverlay.java +++ b/src/main/java/com/verbii/minekov/client/XosViewportOverlay.java @@ -509,6 +509,13 @@ private static boolean panelContains(double mx, double my, Minecraft mc) { /** Horizontal inset from the pill’s right edge for the online (green) / offline (gray) status dot. */ private static final int MINIMIZED_STATUS_DOT_INSET = 10; + /** Private-use codepoints used to preserve modifier-aware editor keys over Java host char events. */ + private static final int XOS_KEY_SHIFT_TAB = 0xE100; + private static final int XOS_KEY_SHIFT_LEFT = 0xE101; + private static final int XOS_KEY_SHIFT_RIGHT = 0xE102; + private static final int XOS_KEY_SHIFT_UP = 0xE103; + private static final int XOS_KEY_SHIFT_DOWN = 0xE104; + private static final int XOS_KEY_TOGGLE_COMMENT = 0xE105; private static boolean shouldRouteKeyboardToXos(Minecraft mc) { if (!(mc.screen instanceof ChatScreen)) { @@ -561,16 +568,17 @@ private static boolean isModifierOrToggleKey(int key) { * {@link ScreenEvent.CharacterTyped.Pre}. Returns -1 for keys that should only come through character * events. */ - private static int navigationKeyToCodePoint(int key) { + private static int navigationKeyToCodePoint(int key, int modifiers) { + boolean shiftDown = (modifiers & GLFW.GLFW_MOD_SHIFT) != 0; return switch (key) { case GLFW.GLFW_KEY_BACKSPACE -> '\b'; case GLFW.GLFW_KEY_DELETE -> 0x7F; case GLFW.GLFW_KEY_ENTER, GLFW.GLFW_KEY_KP_ENTER -> '\n'; - case GLFW.GLFW_KEY_TAB -> '\t'; - case GLFW.GLFW_KEY_LEFT -> 0x2190; - case GLFW.GLFW_KEY_RIGHT -> 0x2192; - case GLFW.GLFW_KEY_UP -> 0x2191; - case GLFW.GLFW_KEY_DOWN -> 0x2193; + case GLFW.GLFW_KEY_TAB -> shiftDown ? XOS_KEY_SHIFT_TAB : '\t'; + case GLFW.GLFW_KEY_LEFT -> shiftDown ? XOS_KEY_SHIFT_LEFT : 0x2190; + case GLFW.GLFW_KEY_RIGHT -> shiftDown ? XOS_KEY_SHIFT_RIGHT : 0x2192; + case GLFW.GLFW_KEY_UP -> shiftDown ? XOS_KEY_SHIFT_UP : 0x2191; + case GLFW.GLFW_KEY_DOWN -> shiftDown ? XOS_KEY_SHIFT_DOWN : 0x2193; default -> -1; }; } @@ -589,6 +597,143 @@ private static int ctrlShortcutToActionCode(int key, int modifiers) { }; } + /** + * GLFW key -> xos binding token used by `self.state.keyboard.keys..is_pressed()`. + * Returns null for unknown keys. + */ + private static String keyCodeToBindingName(int key, int scanCode) { + String named = GLFW.glfwGetKeyName(key, scanCode); + if (named != null && named.length() == 1) { + char ch = Character.toUpperCase(named.charAt(0)); + if (Character.isLetterOrDigit(ch)) { + return String.valueOf(ch); + } + } + return switch (key) { + case GLFW.GLFW_KEY_0 -> "0"; + case GLFW.GLFW_KEY_1 -> "1"; + case GLFW.GLFW_KEY_2 -> "2"; + case GLFW.GLFW_KEY_3 -> "3"; + case GLFW.GLFW_KEY_4 -> "4"; + case GLFW.GLFW_KEY_5 -> "5"; + case GLFW.GLFW_KEY_6 -> "6"; + case GLFW.GLFW_KEY_7 -> "7"; + case GLFW.GLFW_KEY_8 -> "8"; + case GLFW.GLFW_KEY_9 -> "9"; + case GLFW.GLFW_KEY_A -> "A"; + case GLFW.GLFW_KEY_B -> "B"; + case GLFW.GLFW_KEY_C -> "C"; + case GLFW.GLFW_KEY_D -> "D"; + case GLFW.GLFW_KEY_E -> "E"; + case GLFW.GLFW_KEY_F -> "F"; + case GLFW.GLFW_KEY_G -> "G"; + case GLFW.GLFW_KEY_H -> "H"; + case GLFW.GLFW_KEY_I -> "I"; + case GLFW.GLFW_KEY_J -> "J"; + case GLFW.GLFW_KEY_K -> "K"; + case GLFW.GLFW_KEY_L -> "L"; + case GLFW.GLFW_KEY_M -> "M"; + case GLFW.GLFW_KEY_N -> "N"; + case GLFW.GLFW_KEY_O -> "O"; + case GLFW.GLFW_KEY_P -> "P"; + case GLFW.GLFW_KEY_Q -> "Q"; + case GLFW.GLFW_KEY_R -> "R"; + case GLFW.GLFW_KEY_S -> "S"; + case GLFW.GLFW_KEY_T -> "T"; + case GLFW.GLFW_KEY_U -> "U"; + case GLFW.GLFW_KEY_V -> "V"; + case GLFW.GLFW_KEY_W -> "W"; + case GLFW.GLFW_KEY_X -> "X"; + case GLFW.GLFW_KEY_Y -> "Y"; + case GLFW.GLFW_KEY_Z -> "Z"; + case GLFW.GLFW_KEY_SPACE -> "SPACE"; + case GLFW.GLFW_KEY_APOSTROPHE -> "APOSTROPHE"; + case GLFW.GLFW_KEY_COMMA -> "COMMA"; + case GLFW.GLFW_KEY_MINUS -> "MINUS"; + case GLFW.GLFW_KEY_PERIOD -> "PERIOD"; + case GLFW.GLFW_KEY_SLASH -> "SLASH"; + case GLFW.GLFW_KEY_SEMICOLON -> "SEMICOLON"; + case GLFW.GLFW_KEY_EQUAL -> "EQUAL"; + case GLFW.GLFW_KEY_LEFT_BRACKET -> "LEFT_BRACKET"; + case GLFW.GLFW_KEY_BACKSLASH -> "BACKSLASH"; + case GLFW.GLFW_KEY_RIGHT_BRACKET -> "RIGHT_BRACKET"; + case GLFW.GLFW_KEY_GRAVE_ACCENT -> "BACKQUOTE"; + case GLFW.GLFW_KEY_WORLD_1 -> "WORLD_1"; + case GLFW.GLFW_KEY_WORLD_2 -> "WORLD_2"; + case GLFW.GLFW_KEY_ESCAPE -> "ESCAPE"; + case GLFW.GLFW_KEY_ENTER -> "ENTER"; + case GLFW.GLFW_KEY_TAB -> "TAB"; + case GLFW.GLFW_KEY_BACKSPACE -> "BACKSPACE"; + case GLFW.GLFW_KEY_INSERT -> "INSERT"; + case GLFW.GLFW_KEY_DELETE -> "DELETE"; + case GLFW.GLFW_KEY_RIGHT -> "ARROW_RIGHT"; + case GLFW.GLFW_KEY_LEFT -> "ARROW_LEFT"; + case GLFW.GLFW_KEY_DOWN -> "ARROW_DOWN"; + case GLFW.GLFW_KEY_UP -> "ARROW_UP"; + case GLFW.GLFW_KEY_PAGE_UP -> "PAGE_UP"; + case GLFW.GLFW_KEY_PAGE_DOWN -> "PAGE_DOWN"; + case GLFW.GLFW_KEY_HOME -> "HOME"; + case GLFW.GLFW_KEY_END -> "END"; + case GLFW.GLFW_KEY_CAPS_LOCK -> "CAPS_LOCK"; + case GLFW.GLFW_KEY_SCROLL_LOCK -> "SCROLL_LOCK"; + case GLFW.GLFW_KEY_NUM_LOCK -> "NUM_LOCK"; + case GLFW.GLFW_KEY_PRINT_SCREEN -> "PRINT_SCREEN"; + case GLFW.GLFW_KEY_PAUSE -> "PAUSE"; + case GLFW.GLFW_KEY_F1 -> "F1"; + case GLFW.GLFW_KEY_F2 -> "F2"; + case GLFW.GLFW_KEY_F3 -> "F3"; + case GLFW.GLFW_KEY_F4 -> "F4"; + case GLFW.GLFW_KEY_F5 -> "F5"; + case GLFW.GLFW_KEY_F6 -> "F6"; + case GLFW.GLFW_KEY_F7 -> "F7"; + case GLFW.GLFW_KEY_F8 -> "F8"; + case GLFW.GLFW_KEY_F9 -> "F9"; + case GLFW.GLFW_KEY_F10 -> "F10"; + case GLFW.GLFW_KEY_F11 -> "F11"; + case GLFW.GLFW_KEY_F12 -> "F12"; + case GLFW.GLFW_KEY_F13 -> "F13"; + case GLFW.GLFW_KEY_F14 -> "F14"; + case GLFW.GLFW_KEY_F15 -> "F15"; + case GLFW.GLFW_KEY_F16 -> "F16"; + case GLFW.GLFW_KEY_F17 -> "F17"; + case GLFW.GLFW_KEY_F18 -> "F18"; + case GLFW.GLFW_KEY_F19 -> "F19"; + case GLFW.GLFW_KEY_F20 -> "F20"; + case GLFW.GLFW_KEY_F21 -> "F21"; + case GLFW.GLFW_KEY_F22 -> "F22"; + case GLFW.GLFW_KEY_F23 -> "F23"; + case GLFW.GLFW_KEY_F24 -> "F24"; + case GLFW.GLFW_KEY_F25 -> "F25"; + case GLFW.GLFW_KEY_KP_0 -> "KP_0"; + case GLFW.GLFW_KEY_KP_1 -> "KP_1"; + case GLFW.GLFW_KEY_KP_2 -> "KP_2"; + case GLFW.GLFW_KEY_KP_3 -> "KP_3"; + case GLFW.GLFW_KEY_KP_4 -> "KP_4"; + case GLFW.GLFW_KEY_KP_5 -> "KP_5"; + case GLFW.GLFW_KEY_KP_6 -> "KP_6"; + case GLFW.GLFW_KEY_KP_7 -> "KP_7"; + case GLFW.GLFW_KEY_KP_8 -> "KP_8"; + case GLFW.GLFW_KEY_KP_9 -> "KP_9"; + case GLFW.GLFW_KEY_KP_DECIMAL -> "KP_DECIMAL"; + case GLFW.GLFW_KEY_KP_DIVIDE -> "KP_DIVIDE"; + case GLFW.GLFW_KEY_KP_MULTIPLY -> "KP_MULTIPLY"; + case GLFW.GLFW_KEY_KP_SUBTRACT -> "KP_SUBTRACT"; + case GLFW.GLFW_KEY_KP_ADD -> "KP_ADD"; + case GLFW.GLFW_KEY_KP_ENTER -> "KP_ENTER"; + case GLFW.GLFW_KEY_KP_EQUAL -> "KP_EQUAL"; + case GLFW.GLFW_KEY_LEFT_SHIFT -> "LEFT_SHIFT"; + case GLFW.GLFW_KEY_LEFT_CONTROL -> "LEFT_CONTROL"; + case GLFW.GLFW_KEY_LEFT_ALT -> "LEFT_ALT"; + case GLFW.GLFW_KEY_LEFT_SUPER -> "LEFT_SUPER"; + case GLFW.GLFW_KEY_RIGHT_SHIFT -> "RIGHT_SHIFT"; + case GLFW.GLFW_KEY_RIGHT_CONTROL -> "RIGHT_CONTROL"; + case GLFW.GLFW_KEY_RIGHT_ALT -> "RIGHT_ALT"; + case GLFW.GLFW_KEY_RIGHT_SUPER -> "RIGHT_SUPER"; + case GLFW.GLFW_KEY_MENU -> "MENU"; + default -> null; + }; + } + private static double scaledMouseX(Minecraft mc, int sw) { return mc.mouseHandler.xpos() * (double) sw / (double) mc.getWindow().getScreenWidth(); } @@ -691,9 +836,18 @@ public static void onScreenKeyPressed(ScreenEvent.KeyPressed.Pre event) { return; } int key = event.getKeyCode(); + String bindingName = keyCodeToBindingName(key, event.getScanCode()); + if (bindingName != null) { + XosViewportRuntime.sendKeyDownToEngine(bindingName); + } int modifiers = event.getModifiers(); - boolean ctrlDown = (modifiers & GLFW.GLFW_MOD_CONTROL) != 0; - if (ctrlDown) { + boolean commandDown = (modifiers & (GLFW.GLFW_MOD_CONTROL | GLFW.GLFW_MOD_SUPER)) != 0; + if (commandDown && (key == GLFW.GLFW_KEY_SLASH || key == GLFW.GLFW_KEY_KP_DIVIDE)) { + XosViewportRuntime.sendKeyCharToEngine(XOS_KEY_TOGGLE_COMMENT); + event.setCanceled(true); + return; + } + if (commandDown) { int actionCode = ctrlShortcutToActionCode(key, modifiers); if (actionCode != -1) { XosNative.onShortcut(actionCode); @@ -702,6 +856,7 @@ public static void onScreenKeyPressed(ScreenEvent.KeyPressed.Pre event) { } } if (key == GLFW.GLFW_KEY_ESCAPE) { + event.setCanceled(true); return; } if (key == GLFW.GLFW_KEY_F3) { @@ -713,7 +868,7 @@ public static void onScreenKeyPressed(ScreenEvent.KeyPressed.Pre event) { event.setCanceled(true); return; } - int nav = navigationKeyToCodePoint(key); + int nav = navigationKeyToCodePoint(key, modifiers); if (nav != -1) { XosViewportRuntime.sendKeyCharToEngine(nav); event.setCanceled(true); @@ -722,6 +877,28 @@ public static void onScreenKeyPressed(ScreenEvent.KeyPressed.Pre event) { event.setCanceled(true); } + @SubscribeEvent + public static void onScreenKeyReleased(ScreenEvent.KeyReleased.Pre event) { + if (!(event.getScreen() instanceof ChatScreen)) { + return; + } + Minecraft mc = Minecraft.getInstance(); + if (!shouldRouteKeyboardToXos(mc)) { + return; + } + int key = event.getKeyCode(); + if (key == GLFW.GLFW_KEY_ESCAPE) { + XosViewportRuntime.sendKeyUpToEngine("ESCAPE"); + event.setCanceled(true); + return; + } + String bindingName = keyCodeToBindingName(key, event.getScanCode()); + if (bindingName != null) { + XosViewportRuntime.sendKeyUpToEngine(bindingName); + } + event.setCanceled(true); + } + @SubscribeEvent public static void onScreenCharTyped(ScreenEvent.CharacterTyped.Pre event) { if (!(event.getScreen() instanceof ChatScreen)) { @@ -777,7 +954,12 @@ public static void onScreenRenderPost(ScreenEvent.Render.Post event) { boolean hover = panelContains(mx, my, mc); XosViewportRuntime.setPanelHovered(hover); - XosViewportRuntime.pumpFrame(mc, panelW, contentH); + XosViewportRuntime.setOverlayRenderPassActive(true); + try { + XosViewportRuntime.pumpFrame(mc, panelW, contentH); + } finally { + XosViewportRuntime.setOverlayRenderPassActive(false); + } float drawA = hover ? ALPHA_HOVER : ALPHA_IDLE; int neon = neonArgb(drawA); int neonBorder = neonBorderArgb(drawA); @@ -789,6 +971,11 @@ public static void onScreenRenderPost(ScreenEvent.Render.Post event) { // Reset tint so blit() uses framebuffer α as-uploaded (60% idle / 80% hover), not × leftover shader. RenderSystem.setShaderColor(1.0F, 1.0F, 1.0F, 1.0F); g.setColor(1.0F, 1.0F, 1.0F, 1.0F); + // Vision's offscreen world pass can leave GL state dirty (depth/scissor/cull). Force GUI-safe state. + RenderSystem.disableDepthTest(); + RenderSystem.depthMask(true); + RenderSystem.disableCull(); + RenderSystem.disableScissor(); if (minimized) { int bx = MINIMIZED_BTN_MARGIN; @@ -831,7 +1018,9 @@ public static void onScreenRenderPost(ScreenEvent.Render.Post event) { RenderSystem.disableBlend(); - updateResizeCursor(mc, sw, sh, mx, my); + if (!XosViewportRuntime.isMouseCaptureActive()) { + updateResizeCursor(mc, sw, sh, mx, my); + } return; } @@ -899,7 +1088,9 @@ public static void onScreenRenderPost(ScreenEvent.Render.Post event) { RenderSystem.disableBlend(); - updateResizeCursor(mc, sw, sh, mx, my); + if (!XosViewportRuntime.isMouseCaptureActive()) { + updateResizeCursor(mc, sw, sh, mx, my); + } } @SubscribeEvent @@ -1081,6 +1272,7 @@ public static void onRenderTickBackgroundPump(TickEvent.RenderTickEvent event) { return; } Minecraft mc = Minecraft.getInstance(); + XosViewportRuntime.flushDeferredVisionCaptures(mc); if (mc.screen instanceof ChatScreen) { return; } diff --git a/src/main/java/com/verbii/minekov/client/XosViewportRuntime.java b/src/main/java/com/verbii/minekov/client/XosViewportRuntime.java index 02997fe..ae44497 100644 --- a/src/main/java/com/verbii/minekov/client/XosViewportRuntime.java +++ b/src/main/java/com/verbii/minekov/client/XosViewportRuntime.java @@ -2,19 +2,24 @@ import ai.xlate.xos.XosNative; +import com.google.gson.JsonElement; import com.mojang.blaze3d.platform.NativeImage; +import com.mojang.serialization.JsonOps; import com.verbii.minekov.Minekov; import com.verbii.minekov.entities.RLOperator; +import com.verbii.minekov.pymodule.McPyModule; import net.minecraft.client.Minecraft; import net.minecraft.client.gui.GuiGraphics; import net.minecraft.client.renderer.texture.DynamicTexture; import net.minecraft.core.registries.BuiltInRegistries; +import net.minecraft.nbt.NbtOps; import net.minecraft.network.chat.Component; import net.minecraft.resources.ResourceLocation; import net.minecraft.world.entity.Entity; import net.minecraft.world.entity.LivingEntity; import net.minecraft.world.entity.ai.attributes.Attributes; import net.minecraft.world.entity.player.Inventory; +import net.minecraft.world.InteractionHand; import net.minecraft.world.item.ItemStack; import net.minecraft.world.level.storage.LevelResource; import net.minecraft.world.phys.Vec3; @@ -36,11 +41,13 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Comparator; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import org.lwjgl.system.MemoryUtil; +import org.lwjgl.glfw.GLFW; /** * Runs the xos engine (ball app via JNI) when {@link #setRunSession(boolean) run session} is on; @@ -73,6 +80,7 @@ public final class XosViewportRuntime { List.of( "balls_many.py", "demo_mod.py", + "controller.py", "inventory.py", "agent_controller.py", "look_at_me.py", @@ -121,6 +129,8 @@ public final class XosViewportRuntime { /** Set each frame before {@link #pumpFrame} (chat) or background pump (not hovered). */ private static boolean panelHovered; + /** True only while chat overlay draw is executing. Used to avoid re-entrant offscreen world renders. */ + private static volatile boolean overlayRenderPassActive; /** * Caps how often {@link #pumpFrame} runs the native {@link XosNative#tick} + texture upload. @@ -137,7 +147,16 @@ public final class XosViewportRuntime { private static volatile int maxPumpsPerSecond; private static long lastPumpNanos; - private static boolean hostBindingsRegistered; + private static int nativeCursorMode = GLFW.GLFW_CURSOR_NORMAL; + private static boolean nativeMouseLocked; + private static boolean nativeMouseHidden; + private static boolean lockAnchorInitialized; + private static double lockAnchorWindowX; + private static double lockAnchorWindowY; + private static float lockSyntheticLocalX; + private static float lockSyntheticLocalY; + private static final double[] CURSOR_X = new double[1]; + private static final double[] CURSOR_Y = new double[1]; /** * Rotation lock requested from mc Python bindings. Values are [yaw, pitch] and are re-applied each * frame so Minecraft AI/network updates do not drift agents off target. @@ -166,11 +185,24 @@ public static boolean isRunSession() { return runSession; } + static void setOverlayRenderPassActive(boolean active) { + overlayRenderPassActive = active; + } + + static boolean isOverlayRenderPassActive() { + return overlayRenderPassActive; + } + /** Native library loaded and {@link XosNative#init} completed (e.g. after {@link #prewarmEngine}). */ public static boolean isEngineReady() { return libraryOk && engineRunning; } + /** True when xos requests host-level cursor capture/hidden behavior. */ + public static boolean isMouseCaptureActive() { + return nativeMouseLocked || nativeMouseHidden; + } + /** * Starts or pauses the xos simulation. Pausing does not tear down the native * engine (library stays loaded, {@link XosNative#init} state kept) so Run resumes instantly. @@ -179,10 +211,44 @@ public static boolean isEngineReady() { public static void setRunSession(boolean active) { if (!active && runSession) { stopActiveExecution(); + releaseNativeMouseCapture(Minecraft.getInstance()); } runSession = active; } + private static void releaseNativeMouseCapture(Minecraft mc) { + if (mc == null || mc.getWindow() == null) { + nativeCursorMode = GLFW.GLFW_CURSOR_NORMAL; + nativeMouseLocked = false; + nativeMouseHidden = false; + lockAnchorInitialized = false; + return; + } + long win = mc.getWindow().getWindow(); + GLFW.glfwSetInputMode(win, GLFW.GLFW_CURSOR, GLFW.GLFW_CURSOR_NORMAL); + nativeCursorMode = GLFW.GLFW_CURSOR_NORMAL; + nativeMouseLocked = false; + nativeMouseHidden = false; + lockAnchorInitialized = false; + } + + private static void applyNativeMouseCaptureState(Minecraft mc, boolean locked, boolean hidden) { + if (mc == null || mc.getWindow() == null) { + return; + } + long win = mc.getWindow().getWindow(); + int mode = (locked || hidden) ? GLFW.GLFW_CURSOR_HIDDEN : GLFW.GLFW_CURSOR_NORMAL; + if (mode != nativeCursorMode) { + GLFW.glfwSetInputMode(win, GLFW.GLFW_CURSOR, mode); + nativeCursorMode = mode; + } + if (!locked) { + lockAnchorInitialized = false; + } + nativeMouseLocked = locked; + nativeMouseHidden = hidden; + } + /** Stops currently running Coder execution(s) inside xos, if engine is initialized. */ public static void stopActiveExecution() { if (!libraryOk || !engineRunning) { @@ -273,6 +339,9 @@ public static void pumpFrame(Minecraft mc, int guiBodyW, int guiBodyH) { long minNs = 1_000_000_000L / (long) cap; long now = System.nanoTime(); if (lastPumpNanos != 0L && now - lastPumpNanos < minNs) { + nativeMouseLocked = XosNative.isMouseLocked(); + nativeMouseHidden = XosNative.isMouseCursorHidden(); + applyNativeMouseCaptureState(mc, nativeMouseLocked, nativeMouseHidden || nativeMouseLocked); return; } lastPumpNanos = now; @@ -283,6 +352,9 @@ public static void pumpFrame(Minecraft mc, int guiBodyW, int guiBodyH) { applyForcedAgentRotations(mc); XosNative.setMinecraftViewportAlpha(panelHovered ? VIEWPORT_ALPHA_HOVER : VIEWPORT_ALPHA_IDLE); XosNative.tick(); + nativeMouseLocked = XosNative.isMouseLocked(); + nativeMouseHidden = XosNative.isMouseCursorHidden(); + applyNativeMouseCaptureState(mc, nativeMouseLocked, nativeMouseHidden || nativeMouseLocked); ByteBuffer buf = cachedPackedFrameBuffer; if (buf == null || nativeImage == null) { @@ -321,6 +393,28 @@ public static void syncPointer( if (!libraryOk || !engineRunning || !runSession || bodyW < 1 || bodyH < 1) { return; } + if (nativeMouseLocked && mc != null && mc.getWindow() != null) { + float s = (float) effectiveGuiScale(mc); + long win = mc.getWindow().getWindow(); + GLFW.glfwGetCursorPos(win, CURSOR_X, CURSOR_Y); + if (!lockAnchorInitialized) { + lockAnchorWindowX = CURSOR_X[0]; + lockAnchorWindowY = CURSOR_Y[0]; + lockSyntheticLocalX = (float) (mouseX - bodyLeft) * s; + lockSyntheticLocalY = (float) (mouseY - bodyTop) * s; + lockAnchorInitialized = true; + } else { + double dx = CURSOR_X[0] - lockAnchorWindowX; + double dy = CURSOR_Y[0] - lockAnchorWindowY; + if (Math.abs(dx) > 0.001 || Math.abs(dy) > 0.001) { + lockSyntheticLocalX += (float) dx * s; + lockSyntheticLocalY += (float) dy * s; + XosNative.onMouseMove(lockSyntheticLocalX, lockSyntheticLocalY); + GLFW.glfwSetCursorPos(win, lockAnchorWindowX, lockAnchorWindowY); + } + } + return; + } if (mouseX >= bodyLeft && mouseX < bodyLeft + bodyW && mouseY >= bodyTop @@ -360,6 +454,34 @@ public static void sendKeyCharToEngine(int codepoint) { XosNative.onKeyChar(codepoint); } + /** + * Forward low-latency key pressed state into xos (`self.state.keyboard.keys..is_pressed()`). + */ + public static void sendKeyDownToEngine(String bindingName) { + if (!runSession || bindingName == null || bindingName.isEmpty()) { + return; + } + tryLoadLibrary(); + if (!libraryOk || !engineRunning) { + return; + } + XosNative.onKeyDown(bindingName); + } + + /** + * Clear low-latency key pressed state in xos. + */ + public static void sendKeyUpToEngine(String bindingName) { + if (!runSession || bindingName == null || bindingName.isEmpty()) { + return; + } + tryLoadLibrary(); + if (!libraryOk || !engineRunning) { + return; + } + XosNative.onKeyUp(bindingName); + } + /** F3 → global FPS overlay toggle (matches desktop xos; not a text character). */ public static void sendF3ToEngine() { if (!runSession) { @@ -399,6 +521,7 @@ public static void disposeEngine() { stopActiveExecution(); runSession = false; if (!libraryOk || !engineRunning) { + releaseNativeMouseCapture(Minecraft.getInstance()); return; } Minecraft mc = Minecraft.getInstance(); @@ -418,6 +541,7 @@ public static void disposeEngine() { panelHovered = false; prewarmAttempted = false; lastPumpNanos = 0L; + releaseNativeMouseCapture(mc); } private static Path resolvePlayerCoderScriptsDirectory(Minecraft mc) { @@ -521,7 +645,15 @@ private static String encodeItemStack(ItemStack stack) { if (stack.isDamageableItem()) { durability = Integer.toString(Math.max(0, stack.getMaxDamage() - stack.getDamageValue())); } - String meta = stack.getTag() != null ? stack.getTag().toString() : "{}"; + String meta = "{}"; + if (stack.getTag() != null) { + try { + JsonElement asJson = NbtOps.INSTANCE.convertTo(JsonOps.INSTANCE, stack.getTag()); + meta = asJson.toString(); + } catch (Throwable ignored) { + meta = stack.getTag().toString(); + } + } return sanitizeItemWireValue(type) + ITEM_FIELD_SEPARATOR + count @@ -598,6 +730,20 @@ private static String encodePlayerInventoryEnderChest(Minecraft mc) { return out.toString(); } + private static String renderAgentViewRgbaHex(Minecraft mc, RLOperator agent, int reqW, int reqH) { + // Toggle renderer by swapping these calls. + // return TexturedWorldVision.renderAgentViewRgbaHex(mc, agent, reqW, reqH); + return BasicWorldVision.renderAgentViewRgbaHex(mc, agent, reqW, reqH); + // return EntityVision.renderAgentViewRgbaHex(mc, agent, reqW, reqH); + } + + static void flushDeferredVisionCaptures(Minecraft mc) { + // Keep paired with the selected renderer above. + // TexturedWorldVision.flushDeferredCaptures(mc); + BasicWorldVision.flushDeferredCaptures(mc); + // EntityVision.flushDeferredCaptures(mc); + } + private static double[] parseTriple(String raw, double fallbackX, double fallbackY, double fallbackZ) { if (raw == null || raw.isBlank()) { return new double[] {fallbackX, fallbackY, fallbackZ}; @@ -711,26 +857,7 @@ private static void setEntityVelocityWithServerMirror(Minecraft mc, Entity entit } private static void applyMovementImpulse(Entity entity, double forward, double strafe) { - if (entity instanceof RLOperator rl) { - double speed = movementImpulseFor(rl); - if (forward > 0.0) { - rl.moveTowards(180.0f, (float) speed); - return; - } - if (forward < 0.0) { - rl.moveTowards(0.0f, (float) speed); - return; - } - if (strafe < 0.0) { - rl.moveTowards(-90.0f, (float) speed); - return; - } - if (strafe > 0.0) { - rl.moveTowards(90.0f, (float) speed); - return; - } - } - + // Keep WASD strictly look-relative for every controllable entity (player + agents). double speed = movementImpulseFor(entity); double yawRad = Math.toRadians(entity.getYRot()); double fx = -Math.sin(yawRad); @@ -739,8 +866,10 @@ private static void applyMovementImpulse(Entity entity, double forward, double s double lz = fx; Vec3 cur = entity.getDeltaMovement(); - double vx = cur.x + (fx * forward + lx * strafe) * speed; - double vz = cur.z + (fz * forward + lz * strafe) * speed; + // Treat scripted WASD as desired movement for this tick (no momentum accumulation), + // so heading updates from mc.agents.rotations immediately affect travel direction. + double vx = (fx * forward + lx * strafe) * speed; + double vz = (fz * forward + lz * strafe) * speed; entity.setDeltaMovement(vx, cur.y, vz); entity.hurtMarked = true; } @@ -758,8 +887,15 @@ private static void applyLookDelta(Entity entity, float pitchDelta, float yawDel private static void applyEntityActionWithServerMirror( Minecraft mc, Entity entity, String action, String payload) { - applyEntityAction(entity, action, payload); Entity mirror = resolveServerMirrorEntity(mc, entity); + boolean preferServerMirror = + "left_click".equals(action) || "right_click".equals(action); + if (preferServerMirror && mirror != null && mirror != entity) { + applyEntityAction(mirror, action, payload); + return; + } + + applyEntityAction(entity, action, payload); if (mirror != null && mirror != entity) { applyEntityAction(mirror, action, payload); } @@ -780,6 +916,8 @@ private static void applyEntityAction(Entity entity, String action, String paylo entity.hurtMarked = true; } } + case "left_click" -> applyLeftClickAction(entity); + case "right_click" -> applyRightClickAction(entity); case "look" -> { float[] delta = parsePair(payload, 0.0f, 0.0f); // API order is look(pitch, yaw) @@ -789,6 +927,26 @@ private static void applyEntityAction(Entity entity, String action, String paylo } } + private static void applyLeftClickAction(Entity entity) { + if (entity instanceof RLOperator rl) { + // Try gun-fire path first (works for TACZ loadouts). + rl.shootForward(); + } + if (entity instanceof LivingEntity living) { + living.swing(InteractionHand.MAIN_HAND, true); + } + } + + private static void applyRightClickAction(Entity entity) { + if (entity instanceof RLOperator rl) { + rl.aim(true); + return; + } + if (entity instanceof LivingEntity living) { + living.swing(InteractionHand.OFF_HAND, true); + } + } + private static RLOperator findAgentById(Minecraft mc, String id) { if (mc == null || mc.level == null || id == null || id.isBlank()) { return null; @@ -808,6 +966,18 @@ private static void applyAgentRotation(RLOperator agent, float yaw, float pitch) agent.setYBodyRot(yaw); } + private static void applyAgentRotationWithServerMirror( + Minecraft mc, RLOperator agent, float yaw, float pitch) { + if (agent == null) { + return; + } + applyAgentRotation(agent, yaw, pitch); + Entity mirrorEntity = resolveServerMirrorEntity(mc, agent); + if (mirrorEntity instanceof RLOperator mirror && mirror != agent) { + applyAgentRotation(mirror, yaw, pitch); + } + } + private static void applyForcedAgentRotations(Minecraft mc) { if (forcedAgentRotations.isEmpty()) { return; @@ -817,7 +987,7 @@ private static void applyForcedAgentRotations(Minecraft mc) { if (rot == null || rot.length < 2) { continue; } - applyAgentRotation(agent, rot[0], rot[1]); + applyAgentRotationWithServerMirror(mc, agent, rot[0], rot[1]); } } @@ -841,6 +1011,9 @@ private static List collectAgents(Minecraft mc) { out.add(operator); } } + // Keep index-based Python APIs deterministic across calls: + // mc.agents.positions / rotations / move / look all rely on the same ordering. + out.sort(Comparator.comparing(RLOperator::getStringUUID)); return out; } @@ -1093,9 +1266,29 @@ def _parse_item(raw): durability = int(durability_text) except Exception: durability = None - meta = parts[3] if parts[3] else "{}" + meta_raw = parts[3] if parts[3] else "{}" + try: + meta_obj = xos.json.loads(meta_raw) + except Exception: + meta_obj = {} + meta = _normalize_meta(meta_obj) return Item(item_type=item_type, count=count, durability=durability, meta=meta) +def _normalize_meta(value): + if isinstance(value, dict): + out = {} + for key, sub in value.items(): + out[str(key)] = _normalize_meta(sub) + return out + if isinstance(value, list): + out = {} + for i, sub in enumerate(value): + out[str(i)] = _normalize_meta(sub) + return out + if value is None: + return "null" + return str(value) + def _parse_item_list(raw): if raw is None: return [] @@ -1104,6 +1297,31 @@ def _parse_item_list(raw): return [] return [_parse_item(part) for part in text.split("\\x1e")] +def _decode_rgba_hex_to_tensor(raw_hex, w, h): + if raw_hex is None: + return xos.zeros((h, w, 4), dtype="uint8") + text = str(raw_hex).strip() + need = int(w) * int(h) * 4 * 2 + if len(text) < need: + return xos.zeros((h, w, 4), dtype="uint8") + vals = [] + try: + for i in range(0, need, 2): + vals.append(int(text[i:i + 2], 16)) + except Exception: + return xos.zeros((h, w, 4), dtype="uint8") + ww = int(w) + hh = int(h) + row_stride = ww * 4 + flipped = [] + for y in range(hh): + row = vals[y * row_stride:(y + 1) * row_stride] + for x in range(ww - 1, -1, -1): + i = x * 4 + flipped.extend(row[i:i + 4]) + vals = flipped + return xos.tensor(vals, (h, w, 4), dtype="uint8") + class Actions: def __init__(self, entity_kind, entity_id=""): self._kind = str(entity_kind) @@ -1128,15 +1346,21 @@ def d(self): def jump(self): self._call("jump", "") + def left_click(self): + self._call("left_click", "") + + def right_click(self): + self._call("right_click", "") + def look(self, pitch, yaw): self._call("look", f"{float(pitch)},{float(yaw)}") class Item: - def __init__(self, item_type="minecraft:air", count=0, durability=None, meta="{}"): + def __init__(self, item_type="minecraft:air", count=0, durability=None, meta=None): self.type = str(item_type) self.count = int(count) self.durability = durability - self.meta = str(meta) if meta is not None else "{}" + self.meta = meta if isinstance(meta, dict) else {} def __repr__(self): return f"mc.Item(type={{{self.type}}}, count={self.count}, meta={self.meta})" @@ -1301,6 +1525,12 @@ def looking_velocity(self, value): def actions(self): return self._actions + def visual(self, h=128, w=128): + hh = int(h) + ww = int(w) + raw = __module__._host_call("agent_view_rgba", f"{self._id};{ww};{hh}") + return _decode_rgba_hex_to_tensor(raw, ww, hh) + class Agents: def _list(self): raw = __module__._host_call("agent_ids", "") @@ -1549,7 +1779,7 @@ private static String invokeHostBinding(Minecraft mc, String moduleName, String return; } float[] yp = parsePair(ypRaw, agent.getYRot(), agent.getXRot()); - applyAgentRotation(agent, yp[0], yp[1]); + applyAgentRotationWithServerMirror(mc, agent, yp[0], yp[1]); forcedAgentRotations.put(agent.getStringUUID(), new float[] {yp[0], yp[1]}); }); yield null; @@ -1567,8 +1797,9 @@ private static String invokeHostBinding(Minecraft mc, String moduleName, String return; } float yaw = parsePair(yawRaw + ",0", agent.getYRot(), agent.getXRot())[0]; - applyAgentRotation(agent, yaw, agent.getXRot()); - forcedAgentRotations.put(agent.getStringUUID(), new float[] {yaw, agent.getXRot()}); + float pitch = agent.getXRot(); + applyAgentRotationWithServerMirror(mc, agent, yaw, pitch); + forcedAgentRotations.put(agent.getStringUUID(), new float[] {yaw, pitch}); }); yield null; } @@ -1585,8 +1816,9 @@ private static String invokeHostBinding(Minecraft mc, String moduleName, String return; } float pitch = parsePair("0," + pitchRaw, agent.getYRot(), agent.getXRot())[1]; - applyAgentRotation(agent, agent.getYRot(), pitch); - forcedAgentRotations.put(agent.getStringUUID(), new float[] {agent.getYRot(), pitch}); + float yaw = agent.getYRot(); + applyAgentRotationWithServerMirror(mc, agent, yaw, pitch); + forcedAgentRotations.put(agent.getStringUUID(), new float[] {yaw, pitch}); }); yield null; } @@ -1647,6 +1879,30 @@ private static String invokeHostBinding(Minecraft mc, String moduleName, String }); yield null; } + case "agent_view_rgba" -> { + AtomicReference viewRef = new AtomicReference<>(""); + runOnClientThreadSync(mc, () -> { + String[] parts = arg.split(";", 3); + if (parts.length < 3) { + return; + } + String id = parts[0].trim(); + int w; + int h; + try { + w = Integer.parseInt(parts[1].trim()); + h = Integer.parseInt(parts[2].trim()); + } catch (Exception ignored) { + return; + } + RLOperator agent = findAgentById(mc, id); + if (agent == null) { + return; + } + viewRef.set(renderAgentViewRgbaHex(mc, agent, w, h)); + }); + yield viewRef.get(); + } case "entity_action" -> { runOnClientThreadSync(mc, () -> { String[] parts = arg.split(";", 4); @@ -1734,7 +1990,7 @@ private static String invokeHostBinding(Minecraft mc, String moduleName, String RLOperator agent = agents.get(i); float yaw = (float) rows.get(i)[0]; float pitch = (float) rows.get(i)[1]; - applyAgentRotation(agent, yaw, pitch); + applyAgentRotationWithServerMirror(mc, agent, yaw, pitch); forcedAgentRotations.put(agent.getStringUUID(), new float[] {yaw, pitch}); } }); @@ -1748,8 +2004,9 @@ private static String invokeHostBinding(Minecraft mc, String moduleName, String for (int i = 0; i < n; i++) { RLOperator agent = agents.get(i); float yaw = vals.get(i); - applyAgentRotation(agent, yaw, agent.getXRot()); - forcedAgentRotations.put(agent.getStringUUID(), new float[] {yaw, agent.getXRot()}); + float pitch = agent.getXRot(); + applyAgentRotationWithServerMirror(mc, agent, yaw, pitch); + forcedAgentRotations.put(agent.getStringUUID(), new float[] {yaw, pitch}); } }); yield null; @@ -1762,8 +2019,9 @@ private static String invokeHostBinding(Minecraft mc, String moduleName, String for (int i = 0; i < n; i++) { RLOperator agent = agents.get(i); float pitch = vals.get(i); - applyAgentRotation(agent, agent.getYRot(), pitch); - forcedAgentRotations.put(agent.getStringUUID(), new float[] {agent.getYRot(), pitch}); + float yaw = agent.getYRot(); + applyAgentRotationWithServerMirror(mc, agent, yaw, pitch); + forcedAgentRotations.put(agent.getStringUUID(), new float[] {yaw, pitch}); } }); yield null; @@ -1779,58 +2037,9 @@ private static String invokeHostBinding(Minecraft mc, String moduleName, String } private static void configureHostBindings(Minecraft mc) { - if (hostBindingsRegistered) { - return; - } - XosNative.setHostBindingCallback( + McPyModule.configureHostBindings( (moduleName, functionName, arg0) -> invokeHostBinding(mc, moduleName, functionName, arg0)); - XosNative.clearHostPythonModules(); - XosNative.registerHostPythonModule( - "mc", - new String[] { - "chat", - "player_position", - "player_set_position", - "player_rotation", - "player_set_rotation", - "player_set_yaw", - "player_set_pitch", - "player_velocity", - "player_set_velocity", - "player_looking_velocity", - "player_set_looking_velocity", - "player_inventory_armor", - "player_inventory_contents", - "player_inventory_hotbar", - "player_inventory_offhand", - "player_inventory_hand", - "player_inventory_enderchest", - "agent_ids", - "agent_position", - "agent_set_position", - "agent_rotation", - "agent_set_rotation", - "agent_set_yaw", - "agent_set_pitch", - "agent_velocity", - "agent_set_velocity", - "agent_looking_velocity", - "agent_set_looking_velocity", - "entity_action", - "agents_look", - "agents_move", - "agents_positions", - "agents_rotations", - "agents_yaws", - "agents_pitches", - "agents_set_positions", - "agents_set_rotations", - "agents_set_yaws", - "agents_set_pitches", - "__bootstrap__" - }); - hostBindingsRegistered = true; } private static boolean configureCoderScriptsDirectory(Minecraft mc) { diff --git a/src/main/java/com/verbii/minekov/entities/AIOperator.java b/src/main/java/com/verbii/minekov/entities/AIOperator.java index 29799c6..cc7e60c 100644 --- a/src/main/java/com/verbii/minekov/entities/AIOperator.java +++ b/src/main/java/com/verbii/minekov/entities/AIOperator.java @@ -79,25 +79,22 @@ public static AttributeSupplier.Builder createAttributes() { @Override public boolean canCollideWith(Entity entity) { - // Use the centralized interaction logic - // return TrainingIsolationHandler.shouldEntitiesInteract(this, entity); - return false; + return super.canCollideWith(entity); } @Override public void push(Entity entity) { - // if (!TrainingIsolationHandler.shouldEntitiesInteract(this, entity)) return; - // super.push(entity); + super.push(entity); } @Override public boolean canBeCollidedWith() { - return false; + return true; } @Override public boolean isPushable() { - return false; + return true; } @Override diff --git a/src/main/java/com/verbii/minekov/pymodule/McPyModule.java b/src/main/java/com/verbii/minekov/pymodule/McPyModule.java new file mode 100644 index 0000000..c0e7057 --- /dev/null +++ b/src/main/java/com/verbii/minekov/pymodule/McPyModule.java @@ -0,0 +1,70 @@ +package com.verbii.minekov.pymodule; + +import ai.xlate.xos.XosNative; + +public final class McPyModule { + private static boolean hostBindingsRegistered; + + private static final String[] EXPORTED_FUNCTIONS = { + "chat", + "player_position", + "player_set_position", + "player_rotation", + "player_set_rotation", + "player_set_yaw", + "player_set_pitch", + "player_velocity", + "player_set_velocity", + "player_looking_velocity", + "player_set_looking_velocity", + "player_inventory_armor", + "player_inventory_contents", + "player_inventory_hotbar", + "player_inventory_offhand", + "player_inventory_hand", + "player_inventory_enderchest", + "agent_ids", + "agent_position", + "agent_set_position", + "agent_rotation", + "agent_set_rotation", + "agent_set_yaw", + "agent_set_pitch", + "agent_velocity", + "agent_set_velocity", + "agent_looking_velocity", + "agent_set_looking_velocity", + "agent_view_rgba", + "entity_action", + "agents_look", + "agents_move", + "agents_positions", + "agents_rotations", + "agents_yaws", + "agents_pitches", + "agents_set_positions", + "agents_set_rotations", + "agents_set_yaws", + "agents_set_pitches", + "__bootstrap__" + }; + + private McPyModule() {} + + @FunctionalInterface + public interface HostBindingInvoker { + String invoke(String moduleName, String functionName, String arg0); + } + + public static void configureHostBindings(HostBindingInvoker invoker) { + if (hostBindingsRegistered) { + return; + } + XosNative.setHostBindingCallback( + (moduleName, functionName, arg0) -> + invoker.invoke(moduleName, functionName, arg0)); + XosNative.clearHostPythonModules(); + XosNative.registerHostPythonModule("mc", EXPORTED_FUNCTIONS); + hostBindingsRegistered = true; + } +} diff --git a/src/main/resources/xos/examples/agent_controller.py b/src/main/resources/xos/examples/agent_controller.py index 3a76a2f..b173863 100644 --- a/src/main/resources/xos/examples/agent_controller.py +++ b/src/main/resources/xos/examples/agent_controller.py @@ -1,6 +1,107 @@ import mc +import xos -print(mc.player.position) +LOOK_SENS = 0.08 +PITCH_MIN = -89.0 +PITCH_MAX = 89.0 +CLICK_REPEAT_SEC = 0.12 -for agent in mc.agents: - mc.chat(agent.position) + +class AgentController(xos.Application): + def __init__(self): + super().__init__(max_fps=60) + self.jump_cooldown_until = 0.0 + self.left_click_cooldown_until = 0.0 + self.right_click_cooldown_until = 0.0 + self.view_w = 128 + self.view_h = 128 + self.look_yaw = None + self.look_pitch = None + self.look_capture_active = False + self.prev_left_click = False + self.prev_right_click = False + self.prev_escape = False + self.last_look_x = None + self.last_look_y = None + + def _now(self): + return xos.time.perf_counter() + + def tick(self): + agents = list(mc.agents) + if len(agents) == 0: + self.frame.clear((10, 10, 10, 255)) + return + + agent = agents[0] + now = self._now() + keys = self.state.keyboard.keys + mouse = self.state.mouse + if self.look_yaw is None or self.look_pitch is None: + self.look_yaw, self.look_pitch = map(float, agent.rotation) + + if keys.W.is_pressed(): + agent.actions.w() + if keys.A.is_pressed(): + agent.actions.a() + if keys.S.is_pressed(): + agent.actions.s() + if keys.D.is_pressed(): + agent.actions.d() + if keys.SPACE.is_pressed() and now >= self.jump_cooldown_until: + agent.actions.jump() + self.jump_cooldown_until = now + 0.25 + + left_click_down = mouse.LEFT_CLICK.is_pressed() + right_click_btn = getattr(mouse, "RIGHT_CLICK", None) + right_click_down = right_click_btn.is_pressed() if right_click_btn is not None else False + escape_down = keys.ESCAPE.is_pressed() + entered_look_this_tick = False + if left_click_down and not self.prev_left_click: + self.look_capture_active = True + entered_look_this_tick = True + self.last_look_x = float(mouse.x) + self.last_look_y = float(mouse.y) + if escape_down and not self.prev_escape: + self.look_capture_active = False + + if self.look_capture_active: + mouse.lock(); mouse.hide() + if self.last_look_x is None or self.last_look_y is None: + self.last_look_x, self.last_look_y = float(mouse.x), float(mouse.y) + mdx = float(mouse.x) - self.last_look_x + mdy = float(mouse.y) - self.last_look_y + self.last_look_x, self.last_look_y = float(mouse.x), float(mouse.y) + self.look_yaw += mdx * LOOK_SENS + self.look_pitch = max(PITCH_MIN, min(PITCH_MAX, self.look_pitch + mdy * LOOK_SENS)) + # Apply latest aim first so click actions use current look direction. + agent.rotation = (self.look_yaw, self.look_pitch) + if not entered_look_this_tick: + left_click_edge = left_click_down and not self.prev_left_click + right_click_edge = right_click_down and not self.prev_right_click + if left_click_down and (left_click_edge or now >= self.left_click_cooldown_until): + agent.actions.left_click() + self.left_click_cooldown_until = now + CLICK_REPEAT_SEC + if right_click_down and (right_click_edge or now >= self.right_click_cooldown_until): + agent.actions.right_click() + self.right_click_cooldown_until = now + CLICK_REPEAT_SEC + else: + mouse.unlock(); mouse.show() + self.last_look_x = None + self.last_look_y = None + self.prev_left_click = left_click_down + self.prev_right_click = right_click_down + self.prev_escape = escape_down + agent.rotation = (self.look_yaw, self.look_pitch) + + view = agent.visual(h=self.view_h, w=self.view_w) + try: + self.frame.clear((0, 0, 0, 255)) + xos.rasterizer._frame_blit_aspect_fit_norm_rect(view, 0.0, 0.0, 1.0, 1.0) + except Exception as e: + print("agent_controller render error:", e) + self.frame.clear((0, 0, 0, 255)) + + +if __name__ == "__main__": + AgentController().run() diff --git a/src/main/resources/xos/examples/controller.py b/src/main/resources/xos/examples/controller.py new file mode 100644 index 0000000..e530c88 --- /dev/null +++ b/src/main/resources/xos/examples/controller.py @@ -0,0 +1,106 @@ +import xos + +BG_COLOR = (8, 10, 16, 255) +BALL_COLOR = (80, 220, 255, 255) +NOSE_COLOR = (255, 255, 255, 255) + +BALL_RADIUS_NORM = 0.024 +NOSE_RADIUS_NORM = 0.008 +NOSE_DISTANCE_NORM = BALL_RADIUS_NORM * 1.6 + +# Units: normalized-screen-units. +THRUST = 2.2 +DRAG_PER_SEC = 2.8 +MAX_SPEED = 0.95 +MAX_DT = 1.0 / 30.0 + + +def _wrap01(value): + return value % 1.0 + + +def _normalized_or_zero(x, y): + mag = xos.math.sqrt(x * x + y * y) + if mag <= 1e-8: + return 0.0, 0.0 + return x / mag, y / mag + + +def _limit_speed(vx, vy, limit): + speed = xos.math.sqrt(vx * vx + vy * vy) + if speed <= limit or speed <= 1e-8: + return vx, vy, speed + scale = limit / speed + return vx * scale, vy * scale, limit + + +class ControllerApp(xos.Application): + def __init__(self): + super().__init__(max_fps=120) + self.x = 0.5 + self.y = 0.5 + self.vx = 0.0 + self.vy = 0.0 + + def _input_axis(self): + keys = self.state.keyboard.keys + mouse = self.state.mouse + ax = 0.0 + ay = 0.0 + if keys.W.is_pressed(): + ay -= 1.0 + if keys.A.is_pressed(): + ax -= 1.0 + if keys.S.is_pressed(): + ay += 1.0 + if keys.D.is_pressed(): + ax += 1.0 + if mouse.LEFT_CLICK.is_pressed(): + mouse.lock(); mouse.hide() + ax += mouse.dx * 0.02 + ay += mouse.dy * 0.02 + else: + mouse.unlock(); mouse.show() + return _normalized_or_zero(ax, ay) + + def tick(self): + dt = min(MAX_DT, max(0.0, self.dt)) + ax, ay = self._input_axis() + + # Apply thrust from current directional intent. + self.vx += ax * THRUST * dt + self.vy += ay * THRUST * dt + + # Linear drag: easier to reason about than exponential damping. + drag = max(0.0, 1.0 - DRAG_PER_SEC * dt) + self.vx *= drag + self.vy *= drag + self.vx, self.vy, speed = _limit_speed(self.vx, self.vy, MAX_SPEED) + + self.x = _wrap01(self.x + self.vx * dt) + self.y = _wrap01(self.y + self.vy * dt) + + self.frame.clear(BG_COLOR) + w, h = self.get_width(), self.get_height() + px = self.x * w + py = self.y * h + scale_px = max(w, h) + + xos.rasterizer.circles(self.frame, [(px, py)], [BALL_RADIUS_NORM * scale_px], BALL_COLOR) + + if speed > 0.02: + nx, ny = self.vx / speed, self.vy / speed + else: + nx, ny = 0.0, -1.0 + + nose_x = _wrap01(self.x + nx * NOSE_DISTANCE_NORM) * w + nose_y = _wrap01(self.y + ny * NOSE_DISTANCE_NORM) * h + xos.rasterizer.circles( + self.frame, + [(nose_x, nose_y)], + [NOSE_RADIUS_NORM * scale_px], + NOSE_COLOR, + ) + +if __name__ == "__main__": + ControllerApp().run() diff --git a/src/main/resources/xos/examples/walk_towards_me.py b/src/main/resources/xos/examples/walk_towards_me.py index 01ef582..3089b7e 100644 --- a/src/main/resources/xos/examples/walk_towards_me.py +++ b/src/main/resources/xos/examples/walk_towards_me.py @@ -1,31 +1,33 @@ import mc import xos + + + while True: agent_positions = mc.agents.positions num_agents = int(agent_positions.shape[0]) if len(agent_positions.shape) > 0 else 0 - if num_agents <= 0: - xos.sleep(0.1) - continue - player_position = mc.player.position - player_positions = xos.tensor( - [player_position[0], player_position[1], player_position[2]], (1, 3) - ).repeat(num_agents, axis=0) + # Batch player position to (N,3) tensor, index-aligned to mc.agents + p = mc.player.position + player_positions = ( + xos.tensor([p[0], p[1], p[2]], (1, 3)) + .repeat(num_agents, axis=0) + ) delta = player_positions - agent_positions dx = delta[:, 0] dy = delta[:, 1] dz = delta[:, 2] - target_yaws = xos.math.degrees(xos.math.atan2(dz, dx)) - 90.0 + # Vectorized look-at in degrees (Minecraft yaw/pitch convention) + yaws = xos.math.degrees(xos.math.atan2(dz, dx)) - 90.0 horizontal = xos.math.sqrt(dx * dx + dz * dz) - target_pitches = -xos.math.degrees(xos.math.atan2(dy, horizontal)) - target_rotations = xos.stack([target_pitches, target_yaws], axis=1) + pitches = -xos.math.degrees(xos.math.atan2(dy, horizontal)) - current_rotations = mc.agents.rotations - look_deltas = target_rotations - current_rotations - mc.agents.look(look_deltas) - mc.agents.move((True, False, False, False)) + rotations_tensor = xos.stack([yaws, pitches], axis=1) + mc.agents.rotations = rotations_tensor - xos.sleep(0.1) + print("updated", num_agents, "agent rotations to look at player") + mc.agents.move((True, False, False, False)) + xos.sleep(0.1) \ No newline at end of file diff --git a/src/xos/examples/agent_controller.py b/src/xos/examples/agent_controller.py index 3a76a2f..291d659 100644 --- a/src/xos/examples/agent_controller.py +++ b/src/xos/examples/agent_controller.py @@ -1,6 +1,110 @@ import mc +import xos -print(mc.player.position) +LOOK_SENS = 0.08 +PITCH_MIN = -89.0 +PITCH_MAX = 89.0 +CLICK_REPEAT_SEC = 0.12 -for agent in mc.agents: - mc.chat(agent.position) + +class AgentController(xos.Application): + def __init__(self): + super().__init__(max_fps=60) + self.jump_cooldown_until = 0.0 + self.left_click_cooldown_until = 0.0 + self.right_click_cooldown_until = 0.0 + self.view_w = 128 + self.view_h = 128 + self.look_yaw = None + self.look_pitch = None + self.look_capture_active = False + self.prev_left_click = False + self.prev_right_click = False + self.prev_escape = False + self.last_look_x = None + self.last_look_y = None + + def _now(self): + return xos.time.perf_counter() + + def tick(self): + agents = list(mc.agents) + if len(agents) == 0: + self.frame.clear((10, 10, 10, 255)) + return + + agent = agents[0] + now = self._now() + keys = self.state.keyboard.keys + mouse = self.state.mouse + if self.look_yaw is None or self.look_pitch is None: + self.look_yaw, self.look_pitch = map(float, agent.rotation) + + # Movement controls (single controllable agent: index 0). + if keys.W.is_pressed(): + agent.actions.w() + if keys.A.is_pressed(): + agent.actions.a() + if keys.S.is_pressed(): + agent.actions.s() + if keys.D.is_pressed(): + agent.actions.d() + if keys.SPACE.is_pressed() and now >= self.jump_cooldown_until: + agent.actions.jump() + self.jump_cooldown_until = now + 0.25 + + left_click_down = mouse.LEFT_CLICK.is_pressed() + right_click_btn = getattr(mouse, "RIGHT_CLICK", None) + right_click_down = right_click_btn.is_pressed() if right_click_btn is not None else False + escape_down = keys.ESCAPE.is_pressed() + entered_look_this_tick = False + if left_click_down and not self.prev_left_click: + self.look_capture_active = True + entered_look_this_tick = True + self.last_look_x = float(mouse.x) + self.last_look_y = float(mouse.y) + if escape_down and not self.prev_escape: + self.look_capture_active = False + + if self.look_capture_active: + mouse.lock(); mouse.hide() + if self.last_look_x is None or self.last_look_y is None: + self.last_look_x, self.last_look_y = float(mouse.x), float(mouse.y) + mdx = float(mouse.x) - self.last_look_x + mdy = float(mouse.y) - self.last_look_y + self.last_look_x, self.last_look_y = float(mouse.x), float(mouse.y) + self.look_yaw += mdx * LOOK_SENS + self.look_pitch = max(PITCH_MIN, min(PITCH_MAX, self.look_pitch + mdy * LOOK_SENS)) + # Apply latest aim first so click actions use current look direction. + agent.rotation = (self.look_yaw, self.look_pitch) + if not entered_look_this_tick: + left_click_edge = left_click_down and not self.prev_left_click + right_click_edge = right_click_down and not self.prev_right_click + if left_click_down and (left_click_edge or now >= self.left_click_cooldown_until): + agent.actions.left_click() + self.left_click_cooldown_until = now + CLICK_REPEAT_SEC + if right_click_down and (right_click_edge or now >= self.right_click_cooldown_until): + agent.actions.right_click() + self.right_click_cooldown_until = now + CLICK_REPEAT_SEC + else: + mouse.unlock(); mouse.show() + self.last_look_x = None + self.last_look_y = None + self.prev_left_click = left_click_down + self.prev_right_click = right_click_down + self.prev_escape = escape_down + agent.rotation = (self.look_yaw, self.look_pitch) + + # Fixed-size viewport from the agent perspective. + view = agent.visual(h=self.view_h, w=self.view_w) + try: + # Fill the live viewport with letterboxed aspect-fit scaling of the 128x128 feed. + self.frame.clear((0, 0, 0, 255)) + xos.rasterizer._frame_blit_aspect_fit_norm_rect(view, 0.0, 0.0, 1.0, 1.0) + except Exception as e: + print("agent_controller render error:", e) + self.frame.clear((0, 0, 0, 255)) + + +if __name__ == "__main__": + AgentController().run() diff --git a/src/xos/examples/controller.py b/src/xos/examples/controller.py new file mode 100644 index 0000000..e530c88 --- /dev/null +++ b/src/xos/examples/controller.py @@ -0,0 +1,106 @@ +import xos + +BG_COLOR = (8, 10, 16, 255) +BALL_COLOR = (80, 220, 255, 255) +NOSE_COLOR = (255, 255, 255, 255) + +BALL_RADIUS_NORM = 0.024 +NOSE_RADIUS_NORM = 0.008 +NOSE_DISTANCE_NORM = BALL_RADIUS_NORM * 1.6 + +# Units: normalized-screen-units. +THRUST = 2.2 +DRAG_PER_SEC = 2.8 +MAX_SPEED = 0.95 +MAX_DT = 1.0 / 30.0 + + +def _wrap01(value): + return value % 1.0 + + +def _normalized_or_zero(x, y): + mag = xos.math.sqrt(x * x + y * y) + if mag <= 1e-8: + return 0.0, 0.0 + return x / mag, y / mag + + +def _limit_speed(vx, vy, limit): + speed = xos.math.sqrt(vx * vx + vy * vy) + if speed <= limit or speed <= 1e-8: + return vx, vy, speed + scale = limit / speed + return vx * scale, vy * scale, limit + + +class ControllerApp(xos.Application): + def __init__(self): + super().__init__(max_fps=120) + self.x = 0.5 + self.y = 0.5 + self.vx = 0.0 + self.vy = 0.0 + + def _input_axis(self): + keys = self.state.keyboard.keys + mouse = self.state.mouse + ax = 0.0 + ay = 0.0 + if keys.W.is_pressed(): + ay -= 1.0 + if keys.A.is_pressed(): + ax -= 1.0 + if keys.S.is_pressed(): + ay += 1.0 + if keys.D.is_pressed(): + ax += 1.0 + if mouse.LEFT_CLICK.is_pressed(): + mouse.lock(); mouse.hide() + ax += mouse.dx * 0.02 + ay += mouse.dy * 0.02 + else: + mouse.unlock(); mouse.show() + return _normalized_or_zero(ax, ay) + + def tick(self): + dt = min(MAX_DT, max(0.0, self.dt)) + ax, ay = self._input_axis() + + # Apply thrust from current directional intent. + self.vx += ax * THRUST * dt + self.vy += ay * THRUST * dt + + # Linear drag: easier to reason about than exponential damping. + drag = max(0.0, 1.0 - DRAG_PER_SEC * dt) + self.vx *= drag + self.vy *= drag + self.vx, self.vy, speed = _limit_speed(self.vx, self.vy, MAX_SPEED) + + self.x = _wrap01(self.x + self.vx * dt) + self.y = _wrap01(self.y + self.vy * dt) + + self.frame.clear(BG_COLOR) + w, h = self.get_width(), self.get_height() + px = self.x * w + py = self.y * h + scale_px = max(w, h) + + xos.rasterizer.circles(self.frame, [(px, py)], [BALL_RADIUS_NORM * scale_px], BALL_COLOR) + + if speed > 0.02: + nx, ny = self.vx / speed, self.vy / speed + else: + nx, ny = 0.0, -1.0 + + nose_x = _wrap01(self.x + nx * NOSE_DISTANCE_NORM) * w + nose_y = _wrap01(self.y + ny * NOSE_DISTANCE_NORM) * h + xos.rasterizer.circles( + self.frame, + [(nose_x, nose_y)], + [NOSE_RADIUS_NORM * scale_px], + NOSE_COLOR, + ) + +if __name__ == "__main__": + ControllerApp().run() diff --git a/src/xos/examples/inventory.py b/src/xos/examples/inventory.py index a05a5f1..7235493 100644 --- a/src/xos/examples/inventory.py +++ b/src/xos/examples/inventory.py @@ -1,12 +1,17 @@ import mc item = mc.player.inventory.hand -mc.chat(str(item)) -print(item) +#mc.chat(str(item)) +#print(item) + +hand = mc.player.inventory.hand +print(hand.type) +print(type(hand.meta)) +#print(hand.meta["GunID"]) +#print("hand:", hand) +#print("offhand:", mc.player.inventory.offhand) +#print("hotbar:", mc.player.inventory.hotbar) +#print("armor:", mc.player.inventory.armor) +#print("contents:", mc.player.inventory.contents) +#print("enderchest:", mc.player.inventory.enderchest) -print("hand:", mc.player.inventory.hand) -print("offhand:", mc.player.inventory.offhand) -print("hotbar:", mc.player.inventory.hotbar) -print("armor:", mc.player.inventory.armor) -print("contents:", mc.player.inventory.contents) -print("enderchest:", mc.player.inventory.enderchest) diff --git a/src/xos/examples/walk_towards_me.py b/src/xos/examples/walk_towards_me.py index 01ef582..3089b7e 100644 --- a/src/xos/examples/walk_towards_me.py +++ b/src/xos/examples/walk_towards_me.py @@ -1,31 +1,33 @@ import mc import xos + + + while True: agent_positions = mc.agents.positions num_agents = int(agent_positions.shape[0]) if len(agent_positions.shape) > 0 else 0 - if num_agents <= 0: - xos.sleep(0.1) - continue - player_position = mc.player.position - player_positions = xos.tensor( - [player_position[0], player_position[1], player_position[2]], (1, 3) - ).repeat(num_agents, axis=0) + # Batch player position to (N,3) tensor, index-aligned to mc.agents + p = mc.player.position + player_positions = ( + xos.tensor([p[0], p[1], p[2]], (1, 3)) + .repeat(num_agents, axis=0) + ) delta = player_positions - agent_positions dx = delta[:, 0] dy = delta[:, 1] dz = delta[:, 2] - target_yaws = xos.math.degrees(xos.math.atan2(dz, dx)) - 90.0 + # Vectorized look-at in degrees (Minecraft yaw/pitch convention) + yaws = xos.math.degrees(xos.math.atan2(dz, dx)) - 90.0 horizontal = xos.math.sqrt(dx * dx + dz * dz) - target_pitches = -xos.math.degrees(xos.math.atan2(dy, horizontal)) - target_rotations = xos.stack([target_pitches, target_yaws], axis=1) + pitches = -xos.math.degrees(xos.math.atan2(dy, horizontal)) - current_rotations = mc.agents.rotations - look_deltas = target_rotations - current_rotations - mc.agents.look(look_deltas) - mc.agents.move((True, False, False, False)) + rotations_tensor = xos.stack([yaws, pitches], axis=1) + mc.agents.rotations = rotations_tensor - xos.sleep(0.1) + print("updated", num_agents, "agent rotations to look at player") + mc.agents.move((True, False, False, False)) + xos.sleep(0.1) \ No newline at end of file