diff --git a/demos/22_vm_demo.py b/demos/22_vm_demo.py index 0fa3384..e8da2cb 100644 --- a/demos/22_vm_demo.py +++ b/demos/22_vm_demo.py @@ -18,6 +18,7 @@ # 4. Run demo: python demos/22_vm_demo.py # 5. Watch: open http://localhost:8430 -> Live VM tab """ + from __future__ import annotations import base64 @@ -34,9 +35,9 @@ from PIL import Image # ── Config ────────────────────────────────────── -VM_IP = os.environ.get("VM_IP", "192.168.64.13") -VNC_USER = os.environ.get("VNC_USER", "admin") -VNC_PASS = os.environ.get("VNC_PASS", "admin") +VM_IP = os.environ.get("VM_IP", "10.0.0.1") +VNC_USER = os.environ.get("VNC_USER", "changeme") +VNC_PASS = os.environ.get("VNC_PASS", "changeme") CUA_URL = f"http://{VM_IP}:8000/cmd" # For run_command only MODEL = "anthropic/claude-sonnet-4.6" LABWORK_URL = os.environ.get("LABWORK_URL", "http://localhost:8430") @@ -59,15 +60,20 @@ # ── VNC helpers (via vncdo CLI) ───────────────── + def _vncdo(*args: str, timeout: int = 15) -> subprocess.CompletedProcess: """Run vncdo command with ARD auth credentials.""" cmd = [ sys.executable.replace("python", "vncdo").replace( - "bin/python", "bin/vncdo", + "bin/python", + "bin/vncdo", ), - "-s", VM_IP, - "--username", VNC_USER, - "--password", VNC_PASS, + "-s", + VM_IP, + "--username", + VNC_USER, + "--password", + VNC_PASS, *args, ] # Fallback: find vncdo next to python @@ -106,17 +112,29 @@ def vnc_type_command(cmd: str) -> None: """ # Triple-click to select all text in command line, then type + Return _vncdo( - "move", str(CMD_X), str(CMD_Y), - "pause", "0.2", - "click", "1", - "pause", "0.05", - "click", "1", - "pause", "0.05", - "click", "1", - "pause", "0.3", - "type", cmd, - "pause", "0.3", - "key", "return", + "move", + str(CMD_X), + str(CMD_Y), + "pause", + "0.2", + "click", + "1", + "pause", + "0.05", + "click", + "1", + "pause", + "0.05", + "click", + "1", + "pause", + "0.3", + "type", + cmd, + "pause", + "0.3", + "key", + "return", ) @@ -125,6 +143,7 @@ def vnc_type_command(cmd: str) -> None: # CUA keyboard/mouse don't reach Java/Swing apps in macOS VMs. # We only use CUA for run_command (launching apps, killing processes). + def _cua_cmd(command: str, params: dict | None = None) -> dict: """Send a command to CUA server, return parsed response.""" body: dict = {"command": command} @@ -145,6 +164,7 @@ def cua_run(command: str) -> dict: # ── Stream helpers ────────────────────────────── + def push_frame(jpeg_bytes: bytes) -> None: """Push a JPEG frame to labwork-web MJPEG stream.""" try: @@ -190,41 +210,47 @@ def screenshot_and_push() -> tuple[str, bytes]: # ── VLM ───────────────────────────────────────── + def ask_sonnet(client: OpenAI, screenshot_b64: str, question: str) -> str: """Send screenshot + question to Sonnet 4.6, get text response.""" response = client.chat.completions.create( model=MODEL, max_tokens=512, - messages=[{ - "role": "user", - "content": [ - { - "type": "image_url", - "image_url": { - "url": f"data:image/png;base64,{screenshot_b64}", + messages=[ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": { + "url": f"data:image/png;base64,{screenshot_b64}", + }, + }, + { + "type": "text", + "text": ( + "This is a 1280x960 screenshot of macOS with " + "Bruker TopSpin 5.0 NMR software.\n" + f"{question}\n\n" + "Return ONLY JSON, no markdown fences." + ), }, - }, - { - "type": "text", - "text": ( - "This is a 1280x960 screenshot of macOS with " - "Bruker TopSpin 5.0 NMR software.\n" - f"{question}\n\n" - "Return ONLY JSON, no markdown fences." - ), - }, - ], - }], + ], + } + ], ) return response.choices[0].message.content.strip() def ask_verify( - client: OpenAI, screenshot_b64: str, check: str, + client: OpenAI, + screenshot_b64: str, + check: str, ) -> dict: """Ask Sonnet to verify a condition.""" text = ask_sonnet( - client, screenshot_b64, + client, + screenshot_b64, f'{check}\nReturn: {{"ok": true/false, "description": "what you see"}}', ) m = re.search(r"\{[^}]+\}", text) @@ -238,6 +264,7 @@ def ask_verify( # ── Pipeline Steps ────────────────────────────── + def step_ensure_topspin(ai: OpenAI) -> bool: """Ensure TopSpin is open and visible.""" push_log(">>> Ensure TopSpin visible", status="operating") @@ -247,7 +274,8 @@ def step_ensure_topspin(ai: OpenAI) -> bool: b64, _ = screenshot_and_push() result = ask_verify( - ai, b64, + ai, + b64, "Is Bruker TopSpin 5.0 open with its main window visible? " "Look for the TopSpin toolbar, spectrum area, and command line.", ) @@ -282,7 +310,8 @@ def step_load_dataset(ai: OpenAI) -> bool: b64, _ = screenshot_and_push() result = ask_verify( - ai, b64, + ai, + b64, "Has NMR data been loaded? Look for a spectrum plot " "(FID or frequency domain) in the main panel, OR dataset " "info in the title area.", @@ -296,7 +325,9 @@ def step_load_dataset(ai: OpenAI) -> bool: time.sleep(5) b64, _ = screenshot_and_push() result = ask_verify( - ai, b64, "Is there ANY spectrum or data displayed in TopSpin?", + ai, + b64, + "Is there ANY spectrum or data displayed in TopSpin?", ) ok = result.get("ok", False) push_log(f" {'OK' if ok else 'WARN'} {result.get('description', '')}") @@ -321,7 +352,8 @@ def step_run_command( if handle_dialog: for _dlg in range(3): dlg = ask_verify( - ai, b64, + ai, + b64, "Is there a dialog/popup window visible in the CENTER? " "NOT a notification. Set ok=true ONLY for centered " "dialogs with Close/OK/Cancel buttons.", @@ -336,8 +368,7 @@ def step_run_command( result = ask_verify(ai, b64, verify_prompt) ok = result.get("ok", False) push_log( - f" {'OK' if ok else 'WARN'} {step_name}: " - f"{result.get('description', '')}", + f" {'OK' if ok else 'WARN'} {step_name}: {result.get('description', '')}", ) return ok @@ -348,7 +379,8 @@ def step_verify_result(ai: OpenAI) -> bool: b64, _ = screenshot_and_push() result = ask_verify( - ai, b64, + ai, + b64, "Describe the NMR spectrum visible in TopSpin. Report ok=true " "if you can see: (1) NMR peaks in the spectrum display, " "(2) a chemical shift axis (ppm) at the bottom, " @@ -362,6 +394,7 @@ def step_verify_result(ai: OpenAI) -> bool: # ── Main ──────────────────────────────────────── + def main() -> int: print( f"\n{B}{'=' * 55}{RST}\n" @@ -375,7 +408,11 @@ def main() -> int: api_key = os.environ.get("OPENROUTER_API_KEY") if not api_key: env_path = os.path.join( - os.path.dirname(__file__), "..", "..", "labwork-web", ".env", + os.path.dirname(__file__), + "..", + "..", + "labwork-web", + ".env", ) if os.path.exists(env_path): with open(env_path) as f: @@ -431,8 +468,7 @@ def main() -> int: cmd="efp", step_name="Fourier Transform", verify_prompt=( - "Has the spectrum changed after Fourier transform? " - "Look for frequency-domain peaks." + "Has the spectrum changed after Fourier transform? Look for frequency-domain peaks." ), ) results.append(("Fourier Transform", ok)) diff --git a/demos/24_vnc_demo.py b/demos/24_vnc_demo.py index 3a7bc64..92c775e 100644 --- a/demos/24_vnc_demo.py +++ b/demos/24_vnc_demo.py @@ -14,6 +14,7 @@ # 3. Run demo: python demos/24_vnc_demo.py # 4. Watch: open http://localhost:8430 → Live VM tab """ + from __future__ import annotations import base64 @@ -52,11 +53,13 @@ # ── VNC helpers ────────────────────────────────── + def get_vnc_info() -> tuple[str, int, str]: """Get VNC connection string and password from lume ls.""" result = subprocess.run( [os.path.expanduser("~/.local/bin/lume"), "ls"], - capture_output=True, text=True, + capture_output=True, + text=True, ) for line in result.stdout.strip().split("\n"): if "topspin-vm" in line and "running" in line: @@ -147,6 +150,7 @@ def vnc_type_command(vnc_addr: str, password: str, cmd: str): # ── Stream helpers ─────────────────────────────── + def push_frame(jpeg_bytes: bytes): """Push a JPEG frame to labwork-web MJPEG stream.""" try: @@ -189,36 +193,41 @@ def screenshot_and_push(vnc_addr: str, password: str) -> tuple[Image.Image, byte # ── VLM ───────────────────────────────────────── + def ask_sonnet(client: OpenAI, screenshot_b64: str, question: str) -> str: """Send screenshot + question to Sonnet 4.6, get text response.""" response = client.chat.completions.create( model=MODEL, max_tokens=512, - messages=[{ - "role": "user", - "content": [ - { - "type": "image_url", - "image_url": {"url": f"data:image/png;base64,{screenshot_b64}"}, - }, - { - "type": "text", - "text": ( - "This is a 2048x1536 pixel screenshot of macOS with Bruker TopSpin 5.0.\n" - f"{question}\n\n" - "Remember: scale perceived coordinates by 1.6 for 2048x1536 space.\n" - "Return ONLY JSON, no markdown fences." - ), - }, - ], - }], + messages=[ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": f"data:image/png;base64,{screenshot_b64}"}, + }, + { + "type": "text", + "text": ( + "This is a 2048x1536 pixel screenshot of macOS with Bruker TopSpin 5.0.\n" + f"{question}\n\n" + "Remember: scale perceived coordinates by 1.6 for 2048x1536 space.\n" + "Return ONLY JSON, no markdown fences." + ), + }, + ], + } + ], ) return response.choices[0].message.content.strip() def ask_sonnet_coords(client: OpenAI, screenshot_b64: str, target: str) -> tuple[int, int] | None: """Ask Sonnet for coordinates of a UI element.""" - text = ask_sonnet(client, screenshot_b64, f'Find the "{target}" button/element. Return: {{"x": N, "y": N}}') + text = ask_sonnet( + client, screenshot_b64, f'Find the "{target}" button/element. Return: {{"x": N, "y": N}}' + ) mx = re.search(r'"x"\s*:\s*(\d+)', text) my = re.search(r'"y"\s*:\s*(\d+)', text) if mx and my: @@ -228,7 +237,11 @@ def ask_sonnet_coords(client: OpenAI, screenshot_b64: str, target: str) -> tuple def ask_sonnet_verify(client: OpenAI, screenshot_b64: str, check: str) -> dict: """Ask Sonnet to verify a condition. Returns {"ok": bool, "description": str}.""" - text = ask_sonnet(client, screenshot_b64, f'{check}\nReturn: {{"ok": true/false, "description": "what you see"}}') + text = ask_sonnet( + client, + screenshot_b64, + f'{check}\nReturn: {{"ok": true/false, "description": "what you see"}}', + ) # Parse JSON from response m = re.search(r"\{[^}]+\}", text) if m: @@ -241,6 +254,7 @@ def ask_sonnet_verify(client: OpenAI, screenshot_b64: str, check: str) -> dict: # ── Pipeline Steps ────────────────────────────── + def step_open_topspin(vnc_addr: str, password: str, ai: OpenAI) -> bool: """Ensure TopSpin is open and visible.""" push_log("▶ Open TopSpin", status="operating") @@ -248,7 +262,11 @@ def step_open_topspin(vnc_addr: str, password: str, ai: OpenAI) -> bool: img, png = screenshot_and_push(vnc_addr, password) b64 = base64.b64encode(png).decode() - result = ask_sonnet_verify(ai, b64, "Is Bruker TopSpin 5.0 open and visible? Look for the TopSpin toolbar and title bar.") + result = ask_sonnet_verify( + ai, + b64, + "Is Bruker TopSpin 5.0 open and visible? Look for the TopSpin toolbar and title bar.", + ) if result.get("ok"): push_log(f" ✓ TopSpin already open: {result.get('description', '')}") return True @@ -269,19 +287,30 @@ def step_open_topspin(vnc_addr: str, password: str, ai: OpenAI) -> bool: # Fallback: activate via SSH (handles cases where dock click didn't work) push_log(" Activating TopSpin via SSH...") subprocess.run( - ["sshpass", "-p", "labclaw", "ssh", "-o", "StrictHostKeyChecking=no", - "labclaw@192.168.64.7", - "open -a '/opt/topspin5.0.0/TopSpin 5.0.0.app'"], - capture_output=True, timeout=10, + [ + "sshpass", + "-p", + os.environ.get("SSH_PASS", "changeme"), + "ssh", + "-o", + "StrictHostKeyChecking=no", + f"{os.environ.get('SSH_USER', 'user')}@{os.environ.get('VM_IP', '10.0.0.1')}", + "open -a '/opt/topspin5.0.0/TopSpin 5.0.0.app'", + ], + capture_output=True, + timeout=10, ) time.sleep(10) # Check for license dialog and accept it img, png = screenshot_and_push(vnc_addr, password) b64 = base64.b64encode(png).decode() - license_check = ask_sonnet_verify(ai, b64, + license_check = ask_sonnet_verify( + ai, + b64, "Is there a license agreement dialog visible? Look for 'I Accept' or 'License' text. " - "Set ok=true if you see a license dialog.") + "Set ok=true if you see a license dialog.", + ) if license_check.get("ok"): push_log(" License dialog — accepting...") coords = ask_sonnet_coords(ai, b64, "'I Accept' button") @@ -313,10 +342,13 @@ def step_load_dataset(vnc_addr: str, password: str, ai: OpenAI) -> bool: img, png = screenshot_and_push(vnc_addr, password) b64 = base64.b64encode(png).decode() - result = ask_sonnet_verify(ai, b64, + result = ask_sonnet_verify( + ai, + b64, "Has NMR data been loaded? Look for: a spectrum plot (FID or frequency domain) " "in the main panel, OR a molecular structure in the structure panel, OR dataset " - "info in the title area. 'No structure available' alone does NOT mean failure.") + "info in the title area. 'No structure available' alone does NOT mean failure.", + ) if result.get("ok"): push_log(f" ✓ Dataset loaded: {result.get('description', '')}") return True @@ -332,9 +364,15 @@ def step_load_dataset(vnc_addr: str, password: str, ai: OpenAI) -> bool: return ok -def step_run_command(vnc_addr: str, password: str, ai: OpenAI, - cmd: str, step_name: str, verify_prompt: str, - handle_dialog: bool = False) -> bool: +def step_run_command( + vnc_addr: str, + password: str, + ai: OpenAI, + cmd: str, + step_name: str, + verify_prompt: str, + handle_dialog: bool = False, +) -> bool: """Run a TopSpin command and verify the result.""" push_log(f"▶ {step_name}", status="operating") @@ -347,10 +385,13 @@ def step_run_command(vnc_addr: str, password: str, ai: OpenAI, if handle_dialog: # Dismiss dialogs in a loop (nested errors may require multiple rounds) for _dlg in range(3): - dialog_check = ask_sonnet_verify(ai, b64, + dialog_check = ask_sonnet_verify( + ai, + b64, "Is there a dialog/popup window visible in the CENTER of the screen? " "NOT a macOS notification in the corner. Set ok=true ONLY for centered " - "dialogs with Close/OK/Cancel buttons.") + "dialogs with Close/OK/Cancel buttons.", + ) if not dialog_check.get("ok"): break push_log(f" Dialog detected: {dialog_check.get('description', '')}") @@ -374,13 +415,16 @@ def step_verify_result(vnc_addr: str, password: str, ai: OpenAI) -> bool: img, png = screenshot_and_push(vnc_addr, password) b64 = base64.b64encode(png).decode() - result = ask_sonnet_verify(ai, b64, + result = ask_sonnet_verify( + ai, + b64, "Describe the NMR spectrum visible in TopSpin. Report ok=true if you can see: " "(1) NMR peaks visible in the spectrum display area, " "(2) a chemical shift axis (ppm scale) at the bottom, " "(3) any peak annotations or red markers in the spectrum area. " "Ignore any error dialogs — focus only on the spectrum itself. " - "A dense pattern of red markers across the top IS valid peak picking output.") + "A dense pattern of red markers across the top IS valid peak picking output.", + ) ok = result.get("ok", False) push_log(f" {'✓' if ok else '⚠'} Final: {result.get('description', '')}") return ok @@ -388,6 +432,7 @@ def step_verify_result(vnc_addr: str, password: str, ai: OpenAI) -> bool: # ── Main ──────────────────────────────────────── + def main() -> int: print( f"\n{B}═══════════════════════════════════════════════{RST}\n" @@ -453,10 +498,13 @@ def main() -> int: crop = img.crop((0, 1350, 700, 1410)) crop.save("/tmp/vnc_kbd_test.png") b64_test = base64.b64encode(png).decode() - caps_check = ask_sonnet_verify(ai, b64_test, + caps_check = ask_sonnet_verify( + ai, + b64_test, "Look at the command line text field at the very bottom of the window. " "Does it contain 'abc' (lowercase) or 'ABC' (uppercase)? " - "Set ok=true if lowercase 'abc', ok=false if uppercase 'ABC'.") + "Set ok=true if lowercase 'abc', ok=false if uppercase 'ABC'.", + ) if not caps_check.get("ok"): push_log(" Caps lock detected — toggling off") vnc_key(vnc_addr, password, "caplk") @@ -482,10 +530,13 @@ def main() -> int: time.sleep(0.5) img, png = screenshot_and_push(vnc_addr, password) b64 = base64.b64encode(png).decode() - dialog_check = ask_sonnet_verify(ai, b64, + dialog_check = ask_sonnet_verify( + ai, + b64, "Is there an error dialog, warning popup, license dialog, or modal window visible? " "NOT a regular application window and NOT a macOS notification banner in the corner. " - "Set ok=true ONLY if you see a centered popup/dialog with Close/OK/Cancel/Accept buttons.") + "Set ok=true ONLY if you see a centered popup/dialog with Close/OK/Cancel/Accept buttons.", + ) if not dialog_check.get("ok"): break desc = dialog_check.get("description", "") @@ -522,28 +573,40 @@ def main() -> int: time.sleep(2) # Step 3: Fourier Transform (efp) - ok = step_run_command(vnc_addr, password, ai, + ok = step_run_command( + vnc_addr, + password, + ai, cmd="efp", step_name="Fourier Transform", - verify_prompt="Has the spectrum changed after Fourier transform? Look for frequency-domain peaks.") + verify_prompt="Has the spectrum changed after Fourier transform? Look for frequency-domain peaks.", + ) results.append(("Fourier Transform", ok)) time.sleep(2) # Step 4: Phase Correction (apk) — may show error dialog on some data - ok = step_run_command(vnc_addr, password, ai, + ok = step_run_command( + vnc_addr, + password, + ai, cmd="apk", step_name="Phase Correction", verify_prompt="Has automatic phase correction been applied? Peaks should be upright and baseline flat.", - handle_dialog=True) + handle_dialog=True, + ) results.append(("Phase Correction", ok)) time.sleep(2) # Step 5: Peak Picking (pp) — may show dialog - ok = step_run_command(vnc_addr, password, ai, + ok = step_run_command( + vnc_addr, + password, + ai, cmd="pp", step_name="Peak Picking", verify_prompt="Are peak annotations/labels visible on the spectrum? Look for red markers or numbers above peaks.", - handle_dialog=True) + handle_dialog=True, + ) results.append(("Peak Picking", ok)) time.sleep(2) diff --git a/demos/25_vnc_cu_demo.py b/demos/25_vnc_cu_demo.py index ef61228..f7431f4 100644 --- a/demos/25_vnc_cu_demo.py +++ b/demos/25_vnc_cu_demo.py @@ -5,7 +5,7 @@ Sonnet 4.6 (via OpenRouter) verifies each step via screenshot analysis. Each frame is pushed to labwork-web as MJPEG stream for live viewing. -Tart VM: topspin-sequoia (ARD auth: admin/admin, VNC port 5900). +Tart VM: topspin-sequoia (ARD auth via env vars VNC_USER/VNC_PASS, VNC port 5900). Framebuffer: 2048x1536 (retina for 1024x768 VM display). Sonnet perceives images at ~1280x960 and applies 1.6x scaling. @@ -15,6 +15,7 @@ # 3. Run demo: python demos/25_vnc_cu_demo.py # 4. Watch: open http://localhost:8430 → Live VM tab """ + from __future__ import annotations import base64 @@ -40,9 +41,9 @@ "/opt/topspin5.0.0/examdata/exam_CMCse_1/1", ) -VM_NAME = "topspin-sequoia" -VM_USER = "admin" -VM_PASS = "admin" +VM_NAME = os.environ.get("VM_NAME", "topspin-sequoia") +VM_USER = os.environ.get("VNC_USER", "changeme") +VM_PASS = os.environ.get("VNC_PASS", "changeme") VNC_PORT = 5900 # TopSpin command line position (framebuffer coords, 2048x1536) @@ -59,11 +60,14 @@ # ── VNC helpers ────────────────────────────────── + def get_vnc_info() -> tuple[str, str]: """Get VNC address from tart ip.""" result = subprocess.run( ["tart", "ip", VM_NAME], - capture_output=True, text=True, timeout=10, + capture_output=True, + text=True, + timeout=10, ) ip = result.stdout.strip() if not ip: @@ -160,6 +164,7 @@ def vnc_type_command(vnc_addr: str, cmd: str): # ── Stream helpers ─────────────────────────────── + def push_frame(jpeg_bytes: bytes): """Push a JPEG frame to labwork-web MJPEG stream.""" try: @@ -202,36 +207,41 @@ def screenshot_and_push(vnc_addr: str) -> tuple[Image.Image, bytes]: # ── VLM ───────────────────────────────────────── + def ask_sonnet(client: OpenAI, screenshot_b64: str, question: str) -> str: """Send screenshot + question to Sonnet 4.6, get text response.""" response = client.chat.completions.create( model=MODEL, max_tokens=512, - messages=[{ - "role": "user", - "content": [ - { - "type": "image_url", - "image_url": {"url": f"data:image/png;base64,{screenshot_b64}"}, - }, - { - "type": "text", - "text": ( - "This is a 2048x1536 pixel screenshot of macOS with Bruker TopSpin 5.0.\n" - f"{question}\n\n" - "Remember: scale perceived coordinates by 1.6 for 2048x1536 space.\n" - "Return ONLY JSON, no markdown fences." - ), - }, - ], - }], + messages=[ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": f"data:image/png;base64,{screenshot_b64}"}, + }, + { + "type": "text", + "text": ( + "This is a 2048x1536 pixel screenshot of macOS with Bruker TopSpin 5.0.\n" + f"{question}\n\n" + "Remember: scale perceived coordinates by 1.6 for 2048x1536 space.\n" + "Return ONLY JSON, no markdown fences." + ), + }, + ], + } + ], ) return response.choices[0].message.content.strip() def ask_sonnet_coords(client: OpenAI, screenshot_b64: str, target: str) -> tuple[int, int] | None: """Ask Sonnet for coordinates of a UI element.""" - text = ask_sonnet(client, screenshot_b64, f'Find the "{target}" button/element. Return: {{"x": N, "y": N}}') + text = ask_sonnet( + client, screenshot_b64, f'Find the "{target}" button/element. Return: {{"x": N, "y": N}}' + ) mx = re.search(r'"x"\s*:\s*(\d+)', text) my = re.search(r'"y"\s*:\s*(\d+)', text) if mx and my: @@ -241,7 +251,11 @@ def ask_sonnet_coords(client: OpenAI, screenshot_b64: str, target: str) -> tuple def ask_sonnet_verify(client: OpenAI, screenshot_b64: str, check: str) -> dict: """Ask Sonnet to verify a condition. Returns {"ok": bool, "description": str}.""" - text = ask_sonnet(client, screenshot_b64, f'{check}\nReturn: {{"ok": true/false, "description": "what you see"}}') + text = ask_sonnet( + client, + screenshot_b64, + f'{check}\nReturn: {{"ok": true/false, "description": "what you see"}}', + ) # Parse JSON from response m = re.search(r"\{[^}]+\}", text) if m: @@ -254,6 +268,7 @@ def ask_sonnet_verify(client: OpenAI, screenshot_b64: str, check: str) -> dict: # ── Pipeline Steps ────────────────────────────── + def step_open_topspin(vnc_addr: str, ai: OpenAI) -> bool: """Ensure TopSpin is open and visible.""" push_log("▶ Open TopSpin", status="operating") @@ -261,7 +276,11 @@ def step_open_topspin(vnc_addr: str, ai: OpenAI) -> bool: img, png = screenshot_and_push(vnc_addr) b64 = base64.b64encode(png).decode() - result = ask_sonnet_verify(ai, b64, "Is Bruker TopSpin 5.0 open and visible? Look for the TopSpin toolbar and title bar.") + result = ask_sonnet_verify( + ai, + b64, + "Is Bruker TopSpin 5.0 open and visible? Look for the TopSpin toolbar and title bar.", + ) if result.get("ok"): push_log(f" ✓ TopSpin already open: {result.get('description', '')}") return True @@ -282,19 +301,30 @@ def step_open_topspin(vnc_addr: str, ai: OpenAI) -> bool: # Fallback: activate via SSH (handles cases where dock click didn't work) push_log(" Activating TopSpin via SSH...") subprocess.run( - ["sshpass", "-p", VM_PASS, "ssh", "-o", "StrictHostKeyChecking=no", - f"{VM_USER}@{_VM_IP}", - "open -a '/opt/topspin5.0.0/TopSpin 5.0.0.app'"], - capture_output=True, timeout=10, + [ + "sshpass", + "-p", + VM_PASS, + "ssh", + "-o", + "StrictHostKeyChecking=no", + f"{VM_USER}@{_VM_IP}", + "open -a '/opt/topspin5.0.0/TopSpin 5.0.0.app'", + ], + capture_output=True, + timeout=10, ) time.sleep(10) # Check for license dialog and accept it img, png = screenshot_and_push(vnc_addr) b64 = base64.b64encode(png).decode() - license_check = ask_sonnet_verify(ai, b64, + license_check = ask_sonnet_verify( + ai, + b64, "Is there a license agreement dialog visible? Look for 'I Accept' or 'License' text. " - "Set ok=true if you see a license dialog.") + "Set ok=true if you see a license dialog.", + ) if license_check.get("ok"): push_log(" License dialog — accepting...") coords = ask_sonnet_coords(ai, b64, "'I Accept' button") @@ -326,10 +356,13 @@ def step_load_dataset(vnc_addr: str, ai: OpenAI) -> bool: img, png = screenshot_and_push(vnc_addr) b64 = base64.b64encode(png).decode() - result = ask_sonnet_verify(ai, b64, + result = ask_sonnet_verify( + ai, + b64, "Has NMR data been loaded? Look for: a spectrum plot (FID or frequency domain) " "in the main panel, OR a molecular structure in the structure panel, OR dataset " - "info in the title area. 'No structure available' alone does NOT mean failure.") + "info in the title area. 'No structure available' alone does NOT mean failure.", + ) if result.get("ok"): push_log(f" ✓ Dataset loaded: {result.get('description', '')}") return True @@ -345,9 +378,14 @@ def step_load_dataset(vnc_addr: str, ai: OpenAI) -> bool: return ok -def step_run_command(vnc_addr: str, ai: OpenAI, - cmd: str, step_name: str, verify_prompt: str, - handle_dialog: bool = False) -> bool: +def step_run_command( + vnc_addr: str, + ai: OpenAI, + cmd: str, + step_name: str, + verify_prompt: str, + handle_dialog: bool = False, +) -> bool: """Run a TopSpin command and verify the result.""" push_log(f"▶ {step_name}", status="operating") @@ -360,10 +398,13 @@ def step_run_command(vnc_addr: str, ai: OpenAI, if handle_dialog: # Dismiss dialogs in a loop (nested errors may require multiple rounds) for _dlg in range(3): - dialog_check = ask_sonnet_verify(ai, b64, + dialog_check = ask_sonnet_verify( + ai, + b64, "Is there a dialog/popup window visible in the CENTER of the screen? " "NOT a macOS notification in the corner. Set ok=true ONLY for centered " - "dialogs with Close/OK/Cancel buttons.") + "dialogs with Close/OK/Cancel buttons.", + ) if not dialog_check.get("ok"): break push_log(f" Dialog detected: {dialog_check.get('description', '')}") @@ -387,13 +428,16 @@ def step_verify_result(vnc_addr: str, ai: OpenAI) -> bool: img, png = screenshot_and_push(vnc_addr) b64 = base64.b64encode(png).decode() - result = ask_sonnet_verify(ai, b64, + result = ask_sonnet_verify( + ai, + b64, "Describe the NMR spectrum visible in TopSpin. Report ok=true if you can see: " "(1) NMR peaks visible in the spectrum display area, " "(2) a chemical shift axis (ppm scale) at the bottom, " "(3) any peak annotations or red markers in the spectrum area. " "Ignore any error dialogs — focus only on the spectrum itself. " - "A dense pattern of red markers across the top IS valid peak picking output.") + "A dense pattern of red markers across the top IS valid peak picking output.", + ) ok = result.get("ok", False) push_log(f" {'✓' if ok else '⚠'} Final: {result.get('description', '')}") return ok @@ -473,10 +517,13 @@ def main() -> int: crop = img.crop((0, 930, 700, 990)) crop.save("/tmp/vnc_kbd_test.png") b64_test = base64.b64encode(png).decode() - caps_check = ask_sonnet_verify(ai, b64_test, + caps_check = ask_sonnet_verify( + ai, + b64_test, "Look at the command line text field at the very bottom of the window. " "Does it contain 'abc' (lowercase) or 'ABC' (uppercase)? " - "Set ok=true if lowercase 'abc', ok=false if uppercase 'ABC'.") + "Set ok=true if lowercase 'abc', ok=false if uppercase 'ABC'.", + ) if not caps_check.get("ok"): push_log(" Caps lock detected — toggling off") vnc_key(vnc_addr, "caplk") @@ -502,10 +549,13 @@ def main() -> int: time.sleep(0.5) img, png = screenshot_and_push(vnc_addr) b64 = base64.b64encode(png).decode() - dialog_check = ask_sonnet_verify(ai, b64, + dialog_check = ask_sonnet_verify( + ai, + b64, "Is there an error dialog, warning popup, license dialog, or modal window visible? " "NOT a regular application window and NOT a macOS notification banner in the corner. " - "Set ok=true ONLY if you see a centered popup/dialog with Close/OK/Cancel/Accept buttons.") + "Set ok=true ONLY if you see a centered popup/dialog with Close/OK/Cancel/Accept buttons.", + ) if not dialog_check.get("ok"): break desc = dialog_check.get("description", "") @@ -542,28 +592,37 @@ def main() -> int: time.sleep(2) # Step 3: Fourier Transform (efp) - ok = step_run_command(vnc_addr, ai, + ok = step_run_command( + vnc_addr, + ai, cmd="efp", step_name="Fourier Transform", - verify_prompt="Has the spectrum changed after Fourier transform? Look for frequency-domain peaks.") + verify_prompt="Has the spectrum changed after Fourier transform? Look for frequency-domain peaks.", + ) results.append(("Fourier Transform", ok)) time.sleep(2) # Step 4: Phase Correction (apk) — may show error dialog on some data - ok = step_run_command(vnc_addr, ai, + ok = step_run_command( + vnc_addr, + ai, cmd="apk", step_name="Phase Correction", verify_prompt="Has automatic phase correction been applied? Peaks should be upright and baseline flat.", - handle_dialog=True) + handle_dialog=True, + ) results.append(("Phase Correction", ok)) time.sleep(2) # Step 5: Peak Picking (pp) — may show dialog - ok = step_run_command(vnc_addr, ai, + ok = step_run_command( + vnc_addr, + ai, cmd="pp", step_name="Peak Picking", verify_prompt="Are peak annotations/labels visible on the spectrum? Look for red markers or numbers above peaks.", - handle_dialog=True) + handle_dialog=True, + ) results.append(("Peak Picking", ok)) time.sleep(2) diff --git a/demos/record_full_showcase.py b/demos/record_full_showcase.py index 05c2e79..8be9ea3 100644 --- a/demos/record_full_showcase.py +++ b/demos/record_full_showcase.py @@ -38,9 +38,9 @@ _OPENAI_IMPORT_ERROR = exc # ── Config ────────────────────────────────────── -VM_IP = os.environ.get("VM_IP", "192.168.64.13") -VNC_USER = os.environ.get("VNC_USER", "admin") -VNC_PASS = os.environ.get("VNC_PASS", "admin") +VM_IP = os.environ.get("VM_IP", "10.0.0.1") +VNC_USER = os.environ.get("VNC_USER", "changeme") +VNC_PASS = os.environ.get("VNC_PASS", "changeme") MODEL = "anthropic/claude-sonnet-4.6" DATASET = os.environ.get( "TOPSPIN_DATASET", diff --git a/lab-instruments/jpg/IMG_3297.jpg b/lab-instruments/jpg/IMG_3297.jpg index 9e4ff8c..8546572 100644 Binary files a/lab-instruments/jpg/IMG_3297.jpg and b/lab-instruments/jpg/IMG_3297.jpg differ diff --git a/lab-instruments/jpg/IMG_3298.jpg b/lab-instruments/jpg/IMG_3298.jpg index 58a7a85..4ee1635 100644 Binary files a/lab-instruments/jpg/IMG_3298.jpg and b/lab-instruments/jpg/IMG_3298.jpg differ diff --git a/lab-instruments/jpg/IMG_3299.jpg b/lab-instruments/jpg/IMG_3299.jpg index 6e6d187..ff3476d 100644 Binary files a/lab-instruments/jpg/IMG_3299.jpg and b/lab-instruments/jpg/IMG_3299.jpg differ diff --git a/lab-instruments/jpg/IMG_3300.jpg b/lab-instruments/jpg/IMG_3300.jpg index 64a58fc..556d2bc 100644 Binary files a/lab-instruments/jpg/IMG_3300.jpg and b/lab-instruments/jpg/IMG_3300.jpg differ diff --git a/lab-instruments/jpg/IMG_3301.jpg b/lab-instruments/jpg/IMG_3301.jpg index 7312a6b..3e42436 100644 Binary files a/lab-instruments/jpg/IMG_3301.jpg and b/lab-instruments/jpg/IMG_3301.jpg differ diff --git a/lab-instruments/jpg/IMG_3302.jpg b/lab-instruments/jpg/IMG_3302.jpg index f6cddbb..2dff548 100644 Binary files a/lab-instruments/jpg/IMG_3302.jpg and b/lab-instruments/jpg/IMG_3302.jpg differ diff --git a/lab-instruments/jpg/IMG_3303.jpg b/lab-instruments/jpg/IMG_3303.jpg index 99c9bd8..c745160 100644 Binary files a/lab-instruments/jpg/IMG_3303.jpg and b/lab-instruments/jpg/IMG_3303.jpg differ diff --git a/lab-instruments/jpg/IMG_3304.jpg b/lab-instruments/jpg/IMG_3304.jpg index c421e87..a379841 100644 Binary files a/lab-instruments/jpg/IMG_3304.jpg and b/lab-instruments/jpg/IMG_3304.jpg differ diff --git a/lab-instruments/jpg/IMG_3305.jpg b/lab-instruments/jpg/IMG_3305.jpg index 4d71665..1d103c9 100644 Binary files a/lab-instruments/jpg/IMG_3305.jpg and b/lab-instruments/jpg/IMG_3305.jpg differ diff --git a/lab-instruments/jpg/IMG_3306.jpg b/lab-instruments/jpg/IMG_3306.jpg index eba07fa..cc5f73d 100644 Binary files a/lab-instruments/jpg/IMG_3306.jpg and b/lab-instruments/jpg/IMG_3306.jpg differ diff --git a/lab-instruments/jpg/IMG_3307.jpg b/lab-instruments/jpg/IMG_3307.jpg index 0387e35..80f1279 100644 Binary files a/lab-instruments/jpg/IMG_3307.jpg and b/lab-instruments/jpg/IMG_3307.jpg differ diff --git a/lab-instruments/jpg/IMG_3308.jpg b/lab-instruments/jpg/IMG_3308.jpg index b665ea4..c4c1a0f 100644 Binary files a/lab-instruments/jpg/IMG_3308.jpg and b/lab-instruments/jpg/IMG_3308.jpg differ diff --git a/lab-instruments/jpg/IMG_3309.jpg b/lab-instruments/jpg/IMG_3309.jpg index 29af43b..a76e633 100644 Binary files a/lab-instruments/jpg/IMG_3309.jpg and b/lab-instruments/jpg/IMG_3309.jpg differ diff --git a/lab-instruments/jpg/IMG_3310.jpg b/lab-instruments/jpg/IMG_3310.jpg index d8785ea..9778638 100644 Binary files a/lab-instruments/jpg/IMG_3310.jpg and b/lab-instruments/jpg/IMG_3310.jpg differ diff --git a/lab-instruments/jpg/IMG_3311.jpg b/lab-instruments/jpg/IMG_3311.jpg index 3b9fa39..5bb68a9 100644 Binary files a/lab-instruments/jpg/IMG_3311.jpg and b/lab-instruments/jpg/IMG_3311.jpg differ diff --git a/lab-instruments/jpg/IMG_3312.jpg b/lab-instruments/jpg/IMG_3312.jpg index fef87e5..18fd147 100644 Binary files a/lab-instruments/jpg/IMG_3312.jpg and b/lab-instruments/jpg/IMG_3312.jpg differ diff --git a/lab-instruments/jpg/IMG_3313.jpg b/lab-instruments/jpg/IMG_3313.jpg index d3d08c5..def994e 100644 Binary files a/lab-instruments/jpg/IMG_3313.jpg and b/lab-instruments/jpg/IMG_3313.jpg differ diff --git a/lab-instruments/jpg/IMG_3314.jpg b/lab-instruments/jpg/IMG_3314.jpg index b487d29..7955d97 100644 Binary files a/lab-instruments/jpg/IMG_3314.jpg and b/lab-instruments/jpg/IMG_3314.jpg differ diff --git a/lab-instruments/jpg/IMG_3315.jpg b/lab-instruments/jpg/IMG_3315.jpg index e170ef0..56d71db 100644 Binary files a/lab-instruments/jpg/IMG_3315.jpg and b/lab-instruments/jpg/IMG_3315.jpg differ diff --git a/lab-instruments/jpg/IMG_3316.jpg b/lab-instruments/jpg/IMG_3316.jpg index 9c45d4a..361e3b1 100644 Binary files a/lab-instruments/jpg/IMG_3316.jpg and b/lab-instruments/jpg/IMG_3316.jpg differ diff --git a/lab-instruments/jpg/IMG_3317.jpg b/lab-instruments/jpg/IMG_3317.jpg index 15b15a9..5060eb2 100644 Binary files a/lab-instruments/jpg/IMG_3317.jpg and b/lab-instruments/jpg/IMG_3317.jpg differ diff --git a/lab-instruments/jpg/IMG_3318.jpg b/lab-instruments/jpg/IMG_3318.jpg index f806b68..0e60a77 100644 Binary files a/lab-instruments/jpg/IMG_3318.jpg and b/lab-instruments/jpg/IMG_3318.jpg differ diff --git a/lab-instruments/jpg/IMG_3319.jpg b/lab-instruments/jpg/IMG_3319.jpg index 3796bd9..d94d748 100644 Binary files a/lab-instruments/jpg/IMG_3319.jpg and b/lab-instruments/jpg/IMG_3319.jpg differ diff --git a/lab-instruments/jpg/IMG_3320.jpg b/lab-instruments/jpg/IMG_3320.jpg index 0ae3a27..c517071 100644 Binary files a/lab-instruments/jpg/IMG_3320.jpg and b/lab-instruments/jpg/IMG_3320.jpg differ diff --git a/lab-instruments/jpg/IMG_3321.jpg b/lab-instruments/jpg/IMG_3321.jpg index be3e1a9..618e210 100644 Binary files a/lab-instruments/jpg/IMG_3321.jpg and b/lab-instruments/jpg/IMG_3321.jpg differ diff --git a/lab-instruments/jpg/IMG_3322.jpg b/lab-instruments/jpg/IMG_3322.jpg index 2b13363..80ecfea 100644 Binary files a/lab-instruments/jpg/IMG_3322.jpg and b/lab-instruments/jpg/IMG_3322.jpg differ