feat: expose OpenWork UI control plane and MCP bridge

AGENTS.md

@@ -57,6 +57,7 @@ Read `ARCHITECTURE.md` for runtime flow, server-vs-shell ownership, and architec
 * **Open source**: keep the repo portable; no secrets committed.
 * **Slick and fluid**: 60fps animations, micro-interactions, premium feel.
 * **Mobile-native**: touch targets, gestures, and layouts optimized for small screens.
+* **Provider-neutral control**: expose app actions through OpenWork-owned control surfaces first; model/provider-specific realtime controllers should drive those surfaces rather than hardwiring provider logic into the app UI. Keep preview control surfaces off by default and user-configurable from settings.
 
 ## Task Intake (Required)
 

ARCHITECTURE.md

@@ -200,6 +200,58 @@ These are all opencode primitives you can read the docs to find out exactly how
 
 OpenWork is a client experience that consumes OpenWork server surfaces.
 
+### Provider-neutral app control surface
+
+OpenWork app control mode is owned by the UI runtime. The app exposes a
+provider-neutral action registry through `window.__openworkControl` so external
+controllers can inspect the current route, discover visible/safe actions, and
+request an action by ID without depending on DOM scraping or a specific model
+provider.
+
+Guidelines:
+
+- The app owns visible, screen-local state: which actions are available, which
+  element should be spotlighted, and how actions are choreographed so users can
+  see control happen.
+- Controllers such as OpenAI Realtime, MCP bridges, or test harnesses should
+  call the app control surface instead of reaching into app internals.
+- OpenAI Realtime is one replaceable control driver, not the owner of the
+  control architecture. The generic app-control registry and session actions
+  should remain useful if the voice driver is removed or replaced by tests,
+  scripts, MCP bridges, or other controllers.
+- Provider/API secrets and privileged filesystem or server mutations remain
+  server-owned; the app control surface should route those through OpenWork
+  server APIs rather than adding provider-specific behavior to the UI.
+- `/remote/session` is the OpenWork server endpoint that brokers short-lived
+  remote-control Realtime sessions. It keeps provider API keys server-side and
+  returns only an ephemeral browser client secret.
+- Realtime control is a Feature Preview capability and is off by default. When
+  enabled, users start or stop it from the session status bar instead of a
+  floating overlay.
+- The OpenAI key used for the initial Realtime controller can come from the
+  server process environment or from the OpenWork local environment store via
+  Settings -> Feature Preview; the browser never receives the long-lived key.
+- Realtime remote control captures microphone audio in the app/browser only
+  after the user starts the mode. The first implementation sends audio input to
+  the Realtime session while keeping model output text/tool-call based.
+- Raw screenshot or coordinate-based control is a fallback for uninstrumented
+  surfaces, not the default architecture.
+
+### MCP UI Control profile
+
+OpenWork should standardize external app control through MCP where possible. The
+app-local `window.__openworkControl` registry remains the source of current UI
+affordances, but public integrations should expose those affordances as MCP
+tools that follow `docs/mcp-ui-control-profile.md`:
+
+- `ui.snapshot` for current semantic app state
+- `ui.list_actions` for currently available action metadata and input schemas
+- `ui.execute_action` for running one semantic action by ID
+
+Standalone control clients such as HandsFree should be MCP clients first: they
+can connect to any configured MCP server and call generic MCP tools. OpenWork's
+local UI bridge is an implementation detail behind the OpenWork MCP surface.
+
 OpenWork supports two product runtime modes for users:
 
 - desktop

apps/app/src/app/lib/desktop.ts

@@ -14,6 +14,9 @@ declare global {
         openExternal?: (url: string) => Promise<void>;
         relaunch?: () => Promise<void>;
       };
+      permissions?: {
+        requestMicrophone?: () => Promise<{ granted: boolean; status: string }>;
+      };
       migration?: {
         readSnapshot?: () => Promise<unknown>;
         ackSnapshot?: () => Promise<{ ok: boolean; moved: boolean }>;

apps/app/src/app/lib/openwork-server.ts

@@ -217,6 +217,20 @@ export type OpenworkBlueprintSessionsMaterializeResult = {
   openSessionId: string | null;
 };
 
+export type OpenworkRemoteSession = {
+  clientSecret: string;
+  expiresAt: number | null;
+  model: string;
+  voice: string;
+  tools: string[];
+};
+
+export type OpenworkRemoteSessionRequest = {
+  model?: string;
+  voice?: string;
+  instructions?: string;
+};
+
 export type OpenworkArtifactItem = {
   id: string;
   name?: string;
@@ -766,6 +780,24 @@ export function createOpenworkServerClient(options: { baseUrl: string; token?: s
       requestJson<OpenworkRuntimeSnapshot>(baseUrl, "/runtime/versions", { token, hostToken, timeoutMs: timeouts.status }),
     status: () => requestJson<OpenworkServerDiagnostics>(baseUrl, "/status", { token, hostToken, timeoutMs: timeouts.status }),
     capabilities: () => requestJson<OpenworkServerCapabilities>(baseUrl, "/capabilities", { token, hostToken, timeoutMs: timeouts.capabilities }),
+    createRemoteSession: (payload: OpenworkRemoteSessionRequest = {}) =>
+      requestJson<OpenworkRemoteSession>(baseUrl, "/remote/session", {
+        token,
+        hostToken,
+        method: "POST",
+        body: payload,
+        timeoutMs: timeouts.status,
+      }).catch((error) => {
+        if (error instanceof OpenworkServerError && error.status === 404) {
+          throw new OpenworkServerError(
+            404,
+            "remote_session_unavailable",
+            "Realtime control requires a newer OpenWork server. Restart OpenWork so the updated server binary is used, then try Control again.",
+            error.details,
+          );
+        }
+        throw error;
+      }),
     listWorkspaces: () => requestJson<OpenworkWorkspaceList>(baseUrl, "/workspaces", { token, hostToken, timeoutMs: timeouts.listWorkspaces }),
     createLocalWorkspace: (payload: { folderPath: string; name: string; preset: string }) =>
       requestJson<WorkspaceList>(baseUrl, "/workspaces/local", {

apps/app/src/app/types.ts

@@ -175,6 +175,7 @@ export type SettingsTab =
   | "skills"
   | "extensions"
   | "environment"
+  | "feature-preview"
   | "advanced"
   | "appearance"
   | "updates"

apps/app/src/i18n/locales/en.ts

@@ -1716,13 +1716,15 @@ export default {
   "settings.environment.validation_shape": "Use letters, digits, and underscores; do not start with a digit.",
   "settings.environment.value_label": "Value",
   "settings.tab_description_environment": "Save API keys and tokens for local agents, skills, and MCP servers. Secrets stay on this device.",
+  "settings.tab_description_feature_preview": "Try experimental OpenWork capabilities before they graduate into the default product.",
   "settings.tab_description_messaging": "Configure router identities and inbox behavior from workspace settings.",
   "settings.tab_description_model": "Tune the default model, runtime behavior, and assistant output settings.",
   "settings.tab_description_recovery": "Repair migration state, reset workspace defaults, and recover local settings.",
   "settings.tab_description_skills": "Browse, edit, and install skills without leaving settings.",
   "settings.tab_description_updates": "Keep the app current with quiet background checks and install controls.",
   "settings.tab_environment": "Environment",
   "settings.tab_extensions": "Extensions",
+  "settings.tab_feature_preview": "Feature Preview",
   "settings.tab_general": "Settings",
   "settings.tab_messaging": "Messaging",
   "settings.tab_model": "Model",
@@ -1733,6 +1735,23 @@ export default {
   "settings.theme_light": "Light",
   "settings.theme_system": "System",
   "settings.theme_system_hint": "System mode follows your OS preference automatically.",
+  "settings.feature_preview.badge": "Preview",
+  "settings.feature_preview.checking": "Checking…",
+  "settings.feature_preview.configured": "Configured",
+  "settings.feature_preview.connect_server_hint": "Connect to an OpenWork server before saving a key.",
+  "settings.feature_preview.disabled": "Disabled",
+  "settings.feature_preview.enabled": "Enabled",
+  "settings.feature_preview.not_configured": "Not configured",
+  "settings.feature_preview.openai_key_description": "The key is saved in OpenWork's local environment store and used server-side to mint short-lived Realtime sessions. It is not sent to the browser as a long-lived secret.",
+  "settings.feature_preview.openai_key_hint": "Required for Realtime control. Existing shell OPENAI_API_KEY values still work.",
+  "settings.feature_preview.openai_key_label": "OpenAI API key",
+  "settings.feature_preview.openai_key_removed": "OpenAI API key removed.",
+  "settings.feature_preview.openai_key_required": "Enter an OpenAI API key before saving.",
+  "settings.feature_preview.openai_key_saved": "OpenAI API key saved. Realtime control can use it immediately.",
+  "settings.feature_preview.openai_key_title": "OpenAI Realtime key",
+  "settings.feature_preview.realtime_description": "Shows the Realtime control entry in the session status bar. When started, OpenWork captures microphone audio and lets the Realtime model drive registered app actions.",
+  "settings.feature_preview.realtime_title": "Realtime control mode",
+  "settings.feature_preview.replace_key": "Replace OpenAI API key",
   "settings.toolbar_ready_to_install": "Ready to install",
   "settings.update": "Update",
   "settings.update_available": "Update available: v",

apps/app/src/react-app/domains/session/chat/session-page.tsx

@@ -26,6 +26,7 @@ import { WorkspaceSessionList } from "../sidebar/workspace-session-list";
 import { SessionSurface, type SessionSurfaceProps } from "../surface/session-surface";
 import { ShareWorkspaceModal } from "../../workspace/share-workspace-modal";
 import { StatusBar, type StatusBarProps } from "./status-bar";
+import { OpenAIRealtimeActivityPanel } from "../../../shell/control-drivers/openai-realtime/openai-realtime-activity-panel";
 import {
   DEFAULT_WORKSPACE_LEFT_SIDEBAR_WIDTH,
   useWorkspaceShellLayout,
@@ -527,6 +528,8 @@ export function SessionPage(props: SessionPageProps) {
             showSettingsButton={props.statusBar?.showSettingsButton}
           />
         </main>
+
+        <OpenAIRealtimeActivityPanel />
       </div>
 
       {props.providerAuthModal ? <ProviderAuthModal {...props.providerAuthModal} /> : null}

apps/app/src/react-app/domains/session/chat/status-bar.tsx

@@ -1,9 +1,11 @@
 /** @jsxImportSource react */
-import { useEffect, useState } from "react";
+import { useEffect, useMemo, useRef, useState } from "react";
 import { BookOpen, MessageCircle, Settings } from "lucide-react";
 
 import { t } from "../../../../i18n";
 import { usePlatform } from "../../../kernel/platform";
+import { useControlAction, type OpenworkControlAction } from "../../../shell/control/control-provider";
+import { OpenAIRealtimeStatusControl } from "../../../shell/control-drivers/openai-realtime/openai-realtime-status-control";
 import type { OpenworkServerStatus } from "../../../../app/lib/openwork-server";
 
 const DOCS_URL = "https://openworklabs.com/docs";
@@ -103,6 +105,9 @@ function deriveStatusCopy(props: StatusBarProps): StatusCopy {
 
 export function StatusBar(props: StatusBarProps) {
   const platform = usePlatform();
+  const docsButtonRef = useRef<HTMLButtonElement>(null);
+  const feedbackButtonRef = useRef<HTMLButtonElement>(null);
+  const settingsButtonRef = useRef<HTMLButtonElement>(null);
   const [initializing, setInitializing] = useState(
     () => Date.now() - STATUS_BAR_BOOT_STARTED_AT < STATUS_BAR_INITIALIZING_MS,
   );
@@ -118,6 +123,36 @@ export function StatusBar(props: StatusBarProps) {
   }, [initializing]);
 
   const statusCopy = deriveStatusCopy({ ...props, initializing });
+  const docsControlAction = useMemo<OpenworkControlAction>(() => ({
+    id: "status.docs.open",
+    label: "Open OpenWork docs",
+    description: "Open the documentation from the status bar.",
+    sideEffect: "external",
+    targetRef: docsButtonRef,
+    execute: () => platform.openLink(DOCS_URL),
+  }), [platform]);
+  useControlAction(docsControlAction);
+
+  const feedbackControlAction = useMemo<OpenworkControlAction>(() => ({
+    id: "status.feedback.open",
+    label: "Send feedback",
+    description: "Open the OpenWork feedback surface from the status bar.",
+    sideEffect: "external",
+    targetRef: feedbackButtonRef,
+    execute: props.onSendFeedback,
+  }), [props.onSendFeedback]);
+  useControlAction(feedbackControlAction);
+
+  const settingsControlAction = useMemo<OpenworkControlAction>(() => ({
+    id: "status.settings.open",
+    label: props.settingsOpen ? "Go back from settings" : "Open settings from the status bar",
+    description: "Use the visible settings button in the status bar.",
+    sideEffect: "navigation",
+    disabled: props.showSettingsButton === false,
+    targetRef: settingsButtonRef,
+    execute: props.onOpenSettings,
+  }), [props.onOpenSettings, props.settingsOpen, props.showSettingsButton]);
+  useControlAction(settingsControlAction);
 
   return (
     <div className="border-t border-dls-border bg-dls-surface">
@@ -142,7 +177,9 @@ export function StatusBar(props: StatusBarProps) {
         </div>
 
         <div className="flex items-center gap-1.5">
+          <OpenAIRealtimeStatusControl />
           <button
+            ref={docsButtonRef}
             type="button"
             className="inline-flex h-8 items-center gap-1.5 rounded-md px-2 text-dls-secondary transition-colors hover:bg-dls-hover hover:text-dls-text"
             onClick={() => platform.openLink(DOCS_URL)}
@@ -153,6 +190,7 @@ export function StatusBar(props: StatusBarProps) {
             <span className="text-[11px] font-medium">{t("status.docs")}</span>
           </button>
           <button
+            ref={feedbackButtonRef}
             type="button"
             className="inline-flex h-8 items-center gap-1.5 rounded-md px-2 text-dls-secondary transition-colors hover:bg-dls-hover hover:text-dls-text"
             onClick={props.onSendFeedback}
@@ -166,6 +204,7 @@ export function StatusBar(props: StatusBarProps) {
           </button>
           {props.showSettingsButton !== false ? (
             <button
+              ref={settingsButtonRef}
               type="button"
               className="flex h-8 w-8 shrink-0 items-center justify-center rounded-md text-dls-secondary transition-colors hover:bg-dls-hover hover:text-dls-text"
               onClick={props.onOpenSettings}