feat(agent-core): guide AI to use ReadMediaFile for video analysis instead of manual frame extraction

packages/agent-core/src/profile/default/system.md

@@ -58,7 +58,7 @@ The user may ask you to research on certain topics, process or generate certain
 - Understand the user's requirements thoroughly, ask for clarification before you start if needed.
 - Make plans before doing deep or wide research, to ensure you are always on track.
 - Search on the Internet if possible, with carefully-designed search queries to improve efficiency and accuracy.
-- Use proper tools or shell commands or Python packages to process or generate images, videos, PDFs, docs, spreadsheets, presentations, or other multimedia files. Detect if there are already such tools in the environment. If you have to install third-party tools/packages, you MUST ensure that they are installed in a virtual/isolated environment.
+- Use proper tools or shell commands or Python packages to process or generate images, videos, PDFs, docs, spreadsheets, presentations, or other multimedia files. **When the user provides a video file for analysis, prefer using the `ReadMediaFile` tool to read it directly rather than writing Python scripts or ffmpeg commands to extract frames manually.** Detect if there are already such tools in the environment. If you have to install third-party tools/packages, you MUST ensure that they are installed in a virtual/isolated environment.
 - Once you generate or edit any images, videos or other media files, try to read it again before proceed, to ensure that the content is as expected.
 - Avoid installing or deleting anything to/from outside of the current working directory. If you have to do so, ask the user for confirmation.
 

packages/agent-core/src/tools/builtin/state/todo-list.md

@@ -19,9 +19,10 @@ Use this tool to maintain a structured TODO list as you work through a multi-ste
 - If no available tool can move any task forward, tell the user where you are stuck instead of repeatedly re-ordering the same todos.
 
 **How to use:**
-- Call with `todos: [...]` to replace the full list. Statuses: pending / in_progress / done.
+- Call with `todos: [...]` to replace the full list. Statuses: `pending` / `in_progress` / `done`.
 - Call with no `todos` argument to retrieve the current list without changing it.
 - Call with `todos: []` to clear the list.
+- **Important:** the status must be exactly `done`, not `completed` or `finished`.
 - Keep titles short and actionable (e.g. "Read session-control.ts", "Add planMode flag to TurnManager").
 - Update statuses as you make progress.
 - When work is underway, keep exactly one task `in_progress`.

packages/agent-core/src/tools/builtin/state/todo-list.ts

@@ -28,7 +28,7 @@ export const TODO_STORE_KEY = 'todo';
 const TODO_LIST_WRITE_REMINDER =
   'Ensure that you continue to use the todo list to track progress. Mark tasks done immediately after finishing them, and keep exactly one task in_progress when work is underway.';
 
-export type TodoStatus = 'pending' | 'in_progress' | 'done';
+export type TodoStatus = 'pending' | 'in_progress' | 'done' | 'completed';
 
 export interface TodoItem {
   readonly title: string;
@@ -45,7 +45,9 @@ declare module '../../store' {
 
 const TodoItemSchema = z.object({
   title: z.string().min(1).describe('Short, actionable title for the todo.'),
-  status: z.enum(['pending', 'in_progress', 'done']).describe('Current status of the todo.'),
+  status: z
+    .preprocess((val) => (val === 'completed' ? 'done' : val), z.enum(['pending', 'in_progress', 'done']))
+    .describe('Current status of the todo. Must be exactly one of: pending, in_progress, done. Do NOT use completed or finished.'),
 });
 
 export interface TodoListInput {
@@ -81,6 +83,7 @@ function statusMarker(status: TodoStatus): string {
     case 'in_progress':
       return '[in_progress]';
     case 'done':
+    case 'completed':
       return '[done]';
     default: {
       const _exhaustive: never = status;
@@ -133,7 +136,10 @@ export class TodoListTool implements BuiltinTool<TodoListInput> {
   private setTodos(todos: readonly TodoItem[]): void {
     this.store.set(
       TODO_STORE_KEY,
-      todos.map((todo) => ({ title: todo.title, status: todo.status })),
+      todos.map((todo) => ({
+        title: todo.title,
+        status: todo.status === 'completed' ? 'done' : todo.status,
+      })),
     );
   }
 }

packages/agent-core/test/tools/todo-list.test.ts

@@ -142,6 +142,23 @@ describe('TodoListTool', () => {
     ]);
   });
 
+  it('accepts "completed" as a status and maps it to "done"', async () => {
+    const { tool, getTodos } = makeTool();
+
+    const result = await executeTool(tool, {
+      turnId: 't1',
+      toolCallId: 'call_1',
+      args: {
+        todos: [{ title: 'done task', status: 'completed' }],
+      },
+      signal,
+    });
+
+    expect(result).toMatchObject({ isError: false });
+    expect(result.output).toContain('[done] done task');
+    expect(getTodos()).toEqual([{ title: 'done task', status: 'done' }]);
+  });
+
   it('renders a done todo with a marker matching the status enum value', async () => {
     const { tool } = makeTool([{ title: 'shipped', status: 'done' }]);