From 1f04cfa5c32e01e5ff003bee6820947fffa7f0ea Mon Sep 17 00:00:00 2001 From: "Li Zexian (External - Orange CyberDefense Sweden AB)" Date: Wed, 12 Aug 2026 12:01:41 +0800 Subject: [PATCH 1/2] Add GitHub Copilot parser and fix incremental timestamps for resumed sessions --- Sensor/README.md | 3 + Sensor/adr_sensor/observer.py | 5 +- Sensor/adr_sensor/parsers/__init__.py | 2 + Sensor/adr_sensor/parsers/claude_parser.py | 2 +- Sensor/adr_sensor/parsers/codex_parser.py | 10 +- Sensor/adr_sensor/parsers/copilot_parser.py | 589 ++++++++++++++++++++ Sensor/tests/test_observer.py | 2 + Sensor/tests/test_parsers.py | 165 ++++++ 8 files changed, 775 insertions(+), 3 deletions(-) create mode 100644 Sensor/adr_sensor/parsers/copilot_parser.py diff --git a/Sensor/README.md b/Sensor/README.md index bd81ecf..6d79f02 100644 --- a/Sensor/README.md +++ b/Sensor/README.md @@ -17,6 +17,7 @@ ADR Sensor is a Python library that collects telemetry from AI coding agents to | **Cline (Claude Dev)** | `cline` | JSON task files | macOS, Linux, Windows | | **Claude Desktop** | `claude_desktop` | JSONL audit logs | macOS, Windows | | **OpenAI Codex CLI** | `codex` | JSONL (`~/.codex/sessions/`) | macOS, Linux, Windows | +| **GitHub Copilot** | `copilot` | JSONL (`~/.copilot/session-state/`) | macOS, Linux, Windows | | **Warp Terminal** | `warp` | SQLite (`warp.sqlite`) | macOS, Windows | | **opencode** | `opencode` | SQLite (`opencode.db`) or JSON tree | macOS, Linux | @@ -110,6 +111,7 @@ adr-sensor adr-sensor --source claude adr-sensor --source cursor adr-sensor --source codex +adr-sensor --source copilot adr-sensor --source claude_desktop adr-sensor --source opencode @@ -349,6 +351,7 @@ adr-sensor/ │ │ ├── cline_parser.py │ │ ├── claude_desktop_parser.py │ │ ├── codex_parser.py +│ │ ├── copilot_parser.py │ │ ├── opencode_parser.py │ │ └── warp_parser.py │ ├── schemas/ diff --git a/Sensor/adr_sensor/observer.py b/Sensor/adr_sensor/observer.py index 97e2109..514f22b 100644 --- a/Sensor/adr_sensor/observer.py +++ b/Sensor/adr_sensor/observer.py @@ -19,6 +19,7 @@ from .parsers.claude_desktop_parser import ClaudeDesktopParser from .parsers.claude_parser import ClaudeParser from .parsers.cline_parser import ClineParser +from .parsers.copilot_parser import CopilotParser from .parsers.codex_parser import CodexParser from .parsers.cursor_parser import CursorParser from .parsers.opencode_parser import OpencodeParser @@ -50,6 +51,7 @@ class AgentObserver: ("cline", "Cline"), ("warp", "Warp Terminal"), ("codex", "Codex"), + ("copilot", "GitHub Copilot"), ("opencode", "opencode"), ) @@ -72,6 +74,7 @@ def __init__(self, output_dir: Optional[Path] = None, max_age_days: Optional[int ClaudeDesktopParser(max_age_days=max_age_days) if max_age_days is not None else ClaudeDesktopParser() ) self.codex_parser = CodexParser() + self.copilot_parser = CopilotParser() self.cline_parser = ClineParser() self.warp_parser = WarpParser(max_age_days=max_age_days) if max_age_days is not None else WarpParser() self.opencode_parser = ( @@ -114,7 +117,7 @@ def ingest_all( Args: source_filter: Which source to ingest. One of 'all', 'claude', 'cursor', - 'claude_desktop', 'cline', 'warp', 'codex', 'opencode'. + 'claude_desktop', 'cline', 'warp', 'codex', 'copilot', 'opencode'. Returns: Tuple of (agent_events, system_configs). diff --git a/Sensor/adr_sensor/parsers/__init__.py b/Sensor/adr_sensor/parsers/__init__.py index a3b1281..c94fff0 100644 --- a/Sensor/adr_sensor/parsers/__init__.py +++ b/Sensor/adr_sensor/parsers/__init__.py @@ -8,6 +8,7 @@ from .claude_desktop_parser import ClaudeDesktopParser from .claude_parser import ClaudeParser from .cline_parser import ClineParser +from .copilot_parser import CopilotParser from .codex_parser import CodexParser from .cursor_parser import CursorParser from .opencode_parser import OpencodeParser @@ -18,6 +19,7 @@ "ClaudeDesktopParser", "ClaudeParser", "ClineParser", + "CopilotParser", "CodexParser", "CursorParser", "OpencodeParser", diff --git a/Sensor/adr_sensor/parsers/claude_parser.py b/Sensor/adr_sensor/parsers/claude_parser.py index 856de38..05000b7 100644 --- a/Sensor/adr_sensor/parsers/claude_parser.py +++ b/Sensor/adr_sensor/parsers/claude_parser.py @@ -127,7 +127,7 @@ def parse_jsonl_file(self, file_path: Path) -> List[AgentEvent]: if "timestamp" in obj: try: ts = normalize_timestamp(obj["timestamp"]) - if sessions[session_id]["timestamp"] is None or ts < sessions[session_id]["timestamp"]: + if sessions[session_id]["timestamp"] is None or ts > sessions[session_id]["timestamp"]: sessions[session_id]["timestamp"] = ts except Exception: pass diff --git a/Sensor/adr_sensor/parsers/codex_parser.py b/Sensor/adr_sensor/parsers/codex_parser.py index 4987c95..42c430f 100644 --- a/Sensor/adr_sensor/parsers/codex_parser.py +++ b/Sensor/adr_sensor/parsers/codex_parser.py @@ -48,6 +48,7 @@ def parse_jsonl_file(self, file_path: Path) -> Optional[AgentEvent]: session_data: Dict[str, Any] = { "id": None, "timestamp": None, + "last_event_timestamp": None, "cwd": None, "model": None, "messages": [], @@ -94,7 +95,7 @@ def parse_jsonl_file(self, file_path: Path) -> Optional[AgentEvent]: ) return AgentEvent( - timestamp=session_data["timestamp"] or datetime.now(timezone.utc), + timestamp=session_data["last_event_timestamp"] or session_data["timestamp"] or datetime.now(timezone.utc), source="codex", session_id=f"codex_{session_data['id']}", project_path=session_data["cwd"], @@ -174,6 +175,13 @@ def _process_event(self, event: Dict[str, Any], session_data: Dict[str, Any]): evt_type = event.get("type") payload = event.get("payload", {}) + event_timestamp = event.get("timestamp") + if event_timestamp: + normalized = normalize_timestamp(event_timestamp) + current = session_data.get("last_event_timestamp") + if current is None or normalized > current: + session_data["last_event_timestamp"] = normalized + if evt_type == "session_meta": session_data["id"] = payload.get("id") if payload.get("timestamp"): diff --git a/Sensor/adr_sensor/parsers/copilot_parser.py b/Sensor/adr_sensor/parsers/copilot_parser.py new file mode 100644 index 0000000..a39845d --- /dev/null +++ b/Sensor/adr_sensor/parsers/copilot_parser.py @@ -0,0 +1,589 @@ +""" +Parser for GitHub Copilot session-state logs. + +Reads per-session event streams from ``~/.copilot/session-state//`` +and normalizes them into ADR's ``AgentEvent`` schema. + +The primary signal lives in ``events.jsonl``. When available, the parser also +enriches sessions with lightweight metadata from ``workspace.yaml`` and +``vscode.metadata.json``. +""" + +import json +import traceback +from collections import Counter +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Dict, List, Optional + +from ..schemas.agent_event_schema import AgentEvent, ChatMessage, ToolUsage +from ..utils.string_utils import truncate_middle +from ..utils.timestamp_utils import normalize_timestamp +from .base_parser import BaseParser + +MAX_STRING_LENGTH = 1000 +EDGE_CHARS = 400 + + +class CopilotParser(BaseParser): + """Parser for GitHub Copilot session-state logs.""" + + def __init__(self, base_path: Optional[Path] = None): + self.base_path = Path(base_path) if base_path else Path.home() / ".copilot" / "session-state" + + def parse_all(self) -> List[AgentEvent]: + """Parse all available Copilot sessions.""" + entries: List[AgentEvent] = [] + + if not self.base_path.exists(): + print(f"[COPILOT] No logs found at {self.base_path}") + return entries + + session_dirs = [path for path in self.base_path.iterdir() if path.is_dir() and (path / "events.jsonl").exists()] + session_dirs.sort(key=lambda path: path.stat().st_mtime, reverse=True) + print(f"[COPILOT] Found {len(session_dirs)} session directories") + + for session_dir in session_dirs: + try: + entry = self.parse_session_dir(session_dir) + if entry and entry.has_meaningful_content(): + entries.append(entry) + except Exception as exc: + print(f"[COPILOT] Error parsing {session_dir}: {exc}") + + return entries + + def parse_session_dir(self, session_dir: Path) -> Optional[AgentEvent]: + """Parse one Copilot session directory.""" + events_path = session_dir / "events.jsonl" + if not events_path.exists(): + return None + + workspace_meta = self._load_workspace_yaml(session_dir / "workspace.yaml") + vscode_meta = self._load_json_file(session_dir / "vscode.metadata.json") + + session_data: Dict[str, Any] = { + "id": session_dir.name, + "timestamp": None, + "cwd": workspace_meta.get("cwd"), + "model": None, + "messages": [], + "pending_tool_calls": {}, + "first_event_at": None, + "last_event_at": None, + "event_counts": Counter(), + "workspace_metadata": workspace_meta, + "vscode_metadata": self._sanitize_for_context(vscode_meta) if isinstance(vscode_meta, dict) else {}, + "model_changes": [], + "usage_checkpoints": [], + "permissions": [], + "skills_invoked": [], + "subagents": [], + "session_info": [], + "mode_changes": [], + "hooks": [], + "system_messages": [], + "system_notifications": [], + "workspace_file_changes": [], + "plan_changes": [], + "transformed_user_messages": [], + } + + try: + with open(events_path, encoding="utf-8") as handle: + for line in handle: + line = line.strip() + if not line: + continue + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + self._process_event(event, session_data) + except Exception as exc: + print(f"[COPILOT] Error reading {events_path}: {exc}") + traceback.print_exc() + return None + + chat_history: List[ChatMessage] = [] + for index, msg_dict in enumerate(session_data["messages"]): + tools = [ + ToolUsage( + tool_name=tool_dict["tool_name"], + tool_type=tool_dict["tool_type"], + arguments=tool_dict["arguments"], + result=tool_dict.get("result"), + status=tool_dict.get("status"), + error=tool_dict.get("error"), + ) + for tool_dict in msg_dict["tools"] + ] + + chat_history.append( + ChatMessage( + role=msg_dict["role"], + content=msg_dict["content"], + tools=tools, + sequence_id=msg_dict.get("sequence_id") or f"{session_data['id']}_msg_{index}", + ) + ) + + if not chat_history: + return None + + timestamp = ( + self._normalize_optional_timestamp(workspace_meta.get("updated_at")) + or self._normalize_optional_timestamp(vscode_meta.get("modified")) + or self._normalize_optional_timestamp(session_data["last_event_at"]) + or session_data["timestamp"] + or self._normalize_optional_timestamp(workspace_meta.get("created_at")) + or self._normalize_optional_timestamp(vscode_meta.get("created")) + or self._normalize_optional_timestamp(session_data["first_event_at"]) + or datetime.now(timezone.utc) + ) + + context = self._build_session_context(session_data) + + return AgentEvent( + timestamp=timestamp, + source="copilot", + session_id=f"copilot_{session_data['id']}", + project_path=session_data["cwd"], + model=session_data["model"], + chat_history=chat_history, + raw_log_path=str(events_path), + session_context=context or None, + ) + + def _process_event(self, event: Dict[str, Any], session_data: Dict[str, Any]): + """Process a single Copilot event from events.jsonl.""" + event_type = event.get("type") + if not event_type: + return + + data = event.get("data", {}) + event_time = self._normalize_optional_timestamp(event.get("timestamp")) + + session_data["event_counts"][event_type] += 1 + if event_time and session_data["first_event_at"] is None: + session_data["first_event_at"] = event_time + if event_time: + session_data["last_event_at"] = event_time + + if event_type == "session.start": + session_id = data.get("sessionId") + if session_id: + session_data["id"] = session_id + session_data["timestamp"] = self._normalize_optional_timestamp(data.get("startTime")) or event_time + session_data["model"] = data.get("selectedModel") or session_data["model"] + context = data.get("context", {}) + if isinstance(context, dict): + session_data["cwd"] = context.get("cwd") or session_data["cwd"] + return + + if event_type == "session.resume": + session_id = data.get("sessionId") + if session_id: + session_data["id"] = session_id + context = data.get("context", {}) + if isinstance(context, dict): + session_data["cwd"] = context.get("cwd") or session_data["cwd"] + return + + if event_type == "user.message": + content = (data.get("content") or "").strip() + transformed = (data.get("transformedContent") or "").strip() + if transformed and transformed != content: + session_data["transformed_user_messages"].append( + { + "sequence_id": event.get("id"), + "content": truncate_middle(transformed, max_length=MAX_STRING_LENGTH, edge_chars=EDGE_CHARS), + } + ) + if content: + session_data["messages"].append( + { + "role": "user", + "content": content, + "tools": [], + "sequence_id": event.get("id"), + } + ) + return + + if event_type == "assistant.message": + content = (data.get("content") or "").strip() + tools = [] + for request in data.get("toolRequests") or []: + tool_call_id = request.get("toolCallId") + tool_dict = { + "tool_name": request.get("name") or "unknown", + "tool_type": request.get("type") or "tool_request", + "arguments": self._truncate_large_arguments(request.get("arguments") or {}), + "result": None, + "status": "pending", + "error": None, + } + tools.append(tool_dict) + if tool_call_id: + session_data["pending_tool_calls"][tool_call_id] = tool_dict + + if content or tools: + session_data["messages"].append( + { + "role": "assistant", + "content": content, + "tools": tools, + "sequence_id": data.get("messageId") or event.get("id"), + } + ) + + session_data["model"] = data.get("model") or session_data["model"] + return + + if event_type == "tool.execution_start": + tool = self._get_or_create_tool( + session_data, + data.get("toolCallId"), + default_tool_name=data.get("toolName"), + ) + if tool is not None: + tool["tool_name"] = data.get("toolName") or tool["tool_name"] + tool["arguments"] = self._truncate_large_arguments(data.get("arguments") or tool["arguments"]) + tool["status"] = "running" + session_data["model"] = data.get("model") or session_data["model"] + return + + if event_type == "tool.execution_complete": + tool = self._get_or_create_tool(session_data, data.get("toolCallId"), default_tool_name=None) + if tool is not None: + success = bool(data.get("success")) + result_text = self._normalize_tool_result(data.get("result")) + tool["result"] = result_text + tool["status"] = "success" if success else "error" + if not success and result_text: + tool["error"] = result_text + session_data["model"] = data.get("model") or session_data["model"] + return + + if event_type == "permission.requested": + permission_record = { + "request_id": data.get("requestId"), + "tool_call_id": self._extract_tool_call_id_from_permission(data), + "permission_kind": self._extract_permission_kind(data), + "command": self._extract_permission_command(data), + "timestamp": event.get("timestamp"), + } + session_data["permissions"].append(self._sanitize_for_context(permission_record)) + + tool = self._get_or_create_tool( + session_data, + permission_record["tool_call_id"], + default_tool_name="permission_request", + ) + if tool is not None and tool.get("status") == "pending": + tool["status"] = "permission_requested" + return + + if event_type == "permission.completed": + permission_result = data.get("result", {}) + permission_record = { + "request_id": data.get("requestId"), + "tool_call_id": data.get("toolCallId"), + "result": permission_result, + "timestamp": event.get("timestamp"), + } + session_data["permissions"].append(self._sanitize_for_context(permission_record)) + + tool = self._get_or_create_tool(session_data, data.get("toolCallId"), default_tool_name=None) + kind = permission_result.get("kind") if isinstance(permission_result, dict) else None + if tool is not None and tool.get("status") in {"pending", "permission_requested"} and kind == "approved": + tool["status"] = "approved" + return + + if event_type == "session.model_change": + session_data["model_changes"].append( + self._sanitize_for_context( + { + "previous_model": data.get("previousModel"), + "new_model": data.get("newModel"), + "previous_reasoning_effort": data.get("previousReasoningEffort"), + "reasoning_effort": data.get("reasoningEffort"), + "timestamp": event.get("timestamp"), + } + ) + ) + session_data["model"] = data.get("newModel") or session_data["model"] + return + + if event_type == "session.usage_checkpoint": + session_data["usage_checkpoints"].append(self._sanitize_for_context(data)) + return + + if event_type == "skill.invoked": + session_data["skills_invoked"].append( + self._sanitize_for_context( + { + "name": data.get("name"), + "path": data.get("path"), + "content": data.get("content"), + } + ) + ) + return + + if event_type == "subagent.started": + session_data["subagents"].append( + self._sanitize_for_context( + { + "agent_id": event.get("agentId"), + "tool_call_id": data.get("toolCallId"), + "agent_name": data.get("agentName"), + "display_name": data.get("agentDisplayName"), + "description": data.get("agentDescription"), + "timestamp": event.get("timestamp"), + } + ) + ) + return + + if event_type == "session.info": + session_data["session_info"].append(self._sanitize_for_context(data)) + return + + if event_type == "session.mode_changed": + session_data["mode_changes"].append(self._sanitize_for_context(data)) + return + + if event_type in {"hook.start", "hook.end"}: + session_data["hooks"].append( + self._sanitize_for_context( + { + "type": event_type, + "timestamp": event.get("timestamp"), + "data": data, + } + ) + ) + return + + if event_type == "system.message": + content = data.get("content") + if isinstance(content, str) and content.strip(): + session_data["system_messages"].append( + truncate_middle(content.strip(), max_length=MAX_STRING_LENGTH, edge_chars=EDGE_CHARS) + ) + return + + if event_type == "system.notification": + session_data["system_notifications"].append(self._sanitize_for_context(data)) + return + + if event_type == "session.workspace_file_changed": + session_data["workspace_file_changes"].append(self._sanitize_for_context(data)) + return + + if event_type == "session.plan_changed": + session_data["plan_changes"].append(self._sanitize_for_context(data)) + + def _get_or_create_tool( + self, + session_data: Dict[str, Any], + tool_call_id: Optional[str], + default_tool_name: Optional[str], + ) -> Optional[Dict[str, Any]]: + """Return a mutable tool dict for a toolCallId, creating an orphan entry if needed.""" + if not tool_call_id: + return None + + existing = session_data["pending_tool_calls"].get(tool_call_id) + if existing is not None: + return existing + + tool_dict = { + "tool_name": default_tool_name or "unknown", + "tool_type": "tool_execution", + "arguments": {}, + "result": None, + "status": "pending", + "error": None, + } + session_data["pending_tool_calls"][tool_call_id] = tool_dict + session_data["messages"].append( + { + "role": "assistant", + "content": "", + "tools": [tool_dict], + "sequence_id": f"{session_data['id']}_tool_{tool_call_id}", + } + ) + return tool_dict + + def _build_session_context(self, session_data: Dict[str, Any]) -> Dict[str, Any]: + """Build session-level context from side-channel event metadata.""" + context = { + "workspace_metadata": session_data["workspace_metadata"], + "vscode_metadata": session_data["vscode_metadata"], + "first_event_at": self._format_datetime(session_data["first_event_at"]), + "last_event_at": self._format_datetime(session_data["last_event_at"]), + "event_counts": dict(session_data["event_counts"]), + "model_changes": session_data["model_changes"], + "usage_checkpoints": session_data["usage_checkpoints"], + "permissions": session_data["permissions"], + "skills_invoked": session_data["skills_invoked"], + "subagents": session_data["subagents"], + "session_info": session_data["session_info"], + "mode_changes": session_data["mode_changes"], + "hooks": session_data["hooks"], + "system_messages": session_data["system_messages"][:5], + "system_notifications": session_data["system_notifications"], + "workspace_file_changes": session_data["workspace_file_changes"], + "plan_changes": session_data["plan_changes"], + "transformed_user_messages": session_data["transformed_user_messages"], + } + + filtered: Dict[str, Any] = {} + for key, value in context.items(): + if isinstance(value, dict) and value: + filtered[key] = value + elif isinstance(value, list) and value: + filtered[key] = value + elif isinstance(value, str) and value: + filtered[key] = value + return filtered + + def _normalize_tool_result(self, result: Any) -> Optional[str]: + """Normalize tool result payloads to a truncated string.""" + if result is None: + return None + + if isinstance(result, str): + text = result + elif isinstance(result, dict): + text = result.get("detailedContent") or result.get("content") or json.dumps(result, ensure_ascii=False) + else: + text = str(result) + + if not text: + return text + + return truncate_middle(text, max_length=MAX_STRING_LENGTH, edge_chars=EDGE_CHARS) + + def _truncate_large_arguments(self, arguments: Dict[str, Any]) -> Dict[str, Any]: + """Truncate large string values within tool argument dictionaries.""" + if not isinstance(arguments, dict): + return {"raw": arguments} + + return {key: self._sanitize_for_context(value) for key, value in arguments.items()} + + def _sanitize_for_context(self, value: Any) -> Any: + """Recursively truncate long strings so session_context stays bounded.""" + if isinstance(value, str): + return truncate_middle(value, max_length=MAX_STRING_LENGTH, edge_chars=EDGE_CHARS) + if isinstance(value, dict): + return {str(key): self._sanitize_for_context(val) for key, val in value.items()} + if isinstance(value, list): + return [self._sanitize_for_context(item) for item in value] + return value + + def _load_json_file(self, path: Path) -> Dict[str, Any]: + """Read a JSON file if it exists, else return an empty dict.""" + if not path.exists(): + return {} + + try: + with open(path, encoding="utf-8") as handle: + value = json.load(handle) + return value if isinstance(value, dict) else {} + except Exception: + return {} + + def _load_workspace_yaml(self, path: Path) -> Dict[str, Any]: + """Parse the simple key/value workspace.yaml emitted by Copilot.""" + if not path.exists(): + return {} + + data: Dict[str, Any] = {} + try: + with open(path, encoding="utf-8") as handle: + for raw_line in handle: + line = raw_line.strip() + if not line or ":" not in line: + continue + key, value = line.split(":", 1) + data[key.strip()] = self._coerce_scalar(value.strip()) + except Exception: + return {} + return data + + def _coerce_scalar(self, value: str) -> Any: + """Coerce simple YAML scalars without pulling in a YAML dependency.""" + if value == "": + return "" + if value.lower() == "true": + return True + if value.lower() == "false": + return False + try: + return int(value) + except ValueError: + return value + + def _normalize_optional_timestamp(self, value: Any) -> Optional[datetime]: + """Normalize a timestamp when possible.""" + if value in (None, ""): + return None + try: + return normalize_timestamp(value) + except Exception: + return None + + @staticmethod + def _format_datetime(value: Optional[datetime]) -> Optional[str]: + """Format datetimes for session_context.""" + return value.isoformat() if isinstance(value, datetime) else None + + @staticmethod + def _extract_tool_call_id_from_permission(data: Dict[str, Any]) -> Optional[str]: + """Extract toolCallId from the permission event payload.""" + if data.get("toolCallId"): + return data.get("toolCallId") + + permission_request = data.get("permissionRequest", {}) + if isinstance(permission_request, dict) and permission_request.get("toolCallId"): + return permission_request.get("toolCallId") + + prompt_request = data.get("promptRequest", {}) + if isinstance(prompt_request, dict): + return prompt_request.get("toolCallId") + + return None + + @staticmethod + def _extract_permission_kind(data: Dict[str, Any]) -> Optional[str]: + """Return the requested permission kind if present.""" + permission_request = data.get("permissionRequest", {}) + if isinstance(permission_request, dict) and permission_request.get("kind"): + return permission_request.get("kind") + + prompt_request = data.get("promptRequest", {}) + if isinstance(prompt_request, dict): + return prompt_request.get("kind") + + return None + + def _extract_permission_command(self, data: Dict[str, Any]) -> Optional[str]: + """Return the shell command associated with a permission request.""" + permission_request = data.get("permissionRequest", {}) + if isinstance(permission_request, dict): + command = permission_request.get("fullCommandText") + if isinstance(command, str) and command.strip(): + return truncate_middle(command.strip(), max_length=MAX_STRING_LENGTH, edge_chars=EDGE_CHARS) + + prompt_request = data.get("promptRequest", {}) + if isinstance(prompt_request, dict): + command = prompt_request.get("fullCommandText") + if isinstance(command, str) and command.strip(): + return truncate_middle(command.strip(), max_length=MAX_STRING_LENGTH, edge_chars=EDGE_CHARS) + + return None diff --git a/Sensor/tests/test_observer.py b/Sensor/tests/test_observer.py index 2854b0f..6802cef 100644 --- a/Sensor/tests/test_observer.py +++ b/Sensor/tests/test_observer.py @@ -16,6 +16,8 @@ def test_init_default(self, tmp_path): """Test default initialization.""" observer = AgentObserver(output_dir=tmp_path) assert observer.output_dir == tmp_path + assert hasattr(observer, "copilot_parser") + assert ("copilot", "GitHub Copilot") in observer.SOURCES def test_display_summary_empty(self, tmp_path, capsys): """Test display summary with no data.""" diff --git a/Sensor/tests/test_parsers.py b/Sensor/tests/test_parsers.py index 4112a14..ec13721 100644 --- a/Sensor/tests/test_parsers.py +++ b/Sensor/tests/test_parsers.py @@ -12,6 +12,7 @@ from adr_sensor.parsers.claude_desktop_parser import ClaudeDesktopParser from adr_sensor.parsers.claude_parser import ClaudeParser from adr_sensor.parsers.cline_parser import ClineParser +from adr_sensor.parsers.copilot_parser import CopilotParser from adr_sensor.parsers.codex_parser import CodexParser from adr_sensor.parsers.cursor_parser import CursorParser from adr_sensor.parsers.opencode_parser import OpencodeParser @@ -101,6 +102,37 @@ def test_truncate_large_arguments(self): assert len(result["long"]) < 2000 assert "[truncated" in result["long"] + def test_uses_latest_timestamp_for_resumed_session(self, tmp_path): + """Incremental export should see a resumed session as updated.""" + jsonl_file = tmp_path / "resumed.jsonl" + messages = [ + { + "type": "user", + "sessionId": "session1", + "timestamp": "2025-06-15T10:00:00Z", + "message": {"content": "first prompt"}, + }, + { + "type": "assistant", + "sessionId": "session1", + "timestamp": "2025-06-15T10:00:01Z", + "message": {"content": [{"type": "text", "text": "first reply"}]}, + }, + { + "type": "user", + "sessionId": "session1", + "timestamp": "2025-06-16T12:30:00Z", + "message": {"content": "continued next day"}, + }, + ] + with open(jsonl_file, "w", encoding="utf-8") as f: + for msg in messages: + f.write(json.dumps(msg) + "\n") + + entries = ClaudeParser().parse_jsonl_file(jsonl_file) + assert len(entries) == 1 + assert entries[0].timestamp == datetime(2025, 6, 16, 12, 30, 0, tzinfo=timezone.utc) + class TestClineParser: def test_parse_cline_log(self, tmp_path): @@ -434,6 +466,139 @@ def test_parse_no_directory(self): entries = parser.parse_all() assert entries == [] + def test_uses_latest_event_timestamp_for_resumed_session(self, tmp_path): + """Incremental export should use the last event, not only session_meta.""" + jsonl_file = tmp_path / "resumed-codex.jsonl" + events = [ + { + "type": "session_meta", + "timestamp": "2025-06-15T10:00:00Z", + "payload": {"id": "sess-resume", "timestamp": "2025-06-15T10:00:00Z", "cwd": "/tmp"}, + }, + { + "type": "response_item", + "timestamp": "2025-06-15T10:00:01Z", + "payload": { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "first question"}], + }, + }, + { + "type": "response_item", + "timestamp": "2025-06-16T12:30:00Z", + "payload": { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "later follow-up"}], + }, + }, + ] + with open(jsonl_file, "w", encoding="utf-8") as f: + for event in events: + f.write(json.dumps(event) + "\n") + + entry = CodexParser().parse_jsonl_file(jsonl_file) + assert entry is not None + assert entry.timestamp == datetime(2025, 6, 16, 12, 30, 0, tzinfo=timezone.utc) + + +class TestCopilotParser: + def test_parse_session_dir(self, tmp_path): + """Parse Copilot session directories into chat history and tool usage.""" + session_dir = tmp_path / "abc-session" + session_dir.mkdir() + + (session_dir / "workspace.yaml").write_text( + "\n".join( + [ + "id: abc-session", + "cwd: C:\\repo", + "client_name: vscode", + "name: Sample session", + "created_at: 2026-08-10T10:00:00.000Z", + "updated_at: 2026-08-10T11:00:00.000Z", + ] + ), + encoding="utf-8", + ) + (session_dir / "vscode.metadata.json").write_text( + json.dumps( + { + "origin": "vscode", + "created": 1785470000000, + "modified": 1785476400000, + "firstUserMessage": "inspect the repo", + } + ), + encoding="utf-8", + ) + + events = [ + { + "type": "session.start", + "timestamp": "2026-08-10T10:00:00.000Z", + "data": { + "sessionId": "abc-session", + "startTime": "2026-08-10T10:00:00.000Z", + "selectedModel": "gpt-5.6-sol", + "context": {"cwd": "C:\\repo"}, + }, + }, + { + "type": "user.message", + "id": "user-1", + "timestamp": "2026-08-10T10:00:05.000Z", + "data": {"content": "inspect the repo", "transformedContent": "...\ninspect the repo"}, + }, + { + "type": "assistant.message", + "id": "assistant-1", + "timestamp": "2026-08-10T10:00:06.000Z", + "data": { + "messageId": "assistant-message-1", + "model": "gpt-5.6-sol", + "content": "I will inspect it.", + "toolRequests": [ + { + "toolCallId": "call-1", + "name": "powershell", + "type": "function", + "arguments": {"command": "Get-ChildItem", "description": "List files"}, + } + ], + }, + }, + { + "type": "tool.execution_complete", + "timestamp": "2026-08-10T11:00:00.000Z", + "data": { + "toolCallId": "call-1", + "model": "gpt-5.6-sol", + "success": True, + "result": {"content": "file1\nfile2"}, + }, + }, + ] + with open(session_dir / "events.jsonl", "w", encoding="utf-8") as f: + for event in events: + f.write(json.dumps(event) + "\n") + + entry = CopilotParser(base_path=tmp_path).parse_session_dir(session_dir) + assert entry is not None + assert entry.source == "copilot" + assert entry.session_id == "copilot_abc-session" + assert entry.project_path == "C:\\repo" + assert entry.model == "gpt-5.6-sol" + assert entry.timestamp == datetime(2026, 8, 10, 11, 0, 0, tzinfo=timezone.utc) + assert [msg.role for msg in entry.chat_history] == ["user", "assistant"] + tool = entry.chat_history[1].tools[0] + assert tool.tool_name == "powershell" + assert tool.result == "file1\nfile2" + assert tool.status == "success" + assert entry.session_context["workspace_metadata"]["updated_at"] == "2026-08-10T11:00:00.000Z" + assert entry.session_context["transformed_user_messages"][0]["sequence_id"] == "user-1" + def _build_warp_db(db_path: Path, conversations: list) -> None: """Create a synthetic warp.sqlite matching the schema WarpParser queries. From 2e889da1e59aa24a80d64cd09371abe5b5fb8569 Mon Sep 17 00:00:00 2001 From: "Li Zexian (External - Orange CyberDefense Sweden AB)" Date: Wed, 12 Aug 2026 12:14:18 +0800 Subject: [PATCH 2/2] Add GitHub Copilot parser and fix incremental timestamps for resumed sessions --- Sensor/adr_sensor/parsers/codex_parser.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/Sensor/adr_sensor/parsers/codex_parser.py b/Sensor/adr_sensor/parsers/codex_parser.py index 42c430f..5d0958a 100644 --- a/Sensor/adr_sensor/parsers/codex_parser.py +++ b/Sensor/adr_sensor/parsers/codex_parser.py @@ -177,10 +177,13 @@ def _process_event(self, event: Dict[str, Any], session_data: Dict[str, Any]): event_timestamp = event.get("timestamp") if event_timestamp: - normalized = normalize_timestamp(event_timestamp) - current = session_data.get("last_event_timestamp") - if current is None or normalized > current: - session_data["last_event_timestamp"] = normalized + try: + normalized = normalize_timestamp(event_timestamp) + current = session_data.get("last_event_timestamp") + if current is None or normalized > current: + session_data["last_event_timestamp"] = normalized + except Exception: + pass if evt_type == "session_meta": session_data["id"] = payload.get("id")