diff --git a/python/packages/github_copilot/agent_framework_github_copilot/_agent.py b/python/packages/github_copilot/agent_framework_github_copilot/_agent.py index 9256e3eb11..1a78fb185f 100644 --- a/python/packages/github_copilot/agent_framework_github_copilot/_agent.py +++ b/python/packages/github_copilot/agent_framework_github_copilot/_agent.py @@ -30,8 +30,12 @@ ) from agent_framework._settings import load_settings from agent_framework._tools import FunctionTool, ToolTypes -from agent_framework._types import AgentRunInputs, normalize_tools -from agent_framework.exceptions import AgentException +from agent_framework._types import ( + AgentRunInputs, + _get_data_bytes_as_str, # pyright: ignore[reportPrivateUsage] + normalize_tools, +) +from agent_framework.exceptions import AgentException, ContentError from agent_framework.observability import AgentTelemetryLayer if sys.version_info >= (3, 11): @@ -47,6 +51,8 @@ from copilot import CopilotClient, CopilotSession, RuntimeConnection from copilot.generated.rpc import PermissionDecisionUserNotAvailable from copilot.session import ( + Attachment, + BlobAttachment, MCPServerConfig, PermissionRequestResult, PreToolUseHandler, @@ -656,10 +662,11 @@ def usage_event_handler(event: SessionEvent) -> None: prompt = "\n".join([message.text for message in context_messages]) if session_context.instructions: prompt = "\n".join(session_context.instructions) + "\n" + prompt + attachments = self._prepare_attachments_for_copilot(context_messages) unsubscribe = copilot_session.on(usage_event_handler) try: - response_event = await copilot_session.send_and_wait(prompt, timeout=timeout) + response_event = await copilot_session.send_and_wait(prompt, attachments=attachments, timeout=timeout) except Exception as ex: raise AgentException(f"GitHub Copilot request failed: {ex}") from ex finally: @@ -762,6 +769,7 @@ async def _stream_updates( prompt = "\n".join([message.text for message in context_messages]) if session_context.instructions: prompt = "\n".join(session_context.instructions) + "\n" + prompt + attachments = self._prepare_attachments_for_copilot(context_messages) queue: asyncio.Queue[AgentResponseUpdate | Exception | None] = asyncio.Queue() @@ -844,7 +852,7 @@ def event_handler(event: SessionEvent) -> None: unsubscribe = copilot_session.on(event_handler) try: - await copilot_session.send(prompt) + await copilot_session.send(prompt, attachments=attachments) while (item := await queue.get()) is not None: if isinstance(item, Exception): @@ -916,6 +924,56 @@ def _prepare_system_message( elif opts_system_message is not None: opts["system_message"] = opts_system_message + @staticmethod + def _prepare_attachments_for_copilot(messages: Sequence[Message]) -> list[Attachment] | None: + """Convert inline binary message content into Copilot SDK attachments. + + Scans the outgoing messages for ``data`` content (binary payloads such as + images or documents carried as base64 data URIs) and maps each one to an + inline ``blob`` attachment understood by the Copilot SDK. + + Only base64 ``data:`` content is forwarded as an attachment. Other content + is not turned into an attachment: text content is already carried in the + prompt, while remote URIs (for example ``https://`` links) and malformed or + non-base64 ``data:`` URIs are skipped -- they are neither attached nor added + to the prompt. + + Args: + messages: The messages being sent to the Copilot session. + + Returns: + A list of Copilot ``Attachment`` objects, or ``None`` when the messages + contain no attachable binary content. + """ + attachments: list[Attachment] = [] + for message in messages: + for content in message.contents: + if content.type != "data": + continue + try: + data_str = _get_data_bytes_as_str(content) + except ContentError: + logger.warning( + "Skipping GitHub Copilot attachment with an unsupported data URI; " + "only base64-encoded 'data:' URIs can be forwarded as attachments." + ) + continue + if not data_str: + continue + if not content.media_type: + logger.warning( + "Dropping GitHub Copilot attachment with no media type; the Copilot SDK " + "requires a MIME type for inline binary content." + ) + continue + blob: BlobAttachment = { + "type": "blob", + "data": data_str, + "mimeType": content.media_type, + } + attachments.append(blob) + return attachments or None + def _prepare_tools( self, tools: Sequence[ToolTypes | CopilotTool], diff --git a/python/packages/github_copilot/tests/test_github_copilot_agent.py b/python/packages/github_copilot/tests/test_github_copilot_agent.py index 9d5a113dd7..559180856f 100644 --- a/python/packages/github_copilot/tests/test_github_copilot_agent.py +++ b/python/packages/github_copilot/tests/test_github_copilot_agent.py @@ -2,6 +2,7 @@ # ruff: noqa: E402 +import base64 import inspect import os import unittest.mock @@ -37,7 +38,7 @@ ) from copilot.tools import ToolInvocation, ToolResult -from agent_framework_github_copilot import GitHubCopilotAgent, GitHubCopilotOptions +from agent_framework_github_copilot import GitHubCopilotAgent, GitHubCopilotOptions, RawGitHubCopilotAgent def copilot_options(options: GitHubCopilotOptions) -> GitHubCopilotOptions: @@ -3351,10 +3352,131 @@ def load_skill(skill_name: str) -> str: assert "load_skill" in tool_names +class TestGitHubCopilotAttachments: + """Tests for forwarding inline binary message content as Copilot attachments.""" + + async def test_data_content_forwarded_as_blob_attachment( + self, + mock_client: MagicMock, + mock_session: MagicMock, + assistant_message_event: SessionEvent, + ) -> None: + """Non-streaming: DataContent is sent to the SDK as an inline blob attachment.""" + mock_session.send_and_wait.return_value = assistant_message_event + image_bytes = b"\x89PNG\r\n\x1a\n-fake-image" + message = Message( + role="user", + contents=[ + Content.from_text("Describe this image"), + Content.from_data(data=image_bytes, media_type="image/png"), + ], + ) + + agent = GitHubCopilotAgent(client=mock_client) + await agent.run(message) + + attachments = mock_session.send_and_wait.call_args.kwargs["attachments"] + assert attachments is not None + assert len(attachments) == 1 + assert attachments[0]["type"] == "blob" + assert attachments[0]["mimeType"] == "image/png" + assert base64.b64decode(attachments[0]["data"]) == image_bytes + + async def test_data_content_forwarded_as_blob_attachment_streaming( + self, + mock_client: MagicMock, + mock_session: MagicMock, + session_idle_event: SessionEvent, + ) -> None: + """Streaming: DataContent is sent to the SDK as an inline blob attachment.""" + + def mock_on(handler: Any) -> Any: + handler(session_idle_event) + return lambda: None + + mock_session.on = mock_on + image_bytes = b"\x89PNG\r\n\x1a\n-fake-image" + message = Message( + role="user", + contents=[ + Content.from_text("Describe this image"), + Content.from_data(data=image_bytes, media_type="image/png"), + ], + ) + + agent = GitHubCopilotAgent(client=mock_client) + async for _ in agent.run(message, stream=True): + pass + + attachments = mock_session.send.call_args.kwargs["attachments"] + assert attachments is not None + assert len(attachments) == 1 + assert attachments[0]["type"] == "blob" + assert attachments[0]["mimeType"] == "image/png" + assert base64.b64decode(attachments[0]["data"]) == image_bytes + + async def test_text_only_message_sends_no_attachments( + self, + mock_client: MagicMock, + mock_session: MagicMock, + assistant_message_event: SessionEvent, + ) -> None: + """A text-only message results in no attachments being forwarded.""" + mock_session.send_and_wait.return_value = assistant_message_event + + agent = GitHubCopilotAgent(client=mock_client) + await agent.run("Just text, no attachments") + + assert mock_session.send_and_wait.call_args.kwargs["attachments"] is None + + def test_prepare_attachments_skips_data_without_media_type(self) -> None: + """Data content lacking a media type is dropped rather than sent without a MIME type.""" + content = Content.from_data(data=b"payload", media_type="application/octet-stream") + content.media_type = None + message = Message(role="user", contents=[content]) + + attachments = GitHubCopilotAgent._prepare_attachments_for_copilot([message]) + + assert attachments is None + + async def test_non_base64_data_uri_is_skipped_not_raised( + self, + mock_client: MagicMock, + mock_session: MagicMock, + assistant_message_event: SessionEvent, + ) -> None: + """A non-base64 ``data:`` URI is skipped by the send path instead of failing the request. + + Uses ``RawGitHubCopilotAgent`` (no telemetry layer) to isolate the provider's own + attachment handling. The telemetry layer in ``GitHubCopilotAgent`` independently + serializes message content and would trip a separate core limitation on this + contrived input, which is unrelated to attachment forwarding. + """ + mock_session.send_and_wait.return_value = assistant_message_event + # ``Content.from_uri`` classifies this as type="data" but it is not base64-encoded, + # so extracting its bytes raises ContentError internally. + non_base64 = Content.from_uri("data:text/plain,hello") + assert non_base64.type == "data" + message = Message(role="user", contents=[Content.from_text("hi"), non_base64]) + + agent = RawGitHubCopilotAgent(client=mock_client) + # Should complete without raising. + await agent.run(message) + + assert mock_session.send_and_wait.call_args.kwargs["attachments"] is None + + def test_prepare_attachments_skips_non_base64_data_uri(self) -> None: + """The helper drops a non-base64 ``data:`` URI rather than raising ContentError.""" + message = Message(role="user", contents=[Content.from_uri("data:text/plain,hello")]) + + attachments = GitHubCopilotAgent._prepare_attachments_for_copilot([message]) + + assert attachments is None + + # --------------------------------------------------------------------------- # Integration tests — require COPILOT_GITHUB_TOKEN env var # --------------------------------------------------------------------------- - skip_if_copilot_integration_tests_disabled = pytest.mark.skipif( os.getenv("COPILOT_GITHUB_TOKEN", "") == "", reason="No COPILOT_GITHUB_TOKEN provided; skipping integration tests.",