Python: Improve the Python sample validation workflow to increase pass rate

.github/workflows/python-sample-validation.yml

@@ -9,7 +9,7 @@ env:
   # Configure a constant location for the uv cache
   UV_CACHE_DIR: /tmp/.uv-cache
   # GitHub Copilot configuration
-  GITHUB_COPILOT_MODEL: claude-opus-4.6
+  GITHUB_COPILOT_MODEL: claude-opus-4.8
   COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }}
 
 permissions:
@@ -23,8 +23,8 @@ jobs:
     environment: integration
     env:
       # Required configuration for get-started samples
-      FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT || vars.AZURE_AI_PROJECT_ENDPOINT }}
-      FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
+      FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }}
+      FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }}
     defaults:
       run:
         working-directory: python
@@ -61,21 +61,21 @@ jobs:
     environment: integration
     env:
       # Foundry configuration
-      FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT || vars.AZURE_AI_PROJECT_ENDPOINT }}
-      FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
+      FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }}
+      FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }}
       # Azure OpenAI configuration
       AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }}
       AZURE_OPENAI_MODEL: ${{ vars.AZURE_OPENAI_DEPLOYMENT_NAME || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
       AZURE_OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.AZUREOPENAI__CHATDEPLOYMENTNAME }}
       AZURE_OPENAI_CHAT_MODEL: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
       AZURE_OPENAI_EMBEDDING_MODEL: ${{ vars.AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME || vars.AZUREOPENAI__EMBEDDINGDEPLOYMENTNAME }}
       # OpenAI configuration
-      OPENAI_API_KEY: ${{ secrets.OPENAI__APIKEY }}
-      OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.OPENAI__CHATMODELID }}
-      OPENAI_CHAT_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }}
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+      OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.OPENAI_CHAT_MODEL_NAME }}
+      OPENAI_CHAT_MODEL: ${{ vars.OPENAI_REASONING_MODEL_NAME }}
       # GitHub MCP
       GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
-      OPENAI_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }}
+      OPENAI_MODEL: ${{ vars.OPENAI_REASONING_MODEL_NAME }}
       # Observability
       ENABLE_INSTRUMENTATION: "true"
     defaults:
@@ -122,10 +122,10 @@ jobs:
     runs-on: ubuntu-latest
     environment: integration
     env:
-      OPENAI_API_KEY: ${{ secrets.OPENAI__APIKEY }}
-      OPENAI_MODEL: ${{ vars.OPENAI__CHATMODELID }}
-      OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.OPENAI__CHATMODELID }}
-      OPENAI_CHAT_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }}
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+      OPENAI_MODEL: ${{ vars.OPENAI_CHAT_MODEL_NAME }}
+      OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.OPENAI_CHAT_MODEL_NAME }}
+      OPENAI_CHAT_MODEL: ${{ vars.OPENAI_REASONING_MODEL_NAME }}
     defaults:
       run:
         working-directory: python
@@ -329,12 +329,11 @@ jobs:
 
   validate-02-agents-foundry:
     name: Validate 02-agents/providers/foundry
-    if: false # Temporarily disabled - provider folder also contains the local Foundry sample
     runs-on: ubuntu-latest
     environment: integration
     env:
-      FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT || vars.AZURE_AI_PROJECT_ENDPOINT }}
-      FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
+      FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }}
+      FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }}
       FOUNDRY_AGENT_NAME: ${{ vars.FOUNDRY_AGENT_NAME || '' }}
       FOUNDRY_AGENT_VERSION: ${{ vars.FOUNDRY_AGENT_VERSION || '' }}
     defaults:
@@ -445,8 +444,8 @@ jobs:
     runs-on: ubuntu-latest
     environment: integration
     env:
-      FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT || vars.AZURE_AI_PROJECT_ENDPOINT }}
-      FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
+      FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }}
+      FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }}
     defaults:
       run:
         working-directory: python
@@ -479,12 +478,11 @@ jobs:
 
   validate-04-hosting:
     name: Validate 04-hosting
-    if: false # Temporarily disabled because of sample complexity
     runs-on: ubuntu-latest
     environment: integration
     env:
-      FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT || vars.AZURE_AI_PROJECT_ENDPOINT }}
-      FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
+      FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }}
+      FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }}
       # A2A configuration
       A2A_AGENT_HOST: http://localhost:5001/
     defaults:
@@ -518,8 +516,8 @@ jobs:
     runs-on: ubuntu-latest
     environment: integration
     env:
-      FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT || vars.AZURE_AI_PROJECT_ENDPOINT }}
-      FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
+      FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }}
+      FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }}
       # Azure OpenAI configuration
       AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }}
       AZURE_OPENAI_MODEL: ${{ vars.AZURE_OPENAI_DEPLOYMENT_NAME || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
@@ -560,16 +558,16 @@ jobs:
     runs-on: ubuntu-latest
     environment: integration
     env:
-      FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT || vars.AZURE_AI_PROJECT_ENDPOINT }}
-      FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
+      FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }}
+      FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }}
       # Azure OpenAI configuration
       AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }}
       AZURE_OPENAI_MODEL: ${{ vars.AZURE_OPENAI_DEPLOYMENT_NAME || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
       # OpenAI configuration
-      OPENAI_API_KEY: ${{ secrets.OPENAI__APIKEY }}
-      OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.OPENAI__CHATMODELID }}
-      OPENAI_CHAT_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }}
-      OPENAI_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }}
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+      OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.OPENAI_CHAT_MODEL_NAME }}
+      OPENAI_CHAT_MODEL: ${{ vars.OPENAI_REASONING_MODEL_NAME }}
+      OPENAI_MODEL: ${{ vars.OPENAI_REASONING_MODEL_NAME }}
     defaults:
       run:
         working-directory: python
@@ -610,20 +608,21 @@ jobs:
     runs-on: ubuntu-latest
     environment: integration
     env:
-      FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT || vars.AZURE_AI_PROJECT_ENDPOINT }}
-      FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
+      FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }}
+      FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }}
       # Azure OpenAI configuration for AF
       AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }}
       AZURE_OPENAI_MODEL: ${{ vars.AZURE_OPENAI_DEPLOYMENT_NAME || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
       # Azure OpenAI configuration for SK
       AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZURE_OPENAI_DEPLOYMENT_NAME }}
       # OpenAI key
-      OPENAI_API_KEY: ${{ secrets.OPENAI__APIKEY }}
-      OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.OPENAI__CHATMODELID }}
-      OPENAI_CHAT_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }}
-      OPENAI_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }}
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+      OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.OPENAI_CHAT_MODEL_NAME }}
+      OPENAI_CHAT_MODEL: ${{ vars.OPENAI_REASONING_MODEL_NAME }}
+      OPENAI_MODEL: ${{ vars.OPENAI_REASONING_MODEL_NAME }}
       # OpenAI configuration for SK
-      OPENAI_CHAT_MODEL_ID: ${{ vars.OPENAI__CHATMODELID }}
+      OPENAI_CHAT_MODEL_ID: ${{ vars.OPENAI_CHAT_MODEL_NAME }}
+      OPENAI_RESPONSES_MODEL_ID: ${{ vars.OPENAI__RESPONSESMODELID }}
       # Copilot Studio
       COPILOTSTUDIOAGENT__ENVIRONMENTID: ${{ secrets.COPILOTSTUDIOAGENT__ENVIRONMENTID }}
       COPILOTSTUDIOAGENT__SCHEMANAME: ${{ secrets.COPILOTSTUDIOAGENT__SCHEMANAME }}

python/samples/autogen-migration/single_agent/01_basic_agent.py

@@ -1,12 +1,14 @@
 # /// script
 # requires-python = ">=3.10"
 # dependencies = [
+#     "agent-framework-openai",
 #     "autogen-agentchat",
 #     "autogen-ext[openai]",
+#     "python-dotenv",
 # ]
 # ///
 # Run with any PEP 723 compatible runner, e.g.:
-#   uv run samples/autogen-migration/single_agent/01_basic_assistant_agent.py
+#   uv run samples/autogen-migration/single_agent/01_basic_agent.py
 
 # Copyright (c) Microsoft. All rights reserved.
 

python/samples/semantic-kernel-migration/chat_completion/01_basic_chat_completion.py

@@ -1,6 +1,8 @@
 # /// script
 # requires-python = ">=3.10"
 # dependencies = [
+#     "agent-framework-openai",
+#     "python-dotenv",
 #     "semantic-kernel",
 # ]
 # ///

python/samples/semantic-kernel-migration/chat_completion/02_chat_completion_with_tool.py

@@ -1,6 +1,8 @@
 # /// script
 # requires-python = ">=3.10"
 # dependencies = [
+#     "agent-framework-openai",
+#     "python-dotenv",
 #     "semantic-kernel",
 # ]
 # ///

python/samples/semantic-kernel-migration/chat_completion/03_chat_completion_thread_and_stream.py

@@ -1,6 +1,8 @@
 # /// script
 # requires-python = ">=3.10"
 # dependencies = [
+#     "agent-framework-openai",
+#     "python-dotenv",
 #     "semantic-kernel",
 # ]
 # ///

python/samples/semantic-kernel-migration/copilot_studio/01_basic_copilot_studio_agent.py

@@ -1,11 +1,22 @@
 # /// script
 # requires-python = ">=3.10"
 # dependencies = [
+#     "agent-framework-copilotstudio",
+#     "python-dotenv",
 #     "semantic-kernel",
 # ]
 # ///
 # Run with any PEP 723 compatible runner, e.g.:
 #   uv run samples/semantic-kernel-migration/copilot_studio/01_basic_copilot_studio_agent.py
+#
+# NOTE: The metadata above resolves the Agent Framework half only.
+# The Semantic Kernel half (run_semantic_kernel) requires the older
+# dot-namespace Microsoft Agents SDK (microsoft.agents.copilotstudio.client and
+# microsoft.agents.core, from microsoft-agents-copilotstudio-client<0.3), while
+# Agent Framework requires the newer underscore-namespace SDK
+# (microsoft_agents.copilotstudio.client, from
+# microsoft-agents-copilotstudio-client>=0.3.1). These two generations cannot be
+# installed in the same environment, so run each half in its own isolated env.
 
 # Copyright (c) Microsoft. All rights reserved.
 """Call a Copilot Studio agent with SK and Agent Framework."""

python/samples/semantic-kernel-migration/copilot_studio/02_copilot_studio_streaming.py

@@ -1,11 +1,22 @@
 # /// script
 # requires-python = ">=3.10"
 # dependencies = [
+#     "agent-framework-copilotstudio",
+#     "python-dotenv",
 #     "semantic-kernel",
 # ]
 # ///
 # Run with any PEP 723 compatible runner, e.g.:
 #   uv run samples/semantic-kernel-migration/copilot_studio/02_copilot_studio_streaming.py
+#
+# NOTE: The metadata above resolves the Agent Framework half only.
+# The Semantic Kernel half (run_semantic_kernel) requires the older
+# dot-namespace Microsoft Agents SDK (microsoft.agents.copilotstudio.client and
+# microsoft.agents.core, from microsoft-agents-copilotstudio-client<0.3), while
+# Agent Framework requires the newer underscore-namespace SDK
+# (microsoft_agents.copilotstudio.client, from
+# microsoft-agents-copilotstudio-client>=0.3.1). These two generations cannot be
+# installed in the same environment, so run each half in its own isolated env.
 
 # Copyright (c) Microsoft. All rights reserved.
 """Stream responses from Copilot Studio agents in SK and AF."""

python/samples/semantic-kernel-migration/openai_responses/01_basic_responses_agent.py

@@ -1,6 +1,8 @@
 # /// script
 # requires-python = ">=3.10"
 # dependencies = [
+#     "agent-framework-openai",
+#     "python-dotenv",
 #     "semantic-kernel",
 # ]
 # ///

python/samples/semantic-kernel-migration/openai_responses/02_responses_agent_with_tool.py

@@ -1,6 +1,8 @@
 # /// script
 # requires-python = ">=3.10"
 # dependencies = [
+#     "agent-framework-openai",
+#     "python-dotenv",
 #     "semantic-kernel",
 # ]
 # ///

python/samples/semantic-kernel-migration/openai_responses/03_responses_agent_structured_output.py

@@ -1,6 +1,8 @@
 # /// script
 # requires-python = ">=3.10"
 # dependencies = [
+#     "agent-framework-openai",
+#     "python-dotenv",
 #     "semantic-kernel",
 # ]
 # ///

python/samples/semantic-kernel-migration/orchestrations/concurrent_basic.py

@@ -1,6 +1,9 @@
 # /// script
 # requires-python = ">=3.10"
 # dependencies = [
+#     "agent-framework-openai",
+#     "agent-framework-orchestrations",
+#     "python-dotenv",
 #     "semantic-kernel",
 # ]
 # ///

python/samples/semantic-kernel-migration/orchestrations/group_chat.py

@@ -1,6 +1,9 @@
 # /// script
 # requires-python = ">=3.10"
 # dependencies = [
+#     "agent-framework-openai",
+#     "agent-framework-orchestrations",
+#     "python-dotenv",
 #     "semantic-kernel",
 # ]
 # ///

python/samples/semantic-kernel-migration/orchestrations/handoff.py

@@ -1,6 +1,9 @@
 # /// script
 # requires-python = ">=3.10"
 # dependencies = [
+#     "agent-framework-openai",
+#     "agent-framework-orchestrations",
+#     "python-dotenv",
 #     "semantic-kernel",
 # ]
 # ///

python/samples/semantic-kernel-migration/orchestrations/magentic.py

@@ -1,6 +1,9 @@
 # /// script
 # requires-python = ">=3.10"
 # dependencies = [
+#     "agent-framework-openai",
+#     "agent-framework-orchestrations",
+#     "python-dotenv",
 #     "semantic-kernel",
 # ]
 # ///

python/samples/semantic-kernel-migration/orchestrations/sequential.py

@@ -1,6 +1,9 @@
 # /// script
 # requires-python = ">=3.10"
 # dependencies = [
+#     "agent-framework-openai",
+#     "agent-framework-orchestrations",
+#     "python-dotenv",
 #     "semantic-kernel",
 # ]
 # ///

python/samples/semantic-kernel-migration/processes/fan_out_fan_in_process.py

@@ -1,6 +1,8 @@
 # /// script
 # requires-python = ">=3.10"
 # dependencies = [
+#     "agent-framework-core",
+#     "python-dotenv",
 #     "semantic-kernel",
 # ]
 # ///

python/samples/semantic-kernel-migration/processes/nested_process.py

@@ -1,6 +1,8 @@
 # /// script
 # requires-python = ">=3.10"
 # dependencies = [
+#     "agent-framework-core",
+#     "python-dotenv",
 #     "semantic-kernel",
 # ]
 # ///

python/scripts/sample_validation/create_dynamic_workflow_executor.py

@@ -3,10 +3,12 @@
 import logging
 from collections import deque
 from dataclasses import dataclass
+from pathlib import Path
 
 from agent_framework import (
     Executor,
     Message,
+    SkillsProvider,
     Workflow,
     WorkflowBuilder,
     WorkflowContext,
@@ -31,6 +33,9 @@
 
 logger = logging.getLogger(__name__)
 
+# Directory containing file-based skills used by the validation agents.
+SKILLS_DIR = Path(__file__).parent / "skills"
+
 
 class AgentResponseFormat(BaseModel):
     status: str
@@ -58,7 +63,9 @@ class BatchCompletion:
     "Analyze the sample code and execute it as it is. Based on the execution result, determine "
     "if it runs successfully, fails, or is missing_setup. Use `missing_setup` if the sample reports "
     "missing required environment variables. The environment you're given should contain the necessary "
-    "variables. Don't create new environment variables nor modify the sample code.\n"
+    "variables. Don't create new environment variables nor modify the sample code, unless an available "
+    "skill instructs you to do so for the setup issue you detected. When a skill applies to the problem, "
+    "follow its guidance to resolve the setup and then re-run the sample.\n"
     "Feel free to install any required dependencies if needed.\n"
     "The sample can be interactive. If it is interactive, respond to the sample when prompted "
     "based on your analysis of the code. You do not need to consult human on what to respond.\n"
@@ -289,6 +296,7 @@ async def create(
                 id=agent_id,
                 name=agent_id,
                 instructions=AgentInstruction,
+                context_providers=[SkillsProvider.from_paths(skill_paths=str(SKILLS_DIR))],
                 default_options={
                     "on_permission_request": prompt_permission,
                     "timeout": 120,