From 6b8c24211dbfca82f5b8331e239af5c24de87c8e Mon Sep 17 00:00:00 2001 From: Shahin Saadati Date: Thu, 18 Jun 2026 15:57:15 -0700 Subject: [PATCH 1/6] feat(skills): Add snippet verification skill for documentation guides Introduce the adk-verify-snippets developer skill to automate syntax, import, and runnability validation of Python code blocks inside Markdown documentation guides. --- .agents/skills/adk-verify-snippets/SKILL.md | 66 +++++ .../skills/adk-verify-snippets/scripts/run.py | 177 ++++++++++++++ .../adk-verify-snippets/scripts/verify_md.py | 229 ++++++++++++++++++ 3 files changed, 472 insertions(+) create mode 100644 .agents/skills/adk-verify-snippets/SKILL.md create mode 100644 .agents/skills/adk-verify-snippets/scripts/run.py create mode 100644 .agents/skills/adk-verify-snippets/scripts/verify_md.py diff --git a/.agents/skills/adk-verify-snippets/SKILL.md b/.agents/skills/adk-verify-snippets/SKILL.md new file mode 100644 index 00000000000..e18e85060e3 --- /dev/null +++ b/.agents/skills/adk-verify-snippets/SKILL.md @@ -0,0 +1,66 @@ +--- +name: adk-verify-snippets +description: > + Extracts and verifies the runnability and code coverage of all Python code blocks inside a Markdown file. + Generates a detailed compilation and execution report. +metadata: + author: Antigravity + version: 1.0.0 +--- + +# Verify Markdown Snippets Skill + +This skill allows you to systematically verify the correctness, compile-readiness, and line coverage of Python code snippets embedded within a Markdown document (like tutorials, guides, or READMEs). + +It extracts all ` ```python ` blocks, executes them in process-isolated environments using the bundled `run.py` harness, and generates a structured test report. + +--- + +## ๐Ÿ› ๏ธ How to Use the Skill + +To verify a Markdown file (e.g. `docs/my_guide.md`) and generate its runnability report: + +```bash +uv run --no-sync python .agents/skills/verify-markdown-snippets/scripts/verify_md.py +``` + +For example: +```bash +uv run --no-sync python .agents/skills/verify-markdown-snippets/scripts/verify_md.py docs/my_guide.md +``` + +This will automatically create a detailed report file called **`docs/my_guide_report.md`** in the same directory. + +--- + +## ๐Ÿ“ Snippet Code Conventions + +Each python code snippet inside the Markdown file is verified using our generalized `run.py` contract. For a code block to be fully testable for both loadability and runnability: + +1. **Expose a Global ADK Component (Optional for Runnability)**: + If the snippet instantiates a global `Workflow`, `Agent`, or `App`, the runner will automatically execute it and measure its execution. +2. **Provide a Custom Test Input (Optional)**: + If the snippet defines a global `test_input` variable, the runner will use it during execution. +3. **Basic Python Snippets (Loadability Only)**: + If a code block does not define any runnable ADK components, the runner will verify that it compiles and loads without error, and report a 100% coverage report for the lines present in the block. + +--- + +## ๐Ÿ“Š The Generated Report Structure + +The generated `_report.md` will contain: +1. **Executive Summary**: A high-level table listing every discovered snippet, its preceding Markdown heading, its status (Passed/Failed), and its line coverage. +2. **Detailed Breakdown**: + * The exact extracted python code block. + * The detailed execution stdout logs. + * Any stderr traceback/exceptions (pointing directly to the breaking line of code!). + * A clean, focused 5-line coverage table showing exactly what lines of the snippet were executed. + +--- + +## โš ๏ธ Crucial Behavioral Constraints (For AI Agents) + +* **Strictly Read-Only Workspace**: The skill's operations on the repository are strictly **read-only**. The agent **MUST NOT** modify, create, or delete any existing source files, test files, configuration files, or documentation files in the repository (with the sole exceptions of writing temporary execution files to `.temp_snippets` and generating the final `_report.md` report file). +* **Strictly Report, Do Not Fix**: The sole purpose of this skill is to **identify and report** compile-time and run-time issues within the Markdown document's snippets. +* **No Unsolicited Patches**: When executing this skill, the agent **MUST NOT** attempt to rewrite the source Markdown file, modify its code blocks, or automatically generate code fixes/patches. +* **Focus on the Report**: The agent should run the verification, let the script generate the `_report.md` file, and present the executive summary table to the user. Do not offer solutions or rewrite recommendations unless the user explicitly asks for them. diff --git a/.agents/skills/adk-verify-snippets/scripts/run.py b/.agents/skills/adk-verify-snippets/scripts/run.py new file mode 100644 index 00000000000..284a45aa02c --- /dev/null +++ b/.agents/skills/adk-verify-snippets/scripts/run.py @@ -0,0 +1,177 @@ +import argparse +import asyncio +import importlib.util +import inspect +import os +import sys +import traceback +from pathlib import Path + +# --- Optional Coverage Integration --- +try: + import coverage + HAS_COVERAGE = True +except ImportError: + HAS_COVERAGE = False + +# --- Imports for ADK Inspection --- +from google.adk.agents.base_agent import BaseAgent +from google.adk.apps import App +from google.adk.runners import Runner +from google.adk.sessions.in_memory_session_service import InMemorySessionService +from google.adk.workflow import Workflow +from google.genai import types + +def load_target_module(file_path: Path): + """Dynamically loads a Python file as a module, catching import/compilation/definition errors.""" + module_name = file_path.stem + spec = importlib.util.spec_from_file_location(module_name, file_path) + if spec is None or spec.loader is None: + raise ImportError(f"Could not resolve module spec for file '{file_path.name}'") + + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + + # Executing the module runs all top-level code, which will catch: + # - SyntaxError / IndentationError + # - ImportError (e.g. from google.adk.workflow import build_node) + # - ValidationError (e.g. instantiating Workflow with invalid edges) + spec.loader.exec_module(module) + return module + +def discover_adk_component(module): + """Scans the module namespace to discover runnable ADK components, prioritizing root components.""" + # 1. Search for Workflow + for name, obj in inspect.getmembers(module): + if isinstance(obj, Workflow): + return obj, "Workflow" + + # 2. Search for Agent (filtering out sub-agents to locate the Root Agent) + agents = [] + sub_agent_ids = set() + for name, obj in inspect.getmembers(module): + if isinstance(obj, BaseAgent): + agents.append(obj) + # Track which agents are registered as sub-agents of another agent + if hasattr(obj, "sub_agents") and obj.sub_agents: + for sub in obj.sub_agents: + sub_agent_ids.add(id(sub)) + + # Find the agent(s) that are not sub-agents of any other agent in the module + root_agents = [a for a in agents if id(a) not in sub_agent_ids] + if root_agents: + # Return the first root agent discovered + return root_agents[0], "Agent" + + # 3. Search for App + for name, obj in inspect.getmembers(module): + if isinstance(obj, App): + return obj, "App" + return None, None + +async def run_component(component, component_type, test_input): + """Unified runner to execute the discovered component.""" + print(f"\n๐Ÿ” Discovered ADK {component_type} in target file.") + print(f"๐Ÿš€ Running execution test with input: '{test_input}'...\n") + + runnable_node = component.root_node if component_type == "App" else component + + session_service = InMemorySessionService() + runner = Runner(app_name="runnability_test", node=runnable_node, session_service=session_service) + session = await session_service.create_session(app_name="runnability_test", user_id="tester") + + user_message = types.Content( + parts=[types.Part(text=str(test_input))], + role="user" + ) + + async for event in runner.run_async( + user_id="tester", + session_id=session.id, + new_message=user_message + ): + print(f"๐ŸŽฌ [Event] Author: {event.author}") + if event.output: + print(f"๐Ÿ”น Output: {event.output}") + if hasattr(event, "content") and event.content and event.content.parts: + text = "".join(p.text for p in event.content.parts if p.text) + if text: + print(f"๐Ÿ“ Content Output:\n{'-'*40}\n{text}\n{'-'*40}") + +def main(): + parser = argparse.ArgumentParser(description="Generalized ADK Runnability & Loadability Tester") + parser.add_argument("file", type=str, help="Path to the python file containing the agent/workflow to test") + args = parser.parse_args() + + file_path = Path(args.file).resolve() + if not file_path.exists(): + print(f"โŒ Error: File '{file_path}' does not exist.") + sys.exit(1) + + print(f"๐Ÿ”ฌ Testing file: {file_path.name}") + print("=" * 60) + + # Initialize coverage programmatically to track ONLY the target file + cov = None + if HAS_COVERAGE: + cov = coverage.Coverage( + branch=True, + data_file=None # Keep coverage data in-memory only, no .coverage file needed + ) + cov.start() + else: + print("โ„น๏ธ Install 'coverage' package to enable automated code coverage reporting.") + + try: + # 1. Test Loadability (Imports, Syntax, Instantiation/Validation) + print("๐Ÿ“‹ Phase 1: Loading & Compiling...") + try: + module = load_target_module(file_path) + print(f"โœ… Load Success: '{file_path.name}' compiled and loaded without any issues.") + except Exception as e: + print(f"โŒ Load Failure: Failed to compile/load '{file_path.name}':") + print("-" * 60) + traceback.print_exc(file=sys.stdout) + print("-" * 60) + sys.exit(1) + + # 2. Discover Component + print("\n๐Ÿ“‹ Phase 2: Component Discovery...") + component, comp_type = discover_adk_component(module) + if not component: + print(f"โš ๏ธ No runnable ADK components (Workflow, Agent, or App) found in '{file_path.name}'.") + print(" Loadability check passed, but runnability test was skipped.") + sys.exit(0) + + # Get test input from module, or fallback + test_input = getattr(module, "test_input", "Test input topic") + + # 3. Test Runnability + print(f"\n๐Ÿ“‹ Phase 3: Executing {comp_type}...") + try: + asyncio.run(run_component(component, comp_type, test_input)) + print(f"\nโœ… Run Success: Component '{comp_type}' executed successfully.") + except Exception as e: + print(f"\nโŒ Run Failure: Component failed during execution:") + print("-" * 60) + traceback.print_exc(file=sys.stdout) + print("-" * 60) + sys.exit(1) + + finally: + # Report coverage of the target file at the very end + if cov: + cov.stop() + print("\n๐Ÿ“Š Phase 4: Code Coverage Report (Target File)") + print("=" * 60) + try: + # Report coverage of the target file directly to stdout + cov.report(morfs=[str(file_path)], file=sys.stdout) + except coverage.exceptions.NoDataError: + print("โš ๏ธ No coverage data collected (compilation or execution failed early).") + except Exception as ce: + print(f"โš ๏ธ Failed to generate coverage report: {ce}") + print("=" * 60) + +if __name__ == "__main__": + main() diff --git a/.agents/skills/adk-verify-snippets/scripts/verify_md.py b/.agents/skills/adk-verify-snippets/scripts/verify_md.py new file mode 100644 index 00000000000..f1a531e07aa --- /dev/null +++ b/.agents/skills/adk-verify-snippets/scripts/verify_md.py @@ -0,0 +1,229 @@ +import argparse +from datetime import datetime +import os +import re +import shutil +import subprocess +import sys +from pathlib import Path + +def extract_snippets(md_path: Path): + """Parses a markdown file and extracts python code blocks along with their preceding headings.""" + with open(md_path, "r", encoding="utf-8") as f: + content = f.read() + + lines = content.splitlines() + snippets = [] + current_heading = "Top Level" + in_code_block = False + code_lines = [] + + for line in lines: + # If we are inside a code block, handle it first to preserve comments starting with '#' + if in_code_block: + if line.strip().startswith("```"): + in_code_block = False + code_text = "\n".join(code_lines) + snippets.append({ + "heading": current_heading, + "code": code_text + }) + else: + code_lines.append(line) + continue + + # If we are outside a code block, check for headings or code block starts + if line.startswith("#"): + # Clean up heading markers (e.g., "## Get started" -> "Get started") + current_heading = line.lstrip("#").strip() + continue + + if line.strip().startswith("```python"): + in_code_block = True + code_lines = [] + continue + + return snippets + +def run_snippet(run_py_path: Path, snippet_path: Path): + """Executes run.py on the isolated snippet and returns the result.""" + # Run using the same Python interpreter as this script (which will be the venv's python) + cmd = [sys.executable, str(run_py_path), str(snippet_path)] + + # Ensure GEMINI_API_KEY is preferred if both keys are set in the environment + env = os.environ.copy() + if "GOOGLE_API_KEY" in env and "GEMINI_API_KEY" in env: + env.pop("GOOGLE_API_KEY", None) + + result = subprocess.run( + cmd, + capture_output=True, + text=True, + env=env + ) + + return { + "exit_code": result.returncode, + "stdout": result.stdout, + "stderr": result.stderr + } + +def clean_name(name: str): + """Sanitizes a string to be a safe filename.""" + name = name.lower().replace(" ", "_") + return re.sub(r'[^a-z0-9__]', '', name) + +def main(): + parser = argparse.ArgumentParser(description="Markdown Snippet Verifier") + parser.add_argument("file", type=str, help="Path to the markdown file to verify") + args = parser.parse_args() + + md_path = Path(args.file).resolve() + if not md_path.exists(): + print(f"โŒ Error: Markdown file '{md_path}' does not exist.") + sys.exit(1) + + # Locate run.py bundled inside the same scripts folder as verify_md.py (portable mode!) + run_py_path = Path(__file__).parent / "run.py" + if not run_py_path.exists(): + print(f"โŒ Error: Bundled runner 'run.py' not found at '{run_py_path}'.") + sys.exit(1) + + print(f"๐Ÿ”ฌ Analyzing Markdown: {md_path.name}") + + # 1. Extract snippets + snippets = extract_snippets(md_path) + if not snippets: + print(f"โš ๏ธ No python code blocks found in '{md_path.name}'.") + sys.exit(0) + + print(f"๐Ÿ“‹ Found {len(snippets)} python code snippets to verify.") + + # Create a temp directory in the workspace + temp_dir = Path(".temp_snippets") + temp_dir.mkdir(exist_ok=True) + + results = [] + + # 2. Execute each snippet + for i, snippet in enumerate(snippets, start=1): + heading = snippet["heading"] + code = snippet["code"] + + # Create a unique, sanitized filename for the snippet + safe_heading = clean_name(heading) + temp_file_name = f"snippet_{i}_{safe_heading}.py" + temp_file_path = temp_dir / temp_file_name + + # Write snippet to file + with open(temp_file_path, "w", encoding="utf-8") as f: + f.write(code) + + print(f"๐Ÿงช Testing Snippet {i}/{len(snippets)} under heading '{heading}'...") + + # Run the snippet + run_res = run_snippet(run_py_path, temp_file_path) + + results.append({ + "index": i, + "heading": heading, + "code": code, + "temp_file": temp_file_name, + "exit_code": run_res["exit_code"], + "stdout": run_res["stdout"], + "stderr": run_res["stderr"] + }) + + # Clean up temporary directory + shutil.rmtree(temp_dir, ignore_errors=True) + + # 3. Generate Markdown Report + report_path = md_path.parent / f"{md_path.stem}_report.md" + timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + + with open(report_path, "w", encoding="utf-8") as f: + f.write(f"# ๐Ÿ”ฌ ADK Markdown Snippet Verification Report\n\n") + f.write(f"* **Source File**: [{md_path.name}](file://{md_path})\n") + f.write(f"* **Verified On**: `{timestamp}`\n\n") + + # Write summary table + f.write("## ๐Ÿ“ˆ Executive Summary\n\n") + f.write("| Snippet | preceding Heading | Load Phase | Run Phase | Coverage | Uncovered Issues / Details |\n") + f.write("| :--- | :--- | :---: | :---: | :---: | :--- |\n") + + for r in results: + # Determine Phase 1 (Load) and Phase 3 (Run) statuses + load_status = "โœ… **PASS**" + run_status = "โœ… **PASS**" + coverage_pct = "โ€”" + + stdout_and_stderr = r["stdout"] + "\n" + r["stderr"] + + # 1. Parse Load Phase + if "โŒ Load Failure" in stdout_and_stderr or "Load Failure" in stdout_and_stderr: + load_status = "โŒ **FAIL**" + run_status = "โž– **SKIPPED**" + + # 2. Parse Run Phase + elif "runnability test was skipped" in stdout_and_stderr: + run_status = "โž– **SKIPPED**" + elif "โŒ Run Failure" in stdout_and_stderr or r["exit_code"] != 0: + run_status = "โŒ **FAIL**" + + # 3. Parse Coverage + cov_match = re.search(r"TOTAL\s+\d+\s+\d+\s+\d+\s+\d+\s+(\d+%)", r["stdout"]) + if cov_match and load_status != "โŒ **FAIL**": + coverage_pct = f"`{cov_match.group(1)}`" + + # 4. Formulate details and handle transient 503s + details = "All checks passed successfully." + if load_status == "โŒ **FAIL**": + err_lines = [line for line in stdout_and_stderr.splitlines() if "Error" in line or "Exception" in line] + details = f"`{err_lines[-1]}`" if err_lines else "Failed to compile/load." + elif run_status == "โŒ **FAIL**": + if "503" in stdout_and_stderr and "UNAVAILABLE" in stdout_and_stderr: + details = "โš ๏ธ **Transient 503 from Gemini API (overloaded)**. Code structure is correct." + else: + err_lines = [line for line in stdout_and_stderr.splitlines() if "Error" in line or "Exception" in line] + details = f"`{err_lines[-1]}`" if err_lines else "Failed during execution." + + f.write(f"| **Snippet {r['index']}** | `{r['heading']}` | {load_status} | {run_status} | {coverage_pct} | {details} |\n") + + f.write("\n---\n\n## ๐Ÿ” Detailed Snippet Reports\n\n") + + for r in results: + status_icon = "โœ…" if r["exit_code"] == 0 else "โŒ" + f.write(f"### {status_icon} Snippet {r['index']}: `{r['heading']}`\n\n") + + f.write("#### ๐Ÿ“ Code Block\n") + f.write(f"```python\n{r['code']}\n```\n\n") + + # Write stdout / stderr logs + f.write("#### ๐Ÿ–ฅ๏ธ Loadability & Runnability Logs\n") + f.write("```text\n") + + # Clean up stdout to separate Phase 4 coverage + stdout_clean = r["stdout"] + cov_match = re.search(r"(๐Ÿ“Š Phase 4: Code Coverage Report.*)", r["stdout"], re.DOTALL) + cov_text = cov_match.group(1) if cov_match else None + + if cov_text: + stdout_clean = r["stdout"].replace(cov_text, "").strip() + + f.write(stdout_clean) + if r["stderr"]: + f.write("\n\n=== STDERR/TRACEBACK ===\n") + f.write(r["stderr"].strip()) + f.write("\n```\n\n") + + # Write coverage report if available + if cov_text: + f.write("#### ๐Ÿ“Š Coverage Report\n") + f.write(f"```text\n{cov_text}\n```\n\n") + + f.write("---\n\n") + + print(f"๐ŸŽ‰ Verification complete! Report generated at: {report_path.name}") + +if __name__ == "__main__": + main() From fd5123fe78e32551e08dcfcd34af1fe427c75f61 Mon Sep 17 00:00:00 2001 From: Shahin Saadati Date: Tue, 23 Jun 2026 18:49:27 -0700 Subject: [PATCH 2/6] feat: add snippet skipping, timeout handling, and improved error reporting to verify_md script --- .agents/skills/adk-verify-snippets/SKILL.md | 96 +++- .../skills/adk-verify-snippets/scripts/run.py | 215 ++++++--- .../adk-verify-snippets/scripts/verify_md.py | 436 ++++++++++++------ 3 files changed, 539 insertions(+), 208 deletions(-) diff --git a/.agents/skills/adk-verify-snippets/SKILL.md b/.agents/skills/adk-verify-snippets/SKILL.md index e18e85060e3..6bba3adf515 100644 --- a/.agents/skills/adk-verify-snippets/SKILL.md +++ b/.agents/skills/adk-verify-snippets/SKILL.md @@ -5,7 +5,7 @@ description: > Generates a detailed compilation and execution report. metadata: author: Antigravity - version: 1.0.0 + version: 1.4.0 --- # Verify Markdown Snippets Skill @@ -14,6 +14,40 @@ This skill allows you to systematically verify the correctness, compile-readines It extracts all ` ```python ` blocks, executes them in process-isolated environments using the bundled `run.py` harness, and generates a structured test report. +> [!CAUTION] +> **STRICT READ-ONLY CONSTRAINT โ€” READ THIS BEFORE DOING ANYTHING ELSE** +> +> When executing this skill, the agent operates in a **strictly read-only** mode with respect to the repository. The agent **MUST NOT**, under any circumstances: +> - **Modify** any existing source file, test file, configuration file, documentation file, or skill file (including this SKILL.md). +> - **Delete** any file in the repository. +> - **Create** any new file in the repository other than the two explicit exceptions listed below. +> +> The **only** two write operations permitted are: +> 1. Writing temporary snippet `.py` files to a uniquely-named **system temp directory** (outside the repository). +> 2. Writing the final `_REPORT.md` report file into the **same directory as the source Markdown file**. +> +> Any other file system mutation is a violation of this skill's contract. If in doubt, do not write. + +--- + +## ๐Ÿ”ง Prerequisites + +Before running this skill, ensure the following are in place: + +1. **ADK Python environment**: The skill must be run from the repository root with the project's virtual environment active (via `uv`). +2. **`coverage` package**: Install it to enable per-snippet coverage reporting: + ```bash + uv pip install coverage + ``` + If `coverage` is not installed, the runner degrades gracefully โ€” verification still works, but coverage columns in the report will show `โ€”`. +3. **Gemini API key**: Snippets that instantiate an `Agent`, `App`, or `Workflow` make live calls to the Gemini API. Set one of the following environment variables before running: + ```bash + export GEMINI_API_KEY="your-key-here" + # or + export GOOGLE_API_KEY="your-key-here" + ``` + If both are set, `GEMINI_API_KEY` takes precedence. Snippets that do not expose a runnable ADK component (loadability-only checks) do not require an API key. + --- ## ๐Ÿ› ๏ธ How to Use the Skill @@ -21,46 +55,74 @@ It extracts all ` ```python ` blocks, executes them in process-isolated environm To verify a Markdown file (e.g. `docs/my_guide.md`) and generate its runnability report: ```bash -uv run --no-sync python .agents/skills/verify-markdown-snippets/scripts/verify_md.py +uv run --no-sync python .agents/skills/adk-verify-snippets/scripts/verify_md.py ``` For example: ```bash -uv run --no-sync python .agents/skills/verify-markdown-snippets/scripts/verify_md.py docs/my_guide.md +uv run --no-sync python .agents/skills/adk-verify-snippets/scripts/verify_md.py docs/my_guide.md ``` -This will automatically create a detailed report file called **`docs/my_guide_report.md`** in the same directory. +This will automatically create a detailed report file called **`docs/my_guide_REPORT.md`** in the same directory as the source file. The full path is printed on completion. --- ## ๐Ÿ“ Snippet Code Conventions -Each python code snippet inside the Markdown file is verified using our generalized `run.py` contract. For a code block to be fully testable for both loadability and runnability: +Each Python code block in the Markdown file is verified using the `run.py` contract. Blocks fall into one of four categories: + +1. **Expose a Global ADK Component (Runnability Test)**: + If the snippet instantiates a global `Workflow`, `Agent`, or `App`, the runner will automatically execute it and measure its execution against the Gemini API. + + **Requirement**: The component must be assigned to a **module-level variable** (i.e., defined at the top level of the snippet, not inside a function or class). The variable name does not matter โ€” the runner scans `vars(module)` to find it automatically. For multi-agent snippets, the runner identifies the root agent by filtering out any agents that are listed as `sub_agents` of another agent in the same snippet. + + **If this requirement is not met** (i.e., no module-level `Workflow`, `Agent`, or `App` instance exists), the runnability test is skipped. The report will show `โž– NO ADK COMPONENT` in the Run Phase column and explain that no runnable component was found at module level. -1. **Expose a Global ADK Component (Optional for Runnability)**: - If the snippet instantiates a global `Workflow`, `Agent`, or `App`, the runner will automatically execute it and measure its execution. 2. **Provide a Custom Test Input (Optional)**: - If the snippet defines a global `test_input` variable, the runner will use it during execution. + If the snippet defines a global `test_input` variable, the runner will use that string as the user message during execution instead of the default `"Test input topic"`. 3. **Basic Python Snippets (Loadability Only)**: - If a code block does not define any runnable ADK components, the runner will verify that it compiles and loads without error, and report a 100% coverage report for the lines present in the block. + If a code block does not define any runnable ADK components, the runner will verify that it compiles and loads without error. No API call is made, and coverage is reported as 100% for the lines present. +4. **Skipping Illustrative / Pseudo-code Blocks**: + Place the HTML comment `` on the line **immediately before** the opening ` ```python ` fence to exclude that block from execution entirely. It will appear in the report as `โญ๏ธ SKIPPED`. Use this for conceptual examples, incomplete snippets, or blocks that intentionally require external setup unavailable in CI. + + Example โ€” place the annotation on the line directly above the fence, with no blank line between them: + + ````markdown + + ```python + # This is pseudo-code โ€” not runnable as-is + my_agent = Agent(model="gemini-ultra-hypothetical", ...) + ``` + ```` + +--- + +## โš ๏ธ Known Limitations + +* **No cumulative / dependent snippet support**: Each snippet is executed in a fully isolated subprocess with no shared state. If a snippet depends on imports, variables, or definitions introduced by a *previous* snippet in the same document, it will fail with a `NameError` or `ImportError`. Author each code block to be self-contained, or annotate dependent snippets with ``. +* **120-second execution timeout**: Each snippet subprocess is killed after 120 seconds. Snippets that intentionally block (e.g., long-running servers) must be annotated with ``. +* **`` must be close to the fence**: Blank lines between the annotation and the ` ```python ` fence are tolerated, but any non-blank line (prose, another heading, etc.) between them cancels the annotation. Place the annotation on the line immediately before the opening fence to be safe. +* **No nested fences inside a code block**: The parser closes a Python block only on a bare ` ``` ` line (no language tag). Lines like ` ```bash ` or ` ```text ` inside the block are treated as literal content. However, a bare ` ``` ` that is the closing fence of a *non-Python* outer block will prematurely terminate any open Python block if the nesting depth is mismatched. Annotate such snippets with ``. --- ## ๐Ÿ“Š The Generated Report Structure -The generated `_report.md` will contain: -1. **Executive Summary**: A high-level table listing every discovered snippet, its preceding Markdown heading, its status (Passed/Failed), and its line coverage. +The generated `_REPORT.md` will contain: +1. **Executive Summary**: A high-level table listing every discovered snippet, its preceding Markdown heading, its Load phase status, Run phase status, and line coverage percentage. 2. **Detailed Breakdown**: - * The exact extracted python code block. - * The detailed execution stdout logs. - * Any stderr traceback/exceptions (pointing directly to the breaking line of code!). - * A clean, focused 5-line coverage table showing exactly what lines of the snippet were executed. + * The exact extracted Python code block. + * The detailed execution stdout logs (phases 1โ€“3). + * Any stderr traceback/exceptions (pointing directly to the breaking line of code). + * A full `coverage report` output showing exactly which lines of the snippet were executed. --- ## โš ๏ธ Crucial Behavioral Constraints (For AI Agents) -* **Strictly Read-Only Workspace**: The skill's operations on the repository are strictly **read-only**. The agent **MUST NOT** modify, create, or delete any existing source files, test files, configuration files, or documentation files in the repository (with the sole exceptions of writing temporary execution files to `.temp_snippets` and generating the final `_report.md` report file). +* **Read-Only**: See the caution block at the top of this document. The constraint is absolute โ€” no modifications, no deletions, no new files beyond the two permitted exceptions. * **Strictly Report, Do Not Fix**: The sole purpose of this skill is to **identify and report** compile-time and run-time issues within the Markdown document's snippets. * **No Unsolicited Patches**: When executing this skill, the agent **MUST NOT** attempt to rewrite the source Markdown file, modify its code blocks, or automatically generate code fixes/patches. -* **Focus on the Report**: The agent should run the verification, let the script generate the `_report.md` file, and present the executive summary table to the user. Do not offer solutions or rewrite recommendations unless the user explicitly asks for them. +* **Present the Summary Table Verbatim**: After the script completes, the agent MUST read the generated `_REPORT.md` file and copy the Executive Summary table to the user **exactly as written** โ€” same columns, same column names, same order. The table has exactly these six columns: + `Snippet | Preceding Heading | Load Phase | Run Phase | Coverage | Details` + The agent MUST NOT rename, reorder, merge, or drop any column when presenting results. Do not offer solutions or rewrite recommendations unless the user explicitly asks for them. diff --git a/.agents/skills/adk-verify-snippets/scripts/run.py b/.agents/skills/adk-verify-snippets/scripts/run.py index 284a45aa02c..a4eb8c2b5e9 100644 --- a/.agents/skills/adk-verify-snippets/scripts/run.py +++ b/.agents/skills/adk-verify-snippets/scripts/run.py @@ -1,12 +1,22 @@ import argparse import asyncio import importlib.util -import inspect import os import sys import traceback from pathlib import Path +# Sentinel string used by verify_md.py to locate and split the coverage section +# out of run.py's stdout. Keep in sync with verify_md.py:COV_SECTION_HEADER. +COV_SECTION_HEADER = "๐Ÿ“Š Phase 4: Code Coverage Report" + +# Structured exit codes โ€” consumed by verify_md.py to classify results without +# fragile string/emoji matching. Keep in sync with verify_md.py:EXIT_* constants. +EXIT_SUCCESS = 0 # All phases passed +EXIT_LOAD_FAILURE = 1 # Failed to compile / load the snippet +EXIT_RUN_FAILURE = 2 # Loaded OK but the ADK component failed at runtime +EXIT_NO_COMPONENT = 3 # Loaded OK, no runnable ADK component found (load-only) + # --- Optional Coverage Integration --- try: import coverage @@ -24,49 +34,94 @@ def load_target_module(file_path: Path): """Dynamically loads a Python file as a module, catching import/compilation/definition errors.""" - module_name = file_path.stem + # Use the absolute path string as the key to avoid collisions when multiple + # snippets share the same file stem or when the stem matches an installed package. + module_name = str(file_path) spec = importlib.util.spec_from_file_location(module_name, file_path) if spec is None or spec.loader is None: raise ImportError(f"Could not resolve module spec for file '{file_path.name}'") module = importlib.util.module_from_spec(spec) sys.modules[module_name] = module - + # Executing the module runs all top-level code, which will catch: # - SyntaxError / IndentationError # - ImportError (e.g. from google.adk.workflow import build_node) # - ValidationError (e.g. instantiating Workflow with invalid edges) - spec.loader.exec_module(module) + try: + spec.loader.exec_module(module) + except Exception: + # Remove the partially-initialised module so a broken entry is never + # left in sys.modules for the lifetime of this process. This matters + # when run.py is imported in-process (e.g. from a test harness) rather + # than invoked as a subprocess. + sys.modules.pop(module_name, None) + raise return module def discover_adk_component(module): - """Scans the module namespace to discover runnable ADK components, prioritizing root components.""" - # 1. Search for Workflow - for name, obj in inspect.getmembers(module): - if isinstance(obj, Workflow): - return obj, "Workflow" + """Scans the module namespace to discover runnable ADK components, prioritizing root components. - # 2. Search for Agent (filtering out sub-agents to locate the Root Agent) + Uses two passes to correctly identify root agents regardless of the order + in which names appear in ``vars(module)``: + + * Pass 1 โ€” collect every Workflow, Agent, and App in the module namespace. + * Pass 2 โ€” build the full set of sub-agent IDs from *all* collected agents, + then filter to find agents that are not sub-agents of any other agent. + + Without the two-pass approach, a root agent whose variable name is seen + before its sub-agents (e.g. ``root`` defined above ``child`` in the file) + would be encountered first, before ``child``'s own sub-agents are registered, + causing incorrect root detection. + """ + workflows = [] agents = [] - sub_agent_ids = set() - for name, obj in inspect.getmembers(module): - if isinstance(obj, BaseAgent): + apps = [] + + # Pass 1: collect all candidate components. + # + # Use vars(module) rather than inspect.getmembers(module) because + # getmembers() invokes every attribute getter and silently swallows any + # Exception raised by broken descriptors or properties โ€” a snippet that + # defines an Agent behind a faulty @property would simply be missing from + # the scan with no error or log entry. vars(module) reads the module's + # __dict__ directly, which never triggers descriptors and never suppresses + # exceptions, giving us an accurate view of module-level names. + for obj in vars(module).values(): + if isinstance(obj, Workflow): + workflows.append(obj) + elif isinstance(obj, BaseAgent): agents.append(obj) - # Track which agents are registered as sub-agents of another agent - if hasattr(obj, "sub_agents") and obj.sub_agents: - for sub in obj.sub_agents: - sub_agent_ids.add(id(sub)) - - # Find the agent(s) that are not sub-agents of any other agent in the module + elif isinstance(obj, App): + apps.append(obj) + + # 1. Prefer Workflow + if workflows: + return workflows[0], "Workflow" + + # Pass 2: build the complete sub-agent ID set now that all agents are known, + # then select the root (any agent not listed as a sub-agent of another). + # + # Read sub_agents into a local snapshot rather than calling the attribute + # twice. Calling it twice is unsafe when sub_agents is a non-idempotent + # property: the first call (guard) and the second call (iteration) could + # return different objects, causing id() values to diverge and root + # detection to silently misfire. + sub_agent_ids: set[int] = set() + for agent in agents: + children = getattr(agent, "sub_agents", None) or [] + for sub in children: + sub_agent_ids.add(id(sub)) + + # 2. Find root Agent (not a sub-agent of any other agent in the module) root_agents = [a for a in agents if id(a) not in sub_agent_ids] if root_agents: - # Return the first root agent discovered return root_agents[0], "Agent" - # 3. Search for App - for name, obj in inspect.getmembers(module): - if isinstance(obj, App): - return obj, "App" + # 3. Fall back to App + if apps: + return apps[0], "App" + return None, None async def run_component(component, component_type, test_input): @@ -74,7 +129,15 @@ async def run_component(component, component_type, test_input): print(f"\n๐Ÿ” Discovered ADK {component_type} in target file.") print(f"๐Ÿš€ Running execution test with input: '{test_input}'...\n") - runnable_node = component.root_node if component_type == "App" else component + if component_type == "App": + runnable_node = getattr(component, "root_agent", None) + if runnable_node is None: + raise AttributeError( + f"App instance has no 'root_agent' attribute. " + "Ensure the App is constructed with a root_agent argument." + ) + else: + runnable_node = component session_service = InMemorySessionService() runner = Runner(app_name="runnability_test", node=runnable_node, session_service=session_service) @@ -106,63 +169,102 @@ def main(): file_path = Path(args.file).resolve() if not file_path.exists(): print(f"โŒ Error: File '{file_path}' does not exist.") - sys.exit(1) + sys.exit(EXIT_LOAD_FAILURE) print(f"๐Ÿ”ฌ Testing file: {file_path.name}") print("=" * 60) - # Initialize coverage programmatically to track ONLY the target file + # Initialize coverage programmatically to track ONLY the target file. + # + # Implementation note: snippets are loaded via importlib/exec_module, which + # CPython's sys.settrace-based tracer instruments correctly *only* if the + # tracer is active before the module's code object is compiled and executed. + # Starting coverage here โ€” before load_target_module() โ€” satisfies that + # requirement. The `include` filter ensures no ADK library code is counted. cov = None if HAS_COVERAGE: cov = coverage.Coverage( branch=True, - data_file=None # Keep coverage data in-memory only, no .coverage file needed + data_file=None, # Keep coverage data in-memory only, no .coverage file needed + include=[str(file_path)], # Scope collection to the snippet file only ) cov.start() else: print("โ„น๏ธ Install 'coverage' package to enable automated code coverage reporting.") + # exit_code is set by each phase and consumed inside the finally block so + # that coverage reporting always runs before the process exits. Using a + # mutable list as a simple cell lets the finally clause read the value set + # by any code path (normal completion, early break-out via a flag, or an + # unexpected exception) without requiring nonlocal or a class wrapper. + exit_code = [EXIT_SUCCESS] + try: # 1. Test Loadability (Imports, Syntax, Instantiation/Validation) print("๐Ÿ“‹ Phase 1: Loading & Compiling...") try: module = load_target_module(file_path) print(f"โœ… Load Success: '{file_path.name}' compiled and loaded without any issues.") - except Exception as e: + except Exception: print(f"โŒ Load Failure: Failed to compile/load '{file_path.name}':") print("-" * 60) traceback.print_exc(file=sys.stdout) print("-" * 60) - sys.exit(1) + exit_code[0] = EXIT_LOAD_FAILURE + # Do NOT return here. Fall through to the finally block so that + # coverage is reported and sys.exit() is called with the correct code. + # The module variable is not set, so we skip phases 2โ€“3 via the flag. + else: + # 2. Discover Component (only reached when load succeeded) + print("\n๐Ÿ“‹ Phase 2: Component Discovery...") + component, comp_type = discover_adk_component(module) + if not component: + print(f"โž– NO ADK COMPONENT: No module-level Workflow, Agent, or App instance found in '{file_path.name}'.") + print(" Runnability test skipped. To enable it, assign a Workflow, Agent, or App") + print(" to a module-level variable (e.g. `agent = Agent(...)`). The variable name") + print(" does not matter โ€” the runner detects it automatically via vars(module).") + print("โ„น๏ธ Coverage below reflects load-time execution only (module-level statements).") + exit_code[0] = EXIT_NO_COMPONENT + else: + # Get test input from module, or fallback + test_input = getattr(module, "test_input", "Test input topic") - # 2. Discover Component - print("\n๐Ÿ“‹ Phase 2: Component Discovery...") - component, comp_type = discover_adk_component(module) - if not component: - print(f"โš ๏ธ No runnable ADK components (Workflow, Agent, or App) found in '{file_path.name}'.") - print(" Loadability check passed, but runnability test was skipped.") - sys.exit(0) + # 3. Test Runnability + print(f"\n๐Ÿ“‹ Phase 3: Executing {comp_type}...") + try: + # asyncio.run() creates a fresh event loop each time, so it will raise + # RuntimeError if a loop is already running (e.g. the snippet called + # asyncio.run() at module level without an __main__ guard). + # We catch that specific case and report it clearly, rather than using + # the deprecated asyncio.get_event_loop() API (removed in Python 3.12+). + asyncio.run(run_component(component, comp_type, test_input)) + print(f"\nโœ… Run Success: Component '{comp_type}' executed successfully.") + except RuntimeError as e: + if "event loop" in str(e).lower(): + print(f"\nโŒ Run Failure: An event loop conflict was detected after module load.") + print(" The snippet likely called asyncio.run() at module level, which") + print(" conflicts with the runner's own event loop. Wrap top-level async") + print(" calls in an `if __name__ == '__main__':` guard, or annotate the") + print(" snippet with .") + else: + print(f"\nโŒ Run Failure: Component failed during execution:") + print("-" * 60) + traceback.print_exc(file=sys.stdout) + print("-" * 60) + exit_code[0] = EXIT_RUN_FAILURE + except Exception: + print(f"\nโŒ Run Failure: Component failed during execution:") + print("-" * 60) + traceback.print_exc(file=sys.stdout) + print("-" * 60) + exit_code[0] = EXIT_RUN_FAILURE - # Get test input from module, or fallback - test_input = getattr(module, "test_input", "Test input topic") - - # 3. Test Runnability - print(f"\n๐Ÿ“‹ Phase 3: Executing {comp_type}...") - try: - asyncio.run(run_component(component, comp_type, test_input)) - print(f"\nโœ… Run Success: Component '{comp_type}' executed successfully.") - except Exception as e: - print(f"\nโŒ Run Failure: Component failed during execution:") - print("-" * 60) - traceback.print_exc(file=sys.stdout) - print("-" * 60) - sys.exit(1) - finally: - # Report coverage of the target file at the very end + # Coverage reporting runs here so it is guaranteed to execute on every + # code path: normal completion, load failure, no-component, run failure. if cov: cov.stop() - print("\n๐Ÿ“Š Phase 4: Code Coverage Report (Target File)") + print(f"\n{COV_SECTION_HEADER} (Target File)") print("=" * 60) try: # Report coverage of the target file directly to stdout @@ -172,6 +274,11 @@ def main(): except Exception as ce: print(f"โš ๏ธ Failed to generate coverage report: {ce}") print("=" * 60) + # Only call sys.exit for non-zero codes. If exit_code is EXIT_SUCCESS + # we return normally so that any exception currently propagating out of + # the try block is not silently replaced by a SystemExit raised here. + if exit_code[0] != EXIT_SUCCESS: + sys.exit(exit_code[0]) if __name__ == "__main__": main() diff --git a/.agents/skills/adk-verify-snippets/scripts/verify_md.py b/.agents/skills/adk-verify-snippets/scripts/verify_md.py index f1a531e07aa..5369b989ec9 100644 --- a/.agents/skills/adk-verify-snippets/scripts/verify_md.py +++ b/.agents/skills/adk-verify-snippets/scripts/verify_md.py @@ -5,10 +5,31 @@ import shutil import subprocess import sys +import tempfile from pathlib import Path +SKIP_ANNOTATION = "" +SNIPPET_TIMEOUT = 120 # seconds; adjust if snippets legitimately need longer + +# Must match COV_SECTION_HEADER in run.py exactly โ€” used to split coverage output +# from the main execution log when parsing run.py's stdout. +COV_SECTION_HEADER = "๐Ÿ“Š Phase 4: Code Coverage Report" + +# Structured exit codes from run.py โ€” kept in sync with run.py:EXIT_* constants. +# Using exit codes (not string/emoji matching) makes classification robust to +# future changes in run.py's human-readable output text. +EXIT_SUCCESS = 0 # All phases passed +EXIT_LOAD_FAILURE = 1 # Failed to compile / load the snippet +EXIT_RUN_FAILURE = 2 # Loaded OK but the ADK component failed at runtime +EXIT_NO_COMPONENT = 3 # Loaded OK, no runnable ADK component found (load-only) + def extract_snippets(md_path: Path): - """Parses a markdown file and extracts python code blocks along with their preceding headings.""" + """Parses a markdown file and extracts python code blocks along with their preceding headings. + + A code block immediately preceded by the HTML comment + ```` is recorded but marked as skipped so + that illustrative / pseudo-code examples are excluded from execution. + """ with open(md_path, "r", encoding="utf-8") as f: content = f.read() @@ -17,32 +38,51 @@ def extract_snippets(md_path: Path): current_heading = "Top Level" in_code_block = False code_lines = [] - + skip_next_block = False + for line in lines: # If we are inside a code block, handle it first to preserve comments starting with '#' if in_code_block: - if line.strip().startswith("```"): + stripped = line.strip() + # Close only on a bare closing fence (``` with no language specifier). + # A fenced block of another language (e.g. ```bash) appearing *inside* + # the Python block will not trigger this branch because it carries a + # language tag, so it is appended to code_lines as literal content. + if stripped == "```": in_code_block = False code_text = "\n".join(code_lines) snippets.append({ "heading": current_heading, - "code": code_text + "code": code_text, + "skip": skip_next_block, }) + skip_next_block = False else: code_lines.append(line) continue - + # If we are outside a code block, check for headings or code block starts if line.startswith("#"): # Clean up heading markers (e.g., "## Get started" -> "Get started") current_heading = line.lstrip("#").strip() + # A heading between the annotation and the fence cancels the skip. + skip_next_block = False continue - + + if line.strip() == SKIP_ANNOTATION: + skip_next_block = True + continue + if line.strip().startswith("```python"): in_code_block = True code_lines = [] continue - + + # Any other non-empty line (prose, blank-line-separated text, etc.) between + # the annotation and the fence cancels the skip. + if line.strip(): + skip_next_block = False + return snippets def run_snippet(run_py_path: Path, snippet_path: Path): @@ -54,24 +94,91 @@ def run_snippet(run_py_path: Path, snippet_path: Path): env = os.environ.copy() if "GOOGLE_API_KEY" in env and "GEMINI_API_KEY" in env: env.pop("GOOGLE_API_KEY", None) - - result = subprocess.run( - cmd, - capture_output=True, - text=True, - env=env - ) - - return { - "exit_code": result.returncode, - "stdout": result.stdout, - "stderr": result.stderr - } + + try: + result = subprocess.run( + cmd, + capture_output=True, + text=True, + env=env, + timeout=SNIPPET_TIMEOUT + ) + return { + "exit_code": result.returncode, + "stdout": result.stdout, + "stderr": result.stderr + } + except subprocess.TimeoutExpired: + return { + "exit_code": EXIT_RUN_FAILURE, + "stdout": f"โŒ Run Failure: Snippet execution timed out after {SNIPPET_TIMEOUT} seconds.", + "stderr": f"TimeoutExpired: The snippet process did not complete within the {SNIPPET_TIMEOUT}-second limit." + } + +def extract_error_detail(stdout: str, stderr: str) -> str: + """Extracts the most relevant error line from run.py's output. + + Searches in order: + 1. Last line in stderr that looks like a Python exception (``Error:`` + or ``Exception:``). Scoping to stderr avoids matching runner prose + in stdout (e.g. "โŒ Run Failure: ...") which contains words like "Failure" + but is not an exception line. + 2. Last line in stdout with the same pattern, as a fallback for runtimes that + write tracebacks to stdout instead of stderr. + 3. Last line in stderr matching the generic ``: `` format + (custom exception classes that don't end in Error/Exception). + 4. Fallback string if nothing matches. + """ + # Matches standard Python exception class names: ends in 'Error' or 'Exception', + # followed by a colon and detail text. Anchored to the start of the stripped line + # so runner prose ("โŒ Run Failure: ...") is not matched. + _exception_re = re.compile(r"^[A-Za-z]\w*(?:Error|Exception|Warning):\s*.+") + + for source in (stderr, stdout): + for line in reversed(source.splitlines()): + if _exception_re.match(line.strip()): + return f"`{line.strip()}`" + + # Pass 3: generic ': ' in stderr only + for line in reversed(stderr.splitlines()): + if re.match(r"^[A-Za-z]\w*:.+", line.strip()): + return f"`{line.strip()}`" + + return "Failed to compile/load." + def clean_name(name: str): """Sanitizes a string to be a safe filename.""" name = name.lower().replace(" ", "_") - return re.sub(r'[^a-z0-9__]', '', name) + return re.sub(r'[^a-z0-9_]', '', name) + +def md_cell(value: str) -> str: + """Escapes pipe characters so the value is safe inside a Markdown table cell.""" + return value.replace("|", r"\|") + +def safe_fence(content: str, language: str = "") -> str: + """Returns a Markdown fenced code block that safely wraps *content*. + + Picks the shortest fence (minimum three backticks) that is strictly longer + than any contiguous run of backticks found inside *content*, so the fence + cannot be prematurely closed by content that itself contains backtick runs. + This is the approach recommended by the CommonMark spec. + + Example:: + + safe_fence("x = ```foo```", "python") + # returns: + # ````python + # x = ```foo``` + # ```` + """ + # Find the longest run of backticks inside the content + max_run = max((len(m.group()) for m in re.finditer(r"`+", content)), default=0) + # The outer fence must be strictly longer, and at least 3 characters + fence_len = max(3, max_run + 1) + fence = "`" * fence_len + tag = f"{fence}{language}\n" if language else f"{fence}\n" + return f"{tag}{content}\n{fence}" def main(): parser = argparse.ArgumentParser(description="Markdown Snippet Verifier") @@ -99,131 +206,186 @@ def main(): print(f"๐Ÿ“‹ Found {len(snippets)} python code snippets to verify.") - # Create a temp directory in the workspace - temp_dir = Path(".temp_snippets") - temp_dir.mkdir(exist_ok=True) + # Create a unique temp directory to avoid collisions with concurrent runs + temp_dir = Path(tempfile.mkdtemp(prefix="verify_snippets_")) results = [] - # 2. Execute each snippet - for i, snippet in enumerate(snippets, start=1): - heading = snippet["heading"] - code = snippet["code"] - - # Create a unique, sanitized filename for the snippet - safe_heading = clean_name(heading) - temp_file_name = f"snippet_{i}_{safe_heading}.py" - temp_file_path = temp_dir / temp_file_name - - # Write snippet to file - with open(temp_file_path, "w", encoding="utf-8") as f: - f.write(code) - - print(f"๐Ÿงช Testing Snippet {i}/{len(snippets)} under heading '{heading}'...") - - # Run the snippet - run_res = run_snippet(run_py_path, temp_file_path) - - results.append({ - "index": i, - "heading": heading, - "code": code, - "temp_file": temp_file_name, - "exit_code": run_res["exit_code"], - "stdout": run_res["stdout"], - "stderr": run_res["stderr"] - }) - - # Clean up temporary directory - shutil.rmtree(temp_dir, ignore_errors=True) - - # 3. Generate Markdown Report - report_path = md_path.parent / f"{md_path.stem}_report.md" - timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") - - with open(report_path, "w", encoding="utf-8") as f: - f.write(f"# ๐Ÿ”ฌ ADK Markdown Snippet Verification Report\n\n") - f.write(f"* **Source File**: [{md_path.name}](file://{md_path})\n") - f.write(f"* **Verified On**: `{timestamp}`\n\n") - - # Write summary table - f.write("## ๐Ÿ“ˆ Executive Summary\n\n") - f.write("| Snippet | preceding Heading | Load Phase | Run Phase | Coverage | Uncovered Issues / Details |\n") - f.write("| :--- | :--- | :---: | :---: | :---: | :--- |\n") - - for r in results: - # Determine Phase 1 (Load) and Phase 3 (Run) statuses - load_status = "โœ… **PASS**" - run_status = "โœ… **PASS**" - coverage_pct = "โ€”" - - stdout_and_stderr = r["stdout"] + "\n" + r["stderr"] - - # 1. Parse Load Phase - if "โŒ Load Failure" in stdout_and_stderr or "Load Failure" in stdout_and_stderr: - load_status = "โŒ **FAIL**" - run_status = "โž– **SKIPPED**" - - # 2. Parse Run Phase - elif "runnability test was skipped" in stdout_and_stderr: - run_status = "โž– **SKIPPED**" - elif "โŒ Run Failure" in stdout_and_stderr or r["exit_code"] != 0: - run_status = "โŒ **FAIL**" - - # 3. Parse Coverage - cov_match = re.search(r"TOTAL\s+\d+\s+\d+\s+\d+\s+\d+\s+(\d+%)", r["stdout"]) - if cov_match and load_status != "โŒ **FAIL**": - coverage_pct = f"`{cov_match.group(1)}`" - - # 4. Formulate details and handle transient 503s - details = "All checks passed successfully." - if load_status == "โŒ **FAIL**": - err_lines = [line for line in stdout_and_stderr.splitlines() if "Error" in line or "Exception" in line] - details = f"`{err_lines[-1]}`" if err_lines else "Failed to compile/load." - elif run_status == "โŒ **FAIL**": - if "503" in stdout_and_stderr and "UNAVAILABLE" in stdout_and_stderr: - details = "โš ๏ธ **Transient 503 from Gemini API (overloaded)**. Code structure is correct." - else: - err_lines = [line for line in stdout_and_stderr.splitlines() if "Error" in line or "Exception" in line] - details = f"`{err_lines[-1]}`" if err_lines else "Failed during execution." - - f.write(f"| **Snippet {r['index']}** | `{r['heading']}` | {load_status} | {run_status} | {coverage_pct} | {details} |\n") - - f.write("\n---\n\n## ๐Ÿ” Detailed Snippet Reports\n\n") + # 2. Execute each snippet, then write the report โ€” both inside the try so + # the finally cleanup only runs after the report is fully written. + try: + for i, snippet in enumerate(snippets, start=1): + heading = snippet["heading"] + code = snippet["code"] + is_skipped = snippet.get("skip", False) + + # Create a unique, sanitized filename for the snippet + safe_heading = clean_name(heading) + temp_file_name = f"snippet_{i}_{safe_heading}.py" + temp_file_path = temp_dir / temp_file_name + + if is_skipped: + print(f"โญ๏ธ Skipping Snippet {i}/{len(snippets)} under heading '{heading}' (marked ignore).") + results.append({ + "index": i, + "heading": heading, + "code": code, + "temp_file": temp_file_name, + "exit_code": 0, + "stdout": "", + "stderr": "", + "skipped": True, + }) + continue + + # Write snippet to file + with open(temp_file_path, "w", encoding="utf-8") as f: + f.write(code) + + print(f"๐Ÿงช Testing Snippet {i}/{len(snippets)} under heading '{heading}'...") + + # Run the snippet + run_res = run_snippet(run_py_path, temp_file_path) + + results.append({ + "index": i, + "heading": heading, + "code": code, + "temp_file": temp_file_name, + "exit_code": run_res["exit_code"], + "stdout": run_res["stdout"], + "stderr": run_res["stderr"], + "skipped": False, + }) + + # 3. Generate Markdown Report โ€” inside the try so finally runs after this completes. + # Use clean_name on the stem so the report path is safe on all filesystems. + # If clean_name strips everything (e.g. a fully non-ASCII filename), fall + # back to a hash of the original stem so two such files in the same + # directory never produce the same report path. + safe_stem = clean_name(md_path.stem) or f"report_{abs(hash(md_path.stem))}" + report_path = md_path.parent / f"{safe_stem}_REPORT.md" + timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") - for r in results: - status_icon = "โœ…" if r["exit_code"] == 0 else "โŒ" - f.write(f"### {status_icon} Snippet {r['index']}: `{r['heading']}`\n\n") - - f.write("#### ๐Ÿ“ Code Block\n") - f.write(f"```python\n{r['code']}\n```\n\n") - - # Write stdout / stderr logs - f.write("#### ๐Ÿ–ฅ๏ธ Loadability & Runnability Logs\n") - f.write("```text\n") + with open(report_path, "w", encoding="utf-8") as f: + f.write(f"# ๐Ÿ”ฌ ADK Markdown Snippet Verification Report\n\n") + f.write(f"* **Source File**: [{md_path.name}](file://{md_path})\n") + f.write(f"* **Verified On**: `{timestamp}`\n\n") - # Clean up stdout to separate Phase 4 coverage - stdout_clean = r["stdout"] - cov_match = re.search(r"(๐Ÿ“Š Phase 4: Code Coverage Report.*)", r["stdout"], re.DOTALL) - cov_text = cov_match.group(1) if cov_match else None + # Write summary table + f.write("## ๐Ÿ“ˆ Executive Summary\n\n") + f.write("| Snippet | Preceding Heading | Load Phase | Run Phase | Coverage | Details |\n") + f.write("| :--- | :--- | :---: | :---: | :---: | :--- |\n") - if cov_text: - stdout_clean = r["stdout"].replace(cov_text, "").strip() + for r in results: + # Handle explicitly skipped snippets + if r.get("skipped"): + f.write(f"| **Snippet {r['index']}** | `{md_cell(r['heading'])}` | โญ๏ธ **SKIPPED** | โญ๏ธ **SKIPPED** | โ€” | Marked `{SKIP_ANNOTATION}` โ€” intentionally ignored. |\n") + continue + + # Determine Phase 1 (Load) and Phase 3 (Run) statuses from the + # structured exit code emitted by run.py โ€” no emoji/string matching. + exit_code = r["exit_code"] + load_status = "โœ… **PASS**" + run_status = "โœ… **PASS**" + coverage_pct = "โ€”" + + stdout_and_stderr = r["stdout"] + "\n" + r["stderr"] + + if exit_code == EXIT_LOAD_FAILURE: + load_status = "โŒ **FAIL**" + run_status = "โž– **SKIPPED**" + elif exit_code == EXIT_NO_COMPONENT: + run_status = "โž– **NO ADK COMPONENT**" + elif exit_code == EXIT_RUN_FAILURE: + run_status = "โŒ **FAIL**" + # EXIT_SUCCESS (0): both statuses remain โœ… **PASS** + + # 3. Parse Coverage โ€” anchor to line start to avoid matching prose. + # Handles both branch (5 numeric cols) and non-branch (3 cols) formats. + total_match = re.search(r"^TOTAL(?:\s+\d+)+\s+(\d+)%", r["stdout"], re.MULTILINE) + if total_match and load_status != "โŒ **FAIL**": + coverage_pct = f"`{total_match.group(1)}`" + + # 4. Formulate details and handle transient 503s + details = "All checks passed successfully." + if load_status == "โŒ **FAIL**": + details = extract_error_detail(r["stdout"], r["stderr"]) + elif run_status == "โž– **NO ADK COMPONENT**": + details = "No module-level `Workflow`, `Agent`, or `App` instance found. Assign one to a top-level variable to enable runnability testing." + elif run_status == "โŒ **FAIL**": + if "503" in stdout_and_stderr and "UNAVAILABLE" in stdout_and_stderr: + details = "โš ๏ธ **Transient 503 from Gemini API (overloaded)**. Code structure is correct." + else: + details = extract_error_detail(r["stdout"], r["stderr"]) + + # Store statuses for reuse in the detailed section + r["load_status"] = load_status + r["run_status"] = run_status + + f.write(f"| **Snippet {r['index']}** | `{md_cell(r['heading'])}` | {load_status} | {run_status} | {coverage_pct} | {md_cell(details)} |\n") - f.write(stdout_clean) - if r["stderr"]: - f.write("\n\n=== STDERR/TRACEBACK ===\n") - f.write(r["stderr"].strip()) - f.write("\n```\n\n") + f.write("\n---\n\n## ๐Ÿ” Detailed Snippet Reports\n\n") - # Write coverage report if available - if cov_text: - f.write("#### ๐Ÿ“Š Coverage Report\n") - f.write(f"```text\n{cov_text}\n```\n\n") + for r in results: + if r.get("skipped"): + f.write(f"### โญ๏ธ Snippet {r['index']}: `{r['heading']}` *(ignored)*\n\n") + f.write("#### ๐Ÿ“ Code Block\n") + f.write(safe_fence(r["code"], "python")) + f.write("\n\n") + f.write(f"> This snippet was skipped because it is annotated with `{SKIP_ANNOTATION}`.\n\n") + f.write("---\n\n") + continue + + l_stat = r.get("load_status", "โœ… **PASS**") + r_stat = r.get("run_status", "โœ… **PASS**") + if l_stat == "โŒ **FAIL**" or r_stat == "โŒ **FAIL**": + status_icon = "โŒ" + elif r_stat == "โž– **NO ADK COMPONENT**": + status_icon = "โž–" + else: + status_icon = "โœ…" + f.write(f"### {status_icon} Snippet {r['index']}: `{r['heading']}`\n\n") + + f.write("#### ๐Ÿ“ Code Block\n") + f.write(safe_fence(r["code"], "python")) + f.write("\n\n") + + # Write stdout / stderr logs + # Split run.py stdout into main log and coverage section using the + # shared COV_SECTION_HEADER constant (kept in sync with run.py). + stdout_clean = r["stdout"] + cov_section_match = re.search( + rf"({re.escape(COV_SECTION_HEADER)}.*)", r["stdout"], re.DOTALL + ) + cov_text = cov_section_match.group(1) if cov_section_match else None + + if cov_text: + stdout_clean = r["stdout"].replace(cov_text, "").strip() + + log_content = stdout_clean + if r["stderr"]: + log_content += "\n\n=== STDERR/TRACEBACK ===\n" + r["stderr"].strip() + + f.write("#### ๐Ÿ–ฅ๏ธ Loadability & Runnability Logs\n") + f.write(safe_fence(log_content)) + f.write("\n\n") + + # Write coverage report if available + if cov_text: + f.write("#### ๐Ÿ“Š Coverage Report\n") + f.write(safe_fence(cov_text)) + f.write("\n\n") + + f.write("---\n\n") - f.write("---\n\n") - - print(f"๐ŸŽ‰ Verification complete! Report generated at: {report_path.name}") + print(f"๐ŸŽ‰ Verification complete! Report generated at: {report_path}") + + finally: + # Always clean up the temp directory, even on Ctrl+C or unexpected errors. + # This runs after report generation completes (or if it raises), ensuring + # temp files are never left behind. + shutil.rmtree(temp_dir, ignore_errors=True) if __name__ == "__main__": main() From 8aaaa3fed59347c06da1a2f0609c3e2c892f0c2b Mon Sep 17 00:00:00 2001 From: Shahin Saadati Date: Tue, 23 Jun 2026 19:01:48 -0700 Subject: [PATCH 3/6] docs: update ADK snippet verification documentation and ignore coverage reports in .gitignore --- .agents/skills/adk-verify-snippets/SKILL.md | 131 ++++++++------------ .gitignore | 4 + 2 files changed, 59 insertions(+), 76 deletions(-) diff --git a/.agents/skills/adk-verify-snippets/SKILL.md b/.agents/skills/adk-verify-snippets/SKILL.md index 6bba3adf515..8ef04913652 100644 --- a/.agents/skills/adk-verify-snippets/SKILL.md +++ b/.agents/skills/adk-verify-snippets/SKILL.md @@ -10,119 +10,98 @@ metadata: # Verify Markdown Snippets Skill -This skill allows you to systematically verify the correctness, compile-readiness, and line coverage of Python code snippets embedded within a Markdown document (like tutorials, guides, or READMEs). - -It extracts all ` ```python ` blocks, executes them in process-isolated environments using the bundled `run.py` harness, and generates a structured test report. +This skill extracts all ` ```python ` blocks from a Markdown file, executes each one in a process-isolated environment using the bundled `run.py` harness, and generates a structured report covering load status, run status, and line coverage. > [!CAUTION] > **STRICT READ-ONLY CONSTRAINT โ€” READ THIS BEFORE DOING ANYTHING ELSE** > -> When executing this skill, the agent operates in a **strictly read-only** mode with respect to the repository. The agent **MUST NOT**, under any circumstances: -> - **Modify** any existing source file, test file, configuration file, documentation file, or skill file (including this SKILL.md). +> This skill is **read-only**. The agent **MUST NOT**: +> - **Modify** any file in the repository (source, test, config, docs, or skill files โ€” including this SKILL.md). > - **Delete** any file in the repository. -> - **Create** any new file in the repository other than the two explicit exceptions listed below. +> - **Create** any new file in the repository. > -> The **only** two write operations permitted are: -> 1. Writing temporary snippet `.py` files to a uniquely-named **system temp directory** (outside the repository). -> 2. Writing the final `_REPORT.md` report file into the **same directory as the source Markdown file**. +> The **only two write operations permitted** are: +> 1. Writing temporary `.py` snippet files to a **system temp directory outside the repository**. +> 2. Writing the final `_REPORT.md` into the **same directory as the source Markdown file**. > -> Any other file system mutation is a violation of this skill's contract. If in doubt, do not write. +> If in doubt, do not write. Any other mutation is a violation of this skill's contract. --- ## ๐Ÿ”ง Prerequisites -Before running this skill, ensure the following are in place: - -1. **ADK Python environment**: The skill must be run from the repository root with the project's virtual environment active (via `uv`). -2. **`coverage` package**: Install it to enable per-snippet coverage reporting: - ```bash - uv pip install coverage - ``` - If `coverage` is not installed, the runner degrades gracefully โ€” verification still works, but coverage columns in the report will show `โ€”`. -3. **Gemini API key**: Snippets that instantiate an `Agent`, `App`, or `Workflow` make live calls to the Gemini API. Set one of the following environment variables before running: - ```bash - export GEMINI_API_KEY="your-key-here" - # or - export GOOGLE_API_KEY="your-key-here" - ``` - If both are set, `GEMINI_API_KEY` takes precedence. Snippets that do not expose a runnable ADK component (loadability-only checks) do not require an API key. +1. **ADK Python environment**: Run from the repository root with the `uv` virtual environment active. +2. **`coverage` package** *(optional)*: Enables per-snippet coverage reporting. Without it, coverage columns show `โ€”`. + ```bash + uv pip install coverage + ``` +3. **Gemini API key**: Required only for snippets that instantiate an `Agent`, `App`, or `Workflow` (which make live Gemini API calls). Set one of: + ```bash + export GEMINI_API_KEY="your-key-here" + # or + export GOOGLE_API_KEY="your-key-here" + ``` + If both are set, `GEMINI_API_KEY` takes precedence. --- -## ๐Ÿ› ๏ธ How to Use the Skill - -To verify a Markdown file (e.g. `docs/my_guide.md`) and generate its runnability report: +## ๐Ÿ› ๏ธ Usage ```bash uv run --no-sync python .agents/skills/adk-verify-snippets/scripts/verify_md.py ``` -For example: -```bash -uv run --no-sync python .agents/skills/adk-verify-snippets/scripts/verify_md.py docs/my_guide.md -``` +The script prints progress for each snippet, then writes a report to **`_REPORT.md`** in the same directory as the source file and prints the full path on completion. -This will automatically create a detailed report file called **`docs/my_guide_REPORT.md`** in the same directory as the source file. The full path is printed on completion. +**Report contents:** +- **Executive Summary table** โ€” one row per snippet: preceding heading, Load phase status, Run phase status, coverage %, and error detail. +- **Detailed section** โ€” for each snippet: the extracted code block, full execution logs (stdout + stderr/traceback), and the coverage report. --- -## ๐Ÿ“ Snippet Code Conventions - -Each Python code block in the Markdown file is verified using the `run.py` contract. Blocks fall into one of four categories: +## ๐Ÿ“ How Snippets Are Classified -1. **Expose a Global ADK Component (Runnability Test)**: - If the snippet instantiates a global `Workflow`, `Agent`, or `App`, the runner will automatically execute it and measure its execution against the Gemini API. +Each ` ```python ` block falls into one of these categories: - **Requirement**: The component must be assigned to a **module-level variable** (i.e., defined at the top level of the snippet, not inside a function or class). The variable name does not matter โ€” the runner scans `vars(module)` to find it automatically. For multi-agent snippets, the runner identifies the root agent by filtering out any agents that are listed as `sub_agents` of another agent in the same snippet. +### 1. Runnability Test (has a module-level ADK component) +If the snippet assigns a `Workflow`, `Agent`, or `App` to a **module-level variable**, the runner executes it against the Gemini API. - **If this requirement is not met** (i.e., no module-level `Workflow`, `Agent`, or `App` instance exists), the runnability test is skipped. The report will show `โž– NO ADK COMPONENT` in the Run Phase column and explain that no runnable component was found at module level. +- The variable name does not matter โ€” the runner finds it automatically via `vars(module)`. +- For multi-agent snippets, the runner identifies the root agent by excluding any agent that appears in another agent's `sub_agents` list. +- To use a custom test prompt instead of the default `"Test input topic"`, define a module-level `test_input` string in the snippet. -2. **Provide a Custom Test Input (Optional)**: - If the snippet defines a global `test_input` variable, the runner will use that string as the user message during execution instead of the default `"Test input topic"`. -3. **Basic Python Snippets (Loadability Only)**: - If a code block does not define any runnable ADK components, the runner will verify that it compiles and loads without error. No API call is made, and coverage is reported as 100% for the lines present. -4. **Skipping Illustrative / Pseudo-code Blocks**: - Place the HTML comment `` on the line **immediately before** the opening ` ```python ` fence to exclude that block from execution entirely. It will appear in the report as `โญ๏ธ SKIPPED`. Use this for conceptual examples, incomplete snippets, or blocks that intentionally require external setup unavailable in CI. +If no module-level ADK component is found, the run phase is skipped and the report shows `โž– NO ADK COMPONENT`. - Example โ€” place the annotation on the line directly above the fence, with no blank line between them: +### 2. Loadability-Only (no ADK component) +The runner verifies the snippet compiles and imports without error. No API call is made. - ````markdown - - ```python - # This is pseudo-code โ€” not runnable as-is - my_agent = Agent(model="gemini-ultra-hypothetical", ...) - ``` - ```` +### 3. Skipped (annotated with ignore) +Place `` immediately before the opening ` ```python ` fence to exclude a block entirely. Use this for pseudo-code, illustrative examples, or snippets that require external setup. ---- - -## โš ๏ธ Known Limitations +````markdown + +```python +# pseudo-code โ€” not runnable as-is +my_agent = Agent(model="gemini-ultra-hypothetical", ...) +``` +```` -* **No cumulative / dependent snippet support**: Each snippet is executed in a fully isolated subprocess with no shared state. If a snippet depends on imports, variables, or definitions introduced by a *previous* snippet in the same document, it will fail with a `NameError` or `ImportError`. Author each code block to be self-contained, or annotate dependent snippets with ``. -* **120-second execution timeout**: Each snippet subprocess is killed after 120 seconds. Snippets that intentionally block (e.g., long-running servers) must be annotated with ``. -* **`` must be close to the fence**: Blank lines between the annotation and the ` ```python ` fence are tolerated, but any non-blank line (prose, another heading, etc.) between them cancels the annotation. Place the annotation on the line immediately before the opening fence to be safe. -* **No nested fences inside a code block**: The parser closes a Python block only on a bare ` ``` ` line (no language tag). Lines like ` ```bash ` or ` ```text ` inside the block are treated as literal content. However, a bare ` ``` ` that is the closing fence of a *non-Python* outer block will prematurely terminate any open Python block if the nesting depth is mismatched. Annotate such snippets with ``. +The report shows these as `โญ๏ธ SKIPPED`. --- -## ๐Ÿ“Š The Generated Report Structure +## โš ๏ธ Known Limitations -The generated `_REPORT.md` will contain: -1. **Executive Summary**: A high-level table listing every discovered snippet, its preceding Markdown heading, its Load phase status, Run phase status, and line coverage percentage. -2. **Detailed Breakdown**: - * The exact extracted Python code block. - * The detailed execution stdout logs (phases 1โ€“3). - * Any stderr traceback/exceptions (pointing directly to the breaking line of code). - * A full `coverage report` output showing exactly which lines of the snippet were executed. +- **No shared state between snippets**: Each snippet runs in a fresh subprocess with no imports or variables carried over from previous snippets. A snippet that depends on code from an earlier block will fail with `NameError` or `ImportError`. Make each snippet self-contained, or annotate it with ``. +- **120-second timeout**: Each snippet is killed after 120 seconds. Annotate long-running or blocking snippets with ``. +- **Ignore annotation placement**: The `` annotation applies to the next ` ```python ` fence encountered. Blank lines between the annotation and the fence are tolerated, but any non-blank line (prose or a heading) cancels the annotation. +- **Bare ` ``` ` closes the block**: The parser closes a Python block on the first bare ` ``` ` line (no language tag). A bare ` ``` ` appearing as content inside a snippet (e.g. to demonstrate Markdown syntax) will prematurely close the block. Annotate such snippets with ``. --- -## โš ๏ธ Crucial Behavioral Constraints (For AI Agents) +## โš ๏ธ Behavioral Constraints (For AI Agents) -* **Read-Only**: See the caution block at the top of this document. The constraint is absolute โ€” no modifications, no deletions, no new files beyond the two permitted exceptions. -* **Strictly Report, Do Not Fix**: The sole purpose of this skill is to **identify and report** compile-time and run-time issues within the Markdown document's snippets. -* **No Unsolicited Patches**: When executing this skill, the agent **MUST NOT** attempt to rewrite the source Markdown file, modify its code blocks, or automatically generate code fixes/patches. -* **Present the Summary Table Verbatim**: After the script completes, the agent MUST read the generated `_REPORT.md` file and copy the Executive Summary table to the user **exactly as written** โ€” same columns, same column names, same order. The table has exactly these six columns: - `Snippet | Preceding Heading | Load Phase | Run Phase | Coverage | Details` - The agent MUST NOT rename, reorder, merge, or drop any column when presenting results. Do not offer solutions or rewrite recommendations unless the user explicitly asks for them. +- **Read-only**: See the caution block at the top. The constraint is absolute. +- **Report only, do not fix**: The agent MUST NOT rewrite the source Markdown, modify code blocks, or generate patches. Present the summary table to the user and stop. +- **Present the summary table verbatim**: After the script completes, read the generated `_REPORT.md` and copy the Executive Summary table to the user **exactly as written** โ€” same six columns, same order, no renaming or dropping: + `Snippet | Preceding Heading | Load Phase | Run Phase | Coverage | Details` diff --git a/.gitignore b/.gitignore index c3ddc7ea990..4e16e078fe4 100644 --- a/.gitignore +++ b/.gitignore @@ -121,3 +121,7 @@ CLAUDE.md # Conformance test outputs (timestamped folders from --test mode) **/conformance/20*-*-*_*-*-*/ + +# adk-verify-snippets skill โ€” generated report files (never commit these) +*_REPORT.md +*_report.md From c8a6eac6be74c2bb29087a877477e3849657f515 Mon Sep 17 00:00:00 2001 From: Shahin Saadati Date: Wed, 24 Jun 2026 14:56:39 -0700 Subject: [PATCH 4/6] docs: Undid the changes in .gitignore --- .gitignore | 4 - .../agents/llm_agent/single_turn_REPORT.md | 261 ++++++++++++++++++ docs/guides/agents/llm_agent/task_REPORT.md | 101 +++++++ 3 files changed, 362 insertions(+), 4 deletions(-) create mode 100644 docs/guides/agents/llm_agent/single_turn_REPORT.md create mode 100644 docs/guides/agents/llm_agent/task_REPORT.md diff --git a/.gitignore b/.gitignore index 4e16e078fe4..c3ddc7ea990 100644 --- a/.gitignore +++ b/.gitignore @@ -121,7 +121,3 @@ CLAUDE.md # Conformance test outputs (timestamped folders from --test mode) **/conformance/20*-*-*_*-*-*/ - -# adk-verify-snippets skill โ€” generated report files (never commit these) -*_REPORT.md -*_report.md diff --git a/docs/guides/agents/llm_agent/single_turn_REPORT.md b/docs/guides/agents/llm_agent/single_turn_REPORT.md new file mode 100644 index 00000000000..5cf4953ab13 --- /dev/null +++ b/docs/guides/agents/llm_agent/single_turn_REPORT.md @@ -0,0 +1,261 @@ +# ๐Ÿ”ฌ ADK Markdown Snippet Verification Report + +* **Source File**: [single_turn.md](file:///Users/shahins/projects/mine/adk-python/docs/guides/agents/llm_agent/single_turn.md) +* **Verified On**: `2026-06-23 19:02:40` + +## ๐Ÿ“ˆ Executive Summary + +| Snippet | Preceding Heading | Load Phase | Run Phase | Coverage | Details | +| :--- | :--- | :---: | :---: | :---: | :--- | +| **Snippet 1** | `Example` | โŒ **FAIL** | โž– **SKIPPED** | โ€” | `ImportError: cannot import name 'build_node' from 'google.adk.workflow' (/Users/shahins/projects/mine/adk-python/src/google/adk/workflow/__init__.py)` | +| **Snippet 2** | `Example` | โœ… **PASS** | โœ… **PASS** | `100` | All checks passed successfully. | +| **Snippet 3** | `Context-Aware Sub-Agent Example` | โŒ **FAIL** | โž– **SKIPPED** | โ€” | `NameError: name 'LlmAgent' is not defined` | + +--- + +## ๐Ÿ” Detailed Snippet Reports + +### โŒ Snippet 1: `Example` + +#### ๐Ÿ“ Code Block +```python +from google.adk.agents import LlmAgent +from google.adk.workflow import Workflow, build_node + +# Defaults to mode="single_turn" when run as a node +writer_agent = LlmAgent( + name="writer", + instruction="Write a short story about the input topic." +) + +writer_node = build_node(writer_agent) + +wf = Workflow( + name="story_generator", + edges=[ + ("START", writer_node), + (writer_node, "END") + ] +) +``` + +#### ๐Ÿ–ฅ๏ธ Loadability & Runnability Logs +``` +๐Ÿ”ฌ Testing file: snippet_1_example.py +============================================================ +๐Ÿ“‹ Phase 1: Loading & Compiling... +โŒ Load Failure: Failed to compile/load 'snippet_1_example.py': +------------------------------------------------------------ +Traceback (most recent call last): + File "/Users/shahins/projects/mine/adk-python/.agents/skills/adk-verify-snippets/scripts/run.py", line 206, in main + module = load_target_module(file_path) + File "/Users/shahins/projects/mine/adk-python/.agents/skills/adk-verify-snippets/scripts/run.py", line 52, in load_target_module + spec.loader.exec_module(module) + ~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^ + File "", line 762, in exec_module + File "", line 491, in _call_with_frames_removed + File "/private/var/folders/cm/82x0j39d2076xcfjpm5qcxxr00gp60/T/verify_snippets_09387wsx/snippet_1_example.py", line 2, in + from google.adk.workflow import Workflow, build_node +ImportError: cannot import name 'build_node' from 'google.adk.workflow' (/Users/shahins/projects/mine/adk-python/src/google/adk/workflow/__init__.py) +------------------------------------------------------------ +``` + +#### ๐Ÿ“Š Coverage Report +``` +๐Ÿ“Š Phase 4: Code Coverage Report (Target File) +============================================================ +Name Stmts Miss Branch BrPart Cover +------------------------------------------------------------------------------------------------------------------------------------------ +/private/var/folders/cm/82x0j39d2076xcfjpm5qcxxr00gp60/T/verify_snippets_09387wsx/snippet_1_example.py 5 3 0 0 40% +------------------------------------------------------------------------------------------------------------------------------------------ +TOTAL 5 3 0 0 40% +============================================================ + +``` + +--- + +### โœ… Snippet 2: `Example` + +#### ๐Ÿ“ Code Block +```python +from google.adk.agents import LlmAgent + +# Define a specialized single-turn sub-agent +translator_agent = LlmAgent( + name="translator", + instruction="Translate the input text to Spanish.", + mode="single_turn" # Must be explicit if not auto-wrapped in workflow +) + +# Define the parent agent and assign the sub-agent +bilingual_writer = LlmAgent( + name="bilingual_writer", + instruction="Write a poem about the topic, then use the translator tool to translate it.", + sub_agents=[translator_agent] # Exposes 'translator' as a tool to bilingual_writer +) +``` + +#### ๐Ÿ–ฅ๏ธ Loadability & Runnability Logs +``` +๐Ÿ”ฌ Testing file: snippet_2_example.py +============================================================ +๐Ÿ“‹ Phase 1: Loading & Compiling... +โœ… Load Success: 'snippet_2_example.py' compiled and loaded without any issues. + +๐Ÿ“‹ Phase 2: Component Discovery... + +๐Ÿ“‹ Phase 3: Executing Agent... + +๐Ÿ” Discovered ADK Agent in target file. +๐Ÿš€ Running execution test with input: 'Test input topic'... + +๐ŸŽฌ [Event] Author: bilingual_writer +๐Ÿ“ Content Output: +---------------------------------------- +### Test Input + +A lonely string of text arrives, +To see if the machine survives. +A test input, a simple key, +To unlock what the code might be. + +Through functions, loops, and logic gates, +It travels where the program waits. +Will it pass, or will it break? +What difference will a symbol make? + +A gentle pulse, a quiet spark, +A guiding light within the dark. +The test succeeds, the output clear, +No bugs to find, no faults to fear. + +*** + +Now, translating this poem into Spanish... +---------------------------------------- +๐ŸŽฌ [Event] Author: translator +๐ŸŽฌ [Event] Author: translator +๐ŸŽฌ [Event] Author: bilingual_writer +๐Ÿ“ Content Output: +---------------------------------------- +Aquรญ tienes la traducciรณn del poema al espaรฑol: + +### Entrada de Prueba + +Una solitaria cadena de texto llega, +Para ver si la mรกquina sobrevive. +Una entrada de prueba, una simple clave, +Para desbloquear lo que el cรณdigo podrรญa ser. + +A travรฉs de funciones, bucles y puertas lรณgicas, +Viaja hacia donde el programa espera. +ยฟPasarรก, o se romperรก? +ยฟQuรฉ diferencia harรก un sรญmbolo? + +Un suave pulso, una chispa silenciosa, +Una luz guรญa en la oscuridad. +La prueba tiene รฉxito, el resultado es claro, +Sin errores que encontrar, ni fallos que temer. +---------------------------------------- +๐ŸŽฌ [Event] Author: bilingual_writer +๐ŸŽฌ [Event] Author: bilingual_writer +๐Ÿ“ Content Output: +---------------------------------------- +Here is the translation of the poem: + +### Entrada de Prueba + +Una solitaria cadena de texto llega, +Para ver si la mรกquina sobrevive. +Una entrada de prueba, una simple clave, +Para desbloquear lo que el cรณdigo podrรญa ser. + +A travรฉs de funciones, bucles y puertas lรณgicas, +Viaja hacia donde el programa espera. +ยฟPasarรก, o se romperรก? +ยฟQuรฉ diferencia harรก un sรญmbolo? + +Un suave pulso, una chispa silenciosa, +Una luz guรญa en la oscuridad. +La prueba tiene รฉxito, el resultado es claro, +Sin errores que encontrar, ni fallos que temer. +---------------------------------------- + +โœ… Run Success: Component 'Agent' executed successfully. + +=== STDERR/TRACEBACK === +/Users/shahins/projects/mine/adk-python/src/google/adk/models/llm_request.py:256: UserWarning: [EXPERIMENTAL] feature FeatureName.JSON_SCHEMA_FOR_FUNC_DECL is enabled. + declaration = tool._get_declaration() +``` + +#### ๐Ÿ“Š Coverage Report +``` +๐Ÿ“Š Phase 4: Code Coverage Report (Target File) +============================================================ +Name Stmts Miss Branch BrPart Cover +------------------------------------------------------------------------------------------------------------------------------------------ +/private/var/folders/cm/82x0j39d2076xcfjpm5qcxxr00gp60/T/verify_snippets_09387wsx/snippet_2_example.py 3 0 0 0 100% +------------------------------------------------------------------------------------------------------------------------------------------ +TOTAL 3 0 0 0 100% +============================================================ + +``` + +--- + +### โŒ Snippet 3: `Context-Aware Sub-Agent Example` + +#### ๐Ÿ“ Code Block +```python +verifier_agent = LlmAgent( + name="verifier", + instruction="Verify that the draft meets all constraints discussed in the chat.", + mode="single_turn", + include_contents="default" # Allows the sub-agent to see the parent's conversation history +) + +editor_agent = LlmAgent( + name="editor", + instruction="Discuss the draft with the user and use verifier to check constraints.", + sub_agents=[verifier_agent] +) +``` + +#### ๐Ÿ–ฅ๏ธ Loadability & Runnability Logs +``` +๐Ÿ”ฌ Testing file: snippet_3_contextaware_subagent_example.py +============================================================ +๐Ÿ“‹ Phase 1: Loading & Compiling... +โŒ Load Failure: Failed to compile/load 'snippet_3_contextaware_subagent_example.py': +------------------------------------------------------------ +Traceback (most recent call last): + File "/Users/shahins/projects/mine/adk-python/.agents/skills/adk-verify-snippets/scripts/run.py", line 206, in main + module = load_target_module(file_path) + File "/Users/shahins/projects/mine/adk-python/.agents/skills/adk-verify-snippets/scripts/run.py", line 52, in load_target_module + spec.loader.exec_module(module) + ~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^ + File "", line 762, in exec_module + File "", line 491, in _call_with_frames_removed + File "/private/var/folders/cm/82x0j39d2076xcfjpm5qcxxr00gp60/T/verify_snippets_09387wsx/snippet_3_contextaware_subagent_example.py", line 1, in + verifier_agent = LlmAgent( + ^^^^^^^^ +NameError: name 'LlmAgent' is not defined +------------------------------------------------------------ +``` + +#### ๐Ÿ“Š Coverage Report +``` +๐Ÿ“Š Phase 4: Code Coverage Report (Target File) +============================================================ +Name Stmts Miss Branch BrPart Cover +---------------------------------------------------------------------------------------------------------------------------------------------------------------- +/private/var/folders/cm/82x0j39d2076xcfjpm5qcxxr00gp60/T/verify_snippets_09387wsx/snippet_3_contextaware_subagent_example.py 2 1 0 0 50% +---------------------------------------------------------------------------------------------------------------------------------------------------------------- +TOTAL 2 1 0 0 50% +============================================================ + +``` + +--- + diff --git a/docs/guides/agents/llm_agent/task_REPORT.md b/docs/guides/agents/llm_agent/task_REPORT.md new file mode 100644 index 00000000000..6c46309d765 --- /dev/null +++ b/docs/guides/agents/llm_agent/task_REPORT.md @@ -0,0 +1,101 @@ +# ๐Ÿ”ฌ ADK Markdown Snippet Verification Report + +* **Source File**: [task.md](file:///Users/shahins/projects/mine/adk-python/docs/guides/agents/llm_agent/task.md) +* **Verified On**: `2026-06-23 19:02:28` + +## ๐Ÿ“ˆ Executive Summary + +| Snippet | Preceding Heading | Load Phase | Run Phase | Coverage | Details | +| :--- | :--- | :---: | :---: | :---: | :--- | +| **Snippet 1** | `Example` | โŒ **FAIL** | โž– **SKIPPED** | โ€” | Failed to compile/load. | + +--- + +## ๐Ÿ” Detailed Snippet Reports + +### โŒ Snippet 1: `Example` + +#### ๐Ÿ“ Code Block +```python +from google.adk.agents import LlmAgent +from pydantic import BaseModel, Field + +# 1. Define schemas for Input and Output +class ResearchInput(BaseModel): + topic: str = Field(description="The topic to research.") + depth: str = Field(default="brief", description="Depth of research: brief or detailed.") + +class ResearchOutput(BaseModel): + summary: str = Field(description="A summary of the findings.") + sources: list[str] = Field(description="List of sources used.") + +# 2. Define the Task Agent +researcher_agent = LlmAgent( + name="researcher", + instruction="Research the given topic and provide a structured summary.", + mode="task", + input_schema=ResearchInput, + output_schema=ResearchOutput, + # Add tools needed for the task + tools=[...] +) + +# 3. Define the Parent Agent +writer_agent = LlmAgent( + name="writer", + instruction="Write a blog post. Use the researcher agent to get info on the topic.", + sub_agents=[researcher_agent] # Exposes 'researcher' agent to writer +) +``` + +#### ๐Ÿ–ฅ๏ธ Loadability & Runnability Logs +``` +๐Ÿ”ฌ Testing file: snippet_1_example.py +============================================================ +๐Ÿ“‹ Phase 1: Loading & Compiling... +โŒ Load Failure: Failed to compile/load 'snippet_1_example.py': +------------------------------------------------------------ +Traceback (most recent call last): + File "/Users/shahins/projects/mine/adk-python/.agents/skills/adk-verify-snippets/scripts/run.py", line 206, in main + module = load_target_module(file_path) + File "/Users/shahins/projects/mine/adk-python/.agents/skills/adk-verify-snippets/scripts/run.py", line 52, in load_target_module + spec.loader.exec_module(module) + ~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^ + File "", line 762, in exec_module + File "", line 491, in _call_with_frames_removed + File "/private/var/folders/cm/82x0j39d2076xcfjpm5qcxxr00gp60/T/verify_snippets__vng7hxw/snippet_1_example.py", line 14, in + researcher_agent = LlmAgent( + name="researcher", + ...<5 lines>... + tools=[...] + ) + File "/Users/shahins/projects/mine/adk-python/.venv/lib/python3.14/site-packages/pydantic/main.py", line 250, in __init__ + validated_self = self.__pydantic_validator__.validate_python(data, self_instance=self) +pydantic_core._pydantic_core.ValidationError: 3 validation errors for LlmAgent +tools.0.callable + Input should be callable [type=callable_type, input_value=Ellipsis, input_type=ellipsis] + For further information visit https://errors.pydantic.dev/2.12/v/callable_type +tools.0.is-instance[BaseTool] + Input should be an instance of BaseTool [type=is_instance_of, input_value=Ellipsis, input_type=ellipsis] + For further information visit https://errors.pydantic.dev/2.12/v/is_instance_of +tools.0.is-instance[BaseToolset] + Input should be an instance of BaseToolset [type=is_instance_of, input_value=Ellipsis, input_type=ellipsis] + For further information visit https://errors.pydantic.dev/2.12/v/is_instance_of +------------------------------------------------------------ +``` + +#### ๐Ÿ“Š Coverage Report +``` +๐Ÿ“Š Phase 4: Code Coverage Report (Target File) +============================================================ +Name Stmts Miss Branch BrPart Cover +------------------------------------------------------------------------------------------------------------------------------------------ +/private/var/folders/cm/82x0j39d2076xcfjpm5qcxxr00gp60/T/verify_snippets__vng7hxw/snippet_1_example.py 10 1 0 0 90% +------------------------------------------------------------------------------------------------------------------------------------------ +TOTAL 10 1 0 0 90% +============================================================ + +``` + +--- + From e552e1ce11a032651ff91c63b95242d01591ceec Mon Sep 17 00:00:00 2001 From: Shahin Saadati Date: Wed, 24 Jun 2026 14:58:39 -0700 Subject: [PATCH 5/6] chore: remove obsolete documentation verification reports --- .../agents/llm_agent/single_turn_REPORT.md | 261 ------------------ docs/guides/agents/llm_agent/task_REPORT.md | 101 ------- 2 files changed, 362 deletions(-) delete mode 100644 docs/guides/agents/llm_agent/single_turn_REPORT.md delete mode 100644 docs/guides/agents/llm_agent/task_REPORT.md diff --git a/docs/guides/agents/llm_agent/single_turn_REPORT.md b/docs/guides/agents/llm_agent/single_turn_REPORT.md deleted file mode 100644 index 5cf4953ab13..00000000000 --- a/docs/guides/agents/llm_agent/single_turn_REPORT.md +++ /dev/null @@ -1,261 +0,0 @@ -# ๐Ÿ”ฌ ADK Markdown Snippet Verification Report - -* **Source File**: [single_turn.md](file:///Users/shahins/projects/mine/adk-python/docs/guides/agents/llm_agent/single_turn.md) -* **Verified On**: `2026-06-23 19:02:40` - -## ๐Ÿ“ˆ Executive Summary - -| Snippet | Preceding Heading | Load Phase | Run Phase | Coverage | Details | -| :--- | :--- | :---: | :---: | :---: | :--- | -| **Snippet 1** | `Example` | โŒ **FAIL** | โž– **SKIPPED** | โ€” | `ImportError: cannot import name 'build_node' from 'google.adk.workflow' (/Users/shahins/projects/mine/adk-python/src/google/adk/workflow/__init__.py)` | -| **Snippet 2** | `Example` | โœ… **PASS** | โœ… **PASS** | `100` | All checks passed successfully. | -| **Snippet 3** | `Context-Aware Sub-Agent Example` | โŒ **FAIL** | โž– **SKIPPED** | โ€” | `NameError: name 'LlmAgent' is not defined` | - ---- - -## ๐Ÿ” Detailed Snippet Reports - -### โŒ Snippet 1: `Example` - -#### ๐Ÿ“ Code Block -```python -from google.adk.agents import LlmAgent -from google.adk.workflow import Workflow, build_node - -# Defaults to mode="single_turn" when run as a node -writer_agent = LlmAgent( - name="writer", - instruction="Write a short story about the input topic." -) - -writer_node = build_node(writer_agent) - -wf = Workflow( - name="story_generator", - edges=[ - ("START", writer_node), - (writer_node, "END") - ] -) -``` - -#### ๐Ÿ–ฅ๏ธ Loadability & Runnability Logs -``` -๐Ÿ”ฌ Testing file: snippet_1_example.py -============================================================ -๐Ÿ“‹ Phase 1: Loading & Compiling... -โŒ Load Failure: Failed to compile/load 'snippet_1_example.py': ------------------------------------------------------------- -Traceback (most recent call last): - File "/Users/shahins/projects/mine/adk-python/.agents/skills/adk-verify-snippets/scripts/run.py", line 206, in main - module = load_target_module(file_path) - File "/Users/shahins/projects/mine/adk-python/.agents/skills/adk-verify-snippets/scripts/run.py", line 52, in load_target_module - spec.loader.exec_module(module) - ~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^ - File "", line 762, in exec_module - File "", line 491, in _call_with_frames_removed - File "/private/var/folders/cm/82x0j39d2076xcfjpm5qcxxr00gp60/T/verify_snippets_09387wsx/snippet_1_example.py", line 2, in - from google.adk.workflow import Workflow, build_node -ImportError: cannot import name 'build_node' from 'google.adk.workflow' (/Users/shahins/projects/mine/adk-python/src/google/adk/workflow/__init__.py) ------------------------------------------------------------- -``` - -#### ๐Ÿ“Š Coverage Report -``` -๐Ÿ“Š Phase 4: Code Coverage Report (Target File) -============================================================ -Name Stmts Miss Branch BrPart Cover ------------------------------------------------------------------------------------------------------------------------------------------- -/private/var/folders/cm/82x0j39d2076xcfjpm5qcxxr00gp60/T/verify_snippets_09387wsx/snippet_1_example.py 5 3 0 0 40% ------------------------------------------------------------------------------------------------------------------------------------------- -TOTAL 5 3 0 0 40% -============================================================ - -``` - ---- - -### โœ… Snippet 2: `Example` - -#### ๐Ÿ“ Code Block -```python -from google.adk.agents import LlmAgent - -# Define a specialized single-turn sub-agent -translator_agent = LlmAgent( - name="translator", - instruction="Translate the input text to Spanish.", - mode="single_turn" # Must be explicit if not auto-wrapped in workflow -) - -# Define the parent agent and assign the sub-agent -bilingual_writer = LlmAgent( - name="bilingual_writer", - instruction="Write a poem about the topic, then use the translator tool to translate it.", - sub_agents=[translator_agent] # Exposes 'translator' as a tool to bilingual_writer -) -``` - -#### ๐Ÿ–ฅ๏ธ Loadability & Runnability Logs -``` -๐Ÿ”ฌ Testing file: snippet_2_example.py -============================================================ -๐Ÿ“‹ Phase 1: Loading & Compiling... -โœ… Load Success: 'snippet_2_example.py' compiled and loaded without any issues. - -๐Ÿ“‹ Phase 2: Component Discovery... - -๐Ÿ“‹ Phase 3: Executing Agent... - -๐Ÿ” Discovered ADK Agent in target file. -๐Ÿš€ Running execution test with input: 'Test input topic'... - -๐ŸŽฌ [Event] Author: bilingual_writer -๐Ÿ“ Content Output: ----------------------------------------- -### Test Input - -A lonely string of text arrives, -To see if the machine survives. -A test input, a simple key, -To unlock what the code might be. - -Through functions, loops, and logic gates, -It travels where the program waits. -Will it pass, or will it break? -What difference will a symbol make? - -A gentle pulse, a quiet spark, -A guiding light within the dark. -The test succeeds, the output clear, -No bugs to find, no faults to fear. - -*** - -Now, translating this poem into Spanish... ----------------------------------------- -๐ŸŽฌ [Event] Author: translator -๐ŸŽฌ [Event] Author: translator -๐ŸŽฌ [Event] Author: bilingual_writer -๐Ÿ“ Content Output: ----------------------------------------- -Aquรญ tienes la traducciรณn del poema al espaรฑol: - -### Entrada de Prueba - -Una solitaria cadena de texto llega, -Para ver si la mรกquina sobrevive. -Una entrada de prueba, una simple clave, -Para desbloquear lo que el cรณdigo podrรญa ser. - -A travรฉs de funciones, bucles y puertas lรณgicas, -Viaja hacia donde el programa espera. -ยฟPasarรก, o se romperรก? -ยฟQuรฉ diferencia harรก un sรญmbolo? - -Un suave pulso, una chispa silenciosa, -Una luz guรญa en la oscuridad. -La prueba tiene รฉxito, el resultado es claro, -Sin errores que encontrar, ni fallos que temer. ----------------------------------------- -๐ŸŽฌ [Event] Author: bilingual_writer -๐ŸŽฌ [Event] Author: bilingual_writer -๐Ÿ“ Content Output: ----------------------------------------- -Here is the translation of the poem: - -### Entrada de Prueba - -Una solitaria cadena de texto llega, -Para ver si la mรกquina sobrevive. -Una entrada de prueba, una simple clave, -Para desbloquear lo que el cรณdigo podrรญa ser. - -A travรฉs de funciones, bucles y puertas lรณgicas, -Viaja hacia donde el programa espera. -ยฟPasarรก, o se romperรก? -ยฟQuรฉ diferencia harรก un sรญmbolo? - -Un suave pulso, una chispa silenciosa, -Una luz guรญa en la oscuridad. -La prueba tiene รฉxito, el resultado es claro, -Sin errores que encontrar, ni fallos que temer. ----------------------------------------- - -โœ… Run Success: Component 'Agent' executed successfully. - -=== STDERR/TRACEBACK === -/Users/shahins/projects/mine/adk-python/src/google/adk/models/llm_request.py:256: UserWarning: [EXPERIMENTAL] feature FeatureName.JSON_SCHEMA_FOR_FUNC_DECL is enabled. - declaration = tool._get_declaration() -``` - -#### ๐Ÿ“Š Coverage Report -``` -๐Ÿ“Š Phase 4: Code Coverage Report (Target File) -============================================================ -Name Stmts Miss Branch BrPart Cover ------------------------------------------------------------------------------------------------------------------------------------------- -/private/var/folders/cm/82x0j39d2076xcfjpm5qcxxr00gp60/T/verify_snippets_09387wsx/snippet_2_example.py 3 0 0 0 100% ------------------------------------------------------------------------------------------------------------------------------------------- -TOTAL 3 0 0 0 100% -============================================================ - -``` - ---- - -### โŒ Snippet 3: `Context-Aware Sub-Agent Example` - -#### ๐Ÿ“ Code Block -```python -verifier_agent = LlmAgent( - name="verifier", - instruction="Verify that the draft meets all constraints discussed in the chat.", - mode="single_turn", - include_contents="default" # Allows the sub-agent to see the parent's conversation history -) - -editor_agent = LlmAgent( - name="editor", - instruction="Discuss the draft with the user and use verifier to check constraints.", - sub_agents=[verifier_agent] -) -``` - -#### ๐Ÿ–ฅ๏ธ Loadability & Runnability Logs -``` -๐Ÿ”ฌ Testing file: snippet_3_contextaware_subagent_example.py -============================================================ -๐Ÿ“‹ Phase 1: Loading & Compiling... -โŒ Load Failure: Failed to compile/load 'snippet_3_contextaware_subagent_example.py': ------------------------------------------------------------- -Traceback (most recent call last): - File "/Users/shahins/projects/mine/adk-python/.agents/skills/adk-verify-snippets/scripts/run.py", line 206, in main - module = load_target_module(file_path) - File "/Users/shahins/projects/mine/adk-python/.agents/skills/adk-verify-snippets/scripts/run.py", line 52, in load_target_module - spec.loader.exec_module(module) - ~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^ - File "", line 762, in exec_module - File "", line 491, in _call_with_frames_removed - File "/private/var/folders/cm/82x0j39d2076xcfjpm5qcxxr00gp60/T/verify_snippets_09387wsx/snippet_3_contextaware_subagent_example.py", line 1, in - verifier_agent = LlmAgent( - ^^^^^^^^ -NameError: name 'LlmAgent' is not defined ------------------------------------------------------------- -``` - -#### ๐Ÿ“Š Coverage Report -``` -๐Ÿ“Š Phase 4: Code Coverage Report (Target File) -============================================================ -Name Stmts Miss Branch BrPart Cover ----------------------------------------------------------------------------------------------------------------------------------------------------------------- -/private/var/folders/cm/82x0j39d2076xcfjpm5qcxxr00gp60/T/verify_snippets_09387wsx/snippet_3_contextaware_subagent_example.py 2 1 0 0 50% ----------------------------------------------------------------------------------------------------------------------------------------------------------------- -TOTAL 2 1 0 0 50% -============================================================ - -``` - ---- - diff --git a/docs/guides/agents/llm_agent/task_REPORT.md b/docs/guides/agents/llm_agent/task_REPORT.md deleted file mode 100644 index 6c46309d765..00000000000 --- a/docs/guides/agents/llm_agent/task_REPORT.md +++ /dev/null @@ -1,101 +0,0 @@ -# ๐Ÿ”ฌ ADK Markdown Snippet Verification Report - -* **Source File**: [task.md](file:///Users/shahins/projects/mine/adk-python/docs/guides/agents/llm_agent/task.md) -* **Verified On**: `2026-06-23 19:02:28` - -## ๐Ÿ“ˆ Executive Summary - -| Snippet | Preceding Heading | Load Phase | Run Phase | Coverage | Details | -| :--- | :--- | :---: | :---: | :---: | :--- | -| **Snippet 1** | `Example` | โŒ **FAIL** | โž– **SKIPPED** | โ€” | Failed to compile/load. | - ---- - -## ๐Ÿ” Detailed Snippet Reports - -### โŒ Snippet 1: `Example` - -#### ๐Ÿ“ Code Block -```python -from google.adk.agents import LlmAgent -from pydantic import BaseModel, Field - -# 1. Define schemas for Input and Output -class ResearchInput(BaseModel): - topic: str = Field(description="The topic to research.") - depth: str = Field(default="brief", description="Depth of research: brief or detailed.") - -class ResearchOutput(BaseModel): - summary: str = Field(description="A summary of the findings.") - sources: list[str] = Field(description="List of sources used.") - -# 2. Define the Task Agent -researcher_agent = LlmAgent( - name="researcher", - instruction="Research the given topic and provide a structured summary.", - mode="task", - input_schema=ResearchInput, - output_schema=ResearchOutput, - # Add tools needed for the task - tools=[...] -) - -# 3. Define the Parent Agent -writer_agent = LlmAgent( - name="writer", - instruction="Write a blog post. Use the researcher agent to get info on the topic.", - sub_agents=[researcher_agent] # Exposes 'researcher' agent to writer -) -``` - -#### ๐Ÿ–ฅ๏ธ Loadability & Runnability Logs -``` -๐Ÿ”ฌ Testing file: snippet_1_example.py -============================================================ -๐Ÿ“‹ Phase 1: Loading & Compiling... -โŒ Load Failure: Failed to compile/load 'snippet_1_example.py': ------------------------------------------------------------- -Traceback (most recent call last): - File "/Users/shahins/projects/mine/adk-python/.agents/skills/adk-verify-snippets/scripts/run.py", line 206, in main - module = load_target_module(file_path) - File "/Users/shahins/projects/mine/adk-python/.agents/skills/adk-verify-snippets/scripts/run.py", line 52, in load_target_module - spec.loader.exec_module(module) - ~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^ - File "", line 762, in exec_module - File "", line 491, in _call_with_frames_removed - File "/private/var/folders/cm/82x0j39d2076xcfjpm5qcxxr00gp60/T/verify_snippets__vng7hxw/snippet_1_example.py", line 14, in - researcher_agent = LlmAgent( - name="researcher", - ...<5 lines>... - tools=[...] - ) - File "/Users/shahins/projects/mine/adk-python/.venv/lib/python3.14/site-packages/pydantic/main.py", line 250, in __init__ - validated_self = self.__pydantic_validator__.validate_python(data, self_instance=self) -pydantic_core._pydantic_core.ValidationError: 3 validation errors for LlmAgent -tools.0.callable - Input should be callable [type=callable_type, input_value=Ellipsis, input_type=ellipsis] - For further information visit https://errors.pydantic.dev/2.12/v/callable_type -tools.0.is-instance[BaseTool] - Input should be an instance of BaseTool [type=is_instance_of, input_value=Ellipsis, input_type=ellipsis] - For further information visit https://errors.pydantic.dev/2.12/v/is_instance_of -tools.0.is-instance[BaseToolset] - Input should be an instance of BaseToolset [type=is_instance_of, input_value=Ellipsis, input_type=ellipsis] - For further information visit https://errors.pydantic.dev/2.12/v/is_instance_of ------------------------------------------------------------- -``` - -#### ๐Ÿ“Š Coverage Report -``` -๐Ÿ“Š Phase 4: Code Coverage Report (Target File) -============================================================ -Name Stmts Miss Branch BrPart Cover ------------------------------------------------------------------------------------------------------------------------------------------- -/private/var/folders/cm/82x0j39d2076xcfjpm5qcxxr00gp60/T/verify_snippets__vng7hxw/snippet_1_example.py 10 1 0 0 90% ------------------------------------------------------------------------------------------------------------------------------------------- -TOTAL 10 1 0 0 90% -============================================================ - -``` - ---- - From 7ad98eb99e0155d513426790237a58cdb9ec64b8 Mon Sep 17 00:00:00 2001 From: Shahin Saadati Date: Thu, 25 Jun 2026 09:15:42 -0700 Subject: [PATCH 6/6] style: reformat scripts with consistent indentation and add copyright headers --- .../skills/adk-verify-snippets/scripts/run.py | 550 +++++++------ .../adk-verify-snippets/scripts/verify_md.py | 769 ++++++++++-------- 2 files changed, 725 insertions(+), 594 deletions(-) diff --git a/.agents/skills/adk-verify-snippets/scripts/run.py b/.agents/skills/adk-verify-snippets/scripts/run.py index a4eb8c2b5e9..33024d0a1fb 100644 --- a/.agents/skills/adk-verify-snippets/scripts/run.py +++ b/.agents/skills/adk-verify-snippets/scripts/run.py @@ -1,10 +1,26 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + import argparse import asyncio import importlib.util import os +from pathlib import Path import sys import traceback -from pathlib import Path # Sentinel string used by verify_md.py to locate and split the coverage section # out of run.py's stdout. Keep in sync with verify_md.py:COV_SECTION_HEADER. @@ -12,17 +28,18 @@ # Structured exit codes โ€” consumed by verify_md.py to classify results without # fragile string/emoji matching. Keep in sync with verify_md.py:EXIT_* constants. -EXIT_SUCCESS = 0 # All phases passed -EXIT_LOAD_FAILURE = 1 # Failed to compile / load the snippet -EXIT_RUN_FAILURE = 2 # Loaded OK but the ADK component failed at runtime -EXIT_NO_COMPONENT = 3 # Loaded OK, no runnable ADK component found (load-only) +EXIT_SUCCESS = 0 # All phases passed +EXIT_LOAD_FAILURE = 1 # Failed to compile / load the snippet +EXIT_RUN_FAILURE = 2 # Loaded OK but the ADK component failed at runtime +EXIT_NO_COMPONENT = 3 # Loaded OK, no runnable ADK component found (load-only) # --- Optional Coverage Integration --- try: - import coverage - HAS_COVERAGE = True + import coverage + + HAS_COVERAGE = True except ImportError: - HAS_COVERAGE = False + HAS_COVERAGE = False # --- Imports for ADK Inspection --- from google.adk.agents.base_agent import BaseAgent @@ -32,253 +49,308 @@ from google.adk.workflow import Workflow from google.genai import types + def load_target_module(file_path: Path): - """Dynamically loads a Python file as a module, catching import/compilation/definition errors.""" - # Use the absolute path string as the key to avoid collisions when multiple - # snippets share the same file stem or when the stem matches an installed package. - module_name = str(file_path) - spec = importlib.util.spec_from_file_location(module_name, file_path) - if spec is None or spec.loader is None: - raise ImportError(f"Could not resolve module spec for file '{file_path.name}'") - - module = importlib.util.module_from_spec(spec) - sys.modules[module_name] = module - - # Executing the module runs all top-level code, which will catch: - # - SyntaxError / IndentationError - # - ImportError (e.g. from google.adk.workflow import build_node) - # - ValidationError (e.g. instantiating Workflow with invalid edges) - try: - spec.loader.exec_module(module) - except Exception: - # Remove the partially-initialised module so a broken entry is never - # left in sys.modules for the lifetime of this process. This matters - # when run.py is imported in-process (e.g. from a test harness) rather - # than invoked as a subprocess. - sys.modules.pop(module_name, None) - raise - return module + """Dynamically loads a Python file as a module, catching import/compilation/definition errors.""" + # Use the absolute path string as the key to avoid collisions when multiple + # snippets share the same file stem or when the stem matches an installed package. + module_name = str(file_path) + spec = importlib.util.spec_from_file_location(module_name, file_path) + if spec is None or spec.loader is None: + raise ImportError( + f"Could not resolve module spec for file '{file_path.name}'" + ) + + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + + # Executing the module runs all top-level code, which will catch: + # - SyntaxError / IndentationError + # - ImportError (e.g. from google.adk.workflow import build_node) + # - ValidationError (e.g. instantiating Workflow with invalid edges) + try: + spec.loader.exec_module(module) + except Exception: + # Remove the partially-initialised module so a broken entry is never + # left in sys.modules for the lifetime of this process. This matters + # when run.py is imported in-process (e.g. from a test harness) rather + # than invoked as a subprocess. + sys.modules.pop(module_name, None) + raise + return module + def discover_adk_component(module): - """Scans the module namespace to discover runnable ADK components, prioritizing root components. - - Uses two passes to correctly identify root agents regardless of the order - in which names appear in ``vars(module)``: - - * Pass 1 โ€” collect every Workflow, Agent, and App in the module namespace. - * Pass 2 โ€” build the full set of sub-agent IDs from *all* collected agents, - then filter to find agents that are not sub-agents of any other agent. - - Without the two-pass approach, a root agent whose variable name is seen - before its sub-agents (e.g. ``root`` defined above ``child`` in the file) - would be encountered first, before ``child``'s own sub-agents are registered, - causing incorrect root detection. - """ - workflows = [] - agents = [] - apps = [] - - # Pass 1: collect all candidate components. - # - # Use vars(module) rather than inspect.getmembers(module) because - # getmembers() invokes every attribute getter and silently swallows any - # Exception raised by broken descriptors or properties โ€” a snippet that - # defines an Agent behind a faulty @property would simply be missing from - # the scan with no error or log entry. vars(module) reads the module's - # __dict__ directly, which never triggers descriptors and never suppresses - # exceptions, giving us an accurate view of module-level names. - for obj in vars(module).values(): - if isinstance(obj, Workflow): - workflows.append(obj) - elif isinstance(obj, BaseAgent): - agents.append(obj) - elif isinstance(obj, App): - apps.append(obj) - - # 1. Prefer Workflow - if workflows: - return workflows[0], "Workflow" - - # Pass 2: build the complete sub-agent ID set now that all agents are known, - # then select the root (any agent not listed as a sub-agent of another). - # - # Read sub_agents into a local snapshot rather than calling the attribute - # twice. Calling it twice is unsafe when sub_agents is a non-idempotent - # property: the first call (guard) and the second call (iteration) could - # return different objects, causing id() values to diverge and root - # detection to silently misfire. - sub_agent_ids: set[int] = set() - for agent in agents: - children = getattr(agent, "sub_agents", None) or [] - for sub in children: - sub_agent_ids.add(id(sub)) - - # 2. Find root Agent (not a sub-agent of any other agent in the module) - root_agents = [a for a in agents if id(a) not in sub_agent_ids] - if root_agents: - return root_agents[0], "Agent" - - # 3. Fall back to App - if apps: - return apps[0], "App" - - return None, None + """Scans the module namespace to discover runnable ADK components, prioritizing root components. + + Uses two passes to correctly identify root agents regardless of the order + in which names appear in ``vars(module)``: + + * Pass 1 โ€” collect every Workflow, Agent, and App in the module namespace. + * Pass 2 โ€” build the full set of sub-agent IDs from *all* collected agents, + then filter to find agents that are not sub-agents of any other agent. + + Without the two-pass approach, a root agent whose variable name is seen + before its sub-agents (e.g. ``root`` defined above ``child`` in the file) + would be encountered first, before ``child``'s own sub-agents are registered, + causing incorrect root detection. + """ + workflows = [] + agents = [] + apps = [] + + # Pass 1: collect all candidate components. + # + # Use vars(module) rather than inspect.getmembers(module) because + # getmembers() invokes every attribute getter and silently swallows any + # Exception raised by broken descriptors or properties โ€” a snippet that + # defines an Agent behind a faulty @property would simply be missing from + # the scan with no error or log entry. vars(module) reads the module's + # __dict__ directly, which never triggers descriptors and never suppresses + # exceptions, giving us an accurate view of module-level names. + for obj in vars(module).values(): + if isinstance(obj, Workflow): + workflows.append(obj) + elif isinstance(obj, BaseAgent): + agents.append(obj) + elif isinstance(obj, App): + apps.append(obj) + + # 1. Prefer Workflow + if workflows: + return workflows[0], "Workflow" + + # Pass 2: build the complete sub-agent ID set now that all agents are known, + # then select the root (any agent not listed as a sub-agent of another). + # + # Read sub_agents into a local snapshot rather than calling the attribute + # twice. Calling it twice is unsafe when sub_agents is a non-idempotent + # property: the first call (guard) and the second call (iteration) could + # return different objects, causing id() values to diverge and root + # detection to silently misfire. + sub_agent_ids: set[int] = set() + for agent in agents: + children = getattr(agent, "sub_agents", None) or [] + for sub in children: + sub_agent_ids.add(id(sub)) + + # 2. Find root Agent (not a sub-agent of any other agent in the module) + root_agents = [a for a in agents if id(a) not in sub_agent_ids] + if root_agents: + return root_agents[0], "Agent" + + # 3. Fall back to App + if apps: + return apps[0], "App" + + return None, None + async def run_component(component, component_type, test_input): - """Unified runner to execute the discovered component.""" - print(f"\n๐Ÿ” Discovered ADK {component_type} in target file.") - print(f"๐Ÿš€ Running execution test with input: '{test_input}'...\n") - - if component_type == "App": - runnable_node = getattr(component, "root_agent", None) - if runnable_node is None: - raise AttributeError( - f"App instance has no 'root_agent' attribute. " - "Ensure the App is constructed with a root_agent argument." - ) - else: - runnable_node = component + """Unified runner to execute the discovered component.""" + print(f"\n๐Ÿ” Discovered ADK {component_type} in target file.") + print(f"๐Ÿš€ Running execution test with input: '{test_input}'...\n") - session_service = InMemorySessionService() - runner = Runner(app_name="runnability_test", node=runnable_node, session_service=session_service) - session = await session_service.create_session(app_name="runnability_test", user_id="tester") + if component_type == "App": + runnable_node = getattr(component, "root_agent", None) + if runnable_node is None: + raise AttributeError( + f"App instance has no 'root_agent' attribute. " + f"Ensure the App is constructed with a root_agent argument." + ) + else: + runnable_node = component - user_message = types.Content( - parts=[types.Part(text=str(test_input))], - role="user" - ) + session_service = InMemorySessionService() + runner = Runner( + app_name="runnability_test", + node=runnable_node, + session_service=session_service, + ) + session = await session_service.create_session( + app_name="runnability_test", user_id="tester" + ) + + user_message = types.Content( + parts=[types.Part(text=str(test_input))], role="user" + ) + + async for event in runner.run_async( + user_id="tester", session_id=session.id, new_message=user_message + ): + print(f"๐ŸŽฌ [Event] Author: {event.author}") + if event.output: + print(f"๐Ÿ”น Output: {event.output}") + if hasattr(event, "content") and event.content and event.content.parts: + text = "".join(p.text for p in event.content.parts if p.text) + if text: + print(f"๐Ÿ“ Content Output:\n{'-'*40}\n{text}\n{'-'*40}") - async for event in runner.run_async( - user_id="tester", - session_id=session.id, - new_message=user_message - ): - print(f"๐ŸŽฌ [Event] Author: {event.author}") - if event.output: - print(f"๐Ÿ”น Output: {event.output}") - if hasattr(event, "content") and event.content and event.content.parts: - text = "".join(p.text for p in event.content.parts if p.text) - if text: - print(f"๐Ÿ“ Content Output:\n{'-'*40}\n{text}\n{'-'*40}") def main(): - parser = argparse.ArgumentParser(description="Generalized ADK Runnability & Loadability Tester") - parser.add_argument("file", type=str, help="Path to the python file containing the agent/workflow to test") - args = parser.parse_args() - - file_path = Path(args.file).resolve() - if not file_path.exists(): - print(f"โŒ Error: File '{file_path}' does not exist.") - sys.exit(EXIT_LOAD_FAILURE) - - print(f"๐Ÿ”ฌ Testing file: {file_path.name}") - print("=" * 60) - - # Initialize coverage programmatically to track ONLY the target file. - # - # Implementation note: snippets are loaded via importlib/exec_module, which - # CPython's sys.settrace-based tracer instruments correctly *only* if the - # tracer is active before the module's code object is compiled and executed. - # Starting coverage here โ€” before load_target_module() โ€” satisfies that - # requirement. The `include` filter ensures no ADK library code is counted. - cov = None - if HAS_COVERAGE: - cov = coverage.Coverage( - branch=True, - data_file=None, # Keep coverage data in-memory only, no .coverage file needed - include=[str(file_path)], # Scope collection to the snippet file only - ) - cov.start() - else: - print("โ„น๏ธ Install 'coverage' package to enable automated code coverage reporting.") + parser = argparse.ArgumentParser( + description="Generalized ADK Runnability & Loadability Tester" + ) + parser.add_argument( + "file", + type=str, + help="Path to the python file containing the agent/workflow to test", + ) + args = parser.parse_args() + + file_path = Path(args.file).resolve() + if not file_path.exists(): + print(f"โŒ Error: File '{file_path}' does not exist.") + sys.exit(EXIT_LOAD_FAILURE) - # exit_code is set by each phase and consumed inside the finally block so - # that coverage reporting always runs before the process exits. Using a - # mutable list as a simple cell lets the finally clause read the value set - # by any code path (normal completion, early break-out via a flag, or an - # unexpected exception) without requiring nonlocal or a class wrapper. - exit_code = [EXIT_SUCCESS] + print(f"๐Ÿ”ฌ Testing file: {file_path.name}") + print("=" * 60) + + # Initialize coverage programmatically to track ONLY the target file. + # + # Implementation note: snippets are loaded via importlib/exec_module, which + # CPython's sys.settrace-based tracer instruments correctly *only* if the + # tracer is active before the module's code object is compiled and executed. + # Starting coverage here โ€” before load_target_module() โ€” satisfies that + # requirement. The `include` filter ensures no ADK library code is counted. + cov = None + if HAS_COVERAGE: + cov = coverage.Coverage( + branch=True, + data_file=None, # Keep coverage data in-memory only, no .coverage file needed + include=[str(file_path)], # Scope collection to the snippet file only + ) + cov.start() + else: + print( + "โ„น๏ธ Install 'coverage' package to enable automated code coverage" + " reporting." + ) + # exit_code is set by each phase and consumed inside the finally block so + # that coverage reporting always runs before the process exits. Using a + # mutable list as a simple cell lets the finally clause read the value set + # by any code path (normal completion, early break-out via a flag, or an + # unexpected exception) without requiring nonlocal or a class wrapper. + exit_code = [EXIT_SUCCESS] + + try: + # 1. Test Loadability (Imports, Syntax, Instantiation/Validation) + print("๐Ÿ“‹ Phase 1: Loading & Compiling...") try: - # 1. Test Loadability (Imports, Syntax, Instantiation/Validation) - print("๐Ÿ“‹ Phase 1: Loading & Compiling...") + module = load_target_module(file_path) + print( + f"โœ… Load Success: '{file_path.name}' compiled and loaded without any" + " issues." + ) + except Exception: + print(f"โŒ Load Failure: Failed to compile/load '{file_path.name}':") + print("-" * 60) + traceback.print_exc(file=sys.stdout) + print("-" * 60) + exit_code[0] = EXIT_LOAD_FAILURE + # Do NOT return here. Fall through to the finally block so that + # coverage is reported and sys.exit() is called with the correct code. + # The module variable is not set, so we skip phases 2โ€“3 via the flag. + else: + # 2. Discover Component (only reached when load succeeded) + print("\n๐Ÿ“‹ Phase 2: Component Discovery...") + component, comp_type = discover_adk_component(module) + if not component: + print( + "โž– NO ADK COMPONENT: No module-level Workflow, Agent, or App" + f" instance found in '{file_path.name}'." + ) + print( + " Runnability test skipped. To enable it, assign a Workflow," + " Agent, or App" + ) + print( + " to a module-level variable (e.g. `agent = Agent(...)`). The" + " variable name" + ) + print( + " does not matter โ€” the runner detects it automatically via" + " vars(module)." + ) + print( + "โ„น๏ธ Coverage below reflects load-time execution only (module-level" + " statements)." + ) + exit_code[0] = EXIT_NO_COMPONENT + else: + # Get test input from module, or fallback + test_input = getattr(module, "test_input", "Test input topic") + + # 3. Test Runnability + print(f"\n๐Ÿ“‹ Phase 3: Executing {comp_type}...") try: - module = load_target_module(file_path) - print(f"โœ… Load Success: '{file_path.name}' compiled and loaded without any issues.") - except Exception: - print(f"โŒ Load Failure: Failed to compile/load '{file_path.name}':") + # asyncio.run() creates a fresh event loop each time, so it will raise + # RuntimeError if a loop is already running (e.g. the snippet called + # asyncio.run() at module level without an __main__ guard). + # We catch that specific case and report it clearly, rather than using + # the deprecated asyncio.get_event_loop() API (removed in Python 3.12+). + asyncio.run(run_component(component, comp_type, test_input)) + print( + f"\nโœ… Run Success: Component '{comp_type}' executed" + " successfully." + ) + except RuntimeError as e: + if "event loop" in str(e).lower(): + print( + f"\nโŒ Run Failure: An event loop conflict was detected after" + f" module load." + ) + print( + " The snippet likely called asyncio.run() at module level," + " which" + ) + print( + " conflicts with the runner's own event loop. Wrap top-level" + " async" + ) + print( + " calls in an `if __name__ == '__main__':` guard, or" + " annotate the" + ) + print(" snippet with .") + else: + print(f"\nโŒ Run Failure: Component failed during execution:") print("-" * 60) traceback.print_exc(file=sys.stdout) print("-" * 60) - exit_code[0] = EXIT_LOAD_FAILURE - # Do NOT return here. Fall through to the finally block so that - # coverage is reported and sys.exit() is called with the correct code. - # The module variable is not set, so we skip phases 2โ€“3 via the flag. - else: - # 2. Discover Component (only reached when load succeeded) - print("\n๐Ÿ“‹ Phase 2: Component Discovery...") - component, comp_type = discover_adk_component(module) - if not component: - print(f"โž– NO ADK COMPONENT: No module-level Workflow, Agent, or App instance found in '{file_path.name}'.") - print(" Runnability test skipped. To enable it, assign a Workflow, Agent, or App") - print(" to a module-level variable (e.g. `agent = Agent(...)`). The variable name") - print(" does not matter โ€” the runner detects it automatically via vars(module).") - print("โ„น๏ธ Coverage below reflects load-time execution only (module-level statements).") - exit_code[0] = EXIT_NO_COMPONENT - else: - # Get test input from module, or fallback - test_input = getattr(module, "test_input", "Test input topic") - - # 3. Test Runnability - print(f"\n๐Ÿ“‹ Phase 3: Executing {comp_type}...") - try: - # asyncio.run() creates a fresh event loop each time, so it will raise - # RuntimeError if a loop is already running (e.g. the snippet called - # asyncio.run() at module level without an __main__ guard). - # We catch that specific case and report it clearly, rather than using - # the deprecated asyncio.get_event_loop() API (removed in Python 3.12+). - asyncio.run(run_component(component, comp_type, test_input)) - print(f"\nโœ… Run Success: Component '{comp_type}' executed successfully.") - except RuntimeError as e: - if "event loop" in str(e).lower(): - print(f"\nโŒ Run Failure: An event loop conflict was detected after module load.") - print(" The snippet likely called asyncio.run() at module level, which") - print(" conflicts with the runner's own event loop. Wrap top-level async") - print(" calls in an `if __name__ == '__main__':` guard, or annotate the") - print(" snippet with .") - else: - print(f"\nโŒ Run Failure: Component failed during execution:") - print("-" * 60) - traceback.print_exc(file=sys.stdout) - print("-" * 60) - exit_code[0] = EXIT_RUN_FAILURE - except Exception: - print(f"\nโŒ Run Failure: Component failed during execution:") - print("-" * 60) - traceback.print_exc(file=sys.stdout) - print("-" * 60) - exit_code[0] = EXIT_RUN_FAILURE - - finally: - # Coverage reporting runs here so it is guaranteed to execute on every - # code path: normal completion, load failure, no-component, run failure. - if cov: - cov.stop() - print(f"\n{COV_SECTION_HEADER} (Target File)") - print("=" * 60) - try: - # Report coverage of the target file directly to stdout - cov.report(morfs=[str(file_path)], file=sys.stdout) - except coverage.exceptions.NoDataError: - print("โš ๏ธ No coverage data collected (compilation or execution failed early).") - except Exception as ce: - print(f"โš ๏ธ Failed to generate coverage report: {ce}") - print("=" * 60) - # Only call sys.exit for non-zero codes. If exit_code is EXIT_SUCCESS - # we return normally so that any exception currently propagating out of - # the try block is not silently replaced by a SystemExit raised here. - if exit_code[0] != EXIT_SUCCESS: - sys.exit(exit_code[0]) + exit_code[0] = EXIT_RUN_FAILURE + except Exception: + print(f"\nโŒ Run Failure: Component failed during execution:") + print("-" * 60) + traceback.print_exc(file=sys.stdout) + print("-" * 60) + exit_code[0] = EXIT_RUN_FAILURE + + finally: + # Coverage reporting runs here so it is guaranteed to execute on every + # code path: normal completion, load failure, no-component, run failure. + if cov: + cov.stop() + print(f"\n{COV_SECTION_HEADER} (Target File)") + print("=" * 60) + try: + # Report coverage of the target file directly to stdout + cov.report(morfs=[str(file_path)], file=sys.stdout) + except coverage.exceptions.NoDataError: + print( + "โš ๏ธ No coverage data collected (compilation or execution failed" + " early)." + ) + except Exception as ce: + print(f"โš ๏ธ Failed to generate coverage report: {ce}") + print("=" * 60) + # Only call sys.exit for non-zero codes. If exit_code is EXIT_SUCCESS + # we return normally so that any exception currently propagating out of + # the try block is not silently replaced by a SystemExit raised here. + if exit_code[0] != EXIT_SUCCESS: + sys.exit(exit_code[0]) + if __name__ == "__main__": - main() + main() diff --git a/.agents/skills/adk-verify-snippets/scripts/verify_md.py b/.agents/skills/adk-verify-snippets/scripts/verify_md.py index 5369b989ec9..fe6568da180 100644 --- a/.agents/skills/adk-verify-snippets/scripts/verify_md.py +++ b/.agents/skills/adk-verify-snippets/scripts/verify_md.py @@ -1,12 +1,28 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + import argparse from datetime import datetime import os +from pathlib import Path import re import shutil import subprocess import sys import tempfile -from pathlib import Path SKIP_ANNOTATION = "" SNIPPET_TIMEOUT = 120 # seconds; adjust if snippets legitimately need longer @@ -18,374 +34,417 @@ # Structured exit codes from run.py โ€” kept in sync with run.py:EXIT_* constants. # Using exit codes (not string/emoji matching) makes classification robust to # future changes in run.py's human-readable output text. -EXIT_SUCCESS = 0 # All phases passed -EXIT_LOAD_FAILURE = 1 # Failed to compile / load the snippet -EXIT_RUN_FAILURE = 2 # Loaded OK but the ADK component failed at runtime -EXIT_NO_COMPONENT = 3 # Loaded OK, no runnable ADK component found (load-only) +EXIT_SUCCESS = 0 # All phases passed +EXIT_LOAD_FAILURE = 1 # Failed to compile / load the snippet +EXIT_RUN_FAILURE = 2 # Loaded OK but the ADK component failed at runtime +EXIT_NO_COMPONENT = 3 # Loaded OK, no runnable ADK component found (load-only) + def extract_snippets(md_path: Path): - """Parses a markdown file and extracts python code blocks along with their preceding headings. - - A code block immediately preceded by the HTML comment - ```` is recorded but marked as skipped so - that illustrative / pseudo-code examples are excluded from execution. - """ - with open(md_path, "r", encoding="utf-8") as f: - content = f.read() - - lines = content.splitlines() - snippets = [] - current_heading = "Top Level" - in_code_block = False - code_lines = [] - skip_next_block = False - - for line in lines: - # If we are inside a code block, handle it first to preserve comments starting with '#' - if in_code_block: - stripped = line.strip() - # Close only on a bare closing fence (``` with no language specifier). - # A fenced block of another language (e.g. ```bash) appearing *inside* - # the Python block will not trigger this branch because it carries a - # language tag, so it is appended to code_lines as literal content. - if stripped == "```": - in_code_block = False - code_text = "\n".join(code_lines) - snippets.append({ - "heading": current_heading, - "code": code_text, - "skip": skip_next_block, - }) - skip_next_block = False - else: - code_lines.append(line) - continue - - # If we are outside a code block, check for headings or code block starts - if line.startswith("#"): - # Clean up heading markers (e.g., "## Get started" -> "Get started") - current_heading = line.lstrip("#").strip() - # A heading between the annotation and the fence cancels the skip. - skip_next_block = False - continue - - if line.strip() == SKIP_ANNOTATION: - skip_next_block = True - continue - - if line.strip().startswith("```python"): - in_code_block = True - code_lines = [] - continue - - # Any other non-empty line (prose, blank-line-separated text, etc.) between - # the annotation and the fence cancels the skip. - if line.strip(): - skip_next_block = False - - return snippets + """Parses a markdown file and extracts python code blocks along with their preceding headings. + + A code block immediately preceded by the HTML comment + ```` is recorded but marked as skipped so + that illustrative / pseudo-code examples are excluded from execution. + """ + with open(md_path, "r", encoding="utf-8") as f: + content = f.read() + + lines = content.splitlines() + snippets = [] + current_heading = "Top Level" + in_code_block = False + code_lines = [] + skip_next_block = False + + for line in lines: + # If we are inside a code block, handle it first to preserve comments starting with '#' + if in_code_block: + stripped = line.strip() + # Close only on a bare closing fence (``` with no language specifier). + # A fenced block of another language (e.g. ```bash) appearing *inside* + # the Python block will not trigger this branch because it carries a + # language tag, so it is appended to code_lines as literal content. + if stripped == "```": + in_code_block = False + code_text = "\n".join(code_lines) + snippets.append({ + "heading": current_heading, + "code": code_text, + "skip": skip_next_block, + }) + skip_next_block = False + else: + code_lines.append(line) + continue + + # If we are outside a code block, check for headings or code block starts + if line.startswith("#"): + # Clean up heading markers (e.g., "## Get started" -> "Get started") + current_heading = line.lstrip("#").strip() + # A heading between the annotation and the fence cancels the skip. + skip_next_block = False + continue + + if line.strip() == SKIP_ANNOTATION: + skip_next_block = True + continue + + if line.strip().startswith("```python"): + in_code_block = True + code_lines = [] + continue + + # Any other non-empty line (prose, blank-line-separated text, etc.) between + # the annotation and the fence cancels the skip. + if line.strip(): + skip_next_block = False + + return snippets + def run_snippet(run_py_path: Path, snippet_path: Path): - """Executes run.py on the isolated snippet and returns the result.""" - # Run using the same Python interpreter as this script (which will be the venv's python) - cmd = [sys.executable, str(run_py_path), str(snippet_path)] - - # Ensure GEMINI_API_KEY is preferred if both keys are set in the environment - env = os.environ.copy() - if "GOOGLE_API_KEY" in env and "GEMINI_API_KEY" in env: - env.pop("GOOGLE_API_KEY", None) - - try: - result = subprocess.run( - cmd, - capture_output=True, - text=True, - env=env, - timeout=SNIPPET_TIMEOUT - ) - return { - "exit_code": result.returncode, - "stdout": result.stdout, - "stderr": result.stderr - } - except subprocess.TimeoutExpired: - return { - "exit_code": EXIT_RUN_FAILURE, - "stdout": f"โŒ Run Failure: Snippet execution timed out after {SNIPPET_TIMEOUT} seconds.", - "stderr": f"TimeoutExpired: The snippet process did not complete within the {SNIPPET_TIMEOUT}-second limit." - } + """Executes run.py on the isolated snippet and returns the result.""" + # Run using the same Python interpreter as this script (which will be the venv's python) + cmd = [sys.executable, str(run_py_path), str(snippet_path)] + + # Ensure GEMINI_API_KEY is preferred if both keys are set in the environment + env = os.environ.copy() + if "GOOGLE_API_KEY" in env and "GEMINI_API_KEY" in env: + env.pop("GOOGLE_API_KEY", None) + + try: + result = subprocess.run( + cmd, capture_output=True, text=True, env=env, timeout=SNIPPET_TIMEOUT + ) + return { + "exit_code": result.returncode, + "stdout": result.stdout, + "stderr": result.stderr, + } + except subprocess.TimeoutExpired: + return { + "exit_code": EXIT_RUN_FAILURE, + "stdout": ( + "โŒ Run Failure: Snippet execution timed out after" + f" {SNIPPET_TIMEOUT} seconds." + ), + "stderr": ( + "TimeoutExpired: The snippet process did not complete within the" + f" {SNIPPET_TIMEOUT}-second limit." + ), + } + def extract_error_detail(stdout: str, stderr: str) -> str: - """Extracts the most relevant error line from run.py's output. - - Searches in order: - 1. Last line in stderr that looks like a Python exception (``Error:`` - or ``Exception:``). Scoping to stderr avoids matching runner prose - in stdout (e.g. "โŒ Run Failure: ...") which contains words like "Failure" - but is not an exception line. - 2. Last line in stdout with the same pattern, as a fallback for runtimes that - write tracebacks to stdout instead of stderr. - 3. Last line in stderr matching the generic ``: `` format - (custom exception classes that don't end in Error/Exception). - 4. Fallback string if nothing matches. - """ - # Matches standard Python exception class names: ends in 'Error' or 'Exception', - # followed by a colon and detail text. Anchored to the start of the stripped line - # so runner prose ("โŒ Run Failure: ...") is not matched. - _exception_re = re.compile(r"^[A-Za-z]\w*(?:Error|Exception|Warning):\s*.+") - - for source in (stderr, stdout): - for line in reversed(source.splitlines()): - if _exception_re.match(line.strip()): - return f"`{line.strip()}`" - - # Pass 3: generic ': ' in stderr only - for line in reversed(stderr.splitlines()): - if re.match(r"^[A-Za-z]\w*:.+", line.strip()): - return f"`{line.strip()}`" - - return "Failed to compile/load." + """Extracts the most relevant error line from run.py's output. + + Searches in order: + 1. Last line in stderr that looks like a Python exception (``Error:`` + or ``Exception:``). Scoping to stderr avoids matching runner prose + in stdout (e.g. "โŒ Run Failure: ...") which contains words like "Failure" + but is not an exception line. + 2. Last line in stdout with the same pattern, as a fallback for runtimes that + write tracebacks to stdout instead of stderr. + 3. Last line in stderr matching the generic ``: `` format + (custom exception classes that don't end in Error/Exception). + 4. Fallback string if nothing matches. + """ + # Matches standard Python exception class names: ends in 'Error' or 'Exception', + # followed by a colon and detail text. Anchored to the start of the stripped line + # so runner prose ("โŒ Run Failure: ...") is not matched. + _exception_re = re.compile(r"^[A-Za-z]\w*(?:Error|Exception|Warning):\s*.+") + + for source in (stderr, stdout): + for line in reversed(source.splitlines()): + if _exception_re.match(line.strip()): + return f"`{line.strip()}`" + + # Pass 3: generic ': ' in stderr only + for line in reversed(stderr.splitlines()): + if re.match(r"^[A-Za-z]\w*:.+", line.strip()): + return f"`{line.strip()}`" + + return "Failed to compile/load." def clean_name(name: str): - """Sanitizes a string to be a safe filename.""" - name = name.lower().replace(" ", "_") - return re.sub(r'[^a-z0-9_]', '', name) + """Sanitizes a string to be a safe filename.""" + name = name.lower().replace(" ", "_") + return re.sub(r"[^a-z0-9_]", "", name) + def md_cell(value: str) -> str: - """Escapes pipe characters so the value is safe inside a Markdown table cell.""" - return value.replace("|", r"\|") + """Escapes pipe characters so the value is safe inside a Markdown table cell.""" + return value.replace("|", r"\|") + def safe_fence(content: str, language: str = "") -> str: - """Returns a Markdown fenced code block that safely wraps *content*. - - Picks the shortest fence (minimum three backticks) that is strictly longer - than any contiguous run of backticks found inside *content*, so the fence - cannot be prematurely closed by content that itself contains backtick runs. - This is the approach recommended by the CommonMark spec. - - Example:: - - safe_fence("x = ```foo```", "python") - # returns: - # ````python - # x = ```foo``` - # ```` - """ - # Find the longest run of backticks inside the content - max_run = max((len(m.group()) for m in re.finditer(r"`+", content)), default=0) - # The outer fence must be strictly longer, and at least 3 characters - fence_len = max(3, max_run + 1) - fence = "`" * fence_len - tag = f"{fence}{language}\n" if language else f"{fence}\n" - return f"{tag}{content}\n{fence}" + """Returns a Markdown fenced code block that safely wraps *content*. + + Picks the shortest fence (minimum three backticks) that is strictly longer + than any contiguous run of backticks found inside *content*, so the fence + cannot be prematurely closed by content that itself contains backtick runs. + This is the approach recommended by the CommonMark spec. + + Example:: + + safe_fence("x = ```foo```", "python") + # returns: + # ````python + # x = ```foo``` + # ```` + """ + # Find the longest run of backticks inside the content + max_run = max( + (len(m.group()) for m in re.finditer(r"`+", content)), default=0 + ) + # The outer fence must be strictly longer, and at least 3 characters + fence_len = max(3, max_run + 1) + fence = "`" * fence_len + tag = f"{fence}{language}\n" if language else f"{fence}\n" + return f"{tag}{content}\n{fence}" + def main(): - parser = argparse.ArgumentParser(description="Markdown Snippet Verifier") - parser.add_argument("file", type=str, help="Path to the markdown file to verify") - args = parser.parse_args() - - md_path = Path(args.file).resolve() - if not md_path.exists(): - print(f"โŒ Error: Markdown file '{md_path}' does not exist.") - sys.exit(1) - - # Locate run.py bundled inside the same scripts folder as verify_md.py (portable mode!) - run_py_path = Path(__file__).parent / "run.py" - if not run_py_path.exists(): - print(f"โŒ Error: Bundled runner 'run.py' not found at '{run_py_path}'.") - sys.exit(1) - - print(f"๐Ÿ”ฌ Analyzing Markdown: {md_path.name}") - - # 1. Extract snippets - snippets = extract_snippets(md_path) - if not snippets: - print(f"โš ๏ธ No python code blocks found in '{md_path.name}'.") - sys.exit(0) - - print(f"๐Ÿ“‹ Found {len(snippets)} python code snippets to verify.") - - # Create a unique temp directory to avoid collisions with concurrent runs - temp_dir = Path(tempfile.mkdtemp(prefix="verify_snippets_")) - - results = [] - - # 2. Execute each snippet, then write the report โ€” both inside the try so - # the finally cleanup only runs after the report is fully written. - try: - for i, snippet in enumerate(snippets, start=1): - heading = snippet["heading"] - code = snippet["code"] - is_skipped = snippet.get("skip", False) - - # Create a unique, sanitized filename for the snippet - safe_heading = clean_name(heading) - temp_file_name = f"snippet_{i}_{safe_heading}.py" - temp_file_path = temp_dir / temp_file_name - - if is_skipped: - print(f"โญ๏ธ Skipping Snippet {i}/{len(snippets)} under heading '{heading}' (marked ignore).") - results.append({ - "index": i, - "heading": heading, - "code": code, - "temp_file": temp_file_name, - "exit_code": 0, - "stdout": "", - "stderr": "", - "skipped": True, - }) - continue - - # Write snippet to file - with open(temp_file_path, "w", encoding="utf-8") as f: - f.write(code) - - print(f"๐Ÿงช Testing Snippet {i}/{len(snippets)} under heading '{heading}'...") - - # Run the snippet - run_res = run_snippet(run_py_path, temp_file_path) - - results.append({ - "index": i, - "heading": heading, - "code": code, - "temp_file": temp_file_name, - "exit_code": run_res["exit_code"], - "stdout": run_res["stdout"], - "stderr": run_res["stderr"], - "skipped": False, - }) - - # 3. Generate Markdown Report โ€” inside the try so finally runs after this completes. - # Use clean_name on the stem so the report path is safe on all filesystems. - # If clean_name strips everything (e.g. a fully non-ASCII filename), fall - # back to a hash of the original stem so two such files in the same - # directory never produce the same report path. - safe_stem = clean_name(md_path.stem) or f"report_{abs(hash(md_path.stem))}" - report_path = md_path.parent / f"{safe_stem}_REPORT.md" - timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") - - with open(report_path, "w", encoding="utf-8") as f: - f.write(f"# ๐Ÿ”ฌ ADK Markdown Snippet Verification Report\n\n") - f.write(f"* **Source File**: [{md_path.name}](file://{md_path})\n") - f.write(f"* **Verified On**: `{timestamp}`\n\n") - - # Write summary table - f.write("## ๐Ÿ“ˆ Executive Summary\n\n") - f.write("| Snippet | Preceding Heading | Load Phase | Run Phase | Coverage | Details |\n") - f.write("| :--- | :--- | :---: | :---: | :---: | :--- |\n") - - for r in results: - # Handle explicitly skipped snippets - if r.get("skipped"): - f.write(f"| **Snippet {r['index']}** | `{md_cell(r['heading'])}` | โญ๏ธ **SKIPPED** | โญ๏ธ **SKIPPED** | โ€” | Marked `{SKIP_ANNOTATION}` โ€” intentionally ignored. |\n") - continue - - # Determine Phase 1 (Load) and Phase 3 (Run) statuses from the - # structured exit code emitted by run.py โ€” no emoji/string matching. - exit_code = r["exit_code"] - load_status = "โœ… **PASS**" - run_status = "โœ… **PASS**" - coverage_pct = "โ€”" - - stdout_and_stderr = r["stdout"] + "\n" + r["stderr"] - - if exit_code == EXIT_LOAD_FAILURE: - load_status = "โŒ **FAIL**" - run_status = "โž– **SKIPPED**" - elif exit_code == EXIT_NO_COMPONENT: - run_status = "โž– **NO ADK COMPONENT**" - elif exit_code == EXIT_RUN_FAILURE: - run_status = "โŒ **FAIL**" - # EXIT_SUCCESS (0): both statuses remain โœ… **PASS** - - # 3. Parse Coverage โ€” anchor to line start to avoid matching prose. - # Handles both branch (5 numeric cols) and non-branch (3 cols) formats. - total_match = re.search(r"^TOTAL(?:\s+\d+)+\s+(\d+)%", r["stdout"], re.MULTILINE) - if total_match and load_status != "โŒ **FAIL**": - coverage_pct = f"`{total_match.group(1)}`" - - # 4. Formulate details and handle transient 503s - details = "All checks passed successfully." - if load_status == "โŒ **FAIL**": - details = extract_error_detail(r["stdout"], r["stderr"]) - elif run_status == "โž– **NO ADK COMPONENT**": - details = "No module-level `Workflow`, `Agent`, or `App` instance found. Assign one to a top-level variable to enable runnability testing." - elif run_status == "โŒ **FAIL**": - if "503" in stdout_and_stderr and "UNAVAILABLE" in stdout_and_stderr: - details = "โš ๏ธ **Transient 503 from Gemini API (overloaded)**. Code structure is correct." - else: - details = extract_error_detail(r["stdout"], r["stderr"]) - - # Store statuses for reuse in the detailed section - r["load_status"] = load_status - r["run_status"] = run_status - - f.write(f"| **Snippet {r['index']}** | `{md_cell(r['heading'])}` | {load_status} | {run_status} | {coverage_pct} | {md_cell(details)} |\n") - - f.write("\n---\n\n## ๐Ÿ” Detailed Snippet Reports\n\n") - - for r in results: - if r.get("skipped"): - f.write(f"### โญ๏ธ Snippet {r['index']}: `{r['heading']}` *(ignored)*\n\n") - f.write("#### ๐Ÿ“ Code Block\n") - f.write(safe_fence(r["code"], "python")) - f.write("\n\n") - f.write(f"> This snippet was skipped because it is annotated with `{SKIP_ANNOTATION}`.\n\n") - f.write("---\n\n") - continue - - l_stat = r.get("load_status", "โœ… **PASS**") - r_stat = r.get("run_status", "โœ… **PASS**") - if l_stat == "โŒ **FAIL**" or r_stat == "โŒ **FAIL**": - status_icon = "โŒ" - elif r_stat == "โž– **NO ADK COMPONENT**": - status_icon = "โž–" - else: - status_icon = "โœ…" - f.write(f"### {status_icon} Snippet {r['index']}: `{r['heading']}`\n\n") - - f.write("#### ๐Ÿ“ Code Block\n") - f.write(safe_fence(r["code"], "python")) - f.write("\n\n") - - # Write stdout / stderr logs - # Split run.py stdout into main log and coverage section using the - # shared COV_SECTION_HEADER constant (kept in sync with run.py). - stdout_clean = r["stdout"] - cov_section_match = re.search( - rf"({re.escape(COV_SECTION_HEADER)}.*)", r["stdout"], re.DOTALL - ) - cov_text = cov_section_match.group(1) if cov_section_match else None - - if cov_text: - stdout_clean = r["stdout"].replace(cov_text, "").strip() - - log_content = stdout_clean - if r["stderr"]: - log_content += "\n\n=== STDERR/TRACEBACK ===\n" + r["stderr"].strip() - - f.write("#### ๐Ÿ–ฅ๏ธ Loadability & Runnability Logs\n") - f.write(safe_fence(log_content)) - f.write("\n\n") - - # Write coverage report if available - if cov_text: - f.write("#### ๐Ÿ“Š Coverage Report\n") - f.write(safe_fence(cov_text)) - f.write("\n\n") - - f.write("---\n\n") - - print(f"๐ŸŽ‰ Verification complete! Report generated at: {report_path}") - - finally: - # Always clean up the temp directory, even on Ctrl+C or unexpected errors. - # This runs after report generation completes (or if it raises), ensuring - # temp files are never left behind. - shutil.rmtree(temp_dir, ignore_errors=True) + parser = argparse.ArgumentParser(description="Markdown Snippet Verifier") + parser.add_argument( + "file", type=str, help="Path to the markdown file to verify" + ) + args = parser.parse_args() + + md_path = Path(args.file).resolve() + if not md_path.exists(): + print(f"โŒ Error: Markdown file '{md_path}' does not exist.") + sys.exit(1) + + # Locate run.py bundled inside the same scripts folder as verify_md.py (portable mode!) + run_py_path = Path(__file__).parent / "run.py" + if not run_py_path.exists(): + print(f"โŒ Error: Bundled runner 'run.py' not found at '{run_py_path}'.") + sys.exit(1) + + print(f"๐Ÿ”ฌ Analyzing Markdown: {md_path.name}") + + # 1. Extract snippets + snippets = extract_snippets(md_path) + if not snippets: + print(f"โš ๏ธ No python code blocks found in '{md_path.name}'.") + sys.exit(0) + + print(f"๐Ÿ“‹ Found {len(snippets)} python code snippets to verify.") + + # Create a unique temp directory to avoid collisions with concurrent runs + temp_dir = Path(tempfile.mkdtemp(prefix="verify_snippets_")) + + results = [] + + # 2. Execute each snippet, then write the report โ€” both inside the try so + # the finally cleanup only runs after the report is fully written. + try: + for i, snippet in enumerate(snippets, start=1): + heading = snippet["heading"] + code = snippet["code"] + is_skipped = snippet.get("skip", False) + + # Create a unique, sanitized filename for the snippet + safe_heading = clean_name(heading) + temp_file_name = f"snippet_{i}_{safe_heading}.py" + temp_file_path = temp_dir / temp_file_name + + if is_skipped: + print( + f"โญ๏ธ Skipping Snippet {i}/{len(snippets)} under heading '{heading}'" + " (marked ignore)." + ) + results.append({ + "index": i, + "heading": heading, + "code": code, + "temp_file": temp_file_name, + "exit_code": 0, + "stdout": "", + "stderr": "", + "skipped": True, + }) + continue + + # Write snippet to file + with open(temp_file_path, "w", encoding="utf-8") as f: + f.write(code) + + print( + f"๐Ÿงช Testing Snippet {i}/{len(snippets)} under heading '{heading}'..." + ) + + # Run the snippet + run_res = run_snippet(run_py_path, temp_file_path) + + results.append({ + "index": i, + "heading": heading, + "code": code, + "temp_file": temp_file_name, + "exit_code": run_res["exit_code"], + "stdout": run_res["stdout"], + "stderr": run_res["stderr"], + "skipped": False, + }) + + # 3. Generate Markdown Report โ€” inside the try so finally runs after this completes. + # Use clean_name on the stem so the report path is safe on all filesystems. + # If clean_name strips everything (e.g. a fully non-ASCII filename), fall + # back to a hash of the original stem so two such files in the same + # directory never produce the same report path. + safe_stem = clean_name(md_path.stem) or f"report_{abs(hash(md_path.stem))}" + report_path = md_path.parent / f"{safe_stem}_REPORT.md" + timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + + with open(report_path, "w", encoding="utf-8") as f: + f.write(f"# ๐Ÿ”ฌ ADK Markdown Snippet Verification Report\n\n") + f.write(f"* **Source File**: [{md_path.name}](file://{md_path})\n") + f.write(f"* **Verified On**: `{timestamp}`\n\n") + + # Write summary table + f.write("## ๐Ÿ“ˆ Executive Summary\n\n") + f.write( + "| Snippet | Preceding Heading | Load Phase | Run Phase | Coverage |" + " Details |\n" + ) + f.write("| :--- | :--- | :---: | :---: | :---: | :--- |\n") + + for r in results: + # Handle explicitly skipped snippets + if r.get("skipped"): + f.write( + f"| **Snippet {r['index']}** | `{md_cell(r['heading'])}` | โญ๏ธ" + f" **SKIPPED** | โญ๏ธ **SKIPPED** | โ€” | Marked `{SKIP_ANNOTATION}` โ€”" + " intentionally ignored. |\n" + ) + continue + + # Determine Phase 1 (Load) and Phase 3 (Run) statuses from the + # structured exit code emitted by run.py โ€” no emoji/string matching. + exit_code = r["exit_code"] + load_status = "โœ… **PASS**" + run_status = "โœ… **PASS**" + coverage_pct = "โ€”" + + stdout_and_stderr = r["stdout"] + "\n" + r["stderr"] + + if exit_code == EXIT_LOAD_FAILURE: + load_status = "โŒ **FAIL**" + run_status = "โž– **SKIPPED**" + elif exit_code == EXIT_NO_COMPONENT: + run_status = "โž– **NO ADK COMPONENT**" + elif exit_code == EXIT_RUN_FAILURE: + run_status = "โŒ **FAIL**" + # EXIT_SUCCESS (0): both statuses remain โœ… **PASS** + + # 3. Parse Coverage โ€” anchor to line start to avoid matching prose. + # Handles both branch (5 numeric cols) and non-branch (3 cols) formats. + total_match = re.search( + r"^TOTAL(?:\s+\d+)+\s+(\d+)%", r["stdout"], re.MULTILINE + ) + if total_match and load_status != "โŒ **FAIL**": + coverage_pct = f"`{total_match.group(1)}`" + + # 4. Formulate details and handle transient 503s + details = "All checks passed successfully." + if load_status == "โŒ **FAIL**": + details = extract_error_detail(r["stdout"], r["stderr"]) + elif run_status == "โž– **NO ADK COMPONENT**": + details = ( + "No module-level `Workflow`, `Agent`, or `App` instance found." + " Assign one to a top-level variable to enable runnability" + " testing." + ) + elif run_status == "โŒ **FAIL**": + if "503" in stdout_and_stderr and "UNAVAILABLE" in stdout_and_stderr: + details = ( + "โš ๏ธ **Transient 503 from Gemini API (overloaded)**. Code" + " structure is correct." + ) + else: + details = extract_error_detail(r["stdout"], r["stderr"]) + + # Store statuses for reuse in the detailed section + r["load_status"] = load_status + r["run_status"] = run_status + + f.write( + f"| **Snippet {r['index']}** | `{md_cell(r['heading'])}` |" + f" {load_status} | {run_status} | {coverage_pct} |" + f" {md_cell(details)} |\n" + ) + + f.write("\n---\n\n## ๐Ÿ” Detailed Snippet Reports\n\n") + + for r in results: + if r.get("skipped"): + f.write( + f"### โญ๏ธ Snippet {r['index']}: `{r['heading']}` *(ignored)*\n\n" + ) + f.write("#### ๐Ÿ“ Code Block\n") + f.write(safe_fence(r["code"], "python")) + f.write("\n\n") + f.write( + "> This snippet was skipped because it is annotated with" + f" `{SKIP_ANNOTATION}`.\n\n" + ) + f.write("---\n\n") + continue + + l_stat = r.get("load_status", "โœ… **PASS**") + r_stat = r.get("run_status", "โœ… **PASS**") + if l_stat == "โŒ **FAIL**" or r_stat == "โŒ **FAIL**": + status_icon = "โŒ" + elif r_stat == "โž– **NO ADK COMPONENT**": + status_icon = "โž–" + else: + status_icon = "โœ…" + f.write(f"### {status_icon} Snippet {r['index']}: `{r['heading']}`\n\n") + + f.write("#### ๐Ÿ“ Code Block\n") + f.write(safe_fence(r["code"], "python")) + f.write("\n\n") + + # Write stdout / stderr logs + # Split run.py stdout into main log and coverage section using the + # shared COV_SECTION_HEADER constant (kept in sync with run.py). + stdout_clean = r["stdout"] + cov_section_match = re.search( + rf"({re.escape(COV_SECTION_HEADER)}.*)", r["stdout"], re.DOTALL + ) + cov_text = cov_section_match.group(1) if cov_section_match else None + + if cov_text: + stdout_clean = r["stdout"].replace(cov_text, "").strip() + + log_content = stdout_clean + if r["stderr"]: + log_content += "\n\n=== STDERR/TRACEBACK ===\n" + r["stderr"].strip() + + f.write("#### ๐Ÿ–ฅ๏ธ Loadability & Runnability Logs\n") + f.write(safe_fence(log_content)) + f.write("\n\n") + + # Write coverage report if available + if cov_text: + f.write("#### ๐Ÿ“Š Coverage Report\n") + f.write(safe_fence(cov_text)) + f.write("\n\n") + + f.write("---\n\n") + + print(f"๐ŸŽ‰ Verification complete! Report generated at: {report_path}") + + finally: + # Always clean up the temp directory, even on Ctrl+C or unexpected errors. + # This runs after report generation completes (or if it raises), ensuring + # temp files are never left behind. + shutil.rmtree(temp_dir, ignore_errors=True) + if __name__ == "__main__": - main() + main()