From 4ee008eca8ddf4b21c16f0ee17cb8e68009ae63d Mon Sep 17 00:00:00 2001 From: Maja Massarini Date: Fri, 21 Nov 2025 12:38:53 +0100 Subject: [PATCH] Run a reproducer in a simple way --- Containerfile.mcp | 3 + Makefile | 6 + agents/backport_agent.py | 228 +++++++++++++++++- agents/build_agent.py | 3 + agents/triage_agent.py | 24 +- common/models.py | 14 +- mcp_server/gateway.py | 3 +- mcp_server/testing_farm_tools.py | 84 +++++++ .../tests/unit/test_testing_farm_tools.py | 105 ++++++++ templates/mcp-gateway.env | 2 + 10 files changed, 461 insertions(+), 11 deletions(-) create mode 100644 mcp_server/testing_farm_tools.py create mode 100644 mcp_server/tests/unit/test_testing_farm_tools.py diff --git a/Containerfile.mcp b/Containerfile.mcp index 003b2968..6c3a57d4 100644 --- a/Containerfile.mcp +++ b/Containerfile.mcp @@ -1,5 +1,7 @@ FROM fedora:42 +RUN dnf -y copr enable @testing-farm/stable + # Install system dependencies RUN dnf -y install \ python3 \ @@ -16,6 +18,7 @@ RUN dnf -y install \ krb5-workstation \ centpkg \ git \ + testing-farm \ && dnf clean all # Install FastMCP diff --git a/Makefile b/Makefile index e8c6b0fa..0d7f20a2 100644 --- a/Makefile +++ b/Makefile @@ -69,6 +69,9 @@ run-backport-agent-c9s-standalone: -e DRY_RUN=$(DRY_RUN) \ -e MOCK_JIRA=$(MOCK_JIRA) \ -e CVE_ID=$(CVE_ID) \ + -e REPRODUCER_INFO_REPO_URL=$(REPRODUCER_INFO_REPO_URL) \ + -e REPRODUCER_INFO_REPO_REF=$(REPRODUCER_INFO_REPO_REF) \ + -e REPRODUCER_INFO_TEST=$(REPRODUCER_INFO_TEST) \ backport-agent-c9s .PHONY: run-backport-agent-c10s-standalone @@ -81,6 +84,9 @@ run-backport-agent-c10s-standalone: -e DRY_RUN=$(DRY_RUN) \ -e MOCK_JIRA=$(MOCK_JIRA) \ -e CVE_ID=$(CVE_ID) \ + -e REPRODUCER_INFO_REPO_URL=$(REPRODUCER_INFO_REPO_URL) \ + -e REPRODUCER_INFO_REPO_REF=$(REPRODUCER_INFO_REPO_REF) \ + -e REPRODUCER_INFO_TEST=$(REPRODUCER_INFO_TEST) \ backport-agent-c10s .PHONY: run-backport-agent-standalone diff --git a/agents/backport_agent.py b/agents/backport_agent.py index 77425719..4d73f850 100644 --- a/agents/backport_agent.py +++ b/agents/backport_agent.py @@ -39,6 +39,7 @@ Task, ) from common.utils import redis_client, fix_await +from common.models import ReproducerInfo from constants import I_AM_JOTNAR, CAREFULLY_REVIEW_CHANGES from observability import setup_observability from tools.commands import RunShellCommandTool @@ -292,7 +293,7 @@ def get_prompt() -> str: def get_fix_build_error_prompt() -> str: - return """ + return f""" Your working directory is {{local_clone}}, a clone of dist-git repository of package {{package}}. {{dist_git_branch}} dist-git branch has been checked out. You are working on Jira issue {{jira_issue}} {{#cve_id}}(a.k.a. {{.}}){{/cve_id}}. @@ -389,7 +390,8 @@ def get_fix_build_error_prompt() -> str: - Test if the SRPM builds successfully using the `build_package` tool: * Call build_package with the SRPM path, dist_git_branch, and jira_issue * Wait for build results - * If build PASSES: Report success=true with the SRPM path + * If build PASSES: choose the main package URL (no devel, static, debug packages) from `artifacts_urls` + returned by `build_package` tool and report success=true AND the chosen package URL as `build_url` * If build FAILS: Use `download_artifacts` to get build logs if available * Extract the new error message from the logs: - IMPORTANT: Before viewing log files, check their size using `wc -l` command @@ -402,7 +404,7 @@ def get_fix_build_error_prompt() -> str: * Report success=false with the extracted error Report your results: - - If build passes → Report success=true with the SRPM path + - If build passes → Report success=true with the SRPM path and the chosen package URL as `build_url` - If build fails → Report success=false with the extracted error message - If you can't find a fix → Report success=false explaining why @@ -422,6 +424,81 @@ def get_fix_build_error_prompt() -> str: The upstream repository at {{local_clone}}-upstream is your playground - explore it freely! """ +def get_fix_reproducer_test_case_error_prompt() -> str: + return f""" + Your working directory is {{local_clone}}, a clone of dist-git repository of package {{package}}. + {{dist_git_branch}} dist-git branch has been checked out. You are working on Jira issue {{jira_issue}} + {{#cve_id}}(a.k.a. {{.}}){{/cve_id}}. + + Upstream patches that were backported: + {{#upstream_patches}} + - {{.}} + {{/upstream_patches}} + + The backport of upstream patches was successful since the build passed, + but the reproducer test case failed with the following error: + + {{reproducer_result}} + + Your task is to fix the reproducer test case error by improving the patches - NOT by modifying the spec file. + This includes BOTH compilation errors AND test failures during the check section. + + CRITICAL: The upstream repository ({{local_clone}}-upstream) still exists with all your previous work intact. + DO NOT clone it again. DO NOT reset to base commit. DO NOT modify anything in {{local_clone}} dist-git repository. + Your cherry-picked commits are still there in {{local_clone}}-upstream. + + Follow these steps: + + STEP 1: Analyze the reproducer test case failure {{reproducer_result}}. + And also the test code, the test is located in the {{reproducer_info.git_ref}} branch + of the {{reproducer_info.git_url}} repository at subpath:{{reproducer_info.test}}. + - Identify the specific test failure and the expected behavior + - Identify the root cause of the test failure + - Identify the specific changes you have done at the original patches that caused the test failure + + STEP 2: Manually adapt the code + - Directly edit files in {{local_clone}}-upstream using `str_replace` or `insert` tools + - Make minimal changes to fix the specific test failure + - Commit your changes: `git -C {{local_clone}}-upstream add ` then + `git -C {{local_clone}}-upstream commit -m "Manually backport: "` + + STEP 3: Regenerate the patch + - After making your fixes (cherry-picked or manual), regenerate the patch file + - Use `generate_patch_from_commit` tool with the PATCHED_BASE commit + - This creates a single patch with all changes: original commits + prerequisites/fixes + - Overwrite {{jira_issue}}.patch in {{local_clone}} + + STEP 4: Test the build + - The spec file should already reference {{jira_issue}}.patch + - Run `centpkg --name={{package}} --namespace=rpms --release={{dist_git_branch}} prep` to verify patch applies + - Run `centpkg --name={{package}} --namespace=rpms --release={{dist_git_branch}} srpm` to generate SRPM + - Test if the SRPM builds successfully using the `build_package` tool: + * Call build_package with the SRPM path, dist_git_branch, and jira_issue + * Wait for build results + * If build PASSES: choose the main package URL (no devel, static, debug packages) from `artifacts_urls` + returned by `build_package` tool and report success=true AND the chosen package URL as `build_url` + * If build FAILS: Report success=false with the extracted error message + + Report your results: + - If build PASSED: Report success=true and the chosen package URL as `build_url` + - If build FAILED: Report success=false with the extracted error message + - If you can't find a fix → Report success=false explaining why + + IMPORTANT RULES: + - Work in the EXISTING {{local_clone}}-upstream directory (don't clone again) + - NEVER modify the spec file - reproducer test case failures are caused by incomplete patches, not spec issues + - The ONLY dist-git file you can modify is {{jira_issue}}.patch (by regenerating it from upstream repo) + - Fix reproducer test case failures by adapting the patches to make the reproducer test case pass + - For test failures: backport minimal necessary test helpers/functions to make tests pass + - You can freely explore, edit, cherry-pick, and commit in the upstream repo - it's your workspace + - Use the upstream repo as a rich source of information and examples + - Be creative and pragmatic - the goal is a working reproducer test case, not perfect git history + - Make ONE solid attempt to fix the issue - if the build fails, report the error clearly and explain why you couldn't fix it. + + Remember: Unpacked upstream sources are in {{unpacked_sources}}. + The upstream repository at {{local_clone}}-upstream is your playground - explore it freely! + """ + def create_backport_agent( mcp_tools: list[Tool], local_tool_options: dict[str, Any], include_build_tools: bool = False @@ -535,9 +612,13 @@ class State(PackageUpdateState): attempts_remaining: int = Field(default=max_build_attempts) used_cherry_pick_workflow: bool = Field(default=False) # Track if cherry-pick was used incremental_fix_attempts: int = Field(default=0) # Track how many times we tried incremental fix + build_url: str | None = Field(default=None) + reproducer_success: bool = Field(default=False) + reproducer_result: str | None = Field(default=None) + reproducer_info: ReproducerInfo | None = Field(default=None) async def run_workflow( - package, dist_git_branch, upstream_patches, jira_issue, cve_id, redis_conn=None + package, dist_git_branch, upstream_patches, jira_issue, cve_id, redis_conn=None, reproducer_info=None ): local_tool_options["working_directory"] = None @@ -603,6 +684,10 @@ async def run_backport_agent(state): cve_id=state.cve_id, upstream_patches=state.upstream_patches, build_error=state.build_error, + build_url=state.build_url, + reproducer_success=state.reproducer_success, + reproducer_result=state.reproducer_result, + reproducer_info=state.reproducer_info, ), ), expected_output=BackportOutputSchema, @@ -663,6 +748,10 @@ async def fix_build_error(state): cve_id=state.cve_id, upstream_patches=state.upstream_patches, build_error=state.build_error, + build_url=state.build_url, + reproducer_success=state.reproducer_success, + reproducer_result=state.reproducer_result, + reproducer_info=state.reproducer_info, ), ), expected_output=BackportOutputSchema, @@ -670,6 +759,7 @@ async def fix_build_error(state): ) fix_result = BackportOutputSchema.model_validate_json(response.last_message.text) + state.build_url = fix_result.build_url if fix_result.success: # Build passed! Update state and proceed @@ -677,7 +767,12 @@ async def fix_build_error(state): state.backport_log.append(fix_result.status) logger.info("Incremental fix succeeded with passing build") state.incremental_fix_attempts = 0 # Reset for potential future failures - return "update_release" + if state.reproducer_info: + logger.info(f"Reproducer test case info present for {state.jira_issue}, running reproducer") + return "run_reproducer_test_case" + else: + logger.info(f"No reproducer test case info present for {state.jira_issue}, skipping reproducer test case and updating release") + return "update_release" # Build still failing - update the error for next iteration logger.info(f"Build still failing after fix attempt: {fix_result.error}") @@ -706,6 +801,75 @@ async def fix_build_error(state): state.backport_result.error = f"Exception during incremental fix: {str(e)}" return "comment_in_jira" + async def fix_reproducer_test_case_error(state): + """Try to fix reproducer test case errors by adapting the patches to make the reproducer test case pass. + """ + try: + fix_agent = create_backport_agent(gateway_tools, local_tool_options, include_build_tools=True) + response = await fix_agent.run( + render_prompt( + template=get_fix_reproducer_test_case_error_prompt(), + input=BackportInputSchema( + local_clone=state.local_clone, + unpacked_sources=state.unpacked_sources, + package=state.package, + dist_git_branch=state.dist_git_branch, + jira_issue=state.jira_issue, + cve_id=state.cve_id, + upstream_patches=state.upstream_patches, + build_error=state.build_error, + build_url=state.build_url, + reproducer_success=state.reproducer_success, + reproducer_result=state.reproducer_result, + reproducer_info=state.reproducer_info, + ), + ), + expected_output=BackportOutputSchema, + **get_agent_execution_config(), + ) + fix_result = BackportOutputSchema.model_validate_json(response.last_message.text) + state.build_url = fix_result.build_url + + if fix_result.success: + # Build passed! Update state and proceed + state.backport_result = fix_result + state.backport_log.append(fix_result.status) + logger.info("Incremental fix succeeded with passing build") + state.incremental_fix_attempts = 0 # Reset for potential future failures + if state.reproducer_info: + logger.info(f"Reproducer test case info present for {state.jira_issue}, running reproducer") + return "run_reproducer_test_case" + else: + logger.info(f"No reproducer test case info present for {state.jira_issue}, skipping reproducer test case and updating release") + return "update_release" + + # Build still failing - update the error for next iteration + logger.info(f"Build still failing after fix attempt: {fix_result.error}") + state.build_error = fix_result.error + state.backport_result = fix_result + + # Check if we should try again + state.incremental_fix_attempts += 1 + if state.incremental_fix_attempts < max_incremental_fix_attempts: + logger.info(f"Will retry incremental fix (attempt {state.incremental_fix_attempts + 1}/{max_incremental_fix_attempts})") + return "fix_reproducer_test_case_error" # Try again with the new error + else: + # Exhausted all incremental fix attempts - give up + logger.error(f"Exhausted all {max_incremental_fix_attempts} incremental fix attempts, giving up") + state.backport_result.success = False + state.backport_result.error = ( + f"Unable to fix reproducer test case errors after {max_incremental_fix_attempts} incremental fix attempts. " + f"Last error: {fix_result.error}" + ) + return "comment_in_jira" + + except Exception as e: + # If anything goes wrong in fix_build_error, give up + logger.error(f"Exception during incremental reproducer test case fix: {e}", exc_info=True) + state.backport_result.success = False + state.backport_result.error = f"Exception during incremental reproducer test case fix: {str(e)}" + return "comment_in_jira" + async def run_build_agent(state): # Ensure we have a valid backport result with SRPM path if not state.backport_result or not state.backport_result.srpm_path: @@ -736,7 +900,13 @@ async def run_build_agent(state): if build_result.success: # Build succeeded - reset incremental fix counter for potential future failures state.incremental_fix_attempts = 0 - return "update_release" + state.build_url = build_result.build_url + if state.reproducer_info: + logger.info(f"Reproducer test case info present for {state.jira_issue}, running reproducer") + return "run_reproducer_test_case" + else: + logger.info(f"No reproducer test case info present for {state.jira_issue}, skipping reproducer test case and updating release") + return "update_release" if build_result.is_timeout: logger.info(f"Build timed out for {state.jira_issue}, proceeding") return "update_release" @@ -758,6 +928,13 @@ async def run_build_agent(state): return "fork_and_prepare_dist_git" async def update_release(state): + if state.reproducer_info and state.build_url: + logger.info(f"Reproducer test case info present for {state.jira_issue} and build success, running reproducer test case") + return "run_reproducer_test_case" + elif state.reproducer_info and not state.build_url: + logger.info(f"Reproducer test case info present for {state.jira_issue} but build URL is missing, updating release") + else: + logger.info(f"No reproducer test case info present for {state.jira_issue}, updating release") try: await tasks.update_release( local_clone=state.local_clone, @@ -878,6 +1055,26 @@ async def create_merge_request_checklist(state): async def add_fusa_label(state): return await PackageUpdateStep.add_fusa_label(state, "comment_in_jira", dry_run=dry_run, gateway_tools=gateway_tools) + async def run_reproducer_test_case(state): + if not state.build_url: + logger.error(f"Build URL is not available for {state.jira_issue}, skipping reproducer test case") + return "update_release" + state.reproducer_success, state.reproducer_result = await tasks.run_tool( + "run_testing_farm_test", + git_url=state.reproducer_info.git_url, + git_ref=state.reproducer_info.git_ref, + path_to_test=state.reproducer_info.test, + package=state.build_url, + dist_git_branch=state.dist_git_branch, + available_tools=gateway_tools, + ) + if state.reproducer_success: + logger.info(f"Reproducer test case result: {state.reproducer_result}") + return "update_release" + else: + logger.error(f"Reproducer test case failed: {state.reproducer_result}") + return "run_backport_agent" + async def comment_in_jira(state): if dry_run: return Workflow.END @@ -903,6 +1100,8 @@ async def comment_in_jira(state): workflow.add_step("run_backport_agent", run_backport_agent) workflow.add_step("fix_build_error", fix_build_error) workflow.add_step("run_build_agent", run_build_agent) + workflow.add_step("run_reproducer_test_case", run_reproducer_test_case) + workflow.add_step("fix_reproducer_test_case_error", fix_reproducer_test_case_error) workflow.add_step("update_release", update_release) workflow.add_step("stage_changes", stage_changes) workflow.add_step("run_log_agent", run_log_agent) @@ -919,6 +1118,9 @@ async def comment_in_jira(state): upstream_patches=upstream_patches, jira_issue=jira_issue, cve_id=cve_id, + reproducer_info=reproducer_info, + reproducer_success=False, + reproducer_result=None, ), ) return response.state @@ -931,6 +1133,16 @@ async def comment_in_jira(state): ): upstream_patches = upstream_patches_raw.split(",") logger.info("Running in direct mode with environment variables") + reproducer_info_repo_url = os.getenv("REPRODUCER_INFO_REPO_URL", None) + reproducer_info_repo_ref = os.getenv("REPRODUCER_INFO_REPO_REF", None) + reproducer_info_test = os.getenv("REPRODUCER_INFO_TEST", None) + reproducer_info = None + if reproducer_info_repo_url and reproducer_info_repo_ref and reproducer_info_test: + reproducer_info = ReproducerInfo( + git_url=reproducer_info_repo_url, + git_ref=reproducer_info_repo_ref, + test=reproducer_info_test, + ) state = await run_workflow( package=package, dist_git_branch=branch, @@ -938,6 +1150,7 @@ async def comment_in_jira(state): jira_issue=jira_issue, cve_id=os.getenv("CVE_ID", None), redis_conn=None, + reproducer_info=reproducer_info, ) logger.info(f"Direct run completed: {state.backport_result.model_dump_json(indent=4)}") return @@ -967,7 +1180,7 @@ async def comment_in_jira(state): logger.info( f"Processing backport for package: {backport_data.package}, " f"JIRA: {backport_data.jira_issue}, branch: {dist_git_branch}, " - f"attempt: {task.attempts + 1}" + f"attempt: {task.attempts + 1}, reproducer test case info: {backport_data.reproducer_info}" ) async def retry(task, error): @@ -1000,6 +1213,7 @@ async def retry(task, error): jira_issue=backport_data.jira_issue, cve_id=backport_data.cve_id, redis_conn=redis, + reproducer_info=backport_data.reproducer_info, ) logger.info( f"Backport processing completed for {backport_data.jira_issue}, " f"success: {state.backport_result.success}" diff --git a/agents/build_agent.py b/agents/build_agent.py index 13ad0997..368b1a9d 100644 --- a/agents/build_agent.py +++ b/agents/build_agent.py @@ -35,6 +35,9 @@ def get_instructions() -> str: just return the error message. Otherwise, start with `builder-live.log` and try to identify the build failure. If not found, try the same with `root.log`. Summarize the findings and return them as `error`. If the build failed due to a build timeout, set `is_timeout` to `true` in your output. + If the build succeed, choose the main package URL (no devel, static, debug packages) from `artifacts_urls` returned by `build_package` tool + and return it as `build_url` in your output. + General instructions: diff --git a/agents/triage_agent.py b/agents/triage_agent.py index 4583e4de..019d9e58 100644 --- a/agents/triage_agent.py +++ b/agents/triage_agent.py @@ -253,7 +253,22 @@ def render_prompt(input: InputSchema) -> str: is clarification-needed * This is the correct choice when you are sure a problem exists but cannot find the solution yourself - 2.5 Set the Jira fields as per the instructions below. + 2.5. If the chosen action is backport then search for the issue reproducer test case in the issue comments. + You need to extract the following information to say you have found the issue reproducer: + - Git URL to the issue reproducer; a git repo stored in https://gitlab.com/redhat/rhel/tests/ + - Git reference, it can be omitted and in this case you will use "main" as the reference. + - Test to run, usually a path like folder/test_name + - You need to say you have found the issue reproducer test case and provide the information in the JSON format. + The JSON format should be like this: + ```json + { + "git_url": "https://gitlab.com/redhat/rhel/tests/test_repo.git", + "git_ref": "main", + "test": "folder/test_name" + } + ``` + + 2.6. Set the Jira fields as per the instructions below. 3. **No Action** A No Action decision is appropriate for issues that are NOT bugs or CVEs requiring code fixes: @@ -410,7 +425,12 @@ async def run_triage_analysis(state): "justification": "This patch fixes the bug by doing X, Y, and Z.", "jira_issue": "RHEL-12345", "cve_id": "CVE-1234-98765", - "fix_version": "rhel-X.Y.Z" + "fix_version": "rhel-X.Y.Z", + "reproducer_info": {{ + "git_url": "https://gitlab.com/redhat/rhel/tests/test_repo.git", + "git_ref": "main", + "test": "folder/test_name" + }} }} }} ``` diff --git a/common/models.py b/common/models.py index 95c20d82..5c9ea406 100644 --- a/common/models.py +++ b/common/models.py @@ -88,6 +88,12 @@ class RebaseOutputSchema(BaseModel): # Backport Agent Schemas # ============================================================================ +class ReproducerInfo(BaseModel): + """Data for TMT resolution.""" + git_url: str = Field(description="Git URL to the TMT reproducer") + git_ref: str = Field(description="Git reference to the TMT reproducer") + test: str = Field(description="Test to run the TMT reproducer") + class BackportInputSchema(BaseModel): """Input schema for the backport agent.""" local_clone: Path = Field(description="Path to the local clone of forked dist-git repository") @@ -99,6 +105,10 @@ class BackportInputSchema(BaseModel): upstream_patches: list[str] = Field( description="List of URLs to upstream patches that were validated using the PatchValidator tool") build_error: str | None = Field(description="Error encountered during package build") + build_url: str | None = Field(description="URL to the principal built package", default=None) + reproducer_info: ReproducerInfo | None = Field(description="Information about the TMT reproducer", default=None) + reproducer_success: bool = Field(default=False, description="Whether the reproducer test case passed") + reproducer_result: str | None = Field(description="Result of the reproducer test case", default=None) class BackportOutputSchema(BaseModel): @@ -106,6 +116,7 @@ class BackportOutputSchema(BaseModel): success: bool = Field(description="Whether the backport was successfully completed") status: str = Field(description="Backport status with details of how the potential merge conflicts were resolved") srpm_path: Path | None = Field(description="Absolute path to generated SRPM") + build_url: str | None = Field(description="URL to the principal built package", default=None) error: str | None = Field(description="Specific details about an error") @@ -129,7 +140,6 @@ class RebaseData(BaseModel): jira_issue: str = Field(description="Jira issue identifier") fix_version: str | None = Field(description="Fix version in Jira (e.g., 'rhel-9.8')", default=None) - class BackportData(BaseModel): """Data for backport resolution.""" package: str = Field(description="Package name") @@ -139,6 +149,7 @@ class BackportData(BaseModel): jira_issue: str = Field(description="Jira issue identifier") cve_id: str | None = Field(description="CVE identifier", default=None) fix_version: str | None = Field(description="Fix version in Jira (e.g., 'rhel-9.8')", default=None) + reproducer_info: ReproducerInfo | None = Field(description="Reproducer test case information", default=None) class ClarificationNeededData(BaseModel): @@ -246,6 +257,7 @@ class BuildInputSchema(BaseModel): class BuildOutputSchema(BaseModel): """Output schema for the build agent.""" success: bool = Field(description="Whether the build was successfully completed") + build_url: str | None = Field(description="URL to the principal built package", default=None) error: str | None = Field(description="Specific details about an error") is_timeout: bool = Field(default=False, description="Whether the build failed due to a timeout") diff --git a/mcp_server/gateway.py b/mcp_server/gateway.py index 12395476..b77220a6 100644 --- a/mcp_server/gateway.py +++ b/mcp_server/gateway.py @@ -9,13 +9,14 @@ import gitlab_tools import jira_tools import lookaside_tools +import testing_farm_tools mcp = FastMCP( name="MCP Gateway", tools=[ coroutine - for module in [copr_tools, distgit_tools, gitlab_tools, jira_tools, lookaside_tools] + for module in [copr_tools, distgit_tools, gitlab_tools, jira_tools, lookaside_tools, testing_farm_tools] for name, coroutine in inspect.getmembers(module, inspect.iscoroutinefunction) if coroutine.__module__ == module.__name__ and not name.startswith("_") diff --git a/mcp_server/testing_farm_tools.py b/mcp_server/testing_farm_tools.py new file mode 100644 index 00000000..da3d60eb --- /dev/null +++ b/mcp_server/testing_farm_tools.py @@ -0,0 +1,84 @@ +import asyncio +import logging +import re +import shlex +from typing import Annotated, Tuple + +from fastmcp.exceptions import ToolError +from pydantic import Field + +from common.utils import init_kerberos_ticket, KerberosError + +logger = logging.getLogger(__name__) + +async def get_copr_repo(issue: Annotated[str, Field(description="Jira issue to get the Copr repo for")]) -> str: + """Gets the Copr repo for the package""" + try: + principal = await init_kerberos_ticket() + except KerberosError as e: + raise ToolError(f"Failed to initialize Kerberos ticket: {e}") from e + copr_user = principal.split("@", maxsplit=1)[0] + return f"copr.devel.redhat.com/{copr_user}/{issue}" + +async def get_compose_from_branch(dist_git_branch: Annotated[str, Field(description="Branch to get the compose from")]) -> str: + """Gets the compose from the branch""" + if dist_git_branch == "rhel-8.10": + # There is one more .0 needed only for the RHEL 8.10 compose (not for 10.X branches) + return "RHEL-8.10.0-Nightly" + elif dist_git_branch.startswith("rhel-"): + return dist_git_branch.upper() + "-Nightly" + else: + match = re.match(r'^c(\d+)s$', dist_git_branch) + if match: + number = match.group(1) + return f"CentOS-Stream-{number}" + else: + raise ToolError(f"Invalid branch format, can't get compose from branch: {dist_git_branch}") + +async def run_testing_farm_test( + git_url: Annotated[str, Field(description="Git URL to the test repository")], + git_ref: Annotated[str, Field(description="Git reference to the test repository")], + path_to_test: Annotated[str, Field(description="Path to the test to run")], + package: Annotated[str, Field(description="Package URL to be installed in the test environment")], + dist_git_branch: Annotated[str, Field(description="Dist Git branch to use to get the compose")], +) -> Tuple[bool, str]: + """Runs the specified testing-farm test and returns True if the test passed, False otherwise.""" + + tmt_prepare = f'--insert --how install --package {shlex.quote(package)}' + compose = await get_compose_from_branch(dist_git_branch) + + # Build the command arguments + cmd = [ + "testing-farm", + "request", + "--tmt-prepare", tmt_prepare, + "--compose", compose, + "--git-ref", git_ref, + "--git-url", git_url, + "--test", path_to_test, + ] + + logger.info(f"Running testing-farm command: {' '.join(shlex.quote(arg) for arg in cmd)}") + + try: + process = await asyncio.create_subprocess_exec( + *cmd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + + stdout, stderr = await process.communicate() + + if process.returncode == 0: + msg = f"Testing Farm test passed: \n stdout {'='*60}\n {stdout.decode()}\n {'='*60}\n stderr {'='*60}\n {stderr.decode()}\n {'='*60}" + logger.info(msg) + return True, msg + else: + msg = f"Testing Farm test failed (exit code {process.returncode}): \n stdout {'='*60}\n {stdout.decode()}\n {'='*60}\n stderr {'='*60}\n {stderr.decode()}\n {'='*60}" + msg += f"\n Ran command: {' '.join(shlex.quote(arg) for arg in cmd)}" + logger.error(msg) + return False, msg + + except Exception as e: + logger.error(f"Failed to run testing-farm command: {e}") + raise ToolError(f"Failed to run testing-farm test: {e}") from e diff --git a/mcp_server/tests/unit/test_testing_farm_tools.py b/mcp_server/tests/unit/test_testing_farm_tools.py new file mode 100644 index 00000000..ecd2b743 --- /dev/null +++ b/mcp_server/tests/unit/test_testing_farm_tools.py @@ -0,0 +1,105 @@ +"""Tests for testing_farm_tools module.""" + +import asyncio +import shlex +import pytest +from unittest.mock import AsyncMock, patch +from mcp_server.testing_farm_tools import run_testing_farm_test, get_compose_from_branch + + +def test_tmt_prepare_quoting(): + """Test that package URLs with special characters are properly quoted.""" + # Test with a package URL that has special characters + package = 'http://example.com/package-1.0-1.el9.x86_64.rpm' + tmt_prepare = f'--insert --how install --package {shlex.quote(package)}' + + # Should be properly quoted + assert '--insert' in tmt_prepare + assert '--how' in tmt_prepare + assert 'install' in tmt_prepare + assert '--package' in tmt_prepare + assert package in tmt_prepare + + +@pytest.mark.asyncio +async def test_get_compose_from_branch(): + """Test get_compose_from_branch function with various branch formats.""" + test_cases = [ + ("rhel-9.7.0", "RHEL-9.7.0-Nightly"), + ("rhel-10.2", "RHEL-10.2-Nightly"), + ("rhel-8.10", "RHEL-8.10.0-Nightly"), + ("c8s", "CentOS-Stream-8"), + ] + + for branch, expected_compose in test_cases: + result = await get_compose_from_branch(branch) + assert result == expected_compose, ( + f"Branch '{branch}' should return '{expected_compose}', " + f"but got '{result}'" + ) + + +@pytest.mark.asyncio +async def test_run_testing_farm_test_success(): + """Test run_testing_farm_test with successful test execution.""" + git_url = "https://gitlab.com/redhat/rhel/tests/expat.git" + git_ref = "master" + path_to_test = "Security/RHEL-114639-CVE-2025-59375-expat-libexpat-in-Expat-allows" + package = "http://coprbe.devel.redhat.com/results/mmassari/RHEL-114644/rhel-9-x86_64/00127295-expat/expat-2.5.0-5.el9.x86_64.rpm" + dist_git_branch = "rhel-9.7.0" + + # Mock the subprocess to return success + mock_process = AsyncMock() + mock_process.returncode = 0 + mock_process.communicate = AsyncMock(return_value=(b"Test passed", b"")) + + with patch('mcp_server.testing_farm_tools.asyncio.create_subprocess_exec', return_value=mock_process): + success, result = await run_testing_farm_test( + git_url=git_url, + git_ref=git_ref, + path_to_test=path_to_test, + package=package, + dist_git_branch=dist_git_branch, + ) + + assert success is True + assert "Test passed" in result + assert "Testing Farm test passed" in result + + +@pytest.mark.asyncio +async def test_run_testing_farm_test_failure(): + """Test run_testing_farm_test with failed test execution.""" + git_url = "https://gitlab.com/redhat/rhel/tests/expat.git" + git_ref = "master" + path_to_test = "Security/RHEL-114639-CVE-2025-59375-expat-libexpat-in-Expat-allows" + package = "http://coprbe.devel.redhat.com/results/mmassari/RHEL-114644/rhel-9-x86_64/00127295-expat/expat-2.5.0-5.el9.x86_64.rpm" + dist_git_branch = "rhel-9.7.0" + + # Mock the subprocess to return failure + mock_process = AsyncMock() + mock_process.returncode = 1 + mock_process.communicate = AsyncMock(return_value=(b"", b"Test failed with error")) + + with patch('mcp_server.testing_farm_tools.asyncio.create_subprocess_exec', return_value=mock_process): + success, result = await run_testing_farm_test( + git_url=git_url, + git_ref=git_ref, + path_to_test=path_to_test, + package=package, + dist_git_branch=dist_git_branch, + ) + + assert success is False + assert "Test failed" in result or "exit code 1" in result + assert "Testing Farm test failed" in result + + +if __name__ == '__main__': + # Run tests if executed directly + test_tmt_prepare_quoting() + # Run async tests + asyncio.run(test_get_compose_from_branch()) + asyncio.run(test_run_testing_farm_test_success()) + asyncio.run(test_run_testing_farm_test_failure()) + print("All tests passed!") diff --git a/templates/mcp-gateway.env b/templates/mcp-gateway.env index 2aeeca9b..4f087e52 100644 --- a/templates/mcp-gateway.env +++ b/templates/mcp-gateway.env @@ -6,3 +6,5 @@ GITLAB_TOKEN= JIRA_URL=https://issues.redhat.com # JIRA profile -> Create token JIRA_TOKEN= + +TESTING_FARM_API_TOKEN=