From 5c7656c83e73efa354a2432c4dbb62fc1086f129 Mon Sep 17 00:00:00 2001 From: Zdenek Veleba Date: Mon, 27 Apr 2026 10:55:55 +0200 Subject: [PATCH 1/2] Don't cut longer links in stdout report sender --- libpermian/plugins/stdout/templates/table80.j2 | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/libpermian/plugins/stdout/templates/table80.j2 b/libpermian/plugins/stdout/templates/table80.j2 index 09e9545..813ac24 100644 --- a/libpermian/plugins/stdout/templates/table80.j2 +++ b/libpermian/plugins/stdout/templates/table80.j2 @@ -6,6 +6,11 @@ +----------------------------+-------------------+-----------------------------| | {{ crc.configuration.values()|join(', ')|setlen(26) }} | {{ crc.result.result|setlen(17) }} | {{ crc.result.state|setlen(27) }} | +----------------------------+-------------------+-----------------------------+ -| {{ crc.workflow.groupDisplayStatus(crc.id)|setlen(76) }} | +{%- set status = crc.workflow.groupDisplayStatus(crc.id) %} +{%- if status|length <= 76 %} +| {{ status|setlen(76) }} | +{%- else %} +| {{ status }} +{%- endif %} +==============================================================================+ {%- endfor %} \ No newline at end of file From 3d51eea4d3e03c6f4ba54230341ff74355c15e6a Mon Sep 17 00:00:00 2001 From: Zdenek Veleba Date: Mon, 27 Apr 2026 10:56:24 +0200 Subject: [PATCH 2/2] Add new plugin for running tests in testing farm --- libpermian/plugins/testing_farm/__init__.py | 332 +++++++++++++++++++ libpermian/plugins/testing_farm/settings.ini | 7 + libpermian/plugins/testing_farm/test.py | 297 +++++++++++++++++ 3 files changed, 636 insertions(+) create mode 100644 libpermian/plugins/testing_farm/__init__.py create mode 100644 libpermian/plugins/testing_farm/settings.ini create mode 100644 libpermian/plugins/testing_farm/test.py diff --git a/libpermian/plugins/testing_farm/__init__.py b/libpermian/plugins/testing_farm/__init__.py new file mode 100644 index 0000000..989e3d7 --- /dev/null +++ b/libpermian/plugins/testing_farm/__init__.py @@ -0,0 +1,332 @@ +import json +import logging +import time +import requests +import re + +from .. import api +from ...workflows.isolated import IsolatedWorkflow +from libpermian.result import Result + + +LOGGER = logging.getLogger(__name__) + +# Maps Testing Farm request states to Permian state values +state2state_map = { + 'canceled': 'canceled', + 'new': 'not started', + 'queued': 'queued', + 'running': 'running', + 'complete': 'complete', + 'error': 'DNF', +} + +# Maps terminal Testing Farm states to Permian result values +state2result_map = { + 'canceled': None, + 'error': 'ERROR', +} +# Maps Testing Farm test results to Permian result values +result2result_map = { + 'passed': 'PASS', + 'failed': 'FAIL', +} + + +@api.workflows.register("testingFarm") +class TestingFarmWorkflow(IsolatedWorkflow): + """ + Workflow for executing tests on Testing Farm infrastructure. + """ + + def __init__(self, testRuns, crcList): + """ + Initialize the Testing Farm workflow. + + Loads API credentials, test configuration, and initializes tracking variables. + + Args: + testRuns: Test run instances. + crcList: List of case run configurations. + """ + super().__init__(testRuns, crcList) + + self.api_url = self.settings.get('testingFarm', 'api_url') + self.api_token = self.settings.get('testingFarm', 'api_token') + self.poll_interval = self.settings.getint('testingFarm', 'poll_interval') + self.max_status_retry = self.settings.getint('testingFarm', 'max_status_retry') + default_test_repo = self.settings.get('testingFarm', 'default_test_repo') + default_test_branch = self.settings.get('testingFarm', 'default_test_branch') + + self.test_repo = self.crc.testcase.execution.automation_data.get('test_repo', default_test_repo) + self.test_branch = self.crc.testcase.execution.automation_data.get('test_branch', default_test_branch) + self.tmt_test = self.crc.testcase.execution.automation_data['test'] + + self.variant = self.crc.configuration.get('variant') + self.architecture = self.crc.configuration.get('architecture') + self.tested_compose = self.event.compose.id + self.execution_os = self.crc.testcase.execution.automation_data.get('compose', self.tested_compose) + self.execution_arch = self.crc.testcase.execution.automation_data.get('arch', self.architecture) + + self.provision_weblink = None + self.weblink = None + self.request_id = None + + def setup(self): + """ + Build the test payload and report initial state. + + Constructs the Testing Farm API request payload with environment + variables, test configuration, and compose information. Merges + any custom env_vars from test metadata. Reports 'not started' state. + """ + + env_vars = { + 'TEST_PARAM_COMPOSE': self.tested_compose, + 'TEST_PARAM_VARIANT': self.variant, + 'TEST_PARAM_ARCH': self.architecture + } + if 'env_vars' in self.crc.testcase.execution.automation_data: + env_vars.update(self.crc.testcase.execution.automation_data['env_vars']) + + self.payload = { + "test": { + "fmf": { + "url": self.test_repo, + "ref": self.test_branch, + "test_name": self.tmt_test + } + }, + "environments": [ + { + "arch": self.execution_arch, + "os": { + "compose": self.execution_os + }, + "variables": env_vars + } + ] + } + + self.reportResult(Result('not started')) + + def execute(self): + """ + Execute the test on Testing Farm. + + Submits the test request, polls for status updates, and reports + results. Collects artifacts when the test completes. + """ + self.log(f'Submitting test request to Testing Farm, payload:\n{json.dumps(self.payload, indent=2)}') + + try: + self.request_id = self.submit_test() + except requests.HTTPError as e: + LOGGER.error(f'Can\'t submit test {e}') + self.reportResult(Result('not started', 'ERROR', final=True)) + return + + self.log(f"Test submitted. Request ID: {self.request_id}") + + status_attempts = 0 + status = dict() + + while not self.crc.result.final: + try: + status = self.get_status() + status_attempts = 0 + state = status['state'] + self.log(f"Current state: {state}") + + # Try to get link to a web page with info about provisioning + try: + if self.provision_weblink is None and status['notes'][0]['message'].startswith('http'): + self.provision_weblink = status['notes'][0]['message'] + self.log(f'New link: {self.provision_weblink}') + except (KeyError, IndexError, TypeError): + pass + + # Try to get link to a web page with info about test run + try: + if self.weblink is None: + self.weblink = status['run']['artifacts'] + self.log(f'New link: {self.weblink}') + except (KeyError, IndexError, TypeError): + pass + + # Update result + if state == 'complete': + if status['result']['overall'] in result2result_map: + self.reportResult(Result('complete', + result2result_map[status['result']['overall']], + final=True)) + break + else: + self.reportResult(Result('complete', 'ERROR', final=True)) + break + + elif state in ['error', 'canceled']: + self.reportResult(Result(state2state_map[state], + state2result_map[state], + final=True)) + break + + elif state in state2state_map and self.crc.result.state != state2state_map[state]: + self.reportResult(Result(state2state_map[state])) + + except requests.HTTPError as e: + LOGGER.error(f'Can\'t get test status {e}') + status_attempts += 1 + if status_attempts == self.max_status_retry: + self.reportResult(Result('DNF', 'ERROR', final=True)) + return + + time.sleep(self.poll_interval) + + try: + artifacts_url = status['result'].get('xunit_url', None) + if artifacts_url is not None: + self.collect_artifacts(artifacts_url) + else: + LOGGER.warning('No xunit, can\'t collect artifacts') + except (KeyError, AttributeError): + LOGGER.warning('No result, can\'t collect artifacts') + + def dry_execute(self): + """ + Simulate test execution without actually submitting to Testing Farm. + + Logs the payload that would be sent and reports a successful result. + Used when the pipeline is in dry-run mode. + """ + LOGGER.info(f'DRY RUN: Would request {self.api_url}/requests\n {json.dumps(self.payload, indent=2)}') + self.reportResult(Result('complete', 'PASS', final=True)) + + def terminate(self): + """ + Cancel the running test request. + + Sends a delete request to Testing Farm API to cancel the test. + Called asynchronously when the workflow needs to be stopped. + + Returns: + bool: True if cancellation succeeded, False otherwise. + """ + if self.request_id is None: + LOGGER.warning("No request to cancel") + return True + + try: + self.delete_request() + self.reportResult(Result('canceled', None, final=True)) + return True + except requests.HTTPError as e: + LOGGER.error(f"Test termination failed: {e}") + self.reportResult(Result('canceled', 'ERROR', final=True)) + return False + + def displayStatus(self): + """ + Return markdown-formatted status link for WebUI display. + + Returns: + str: Markdown link to test run or provisioning page, or empty string. + """ + if self.weblink is not None: + return f'[TF testrun]({self.weblink})' + if self.provision_weblink is not None: + return f'[TF provisioning]({self.provision_weblink})' + return '' + + def submit_test(self): + """ + Submit a test request to Testing Farm API. + + Returns: + str: Request ID assigned by Testing Farm. + + Raises: + requests.HTTPError: If the API request fails. + """ + headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {self.api_token}" + } + + response = requests.post( + f"{self.api_url}/requests", + headers=headers, + json=self.payload + ) + response.raise_for_status() + + result = response.json() + return result['id'] + + def get_status(self): + """ + Retrieve current status of the test request from Testing Farm. + + Returns: + dict: Status response containing state, results, and metadata. + + Raises: + requests.HTTPError: If the API request fails. + """ + headers = { + "Authorization": f"Bearer {self.api_token}" + } + + response = requests.get( + f"{self.api_url}/requests/{self.request_id}", + headers=headers + ) + response.raise_for_status() + + return response.json() + + def delete_request(self): + """ + Send DELETE request to cancel the test on Testing Farm. + + Raises: + requests.HTTPError: If the cancellation request fails. + """ + headers = { + "Authorization": f"Bearer {self.api_token}" + } + + response = requests.delete( + f"{self.api_url}/requests/{self.request_id}", + headers=headers + ) + response.raise_for_status() + + def collect_artifacts(self, artifacts_url): + """ + Fetch and add test artifacts from Testing Farm results. + + Parses the artifacts XML/HTML listing and adds log files to the workflow. + + Args: + artifacts_url (str): URL to the artifacts directory listing. + """ + try: + response = requests.get(artifacts_url) + response.raise_for_status() + except requests.HTTPError as e: + LOGGER.error(f"Can\'t collect artifacts: {e}") + return + + xml = response.text + + link_pattern = r'' + matches = re.findall(link_pattern, xml) + + for href, name, _ in matches: + # Skip parent directory and absolute URLs + if name in ['log_dir', 'data']: + continue + if name.startswith('data/'): + name = name.split('/', 1)[1] + self.addLog(name, href) \ No newline at end of file diff --git a/libpermian/plugins/testing_farm/settings.ini b/libpermian/plugins/testing_farm/settings.ini new file mode 100644 index 0000000..bcedd13 --- /dev/null +++ b/libpermian/plugins/testing_farm/settings.ini @@ -0,0 +1,7 @@ +[testingFarm] +default_test_repo= +default_test_branch=main +api_url=https://api.dev.testing-farm.io/v0.1 +api_token= +poll_interval=30 +max_status_retry=50 \ No newline at end of file diff --git a/libpermian/plugins/testing_farm/test.py b/libpermian/plugins/testing_farm/test.py new file mode 100644 index 0000000..00de5be --- /dev/null +++ b/libpermian/plugins/testing_farm/test.py @@ -0,0 +1,297 @@ +"""Unit tests for Testing Farm workflow plugin.""" +import unittest +from unittest.mock import Mock, patch + +from libpermian.events.base import Event +from libpermian.settings import Settings +from libpermian.result import Result + +from . import TestingFarmWorkflow + + +class FakeTestCase: + """Mock test case for testing.""" + def __init__(self): + self.name = 'test-basic' + self.id = 'test-basic-id' + self.execution = Mock() + self.execution.automation_data = { + 'test': '/test-basic', + } + + +class FakeConfiguration: + """Mock configuration for testing.""" + def __init__(self): + self._data = { + 'variant': 'BaseOS', + 'architecture': 'x86_64', + } + + def get(self, key, default=None): + return self._data.get(key, default) + + def items(self): + return self._data.items() + + def values(self): + return self._data.values() + + +class FakeEvent(Event): + """Mock event for testing.""" + def __init__(self, settings): + super().__init__( + settings, + 'testing_farm_test', + compose={ + 'id': 'COMP-9.8.0-20260324.7', + } + ) + + +class TestTestingFarmWorkflow(unittest.TestCase): + """Test TestingFarmWorkflow class.""" + + def setUp(self): + """Set up test fixtures.""" + self.settings = Settings( + cmdline_overrides={ + 'testingFarm': { + 'api_url': 'https://api.testing-farm.io/v0.1', + 'api_token': 'test-token', + 'poll_interval': '1', + 'default_test_repo': 'https://github.com/example/tests.git', + 'default_test_branch': 'main', + } + }, + environment={}, + settings_locations=[], + ) + + self.event = FakeEvent(self.settings) + self.testcase = FakeTestCase() + self.configuration = FakeConfiguration() + + # Create mock CRC + self.crc = Mock() + self.crc.testcase = self.testcase + self.crc.configuration = self.configuration + self.crc.id = 'crc-test-id' + self.crc.result = Result('not started', None, False) + + # Mock openLogfile to return a file-like context manager + mock_file = Mock() + mock_file.__enter__ = Mock(return_value=mock_file) + mock_file.__exit__ = Mock(return_value=False) + self.crc.openLogfile = Mock(return_value=mock_file) + + # Create mock TestRuns + self.testRuns = Mock() + self.testRuns.event = self.event + self.testRuns.settings = self.settings + + self.crcList = [self.crc] + + def test_setup(self): + """Test setup method builds payload correctly.""" + workflow = TestingFarmWorkflow(self.testRuns, self.crcList) + + self.testcase.execution.automation_data['env_vars'] = { + 'CUSTOM_VAR': 'custom_value', + 'TEST_PARAM_COMPOSE': 'OVERRIDE-COMPOSE' + } + + workflow.setup() + + # Check payload structure + self.assertIn('test', workflow.payload) + self.assertIn('environments', workflow.payload) + self.assertEqual(workflow.payload['test']['fmf']['url'], 'https://github.com/example/tests.git') + self.assertEqual(workflow.payload['test']['fmf']['ref'], 'main') + self.assertEqual(workflow.payload['test']['fmf']['test_name'], '/test-basic') + + # Check environment variables + env_vars = workflow.payload['environments'][0]['variables'] + self.assertEqual(env_vars['CUSTOM_VAR'], 'custom_value') + self.assertEqual(env_vars['TEST_PARAM_COMPOSE'], 'OVERRIDE-COMPOSE') + self.assertEqual(env_vars['TEST_PARAM_VARIANT'], 'BaseOS') + self.assertEqual(env_vars['TEST_PARAM_ARCH'], 'x86_64') + + @patch('libpermian.plugins.testing_farm.requests.post') + def test_submit_test_success(self, mock_post): + """Test successful test submission.""" + mock_response = Mock() + mock_response.json.return_value = {'id': 'request-123'} + mock_post.return_value = mock_response + + workflow = TestingFarmWorkflow(self.testRuns, self.crcList) + workflow.setup() + + request_id = workflow.submit_test() + + self.assertEqual(request_id, 'request-123') + mock_post.assert_called_once() + call_args = mock_post.call_args + self.assertEqual(call_args[0][0], 'https://api.testing-farm.io/v0.1/requests') + self.assertEqual(call_args[1]['headers']['Authorization'], 'Bearer test-token') + self.assertEqual(call_args[1]['json'], workflow.payload) + + + @patch('libpermian.plugins.testing_farm.requests.get') + def test_get_status(self, mock_get): + """Test getting status from Testing Farm.""" + mock_response = Mock() + mock_response.json.return_value = { + 'state': 'running', + 'result': None, + } + mock_get.return_value = mock_response + + workflow = TestingFarmWorkflow(self.testRuns, self.crcList) + workflow.request_id = 'request-123' + + status = workflow.get_status() + + self.assertEqual(status['state'], 'running') + mock_get.assert_called_once_with( + 'https://api.testing-farm.io/v0.1/requests/request-123', + headers={'Authorization': 'Bearer test-token'} + ) + + @patch('libpermian.plugins.testing_farm.requests.delete') + def test_delete_request(self, mock_delete): + """Test deleting a request.""" + mock_response = Mock() + mock_delete.return_value = mock_response + + workflow = TestingFarmWorkflow(self.testRuns, self.crcList) + workflow.request_id = 'request-123' + + workflow.delete_request() + + mock_delete.assert_called_once_with( + 'https://api.testing-farm.io/v0.1/requests/request-123', + headers={'Authorization': 'Bearer test-token'} + ) + + @patch('libpermian.plugins.testing_farm.requests.get') + def test_collect_artifacts_success(self, mock_get): + """Test artifact collection.""" + mock_response = Mock() + mock_response.text = ''' + + + + + + + + + + + + + + + + ''' + mock_get.return_value = mock_response + + workflow = TestingFarmWorkflow(self.testRuns, self.crcList) + workflow.addLog = Mock() + + workflow.collect_artifacts('https://artifacts.testing-farm.io/request-123') + + # Check that logs were added + calls = workflow.addLog.call_args_list + self.assertEqual(len(calls), 7) + + # Check that data/ prefix is stripped + log_names = [call[0][0] for call in calls] + self.assertIn('RTT_AUTOBUG_1234abcd.ini', log_names) # data/ prefix stripped + self.assertIn('Setup/output.txt', log_names) + self.assertIn('Test/output.txt', log_names) + self.assertIn('Cleanup/output.txt', log_names) + self.assertIn('failures.yaml', log_names) + self.assertIn('journal.xml', log_names) + self.assertIn('testout.log', log_names) + + def test_dry_execute(self): + """Test dry execution mode.""" + workflow = TestingFarmWorkflow(self.testRuns, self.crcList) + workflow.setup() + + with patch('libpermian.plugins.testing_farm.LOGGER') as mock_logger: + workflow.dry_execute() + + # Check that info was logged + mock_logger.info.assert_called_once() + logged_message = mock_logger.info.call_args[0][0] + self.assertIn('DRY RUN', logged_message) + self.assertIn(workflow.api_url, logged_message) + + @patch('libpermian.plugins.testing_farm.time.sleep') + @patch('libpermian.plugins.testing_farm.requests.get') + @patch('libpermian.plugins.testing_farm.requests.post') + def test_execute(self, mock_post, mock_get, mock_sleep): + """Test execute with full state transitions: new → queued → running → complete (passed).""" + # Mock submit_test response + mock_post_response = Mock() + mock_post_response.json.return_value = {'id': 'request-123'} + mock_post.return_value = mock_post_response + + # Mock status responses - simulate state transitions + status_responses = [ + {'state': 'new', 'result': None,'notes': [], 'run': {}}, + {'state': 'queued', 'result': None,'notes': [], 'run': {}}, + {'state': 'running', 'result': None,'notes': [], 'run': {}}, + {'state': 'running', 'result': None,'notes': [{"level": "info", "message": "http://testing-farm.example.com/request-123"}], 'run': {}}, + {'state': 'running', 'result': None, 'notes': [{"level": "info", "message": "http://testing-farm.example.com/request-123"}], 'run': {'artifacts': 'https://artifacts.example.com/123'}}, + {'state': 'complete', 'result': {'overall': 'passed', 'xunit_url': 'https://artifacts.example.com/123/xunit.xml'}, 'notes': [{"level": "info", "message": "http://testing-farm.example.com/request-123"}], 'run': {'artifacts': 'https://artifacts.example.com/123'}}, + ] + + mock_get_responses = [] + for status in status_responses: + mock_response = Mock() + mock_response.json.return_value = status + mock_get_responses.append(mock_response) + + # For artifacts collection + artifacts_response = Mock() + artifacts_response.text = '' + mock_get_responses.append(artifacts_response) + + mock_get.side_effect = mock_get_responses + + workflow = TestingFarmWorkflow(self.testRuns, self.crcList) + workflow.setup() + + # Track reportResult calls + original_reportResult = workflow.reportResult + reportResult_calls = [] + + def track_reportResult(result): + reportResult_calls.append((result.state, result.result)) + self.crc.result = result + + workflow.reportResult = track_reportResult + + # Execute workflow + workflow.execute() + + # Verify submit was called + self.assertEqual(mock_post.call_count, 1) + + # Verify status was polled 5 times + self.assertEqual(mock_get.call_count, 7) # 5 status checks + 1 artifact collection + + # Check that the request_id was set + self.assertEqual(workflow.request_id, 'request-123') + self.assertEqual(workflow.weblink, 'https://artifacts.example.com/123') + + # Verify the result was updated only when needed and final result is PASS + self.assertEqual(reportResult_calls, [('queued', None), ('running', None), ('complete', 'PASS')]) + +if __name__ == '__main__': + unittest.main()