diff --git a/src/buildkite_test_collector/pytest_plugin/__init__.py b/src/buildkite_test_collector/pytest_plugin/__init__.py index ba1d7af..53a10a0 100644 --- a/src/buildkite_test_collector/pytest_plugin/__init__.py +++ b/src/buildkite_test_collector/pytest_plugin/__init__.py @@ -41,7 +41,7 @@ def pytest_unconfigure(config): if plugin: api = API(os.environ) - numprocesses = config.getoption("numprocesses") + numprocesses = config.getoption("numprocesses", None) xdist_enabled = ( config.pluginmanager.getplugin("xdist") is not None and numprocesses is not None @@ -89,3 +89,10 @@ def pytest_addoption(parser): dest="mergejson", help='merge json output with existing file, if it exists' ) + group.addoption( + '--tag-filters', + default=None, + action='store', + dest="tag_filters", + help='filter tests by execution_tag with `key:value`, e.g. `--tag-filters color:red`' + ) diff --git a/src/buildkite_test_collector/pytest_plugin/buildkite_plugin.py b/src/buildkite_test_collector/pytest_plugin/buildkite_plugin.py index 1ab4271..52d528c 100644 --- a/src/buildkite_test_collector/pytest_plugin/buildkite_plugin.py +++ b/src/buildkite_test_collector/pytest_plugin/buildkite_plugin.py @@ -17,6 +17,17 @@ def __init__(self, payload): self.in_flight = {} self.spans = {} + def pytest_collection_modifyitems(self, config, items): + """pytest_collection_modifyitems hook callback to filter tests by execution_tag markers""" + tag_filter = config.getoption("tag_filters") + if not tag_filter: + return + + filtered_items, unfiltered_items = self._filter_tests_by_tag(items, tag_filter) + + config.hook.pytest_deselected(items=unfiltered_items) + items[:] = filtered_items + def pytest_runtest_logstart(self, nodeid, location): """pytest_runtest_logstart hook callback""" logger.debug('hook=pytest_runtest_logstart nodeid=%s', nodeid) @@ -144,3 +155,32 @@ def save_payload_as_json(self, path, merge=False): with open(path, "w", encoding="utf-8") as f: json.dump(data, f) + + def _filter_tests_by_tag(self, items, tag_filter): + """ + Filters tests based on the tag_filter option. + Supports filtering by a single tag in the format key:value. + Only equality comparison is supported. + Returns a tuple of (filtered_items, unfiltered_items). + """ + key, _, value = tag_filter.partition(":") + + filtered_items = [] + unfiltered_items = [] + for item in items: + # Extract all execution_tag markers and store them in a dict + tags = {} + markers = item.iter_markers("execution_tag") + for tag_marker in markers: + # Ensure the marker has exactly two arguments: key and value + if len(tag_marker.args) != 2: + continue + + tags[tag_marker.args[0]] = tag_marker.args[1] + + if tags.get(key) == value: + filtered_items.append(item) + else: + unfiltered_items.append(item) + + return filtered_items, unfiltered_items diff --git a/tests/buildkite_test_collector/data/test_sample_execution_tag.py b/tests/buildkite_test_collector/data/test_sample_execution_tag.py new file mode 100644 index 0000000..6791919 --- /dev/null +++ b/tests/buildkite_test_collector/data/test_sample_execution_tag.py @@ -0,0 +1,15 @@ +# This is sample test file used for integration testing of execution_tag marker +import pytest + + +@pytest.mark.execution_tag("language.version", "3.12") +@pytest.mark.execution_tag("team", "backend") +def test_with_multiple_tags(): + assert True + +@pytest.mark.execution_tag("team", "frontend") +def test_with_single_tag(): + assert True + +def test_without_tags(): + assert True diff --git a/tests/buildkite_test_collector/data/test_sample_execution_tag_filter.py b/tests/buildkite_test_collector/data/test_sample_execution_tag_filter.py new file mode 100644 index 0000000..3149022 --- /dev/null +++ b/tests/buildkite_test_collector/data/test_sample_execution_tag_filter.py @@ -0,0 +1,29 @@ +# This is sample test file used for integration testing of execution_tag marker +import pytest + + +@pytest.mark.execution_tag("color", "red") +@pytest.mark.execution_tag("size", "medium") +def test_apple(): + assert True + +@pytest.mark.execution_tag("color", "orange") +@pytest.mark.execution_tag("size", "medium") +def test_orange(): + assert True + +@pytest.mark.execution_tag("color", "yellow") +@pytest.mark.execution_tag("size", "large") +def test_banana(): + assert True + +@pytest.mark.execution_tag("color", "purple") +@pytest.mark.execution_tag("size", "small") +def test_grape(): + assert True + +@pytest.mark.execution_tag("color", "red") +@pytest.mark.execution_tag("size", "small") +def test_strawberry(): + assert True + diff --git a/tests/buildkite_test_collector/test_integration.py b/tests/buildkite_test_collector/test_integration.py new file mode 100644 index 0000000..ac68f65 --- /dev/null +++ b/tests/buildkite_test_collector/test_integration.py @@ -0,0 +1,114 @@ +import json +import os +import pytest +import subprocess +import sys + +from pathlib import Path + +def test_add_tag_to_execution_data(tmp_path, fake_env): + """Verify that tags added via the execution_tag marker are correctly captured in the test data.""" + test_file = Path(__file__).parent / "data" / "test_sample_execution_tag.py" + json_output_file = tmp_path / "test_results.json" + + # Run pytest with our plugin on the test file + cmd = [ + sys.executable, "-m", "pytest", + str(test_file), + f"--json={json_output_file}", + ] + + # Run pytest in a subprocess + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + print(result.stdout) + print(result.stderr) + pytest.fail("pytest run failed, see output above") + + assert json_output_file.exists(), "JSON output file was not created" + + with open(json_output_file, 'r') as f: + test_results = json.load(f) + + tests_by_name = {test["name"]: test for test in test_results} + + multi_tag_test = tests_by_name["test_with_multiple_tags"] + assert "tags" in multi_tag_test + assert multi_tag_test["tags"] == { + "language.version": "3.12", + "team": "backend" + } + + single_tag_test = tests_by_name["test_with_single_tag"] + assert "tags" in single_tag_test + assert single_tag_test["tags"] == {"team": "frontend"} + + no_tag_test = tests_by_name["test_without_tags"] + assert "tags" not in no_tag_test or no_tag_test.get("tags") == {} + +class TestTagFiltering: + import subprocess + import sys + + def test_filter_by_single_tag(self,tmp_path, fake_env): + test_file = Path(__file__).parent / "data" / "test_sample_execution_tag_filter.py" + + cmd = [ + sys.executable, "-m", "pytest", "--co", "-q", "--tag-filters", "color:red", + str(test_file), + ] + + # Run pytest in a subprocess + result = subprocess.run(cmd, cwd=str(tmp_path), capture_output=True, text=True) + + assert "2/5 tests collected" in result.stdout, "collect count mismatch" + + # Parse the output to find collected tests + lines = result.stdout.strip().splitlines() + collected_tests = [] + for line in lines: + if "::" in line: + collected_tests.append(line) + + assert "tests/buildkite_test_collector/data/test_sample_execution_tag_filter.py::test_apple" in collected_tests + assert "tests/buildkite_test_collector/data/test_sample_execution_tag_filter.py::test_strawberry" in collected_tests + assert "tests/buildkite_test_collector/data/test_sample_execution_tag_filter.py::test_orange" not in collected_tests + assert "tests/buildkite_test_collector/data/test_sample_execution_tag_filter.py::test_banana" not in collected_tests + assert "tests/buildkite_test_collector/data/test_sample_execution_tag_filter.py::test_grape" not in collected_tests + + def test_wrong_filter_format(self,tmp_path, fake_env): + test_file = Path(__file__).parent / "data" / "test_sample_execution_tag_filter.py" + + cmd = [ + sys.executable, "-m", "pytest", "--co", "-q", "--tag-filters", "foobar", + str(test_file), + ] + + # Run pytest in a subprocess + result = subprocess.run(cmd, cwd=str(tmp_path), capture_output=True, text=True) + + assert "no tests collected" in result.stdout + + def test_no_filter(self,tmp_path, fake_env): + test_file = Path(__file__).parent / "data" / "test_sample_execution_tag_filter.py" + + cmd = [ + sys.executable, "-m", "pytest", "--co", "-q", + str(test_file), + ] + + # Run pytest in a subprocess + result = subprocess.run(cmd, cwd=str(tmp_path), capture_output=True, text=True) + + # Parse the output to find collected tests + lines = result.stdout.strip().splitlines() + collected_tests = [] + for line in lines: + if "::" in line: + collected_tests.append(line) + + assert "tests/buildkite_test_collector/data/test_sample_execution_tag_filter.py::test_apple" in collected_tests + assert "tests/buildkite_test_collector/data/test_sample_execution_tag_filter.py::test_strawberry" in collected_tests + assert "tests/buildkite_test_collector/data/test_sample_execution_tag_filter.py::test_orange" in collected_tests + assert "tests/buildkite_test_collector/data/test_sample_execution_tag_filter.py::test_banana" in collected_tests + assert "tests/buildkite_test_collector/data/test_sample_execution_tag_filter.py::test_grape" in collected_tests