unix: use extended PGO training set

This will run all tests, ensuring maximal training coverage.

As part of this, we had to annotate/ignore every failing test because
test failures would otherwise fail the build.

Author: indygreg

cpython-unix/build-cpython.sh

@@ -52,7 +52,7 @@ PIP_WHEEL="${ROOT}/pip-${PIP_VERSION}-py3-none-any.whl"
 SETUPTOOLS_WHEEL="${ROOT}/setuptools-${SETUPTOOLS_VERSION}-py3-none-any.whl"
 
 # Put critical config files in logs to aid debugging.
-for f in Setup.local Makefile.extra stdlib-test-annotations.json; do
+for f in Setup.local Makefile.extra stdlib-test-annotations.json profiling-training-ignores.txt; do
   echo "BEGIN $f"
   cat $f
   echo "END $f"
@@ -593,7 +593,27 @@ if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_14}"  ]; then
 fi
 
 # Define the base PGO profiling task, which we'll extend below with ignores
-export PROFILE_TASK='-m test --pgo'
+#
+# --pgo-extended implies --pgo. And --pgo mode significantly changes the behavior
+# of the test harness. Notably, it disables richer reporting of test failures,
+# making it vastly more difficult to debug test failures. This behavior undermines
+# our ability to debug failures when running tests for PGO instrumentation.
+#
+# The only material downside to not using --pgo is some tests that self-skip
+# when run under PGO don't do so. When we audited for such tests in 2026-03,
+# all such tests had the reason "PGO isn't useful" or some such. There were
+# ~10 such tests. Since we run the entire test suite anyway, the inclusion of
+# such "worthless" tests isn't impactful. The test failure observability is
+# worth their loss.
+export PROFILE_TASK='-m test'
+
+# Display test output on failure. This helps immensely with debugging PGO
+# failures.
+PROFILE_TASK="${PROFILE_TASK} -W"
+
+# Force kill tests taking too long. This prevents deadlocks that can cause
+# CI jobs to run for potentially hours while doing nothing.
+PROFILE_TASK="${PROFILE_TASK} --timeout 300"
 
 # Run tests in parallel to reduce wall time.
 #
@@ -608,29 +628,26 @@ export PROFILE_TASK='-m test --pgo'
 # and there will be no loss in profile quality.
 PROFILE_TASK="${PROFILE_TASK} -j ${NUM_CPUS}"
 
-# On 3.14+ `test_strftime_y2k` fails when cross-compiling for `x86_64_v2` and `x86_64_v3` targets on
-# Linux, so we ignore it. See https://github.com/python/cpython/issues/128104
-if [[ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_14}" && -n "${CROSS_COMPILING}" && "${PYBUILD_PLATFORM}" != macos* ]]; then
-    PROFILE_TASK="${PROFILE_TASK} --ignore test_strftime_y2k"
+IGNORE_TESTS=()
+
+# Possible race condition.
+IGNORE_TESTS+=("test.test_audit.AuditTest.test_time_fail")
+
+if [[ -n "${CPYTHON_FREETHREADED}" ]]; then
+    IGNORE_TESTS+=("test.test_bytes.FreeThreadingTest.test_free_threading_bytearrayiter")
 fi
 
 # On 3.14+ `test_json.test_recursion.TestCRecursion.test_highly_nested_objects_decoding` fails during
 # PGO due to RecursionError not being raised as expected. See https://github.com/python/cpython/issues/140125
 if [[ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_14}" ]]; then
-    PROFILE_TASK="${PROFILE_TASK} --ignore test_json"
+    IGNORE_TESTS+=("test.test_json.test_recursion.TestCRecursion.test_highly_nested_objects_decoding")
 fi
 
-# PGO optimized / BOLT instrumented binaries segfault in a test_bytes test. Skip it.
-if [[ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_13}" && "${TARGET_TRIPLE}" == x86_64* ]]; then
-    PROFILE_TASK="${PROFILE_TASK} --ignore test.test_bytes.BytesTest.test_from_format"
-fi
+PROFILE_TASK="${PROFILE_TASK} --ignorefile ${ROOT}/profiling-training-ignores.txt"
 
-# Some tests aren't race friendly on freethreaded builds. Disable as a mitigation.
-if [[ -n "${CPYTHON_FREETHREADED}" ]]; then
-    PROFILE_TASK="${PROFILE_TASK} --ignore test.test_bytes.FreeThreadingTest.test_free_threading_bytearrayiter"
-    PROFILE_TASK="${PROFILE_TASK} --ignore test.test_hashlib.HashlibTestCase.test_threaded_hashing_fast"
-    PROFILE_TASK="${PROFILE_TASK} --ignore test.test_itertools.TestBasicOps.test_count_threading"
-    PROFILE_TASK="${PROFILE_TASK} --ignore test.test_itertools.TestBasicOps.test_count_with_step_threading"
+# Exclude whole modules from profiling based on stdlib test annotations.
+if [ -n "${PROFILING_EXCLUDE_MODULES}" ]; then
+    PROFILE_TASK="${PROFILE_TASK} --exclude ${PROFILING_EXCLUDE_MODULES}"
 fi
 
 # ./configure tries to auto-detect whether it can build 128-bit and 256-bit SIMD helpers for HACL,

cpython-unix/build.py

@@ -839,12 +839,28 @@ def build_cpython(
 
             build_env.copy_file(fh.name, dest_name="stdlib-test-annotations.json")
 
+        # Install a file with tests to skip during profile training.
+        # Also collect module excludes for the profiling run of the test harness.
+        profiling_exclude_modules = []
+        with tempfile.NamedTemporaryFile("w", encoding="utf-8") as fh:
+            os.chmod(fh.name, 0o644)
+
+            for ann in test_annotations.annotations:
+                if ann.profile_training_skip():
+                    fh.write(f"{ann.name}\n")
+                if m := ann.profile_training_exclude_module():
+                    profiling_exclude_modules.append(m)
+
+            fh.flush()
+            build_env.copy_file(fh.name, dest_name="profiling-training-ignores.txt")
+
         env = {
             "PIP_VERSION": DOWNLOADS["pip"]["version"],
             "PYTHON_VERSION": python_version,
             "PYTHON_MAJMIN_VERSION": ".".join(python_version.split(".")[0:2]),
             "SETUPTOOLS_VERSION": DOWNLOADS["setuptools"]["version"],
             "TOOLCHAIN": "clang-%s" % host_platform,
+            "PROFILING_EXCLUDE_MODULES": " ".join(profiling_exclude_modules),
         }
 
         # Set environment variables allowing convenient testing for Python

pythonbuild/cpython.py

@@ -197,6 +197,30 @@
                 "required": ["name", "reason"],
             },
         },
+        "profiling-excludes": {
+            "type": "array",
+            "items": {
+                "type": "object",
+                "properties": {
+                    "name": {"type": "string"},
+                    **STDLIB_TEST_ANNOTATION_COMMON_PROPERTIES,
+                },
+                "additionalProperties": False,
+                "required": ["name", "reason"],
+            },
+        },
+        "profiling-skips": {
+            "type": "array",
+            "items": {
+                "type": "object",
+                "properties": {
+                    "name": {"type": "string"},
+                    **STDLIB_TEST_ANNOTATION_COMMON_PROPERTIES,
+                },
+                "additionalProperties": False,
+                "required": ["name", "reason"],
+            },
+        },
     },
 }
 
@@ -817,6 +841,8 @@ def extension_modules_config(yaml_path: pathlib.Path):
 TEST_ANNOTATION_HARNESS_SKIP = "harness-skip"
 TEST_ANNOTATION_MODULE_EXCLUDE = "module-exclude"
 TEST_ANNOTATION_TEST_FAILURE = "test-failure"
+TEST_ANNOTATION_PROFILING_EXCLUDE = "profiling-exclude"
+TEST_ANNOTATION_PROFILING_SKIP = "profiling-skip"
 
 
 @dataclasses.dataclass
@@ -837,6 +863,17 @@ class TestAnnotation:
     # Whether to exclude loading the test module when running tests.
     exclude_testing: bool
 
+    def profile_training_skip(self) -> bool:
+        """Whether to ignore this test during PGO training."""
+        return self.flavor in (
+            TEST_ANNOTATION_TEST_FAILURE,
+            TEST_ANNOTATION_PROFILING_SKIP,
+        )
+
+    def profile_training_exclude_module(self) -> Optional[str]:
+        """Name of module to exclude from profiling."""
+        return self.name if self.flavor == TEST_ANNOTATION_PROFILING_EXCLUDE else None
+
 
 @dataclasses.dataclass
 class TestAnnotations:
@@ -937,6 +974,12 @@ def stdlib_test_annotations(
     for entry in data["expected-failures"]:
         raw_entries.append((TEST_ANNOTATION_TEST_FAILURE, entry))
 
+    for entry in data["profiling-excludes"]:
+        raw_entries.append((TEST_ANNOTATION_PROFILING_EXCLUDE, entry))
+
+    for entry in data["profiling-skips"]:
+        raw_entries.append((TEST_ANNOTATION_PROFILING_SKIP, entry))
+
     for flavor, entry in raw_entries:
         if a := filter_stdlib_test_entry(
             flavor,

pythonbuild/testdist.py

@@ -19,6 +19,8 @@
     TEST_ANNOTATION_HARNESS_SKIP,
     TEST_ANNOTATION_MODULE_EXCLUDE,
     TEST_ANNOTATION_TEST_FAILURE,
+    TEST_ANNOTATION_PROFILING_EXCLUDE,
+    TEST_ANNOTATION_PROFILING_SKIP,
     TestAnnotation,
     meets_python_minimum_version,
 )
@@ -164,6 +166,15 @@ def run_stdlib_tests(
             if annotation.dont_verify:
                 dont_verify.add(name)
 
+        elif flavor == "profiling-exclude":
+            print(f"profiling exclude module ignored {name}")
+            continue
+
+        elif flavor == "profiling-skip":
+            print(f"profiling skip test ignored {name}")
+
+            continue
+
         else:
             raise Exception(f"unhandled test annotation flavor: {flavor}")