Merge remote-tracking branch 'origin/main' into zuban-ignore

Author: davidhalter

.github/workflows/conformance.yml

@@ -0,0 +1,47 @@
+name: Conformance
+
+on:
+  push:
+  pull_request:
+
+permissions:
+  contents: read
+
+jobs:
+  conformance:
+    name: Run conformance suite
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+          cache: "pip"
+
+      - name: Install uv
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install uv
+
+      - name: Run conformance suite
+        working-directory: conformance
+        run: |
+          uv sync --python 3.12 --frozen
+          uv run --python 3.12 --frozen python src/main.py
+
+      - name: Validate conformance invariants
+        working-directory: conformance
+        run: |
+          uv run --python 3.12 --frozen python src/validate_results.py
+
+      - name: Assert conformance results are up to date
+        run: |
+          if [ -n "$(git status --porcelain -- conformance/results)" ]; then
+            git status --short conformance/results
+            git diff -- conformance/results
+            echo "Conformance results are out of date. Run conformance/src/main.py and commit updated results."
+            exit 1
+          fi

conformance/README.md

@@ -12,6 +12,7 @@ This project contains test cases for behaviors defined in the Python typing spec
 
 * [concepts](https://typing.python.org/en/latest/spec/concepts.html)
 * [annotations](https://typing.python.org/en/latest/spec/annotations.html)
+* [typeforms](https://typing.python.org/en/latest/spec/type-forms.html)
 * [specialtypes](https://typing.python.org/en/latest/spec/special-types.html)
 * [generics](https://typing.python.org/en/latest/spec/generics.html)
 * [qualifiers](https://typing.python.org/en/latest/spec/qualifiers.html)
@@ -70,17 +71,17 @@ by the scoring system.
 
 To run the conformance test suite:
 * Clone the https://github.com/python/typing repo.
-* Create and activate a Python 3.12 virtual environment.
-* Switch to the `conformance` subdirectory and install all dependencies (`pip install -r requirements.txt`).
-* Switch to the `src` subdirectory and run `python main.py`.
+* Install [uv](https://docs.astral.sh/uv/) and ensure Python 3.12 is available.
+* Switch to the `conformance` subdirectory and install locked dependencies (`uv sync --python 3.12 --frozen`).
+* Run the conformance tool (`uv run --python 3.12 --frozen python src/main.py`).
 
 Note that some type checkers may not run on some platforms. If a type checker fails to install, tests will be skipped for that type checker.
 
 ## Reporting Conformance Results
 
 Different type checkers report errors in different ways (with different wording in error messages and different line numbers or character ranges for errors). This variation makes it difficult to fully automate test validation given that tests will want to check for both false positive and false negative type errors. Some level of manual inspection will therefore be needed to determine whether a type checker is fully conformant with all tests in any given test file. This "scoring" process is required only when the output of a test changes — e.g. when a new version of that type checker is released and the tests are rerun. We assume that the output of a type checker will be the same from one run to the next unless/until a new version is released that fixes or introduces a bug. In this case, the output will need to be manually inspected and the conformance results re-scored for those tests whose output has changed.
 
-Conformance results are reported and summarized for each supported type checker. Currently, results are reported for mypy, pyrefly, pyright, and zuban. It is the goal and desire to add additional type checkers over time.
+[Conformance results](https://htmlpreview.github.io/?https://github.com/python/typing/blob/main/conformance/results/results.html) are reported and summarized for each supported type checker. Currently, results are reported for mypy, pyrefly, pyright, zuban and ty. It is the goal and desire to add additional type checkers over time.
 
 ## Adding a New Test Case
 
@@ -92,7 +93,22 @@ If a test is updated (augmented or fixed), the process is similar to when adding
 
 ## Updating a Type Checker
 
-If a new version of a type checker is released, re-run the test tool with the new version. If the type checker output has changed for any test cases, the tool will supply the old and new outputs. Examine these to determine whether the conformance status has changed. Once the conformance status has been updated, re-run the test tool again to regenerate the summary report.
+Type checker versions are locked in `uv.lock`.
+
+To bump all supported checkers to their latest released versions:
+
+```bash
+python scripts/bump_type_checkers.py
+```
+
+After bumping, install the new lockfile and rerun conformance:
+
+```bash
+uv sync --python 3.12 --frozen
+uv run --python 3.12 --frozen python src/main.py
+```
+
+If checker output changes for any test cases, examine those deltas to determine whether the conformance status has changed. Once the conformance status has been updated, rerun the tool to regenerate the summary report.
 
 ## Automated Conformance Checking
 

conformance/pyproject.toml

@@ -0,0 +1,16 @@
+[project]
+name = "typing-conformance"
+version = "0.1.0"
+requires-python = "==3.12.*"
+dependencies = [
+    "mypy",
+    "pyrefly",
+    "pyright",
+    "tomli",
+    "tomlkit",
+    "zuban",
+    "ty",
+]
+
+[tool.uv]
+package = false

conformance/requirements.txt

@@ -3,7 +3,7 @@ output = """
 aliases_recursive.py:19: error: Dict entry 1 has incompatible type "str": "complex"; expected "str": "int | str | float | list[Json] | dict[str, Json] | None"  [dict-item]
 aliases_recursive.py:20: error: List item 1 has incompatible type "complex"; expected "int | str | float | list[Json] | dict[str, Json] | None"  [list-item]
 aliases_recursive.py:38: error: Incompatible types in assignment (expression has type "tuple[int, tuple[str, int], tuple[int, tuple[int, list[int]]]]", variable has type "RecursiveTuple")  [assignment]
-aliases_recursive.py:39: error: Name "t6" already defined on line 38  [no-redef]
+aliases_recursive.py:39: error: Incompatible types in assignment (expression has type "tuple[int, list[int]]", variable has type "RecursiveTuple")  [assignment]
 aliases_recursive.py:50: error: Dict entry 0 has incompatible type "str": "list[int]"; expected "str": "str | int | Mapping[str, RecursiveMapping]"  [dict-item]
 aliases_recursive.py:51: error: Dict entry 2 has incompatible type "str": "list[int]"; expected "str": "str | int | Mapping[str, RecursiveMapping]"  [dict-item]
 aliases_recursive.py:52: error: Dict entry 2 has incompatible type "str": "list[int]"; expected "str": "str | int | Mapping[str, RecursiveMapping]"  [dict-item]

conformance/results/mypy/aliases_recursive.toml

@@ -3,7 +3,6 @@ notes = """
 Allows callable without kwargs to be assigned to callable with unpacked kwargs
 """
 output = """
-callables_kwargs.py:22: note: "func1" defined here
 callables_kwargs.py:46: error: Missing named argument "v1" for "func1"  [call-arg]
 callables_kwargs.py:46: error: Missing named argument "v3" for "func1"  [call-arg]
 callables_kwargs.py:51: error: Unexpected keyword argument "v4" for "func1"  [call-arg]
@@ -15,14 +14,14 @@ callables_kwargs.py:63: error: "func1" gets multiple values for keyword argument
 callables_kwargs.py:64: error: "func2" gets multiple values for keyword argument "v3"  [misc]
 callables_kwargs.py:64: error: Argument 1 to "func2" has incompatible type "int"; expected "str"  [arg-type]
 callables_kwargs.py:65: error: "func2" gets multiple values for keyword argument "v1"  [misc]
-callables_kwargs.py:101: error: Incompatible types in assignment (expression has type "def func1(**kwargs: Unpack[TD2]) -> None", variable has type "TDProtocol3")  [assignment]
+callables_kwargs.py:101: error: Incompatible types in assignment (expression has type "def func1(**kwargs: **TD2) -> None", variable has type "TDProtocol3")  [assignment]
 callables_kwargs.py:101: note: "TDProtocol3.__call__" has type "def __call__(self, *, v1: int, v2: int, v3: str) -> None"
-callables_kwargs.py:102: error: Incompatible types in assignment (expression has type "def func1(**kwargs: Unpack[TD2]) -> None", variable has type "TDProtocol4")  [assignment]
+callables_kwargs.py:102: error: Incompatible types in assignment (expression has type "def func1(**kwargs: **TD2) -> None", variable has type "TDProtocol4")  [assignment]
 callables_kwargs.py:102: note: "TDProtocol4.__call__" has type "def __call__(self, *, v1: int) -> None"
-callables_kwargs.py:103: error: Incompatible types in assignment (expression has type "def func1(**kwargs: Unpack[TD2]) -> None", variable has type "TDProtocol5")  [assignment]
+callables_kwargs.py:103: error: Incompatible types in assignment (expression has type "def func1(**kwargs: **TD2) -> None", variable has type "TDProtocol5")  [assignment]
 callables_kwargs.py:103: note: "TDProtocol5.__call__" has type "def __call__(self, v1: int, v3: str) -> None"
-callables_kwargs.py:111: error: Overlap between argument names and ** TypedDict items: "v1"  [misc]
-callables_kwargs.py:122: error: Unpack item in ** argument must be a TypedDict  [misc]
+callables_kwargs.py:111: error: Overlap between parameter names and ** TypedDict items: "v1"  [misc]
+callables_kwargs.py:122: error: Unpack item in ** parameter must be a TypedDict  [misc]
 """
 conformance_automated = "Fail"
 errors_diff = """

conformance/results/mypy/callables_kwargs.toml

@@ -1,4 +1,4 @@
-conformant = "Unupported"
+conformant = "Unsupported"
 notes = """
 Does not honor metaclass __call__ method when evaluating constructor call.
 Does not skip evaluation of __new__ and __init__ if custom metaclass call returns non-class.
@@ -7,14 +7,12 @@ conformance_automated = "Fail"
 errors_diff = """
 Line 26: Unexpected errors ['constructors_call_metaclass.py:26: error: Expression is of type "Class1", not "Never"  [assert-type]', 'constructors_call_metaclass.py:26: error: Missing positional argument "x" in call to "Class1"  [call-arg]']
 Line 39: Unexpected errors ['constructors_call_metaclass.py:39: error: Expression is of type "Class2", not "int | Meta2"  [assert-type]', 'constructors_call_metaclass.py:39: error: Missing positional argument "x" in call to "Class2"  [call-arg]']
-Line 46: Unexpected errors ['constructors_call_metaclass.py:46: error: Argument 2 for "super" not an instance of argument 1  [misc]']
 """
 output = """
 constructors_call_metaclass.py:26: error: Expression is of type "Class1", not "Never"  [assert-type]
 constructors_call_metaclass.py:26: error: Missing positional argument "x" in call to "Class1"  [call-arg]
 constructors_call_metaclass.py:39: error: Expression is of type "Class2", not "int | Meta2"  [assert-type]
 constructors_call_metaclass.py:39: error: Missing positional argument "x" in call to "Class2"  [call-arg]
-constructors_call_metaclass.py:46: error: Argument 2 for "super" not an instance of argument 1  [misc]
 constructors_call_metaclass.py:54: error: Missing positional argument "x" in call to "Class3"  [call-arg]
 constructors_call_metaclass.py:68: error: Missing positional argument "x" in call to "Class4"  [call-arg]
 """

conformance/results/mypy/constructors_call_metaclass.toml

@@ -17,33 +17,33 @@ Line 128: Unexpected errors ['constructors_callable.py:128: error: Expression is
 Line 145: Unexpected errors ['constructors_callable.py:145: error: Expression is of type "Class6Any", not "Any"  [assert-type]', 'constructors_callable.py:145: error: Too few arguments  [call-arg]']
 """
 output = """
-constructors_callable.py:36: note: Revealed type is "def (x: builtins.int) -> constructors_callable.Class1"
+constructors_callable.py:36: note: Revealed type is "def (x: int) -> constructors_callable.Class1"
 constructors_callable.py:38: error: Too few arguments  [call-arg]
 constructors_callable.py:39: error: Unexpected keyword argument "y"  [call-arg]
 constructors_callable.py:49: note: Revealed type is "def () -> constructors_callable.Class2"
 constructors_callable.py:51: error: Too many arguments  [call-arg]
-constructors_callable.py:64: note: Revealed type is "def (x: builtins.int) -> constructors_callable.Class3"
+constructors_callable.py:64: note: Revealed type is "def (x: int) -> constructors_callable.Class3"
 constructors_callable.py:66: error: Too few arguments  [call-arg]
 constructors_callable.py:67: error: Unexpected keyword argument "y"  [call-arg]
 constructors_callable.py:68: error: Too many arguments  [call-arg]
 constructors_callable.py:74: error: Incompatible return type for "__new__" (returns "int", but must return a subtype of "Class4")  [misc]
-constructors_callable.py:79: note: Revealed type is "def (x: builtins.int) -> constructors_callable.Class4"
+constructors_callable.py:79: note: Revealed type is "def (x: int) -> constructors_callable.Class4"
 constructors_callable.py:80: error: Expression is of type "Class4", not "int"  [assert-type]
 constructors_callable.py:81: error: Too few arguments  [call-arg]
 constructors_callable.py:82: error: Unexpected keyword argument "y"  [call-arg]
 constructors_callable.py:99: note: Revealed type is "def (*args: Any, **kwargs: Any) -> constructors_callable.Class5"
 constructors_callable.py:102: error: Expression is of type "Class5", not "Never"  [assert-type]
 constructors_callable.py:107: error: Expression is of type "Class5", not "Never"  [assert-type]
 constructors_callable.py:118: error: Incompatible return type for "__new__" (returns "Class6Proxy", but must return a subtype of "Class6")  [misc]
-constructors_callable.py:127: note: Revealed type is "def (x: builtins.int) -> constructors_callable.Class6"
+constructors_callable.py:127: note: Revealed type is "def (x: int) -> constructors_callable.Class6"
 constructors_callable.py:128: error: Expression is of type "Class6", not "Class6Proxy"  [assert-type]
 constructors_callable.py:128: error: Too few arguments  [call-arg]
-constructors_callable.py:144: note: Revealed type is "def (x: builtins.int) -> constructors_callable.Class6Any"
+constructors_callable.py:144: note: Revealed type is "def (x: int) -> constructors_callable.Class6Any"
 constructors_callable.py:145: error: Expression is of type "Class6Any", not "Any"  [assert-type]
 constructors_callable.py:145: error: Too few arguments  [call-arg]
-constructors_callable.py:164: note: Revealed type is "Overload(def (x: builtins.int) -> constructors_callable.Class7[builtins.int], def (x: builtins.str) -> constructors_callable.Class7[builtins.str])"
-constructors_callable.py:184: note: Revealed type is "def [T] (x: builtins.list[T`1], y: builtins.list[T`1]) -> constructors_callable.Class8[T`1]"
+constructors_callable.py:164: note: Revealed type is "Overload(def (x: int) -> constructors_callable.Class7[int], def (x: str) -> constructors_callable.Class7[str])"
+constructors_callable.py:184: note: Revealed type is "def [T] (x: list[T], y: list[T]) -> constructors_callable.Class8[T]"
 constructors_callable.py:186: error: Cannot infer function type argument  [misc]
-constructors_callable.py:195: note: Revealed type is "def [T] (x: builtins.list[T`-1], y: builtins.list[T`-1]) -> constructors_callable.Class9"
+constructors_callable.py:195: note: Revealed type is "def [T] (x: list[T], y: list[T]) -> constructors_callable.Class9"
 constructors_callable.py:197: error: Cannot infer function type argument  [misc]
 """

conformance/results/mypy/constructors_callable.toml

@@ -1,13 +1,18 @@
 conformant = "Partial"
 notes = """
+Assumes descriptor behavior only when field is assigned in class body.
 Does not correctly evaluate type of descriptor access.
 """
 output = """
+dataclasses_descriptors.py:61: error: Cannot access instance-only attribute "x" on class object  [misc]
+dataclasses_descriptors.py:62: error: Cannot access instance-only attribute "y" on class object  [misc]
 dataclasses_descriptors.py:66: error: Expression is of type "Desc2[int]", not "int"  [assert-type]
 dataclasses_descriptors.py:67: error: Expression is of type "Desc2[str]", not "str"  [assert-type]
 """
 conformance_automated = "Fail"
 errors_diff = """
+Line 61: Unexpected errors ['dataclasses_descriptors.py:61: error: Cannot access instance-only attribute "x" on class object  [misc]']
+Line 62: Unexpected errors ['dataclasses_descriptors.py:62: error: Cannot access instance-only attribute "y" on class object  [misc]']
 Line 66: Unexpected errors ['dataclasses_descriptors.py:66: error: Expression is of type "Desc2[int]", not "int"  [assert-type]']
 Line 67: Unexpected errors ['dataclasses_descriptors.py:67: error: Expression is of type "Desc2[str]", not "str"  [assert-type]']
 """

conformance/results/mypy/dataclasses_descriptors.toml

@@ -1,11 +1,19 @@
-conformant = "Partial"
+conformant = "Unsupported"
 notes = """
+Does not synthesize `__hash__ = None` as a class attribute for unhashable dataclasses.
+Does not report when an unhashable dataclass has `__hash__` called directly on an instance.
 Does not report when dataclass is not compatible with Hashable protocol.
 """
 output = """
+dataclasses_hash.py:14: error: Expression is of type "Callable[[object], int]", not "None"  [assert-type]
+dataclasses_hash.py:36: error: Expression is of type "Callable[[object], int]", not "None"  [assert-type]
 """
 conformance_automated = "Fail"
 errors_diff = """
-Line 15: Expected 1 errors
-Line 32: Expected 1 errors
+Line 17: Expected 1 errors
+Line 18: Expected 1 errors
+Line 39: Expected 1 errors
+Line 40: Expected 1 errors
+Line 14: Unexpected errors ['dataclasses_hash.py:14: error: Expression is of type "Callable[[object], int]", not "None"  [assert-type]']
+Line 36: Unexpected errors ['dataclasses_hash.py:36: error: Expression is of type "Callable[[object], int]", not "None"  [assert-type]']
 """