From 5925ef483ed8bb40d77e9ebe86f1d5a168869a10 Mon Sep 17 00:00:00 2001 From: Mauricio Villegas <5780272+mauvilsa@users.noreply.github.com> Date: Fri, 7 Aug 2026 12:04:33 +0200 Subject: [PATCH] Sort union subtypes when the argument is added instead of while parsing --- CHANGELOG.rst | 9 ++ DOCUMENTATION.rst | 92 +++++++++++++-- jsonargparse/_typehints.py | 108 +++++++++++++---- .../test_postponed_annotations.py | 2 +- jsonargparse_tests/test_subclasses.py | 4 +- jsonargparse_tests/test_typehints.py | 109 +++++++++++++++++- 6 files changed, 284 insertions(+), 40 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e7d4aadd..67ab4990 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -115,6 +115,15 @@ Changed ``ValueError`` when adding the arguments, instead of the parameter being silently skipped. ``Namespace`` is only intended for parsing results (`#948 `__). +- The subtypes of a ``Union`` are now sorted when the argument is added, instead + of only while parsing. This means that the type shown in the help tells the + order in which the subtypes are attempted. The subtypes that accept any value, + i.e. ``Any`` and the ones that can't be validated, are now moved to the end, + so that they no longer prevent the remaining subtypes from being attempted. + The same is done for ``object``, which accepts the import path of any class. + The only sorting that still happens while parsing is for list append, since it + depends on the value. See the new documentation section :ref:`union-types` + (`#949 `__). v4.50.0 (2026-07-22) diff --git a/DOCUMENTATION.rst b/DOCUMENTATION.rst index 468f6e7a..b64b93a0 100644 --- a/DOCUMENTATION.rst +++ b/DOCUMENTATION.rst @@ -521,10 +521,11 @@ Some notes about this support are: :ref:`boolean-arguments`), ``int``, ``float``, ``Decimal``, ``complex``, ``bytes``/``bytearray`` (Base64 encoding), ``range``, ``list`` (more details in :ref:`list-append`), ``Deque``, ``Iterable``, ``Sequence``, ``Any``, - ``Union``, ``Optional``, ``Type``, ``Enum``, ``PathLike``, ``UUID``, - ``timedelta``, restricted types as explained in sections - :ref:`restricted-numbers` and :ref:`restricted-strings` and path and URL types - as explained in sections :ref:`parsing-paths` and :ref:`parsing-urls`. + ``Union``/``Optional`` (more details in :ref:`union-types`), ``Type``, + ``Enum``, ``PathLike``, ``UUID``, ``timedelta``, restricted types as explained + in sections :ref:`restricted-numbers` and :ref:`restricted-strings` and path + and URL types as explained in sections :ref:`parsing-paths` and + :ref:`parsing-urls`. - ``dict``, ``Mapping``, ``MutableMapping``, ``MappingProxyType``, ``OrderedDict``, and ``TypedDict`` are supported but only with ``str`` or @@ -612,6 +613,77 @@ Some notes about this support are: (python 3.12+) and aliases created with ``typing_extensions.TypeAliasType``. +.. _union-types: + +Union types +----------- + +A value given for an argument that has a ``Union`` type is validated against +each of the subtypes, one at a time, and the first subtype that accepts it +decides the parsed value. This means that the order of the subtypes matters. For +example, for ``Union[str, int]`` the command line value ``2`` is parsed as the +``str`` ``"2"``, since any command line value is a valid ``str``, whereas for +``Union[int, str]`` it is parsed as the ``int`` ``2``. + +The subtypes are mostly attempted in the order in which they are written. The +exception are the ones that accept anything, which are sorted to the end when +the argument is added, so that the subtypes that validate get a chance of being +used. From first to last attempted, the groups are: + +1. All types not mentioned below, in the order in which they are written. +2. ``object``, which accepts the import path of any class, making any class + subtype after it unreachable. +3. ``None``, which only accepts ``null``. It is placed second to last so that + ``Optional[]`` reads in the help as it does in the source code. +4. ``Any`` and the types that can't be validated, see :ref:`unvalidated-types`. + These accept any value, so a subtype after them would never be attempted. + +The sorting is stable, meaning that subtypes in the same group keep the relative +order in which they are given. Unions nested inside other types are sorted as +well, e.g. the ``Union`` in ``list[Union[int, Any]]``. + +Be aware that ``typing`` considers two unions equal independent of the order of +the subtypes, and caches the types that it creates. This means that for a union +nested in a ``typing`` type, e.g. ``typing.List[Union[int, str]]``, the order +can end up being the one of an equal union created before somewhere else. The +`PEP 585 `__ types are not cached, so writing +``list[Union[int, str]]`` always gives the order as written. + +Since the sorting is done when the argument is added, the type shown in the +``--help`` is the sorted one. That is, the help always tells the order in which +the subtypes are attempted. For example, an argument added as: + +.. testsetup:: union + + from typing import Any, Union + + parser = ArgumentParser(exit_on_error=False) + +.. testcode:: union + + parser.add_argument("--val", type=Union[Any, int, None]) + +is shown in the help as ``(type: Union[int, null, Any], default: null)`` and +parses values as: + +.. doctest:: union + + >>> parser.parse_args(["--val=2"]) + Namespace(val=2) + >>> parser.parse_args(["--val=null"]) + Namespace(val=None) + >>> parser.parse_args(["--val=abc"]) + Namespace(val='abc') + +There is a single case in which the order is changed while parsing, instead of +when the argument is added. When appending to a list, see :ref:`list-append`, +the subtypes that are a list are moved to the front. This can only be decided +when parsing, since it depends on the value being appended to a previous list +instead of replacing it. For instance, for an argument with type ``Union[int, +list[int]]``, ``--val=1`` is parsed as ``1``, while ``--val+=1`` is parsed as +``[1]``. + + .. _unvalidated-types: Unvalidated types @@ -1016,11 +1088,13 @@ files would first assign a list and then append to this list: - 2 - 3 -Appending works for any type for the list elements. Lists with class type -elements (see :ref:`sub-classes`) are also supported. To append to the list, -first append a new class by using the ``+`` suffix. Then ``init_args`` for this -class are specified like if the type wasn't a list, since the arguments are -applied to the last class in the list. Take for example that an argument is +Appending works for any type for the list elements. When the type is a union +that has a list among its subtypes, appending changes the order in which the +subtypes are attempted, see :ref:`union-types`. Lists with class type elements +(see :ref:`sub-classes`) are also supported. To append to the list, first append +a new class by using the ``+`` suffix. Then ``init_args`` for this class are +specified like if the type wasn't a list, since the arguments are applied to the +last class in the list. Take for example that an argument is added to a parser as: .. testcode:: append diff --git a/jsonargparse/_typehints.py b/jsonargparse/_typehints.py index c4dcb8b1..ff2f2207 100644 --- a/jsonargparse/_typehints.py +++ b/jsonargparse/_typehints.py @@ -14,9 +14,10 @@ from copy import deepcopy from datetime import date, datetime from enum import Enum -from functools import partial +from functools import partial, reduce from importlib import import_module from importlib.util import find_spec +from operator import or_ from types import FunctionType, GenericAlias, MappingProxyType, ModuleType, UnionType from typing import ( Any, @@ -338,7 +339,7 @@ def __init__(self, typehint: type | None = None, enable_path: bool = False, **kw kwargs["logger"].debug(f"Discarding unsupported subtypes {discard} from {typehint}") subtypes = tuple(t for t, s in zip(typehint.__args__, subtype_supported) if s) typehint = Union[subtypes] - self._typehint = typehint + self._typehint = sort_unions_in_typehint(typehint) self._enable_path = False if is_pathlike(typehint) else enable_path elif "_typehint" not in kwargs: raise ValueError("Expected typehint keyword argument.") @@ -978,17 +979,10 @@ def replaced(typehint, reason): for a in args ) if new_args != args: - try: - if hasattr(typehint, "copy_with"): - return typehint.copy_with(new_args) - subscript_args = get_args(typehint) - if subscript_args and isinstance(subscript_args[0], list): - # a Callable that has its parameters flattened in __args__, e.g. the __args__ of - # Callable[[int], str] are (int, str), while subscripting needs them as a list - return get_typehint_origin(typehint)[[*new_args[:-1]], new_args[-1]] - return get_typehint_origin(typehint)[new_args] - except Exception: + rebuilt = rebuild_typehint_args(typehint, new_args) + if rebuilt is typehint: return replaced(typehint, unrebuildable_reason) + return rebuilt if replace_unsupported and not ActionTypeHint.is_supported_typehint(typehint, full=True): return replaced(typehint, unsupported_reason) return typehint @@ -2146,22 +2140,86 @@ def adapt_classes_any(val, serialize, instantiate_classes, sub_add_kwargs, logge return val +def union_subtype_sort_key(subtype) -> int: + """Rank of a union subtype, sorted by which is attempted first when parsing. + + The subtypes that accept the most values get a higher rank, so that sorting + moves them to the end and the ones that validate more strictly get a chance + of being used. Sorting is stable, thus subtypes with the same rank keep the + relative order in which they are given. + """ + if subtype == Any or isinstance(subtype, UnvalidatedType): + return 3 # accept any value, so nothing after them would ever be attempted + if subtype is NoneType: + return 2 # only accepts null, second to last so that Optional[] reads as in the source code + if subtype is object: + return 1 # accepts the import path of any class, so no class subtype after it would be attempted + return 0 + + +def sort_unions_in_typehint(typehint): + """Returns the type hint with the subtypes of all its unions sorted, including nested ones. + + Done when an argument is added, such that the help shows the type hint with + the subtypes of unions in the order in which they are attempted when + parsing, see union_subtype_sort_key. + """ + if get_typehint_origin(typehint) in literal_types: + return typehint # the args of a Literal are values, not types + args = getattr(typehint, "__args__", None) + # only a tuple, since e.g. types.UnionType and types.GenericAlias have __args__ as a + # class level slot descriptor, which is truthy but not the subtypes of an instance + if not isinstance(args, tuple) or not args: + return typehint + new_args = tuple(sort_unions_in_typehint(a) for a in args) + if get_typehint_origin(typehint) == Union: + new_args = tuple(sorted(new_args, key=union_subtype_sort_key)) + # compared by identity, since typing considers Union[int, str] and Union[str, int] equal + if all(new is old for new, old in zip(new_args, args)): + return typehint + return rebuild_typehint_args(typehint, new_args) + + +def rebuild_typehint_args(typehint, new_args): + """Returns the given type hint with its subtypes replaced, or unchanged if not possible.""" + try: + if isinstance(typehint, UnionType): + try: + return reduce(or_, new_args) + except TypeError: + return Union[new_args] # e.g. an UnvalidatedType subtype doesn't support | + if get_typehint_origin(typehint) == Union: + # neither Union[...] nor its copy_with are used because typing caches unions and + # considers two of them equal independent of the order of the subtypes, thus a + # previously created union with the same subtypes in a different order would be + # returned, silently undoing the sorting + return type(typehint)(Union, tuple(new_args), name=getattr(typehint, "_name", None)) + if hasattr(typehint, "copy_with"): + return typehint.copy_with(new_args) + subscript_args = get_args(typehint) + if subscript_args and isinstance(subscript_args[0], list): + # a Callable that has its parameters flattened in __args__, e.g. the __args__ of + # Callable[[int], str] are (int, str), while subscripting needs them as a list + return get_typehint_origin(typehint)[[*new_args[:-1]], new_args[-1]] + return get_typehint_origin(typehint)[new_args] + except Exception: + return typehint + + def sort_subtypes_for_union(subtypes, val, prev_val, append): + """Sorts the subtypes of a union for the parsing of a given value. + + The sort that does not depend on the value is applied first, which is a + no-op for the type hint of an added argument, since it is already sorted, + see sort_unions_in_typehint. It is still needed for type hints resolved + while parsing, e.g. the types of the keys of a TypedDict. Then, only when + appending to a list, the sequence subtypes are moved to the front, since + the value must extend the previous list instead of replacing it. + """ if len(subtypes) > 1: - if isinstance(val, str): - key_fn = lambda x: ( - x != NoneType, - get_typehint_origin(x) not in sequence_or_mapping_origin_types, - ) - else: - key_fn = lambda x: x != NoneType - subtypes = sorted(subtypes, key=key_fn) + subtypes = sorted(subtypes, key=union_subtype_sort_key) if append or (isinstance(prev_val, list) and isinstance(val, NestedArg)): - key_fn = lambda x: ( - x != NoneType, - get_typehint_origin(x) not in sequence_origin_types, - ) - subtypes = sorted(subtypes, key=key_fn) + subtypes = sorted(subtypes, key=lambda x: get_typehint_origin(x) not in sequence_origin_types) return subtypes diff --git a/jsonargparse_tests/test_postponed_annotations.py b/jsonargparse_tests/test_postponed_annotations.py index 7e53dc6e..4cc01fd7 100644 --- a/jsonargparse_tests/test_postponed_annotations.py +++ b/jsonargparse_tests/test_postponed_annotations.py @@ -388,7 +388,7 @@ def test_unresolvable_annotation_help(parser): if sys.version_info < (3, 14): optional = "Optional[Unvalidated]" else: - optional = "Unvalidated | None" + optional = "None | Unvalidated" assert f"--fn.typo TYPO (type: {optional}, default: null)" in help_str diff --git a/jsonargparse_tests/test_subclasses.py b/jsonargparse_tests/test_subclasses.py index 8ff6f508..9ea20e19 100644 --- a/jsonargparse_tests/test_subclasses.py +++ b/jsonargparse_tests/test_subclasses.py @@ -2272,11 +2272,11 @@ def test_subclass_error_indentation_invalid_init_arg(parser): expected = textwrap.dedent(""" Parser key "val": Does not validate against any of the Union subtypes - Subtypes: [, , ] + Subtypes: [, , ] Errors: - - Expected a - Expected a - Expected a + - Expected a Given value type: Given value: abc """).strip() diff --git a/jsonargparse_tests/test_typehints.py b/jsonargparse_tests/test_typehints.py index 2b684091..5f9bc697 100644 --- a/jsonargparse_tests/test_typehints.py +++ b/jsonargparse_tests/test_typehints.py @@ -103,6 +103,15 @@ def test_str_no_strip(parser): assert " " == parser.parse_args(['--cfg={"op":" "}']).op +@parser_modes +@pytest.mark.parametrize("value", ["", " "]) +def test_empty_str_not_loaded_as_null(parser, value): + # an empty string must not be loaded as null, otherwise it would be accepted by the None subtype + parser.add_argument("--op", type=Optional[int]) + with pytest.raises(ArgumentError, match="Does not validate against any of the Union subtypes"): + parser.parse_args([f"--op={value}"]) + + @pytest.mark.parametrize("value", ["2022-04-12", "2022-04-32"]) def test_str_not_timestamp(parser, value): parser.add_argument("foo", type=str) @@ -1529,6 +1538,8 @@ def test_dict_default_ordered_dict(parser): ((str, int), "=2", "2"), ((float, int), "=3", 3.0), ((int, float), "=4", 4), + ((complex, float), "=5.5", complex(5.5)), + ((float, complex), "=6.5", 6.5), ((int, List[int]), "=5", 5), ((List[int], int), "=6", 6), ((int, List[int]), "+=7", [7]), @@ -1543,6 +1554,98 @@ def test_union_subtypes_order(parser, subtypes, arg, expected): assert val == expected +unvalidated_type = UnvalidatedType("some.SomeType") + + +@pytest.mark.parametrize( + # in python 3.14+ typing.Union is types.UnionType, thus shown with the "|" syntax + ["typehint", "expected", "expected_py314"], + [ + # non-validating types last + (Union[Any, int], "Union[int, Any]", "int | Any"), + (Union[unvalidated_type, int], "Union[int, Unvalidated]", "int | Unvalidated"), + ( + Union[Any, unvalidated_type, int], + "Union[int, Any, Unvalidated]", + "int | Any | Unvalidated", + ), + (Union[str, Any], "Union[str, Any]", "str | Any"), + # object last, since it accepts the import path of any class + (Union[object, int], "Union[int, object]", "int | object"), + # None second to last, so that Optional keeps its form + (Optional[str], "Optional[str]", "str | None"), + (Optional[int], "Optional[int]", "int | None"), + (Union[Any, None], "Optional[Any]", "None | Any"), + # relative order otherwise kept + (Union[str, int], "Union[str, int]", "str | int"), + (Union[float, int], "Union[float, int]", "float | int"), + (Union[int, bool, EnumABC], "Union[int, bool, EnumABC]", "int | bool | EnumABC"), + (Union[List[int], Dict[str, int]], "Union[List[int], Dict[str, int]]", "List[int] | Dict[str, int]"), + # nested unions also reordered + (Dict[str, Union[Any, int]], "Dict[str, Union[int, Any]]", "Dict[str, int | Any]"), + (Optional[List[Union[Any, bool]]], "Optional[List[Union[bool, Any]]]", "List[bool | Any] | None"), + ( + Tuple[Union[Any, int], Union[object, int]], + "Tuple[Union[int, Any], Union[int, object]]", + "Tuple[int | Any, int | object]", + ), + ], + ids=str, +) +def test_union_subtypes_sorted_on_add_argument(parser, typehint, expected, expected_py314): + if sys.version_info >= (3, 14): + expected = expected_py314 + parser.add_argument("--val", type=typehint) + action = next(a for a in parser._actions if a.dest == "val") + assert type_to_str(action._typehint) == expected + assert f"(type: {expected}," in get_parser_help(parser) + + +@pytest.mark.parametrize( + ["typehint", "expected"], + [ + (object | int, "int | object"), + (int | None, "int | None"), + (object | int | None, "int | object | None"), + (list[object | int], "list[int | object]"), + ], + ids=str, +) +def test_union_subtypes_sorted_new_syntax(parser, typehint, expected): + # sorting a PEP 604 union keeps it as such, instead of turning it into a typing.Union + parser.add_argument("--val", type=typehint) + action = next(a for a in parser._actions if a.dest == "val") + assert type_to_str(action._typehint) == expected + + +@pytest.mark.parametrize("typehint", [Union[unvalidated_type, EnumABC], Union[Any, EnumABC]], ids=str) +def test_union_subtypes_sorted_accept_any_last_parse(parser, typehint): + # without the sorting the value would be accepted as is by the first subtype + parser.add_argument("--val", type=typehint) + assert EnumABC.A == parser.parse_args(["--val=A"]).val + assert "X" == parser.parse_args(["--val=X"]).val + + +def test_union_subtypes_sorted_nested_parse(parser): + parser.add_argument("--val", type=List[Union[Any, EnumABC]]) + assert [EnumABC.A] == parser.parse_args(['--val=["A"]']).val + + +class TypedDictUnionValue(TypedDict): + key: Union[Any, EnumABC] + + +def test_union_subtypes_sorted_typed_dict_value(parser): + # the type of the key is only resolved when parsing, so it is sorted there + parser.add_argument("--val", type=TypedDictUnionValue) + assert {"key": EnumABC.A} == parser.parse_args(['--val={"key": "A"}']).val + + +def test_union_subtypes_sorted_optional_enum_metavar(parser): + parser.add_argument("--val", type=Optional[EnumABC]) + assert "--val {A,B,C,null}" in get_parser_help(parser) + + def test_union_unsupported_subtype(parser, logger): parser.logger = logger with capture_logs(logger) as logs: @@ -2312,9 +2415,9 @@ def test_unsupported_subtypes_help(parser): help_str = get_parser_help(parser, strip=True) # only the parts that are not supported are shown as unvalidated if sys.version_info < (3, 14): - union = "Union[Unvalidated, int]" + union = "Union[int, Unvalidated]" else: - union = "Unvalidated | int" + union = "int | Unvalidated" assert "type: List[Unvalidated]" in help_str assert f"type: {union}" in help_str @@ -2338,7 +2441,7 @@ def test_unsupported_type_not_required_added(parser): if sys.version_info < (3, 14): optional = "Optional[Unvalidated]" else: - optional = "Unvalidated | None" + optional = "None | Unvalidated" assert f"--fn.p1 P1 (type: {optional}, default: null)" in help_str