diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8c4f0b9b..fb1ab783 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -44,6 +44,9 @@ Added - Support ``Collection``, ``Container`` and ``Reversible``, validated as a list, and ``AbstractSet``, validated as a set (`#950 `__). +- ``shtab`` completion scripts now include the fields of a dataclass-like type + that is not added as a group, e.g. ``Optional[SomeDataclass]`` (`#952 + `__). Fixed ^^^^^ @@ -103,6 +106,23 @@ Fixed input data from keyword arguments``. Now the nearest class docstring in the method resolution order is used, skipping base classes that only provide machinery (`#951 `__). +- A dataclass-like type accepting the ``class_path`` of a subclass when it is + not added as a group, e.g. ``Optional[SomeDataclass]``, even though subclasses + are disabled for these types by default. The error now says how to enable + subclass support for the type, see :ref:`subclasses-disabled` (`#952 + `__). +- Signature parameters typed as a dataclass-like type that is not added as a + group, e.g. ``Optional[SomeDataclass]``, ``list[SomeDataclass]`` and + ``dict[str, SomeDataclass]``, not accepting a path to a sub-config file even + when added with ``sub_configs=True``, failing with e.g. ``No module named + 'data'``. Also, the path of a loaded sub-config was not kept, so ``save`` with + ``multifile=True`` did not write it back to its own file. See + :ref:`sub-config-files` (`#952 + `__). +- The value of an environment variable not validating for a dataclass-like typed + argument of a subcommand, e.g. ``APP_SUB__DATA='{"p1": 2}'`` failing with + ``Not a valid subclass of ...`` (`#952 + `__). Changed ^^^^^^^ @@ -138,6 +158,12 @@ Changed that they no longer prevent the remaining subtypes from being attempted. See the new documentation section :ref:`union-types` (`#949 `__). +- A dataclass-like type added as a group now accepts a subclass spec that has + the ``class_path`` of the type itself, e.g. ``{"class_path": "Data", + "init_args": {...}}``, instead of failing with ``Group 'data' does not accept + option 'init_args....'``. Dataclass-like types now accept the same values + whether or not they are added as a group, see :ref:`subclasses-disabled` + (`#952 `__). v4.50.0 (2026-07-22) diff --git a/DOCUMENTATION.rst b/DOCUMENTATION.rst index 85ad86ac..fc1daf77 100644 --- a/DOCUMENTATION.rst +++ b/DOCUMENTATION.rst @@ -2370,6 +2370,14 @@ moved around without needing to modify them. Furthermore, :meth:`save <.ArgumentParser.save>` with ``multifile=True`` writes back each sub-config to its own file, preserving the original structure. +Dataclass-like types, see :ref:`subclasses-disabled`, also accept a sub-config +file, the difference being that its content are the fields of the type, without +``class_path`` and ``init_args``. Note that this only applies when the type is +not added as an argument group, i.e. when it is part of a larger type, e.g. +``Optional[SomeDataclass]`` or ``list[SomeDataclass]``. When added as a group, +the group's own config argument accepts the path, e.g. ``--data=data.yaml``, +independent of ``sub_configs``. + .. _instance-factories: @@ -2675,6 +2683,16 @@ classes technically support subclassing, subclass support can be enabled as described below. Subclass support has been kept disabled for these types by default to avoid introducing breaking changes. +A type with subclasses disabled is added as an argument group when it is the +entire type of an argument, such that each of its init args is an individual +argument, e.g. ``--data.number``. This is not the case when the type is part of +a larger type, e.g. ``Optional[FinalClass]`` or ``list[FinalClass]``, since then +a single argument must accept the entire value. Independent of this, the +accepted values are the same. A subclass spec is accepted, though only with the +``class_path`` of the type itself, i.e. ``--data={"class_path": "FinalClass", +"init_args": {"number": 8}}``. The ``class_path`` of a subclass is not accepted, +unless subclass support is enabled for the type as described next. + .. _enable-disable-subclasses: diff --git a/jsonargparse/_actions.py b/jsonargparse/_actions.py index 77bd9262..c1502d8a 100644 --- a/jsonargparse/_actions.py +++ b/jsonargparse/_actions.py @@ -229,6 +229,10 @@ def is_print_config_requested(parser): return False +class SubclassesDisabledError(TypeError): + """Raised when a class_path is given for a type that has subclasses disabled.""" + + class _ActionConfigLoad(Action): def __init__(self, basetype: type | None = None, **kwargs): if len(kwargs) == 0: @@ -253,14 +257,56 @@ def __call__(self, *args, **kwargs): namespace[self.dest] = loaded_value return None + def resolve_subclass_spec(self, value): + """Resolves a subclass spec given for a subclasses disabled type, e.g. a dataclass. + + These types are added as a group, such that their init args are individual arguments. Still a + subclass spec is accepted, so that the accepted values are the same as for the optional + counterpart of the type, which is added as a single typehint argument. + """ + from ._typehints import is_subclass_spec, resolve_class_path_by_name, subclasses_disabled_message + + if self.basetype is None or not is_subclasses_disabled(self.basetype): + return value + + def resolve_class(class_path): + try: + return import_object(resolve_class_path_by_name(self.basetype, class_path)) + except Exception: + return None + + if isinstance(value, str): + if resolve_class(value) is None: + return value # not a class path, e.g. a path to a config file + value = Namespace(class_path=value) + elif not is_subclass_spec(value): + return value + class_path = value["class_path"] + if resolve_class(class_path) is not self.basetype: + raise SubclassesDisabledError(subclasses_disabled_message(self.basetype, class_path)) + resolved = Namespace() + for key in ["init_args", "dict_kwargs"]: + sub_value = value.get(key) + if isinstance(sub_value, dict): + sub_value = Namespace(sub_value) + if isinstance(sub_value, Namespace): + resolved.update(sub_value) + return resolved + def _load_config(self, value, parser): try: - cfg, cfg_path = parse_value_or_config(value) - if not isinstance(cfg, dict): + cfg = self.resolve_subclass_spec(value) + cfg_path = None + if cfg is value: + cfg, cfg_path = parse_value_or_config(value) + cfg = self.resolve_subclass_spec(cfg) + if not isinstance(cfg, (dict, Namespace)): raise TypeError(f'Parser key "{self.dest}": Unable to load config "{value}"') with load_config_path_context(cfg_path), change_to_path_dir(cfg_path): cfg = parser._apply_actions(cfg, parent_key=self.dest) return cfg + except SubclassesDisabledError as ex: + raise TypeError(f'Parser key "{self.dest}":\n{indent_text(str(ex))}') from ex except (TypeError,) + get_loader_exceptions() as ex: str_ex = indent_text(f"- {ex}") raise TypeError(f'Parser key "{self.dest}":\nUnable to load config {value!r}\n{str_ex}') from ex diff --git a/jsonargparse/_completions.py b/jsonargparse/_completions.py index 10459054..1c6b8880 100644 --- a/jsonargparse/_completions.py +++ b/jsonargparse/_completions.py @@ -14,7 +14,12 @@ from typing import Literal, Union from ._actions import ActionConfigFile, ActionFail, _ActionConfigLoad, _ActionHelpClassPath, remove_actions -from ._common import NonParsingAction, get_optionals_as_positionals_actions, get_parsing_setting +from ._common import ( + NonParsingAction, + get_optionals_as_positionals_actions, + get_parsing_setting, + is_subclasses_disabled, +) from ._optionals import get_pydantic_path_type from ._parameter_resolvers import get_signature_parameters from ._typehints import ( @@ -23,6 +28,7 @@ get_all_subclass_paths, get_callable_return_type, get_typehint_origin, + is_single_subclass_or_closed_type, is_subclass, type_to_str, ) @@ -347,6 +353,13 @@ def get_choices_state(typehint) -> tuple[list[str], bool, bool]: choices = add_subactions_and_get_subclass_choices(typehint, prefix, parser, skip, added_subclasses) return choices, True, False + if is_single_subclass_or_closed_type(typehint, origin) and is_subclasses_disabled(typehint): + # a closed type, e.g. a dataclass, only inlined as a group when not in a union, + # so its init args need to be added as options for them to be completed + added_subclasses.add(typehint) + add_subactions_and_get_subclass_choices(typehint, prefix, parser, skip, added_subclasses, closed_type=True) + return [], False, True + if origin in callable_origin_types: return_type = get_callable_return_type(typehint) if return_type and ActionTypeHint.is_subclass_typehint(return_type): @@ -363,18 +376,23 @@ def get_choices_state(typehint) -> tuple[list[str], bool, bool]: return choices, require_prefix -def add_subactions_and_get_subclass_choices(typehint, prefix, parser, skip, added_subclasses) -> list[str]: - choices = [] - paths = get_all_subclass_paths(typehint) +def add_subactions_and_get_subclass_choices( + typehint, prefix, parser, skip, added_subclasses, closed_type: bool = False +) -> list[str]: + choices: list[str] = [] init_args = defaultdict(list) subclasses = defaultdict(list) - for path in paths: - choices.append(path) + # a closed type is not a choice, since only its init args are accepted + classes: list = [typehint] if closed_type else get_all_subclass_paths(typehint) + for class_or_path in classes: + name = class_or_path if isinstance(class_or_path, str) else class_or_path.__name__ + if isinstance(class_or_path, str): + choices.append(class_or_path) try: - cls = import_object(path) + cls = import_object(class_or_path) if isinstance(class_or_path, str) else class_or_path params = get_signature_parameters(cls, None, parser._logger) except Exception as ex: - parser._logger.debug(f"Unable to get signature parameters for '{path}': {ex}") + parser._logger.debug(f"Unable to get signature parameters for '{name}': {ex}") continue num_skip = next((s for s in skip if isinstance(s, int)), 0) if num_skip > 0: @@ -382,7 +400,7 @@ def add_subactions_and_get_subclass_choices(typehint, prefix, parser, skip, adde for param in params: if param.name not in skip: init_args[param.name].append(param.annotation) - subclasses[param.name].append(path.rsplit(".", 1)[-1]) + subclasses[param.name].append(name.rsplit(".", 1)[-1]) if prefix is not None: for name, subtypes in init_args.items(): @@ -394,8 +412,9 @@ def add_subactions_and_get_subclass_choices(typehint, prefix, parser, skip, adde subtype, option_string, parser, skip, added_subclasses ) if shtab_shell.get() == "bash": - message = f"Expected type: {type_to_str(subtype)}; " - message += f"Accepted by subclasses: {', '.join(subclasses[name])}" + message = f"Expected type: {type_to_str(subtype)}" + if not closed_type: + message += f"; Accepted by subclasses: {', '.join(subclasses[name])}" add_bash_typehint_completion( parser, action, diff --git a/jsonargparse/_core.py b/jsonargparse/_core.py index 82d8df16..3ccbcf6a 100644 --- a/jsonargparse/_core.py +++ b/jsonargparse/_core.py @@ -367,6 +367,7 @@ def _parse_common( skip_required: bool = False, skip_subcommands: bool = False, fail_no_subcommand: bool = True, + nested_parse: bool = False, ) -> Namespace: """Common parsing code used by other parse methods. @@ -378,6 +379,8 @@ def _parse_common( skip_required: Whether to skip check of required arguments. skip_subcommands: Whether to skip subcommand processing. fail_no_subcommand: Whether to fail if no subcommand given. + nested_parse: Whether the result is to be merged into an ongoing parse of another parser, + in which case the internal representation of values must be preserved. Returns: A config object with all parsed values. @@ -406,7 +409,7 @@ def _parse_common( if not skip_validation: self.validate(cfg, skip_required=skip_required) - if not lenient_check.get(): + if not lenient_check.get() and not nested_parse: cfg = subclasses_disabled_remove_class_path(cfg) return cfg @@ -607,7 +610,9 @@ def parse_env( Raises: ArgumentError: If the parsing fails and ``exit_on_error=False``. """ - skip_validation, skip_subcommands = get_private_kwargs(kwargs, _skip_validation=False, _skip_subcommands=False) + skip_validation, skip_subcommands, nested_parse = get_private_kwargs( + kwargs, _skip_validation=False, _skip_subcommands=False, _nested_parse=False + ) try: cfg = self._parse_defaults_and_environ(defaults, env=True, environ=env) @@ -617,6 +622,7 @@ def parse_env( "defaults": defaults, "skip_validation": skip_validation, "skip_subcommands": skip_subcommands, + "nested_parse": nested_parse, } if skip_validation: kwargs["fail_no_subcommand"] = False @@ -1383,6 +1389,7 @@ def _apply_actions( with parser_context(parent_parser=self, lenient_check=True): value = self._check_value_key(action, value, action_dest, prev_cfg, append=append) if isinstance(action, _ActionConfigLoad): + value = action.resolve_subclass_spec(value) config_keys.add(action_dest) keys.append(action_dest) elif isinstance(action, ActionConfigFile): diff --git a/jsonargparse/_signatures.py b/jsonargparse/_signatures.py index 38346d2c..1177be1f 100644 --- a/jsonargparse/_signatures.py +++ b/jsonargparse/_signatures.py @@ -455,11 +455,12 @@ def _add_signature_parameter( prefix = f"{name}.init_args." nested_skip = {s[len(prefix) :] for s in skip or [] if s.startswith(prefix)} sub_add_kwargs["skip"] = nested_skip + # also_closed since dataclass-like types accept a sub-config when not added as a group enable_path = sub_configs and ( - is_subclass_typehint + ActionTypeHint.is_subclass_typehint(annotation, all_subtypes=False, also_closed=True) or is_return_subclass_typehint or is_list_pathlike(annotation) - or is_subclass_container_typehint(annotation) + or is_subclass_container_typehint(annotation, also_closed=True) ) args = ActionTypeHint.prepare_add_argument( args=args, @@ -588,6 +589,8 @@ def _create_group_if_requested( name = obj.__name__ if nested_key is None else nested_key group = self.add_argument_group(strip_title(doc_group), name=name) if config_load and nested_key is not None: + if config_load_type is None and inspect.isclass(obj): + config_load_type = obj group.add_argument("--" + nested_key, action=_ActionConfigLoad(basetype=config_load_type)) if inspect.isclass(obj) and nested_key is not None and instantiate: group.dest = nested_key.replace("-", "_") diff --git a/jsonargparse/_subcommands.py b/jsonargparse/_subcommands.py index 13a77a7e..3714cd13 100644 --- a/jsonargparse/_subcommands.py +++ b/jsonargparse/_subcommands.py @@ -262,7 +262,9 @@ def handle_subcommands( subnamespace = None key = prefix + subcommand if env: - subnamespace = subparser.parse_env(defaults=defaults, _skip_validation=True) + # nested_parse so that the values keep the same internal representation as the ones + # already in cfg, otherwise merging the two gives a value that is neither + subnamespace = subparser.parse_env(defaults=defaults, _skip_validation=True, _nested_parse=True) elif defaults: subnamespace = subparser.get_defaults(skip_validation=True) diff --git a/jsonargparse/_typehints.py b/jsonargparse/_typehints.py index f5442089..fa57c432 100644 --- a/jsonargparse/_typehints.py +++ b/jsonargparse/_typehints.py @@ -456,7 +456,8 @@ def is_supported_typehint(typehint, full=False): return supported @staticmethod - def is_subclass_typehint(typehint, all_subtypes=True, also_lists=False): + def is_subclass_typehint(typehint, all_subtypes=True, also_lists=False, also_closed=False): + """Whether the type expects a class. also_closed includes types that have subclasses disabled.""" typehint = typehint_from_action(typehint) if typehint is None: return False @@ -465,8 +466,10 @@ def is_subclass_typehint(typehint, all_subtypes=True, also_lists=False): if typehint_origin == Union or (also_lists and typehint_origin in sequence_origin_types): subtypes = [a for a in typehint.__args__ if a != NoneType] test = all if all_subtypes else any - k = {"also_lists": also_lists} + k = {"also_lists": also_lists, "also_closed": also_closed} return test(ActionTypeHint.is_subclass_typehint(s, **k) for s in subtypes) + if also_closed: + return is_single_subclass_or_closed_type(typehint, typehint_origin) return is_single_subclass_type(typehint, typehint_origin) @staticmethod @@ -830,7 +833,7 @@ def is_list_pathlike(typehint) -> bool: return False -def is_subclass_container_typehint(typehint) -> bool: +def is_subclass_container_typehint(typehint, also_closed: bool = False) -> bool: """Whether a container type, e.g. list or dict, has classes as items.""" typehint = get_unaliased_type(typehint) subtypehints = getattr(typehint, "__args__", None) @@ -838,12 +841,12 @@ def is_subclass_container_typehint(typehint) -> bool: return False typehint_origin = get_typehint_origin(typehint) if typehint_origin == Union: - return any(is_subclass_container_typehint(s) for s in subtypehints) + return any(is_subclass_container_typehint(s, also_closed) for s in subtypehints) if typehint_origin in sequence_or_mapping_origin_types: return any( - ActionTypeHint.is_subclass_typehint(s, all_subtypes=False) + ActionTypeHint.is_subclass_typehint(s, all_subtypes=False, also_closed=also_closed) or ActionTypeHint.is_return_subclass_typehint(s) - or is_subclass_container_typehint(s) + or is_subclass_container_typehint(s, also_closed) for s in subtypehints ) return False @@ -1534,6 +1537,12 @@ def adapt_typehints( return val_class # importable instance if is_protocol(val_class): raise_unexpected_value(f"Expected an instantiatable class, but {val['class_path']} is a protocol") + if ( + is_subclasses_disabled(typehint) + and inspect.isclass(val_class) + and val_class is not get_generic_origin(typehint) + ): + raise_unexpected_value(subclasses_disabled_message(typehint, val["class_path"])) subclass = True if not is_subclass_or_implements_protocol(val_class, typehint): subclass = False @@ -2115,6 +2124,15 @@ def adapt_class_type( return _subclasses_disabled_mark(value, typehint) +def subclasses_disabled_message(typehint, class_path) -> str: + name = getattr(typehint, "__name__", str(typehint)) + return ( + f"Subclasses are disabled for {name}, thus {class_path!r} is not accepted as class_path. " + f"Only the class_path of {name} itself is accepted, or its init args given directly. " + f"To accept subclasses use set_parsing_settings(subclasses_enabled=[{name}])." + ) + + def _subclasses_disabled_mark(value, typehint): if is_subclasses_disabled(typehint) and value.class_path == get_import_path(typehint): value[subclasses_disabled_meta_key] = True @@ -2135,7 +2153,10 @@ def subclasses_disabled_remove_class_path(value): value[key] = tuple(subclasses_disabled_remove_class_path(item) for item in val) if value.pop(subclasses_disabled_meta_key, False): - return Namespace({**value.get("init_args", {}), **value.get("dict_kwargs", {})}) + init_args = Namespace({**value.get("init_args", {}), **value.get("dict_kwargs", {})}) + if "__path__" in value: # the value came from a sub-config file + init_args["__path__"] = value["__path__"] + return init_args return value diff --git a/jsonargparse_tests/test_dataclasses.py b/jsonargparse_tests/test_dataclasses.py index ced35778..7f2270f7 100644 --- a/jsonargparse_tests/test_dataclasses.py +++ b/jsonargparse_tests/test_dataclasses.py @@ -265,9 +265,10 @@ def test_add_argument_dataclass_type(parser): def test_add_argument_dataclass_unexpected_keys(parser): parser.add_argument("--b", type=DataClassB) invalid = { - "class_path": f"{__name__}.DataClassB", + "b1": 2.0, + "unexpected": 1, } - with pytest.raises(ArgumentError, match="Group 'b' does not accept option 'class_path'"): + with pytest.raises(ArgumentError, match="Group 'b' does not accept option 'unexpected'"): parser.parse_args([f"--b={json.dumps(invalid)}"]) @@ -850,10 +851,58 @@ def test_dataclass_subclasses_disabled(parser): assert "--data.help" not in help_str config = {"class_path": f"{__name__}.DataSub", "init_args": {"p2": "y"}} - with pytest.raises(ArgumentError, match="Group 'data' does not accept option 'init_args.p2'"): + with pytest.raises(ArgumentError, match="Subclasses are disabled for DataMain"): parser.parse_args([f"--data={json.dumps(config)}"]) +# same capabilities for a dataclass-like type and its optional counterpart + + +data_main_types = [DataMain, Optional[DataMain]] + + +@pytest.mark.parametrize("data_type", data_main_types) +def test_dataclass_optional_symmetry_own_class_path(parser, data_type): + parser.add_argument("--data", type=data_type, default=DataMain(p1=2)) + + config = {"class_path": f"{__name__}.DataMain", "init_args": {"p1": 3}} + cfg = parser.parse_args([f"--data={json.dumps(config)}"]) + assert cfg.data == Namespace(p1=3) + init = parser.instantiate(cfg) + assert init.data == DataMain(p1=3) + assert json_or_yaml_load(parser.dump(cfg))["data"] == {"p1": 3} + + +@pytest.mark.parametrize("data_type", data_main_types) +def test_dataclass_optional_symmetry_class_path_only(parser, data_type): + parser.add_argument("--data", type=data_type, default=DataMain(p1=2)) + + cfg = parser.parse_args([f'--data={{"class_path": "{__name__}.DataMain"}}']) + assert cfg.data == Namespace(p1=2) + + +@pytest.mark.parametrize("data_type", data_main_types) +def test_dataclass_optional_symmetry_subclass_disabled(parser, data_type): + parser.add_argument("--data", type=data_type, default=DataMain(p1=2)) + + config = {"class_path": f"{__name__}.DataSub", "init_args": {"p2": "y"}} + with pytest.raises(ArgumentError, match="Subclasses are disabled for DataMain"): + parser.parse_args([f"--data={json.dumps(config)}"]) + enable_hint = r"set_parsing_settings\(subclasses_enabled=\[DataMain\]\)" + with pytest.raises(ArgumentError, match=enable_hint): + parser.parse_args(["--data=DataSub"]) + + +@pytest.mark.parametrize("data_type", data_main_types) +def test_dataclass_optional_symmetry_subclass_enabled(parser, data_type, enable_subclasses): + parser.add_argument("--data", type=data_type, default=DataMain(p1=2)) + + config = {"class_path": f"{__name__}.DataSub", "init_args": {"p2": "y"}} + cfg = parser.parse_args([f"--data={json.dumps(config)}"]) + init = parser.instantiate(cfg) + assert init.data == DataSub(p1=2, p2="y") + + def test_add_subclass_dataclass_subclasses_disabled(parser): with pytest.raises(ValueError, match="Expected .* a subclass type or a tuple of subclass types"): parser.add_subclass_arguments(DataMain, "data") @@ -955,7 +1004,7 @@ def test_add_argument_dataclass_single_type_subclasses_disabled(parser, enable_s parser.add_argument("--data", type=DataMain, default=DataMain(p1=2)) config = {"class_path": f"{__name__}.DataSub", "init_args": {"p2": "y"}} - with pytest.raises(ArgumentError, match="Group 'data' does not accept option 'init_args.p2'"): + with pytest.raises(ArgumentError, match="Subclasses are disabled for DataMain"): parser.parse_args([f"--data={json.dumps(config)}"]) @@ -969,7 +1018,7 @@ def is_data_main(obj): parser.add_argument("--data", type=DataMain, default=DataMain(p1=2)) config = {"class_path": f"{__name__}.DataSub", "init_args": {"p2": "y"}} - with pytest.raises(ArgumentError, match="Group 'data' does not accept option 'init_args.p2'"): + with pytest.raises(ArgumentError, match="Subclasses are disabled for DataMain"): parser.parse_args([f"--data={json.dumps(config)}"]) @@ -993,10 +1042,29 @@ def test_dataclass_nested_subclasses_disabled(parser): } }, } - with pytest.raises(ArgumentError, match="Group 'data' does not accept option 'init_args.p1'"): + with pytest.raises(ArgumentError, match="Subclasses are disabled for DataMain"): parser.parse_args([f"--parent={json.dumps(config)}"]) +def test_dataclass_nested_own_class_path(parser): + parser.add_argument("--parent", type=ParentData) + + config = { + "class_path": f"{__name__}.ParentData", + "init_args": { + "data": { + "class_path": f"{__name__}.DataMain", + "init_args": {"p1": 3}, + } + }, + } + cfg = parser.parse_args([f"--parent={json.dumps(config)}"]) + assert cfg.parent.init_args.data == Namespace(p1=3) + assert json_or_yaml_load(parser.dump(cfg))["parent"]["init_args"]["data"] == {"p1": 3} + init = parser.instantiate(cfg) + assert init.parent.data == DataMain(p1=3) + + def test_dataclass_nested_subclasses_enabled(parser, enable_subclasses): parser.add_argument("--parent", type=ParentData) diff --git a/jsonargparse_tests/test_paths.py b/jsonargparse_tests/test_paths.py index 595656b6..b3b88c0c 100644 --- a/jsonargparse_tests/test_paths.py +++ b/jsonargparse_tests/test_paths.py @@ -1,5 +1,6 @@ from __future__ import annotations +import dataclasses import json import os import pathlib @@ -739,6 +740,90 @@ def test_sub_configs_dict_subclass_values_from_signature(parser, item_subconfigs assert_items([cfg.main.objects["a"], cfg.main.objects["b"]]) +# sub_configs for dataclass-like types, i.e. subclasses disabled + + +@dataclasses.dataclass +class ItemData: + x: int = 1 + y: str = "-" + + +class ItemDataMain: + def __init__( + self, + single: Optional[ItemData] = None, + objects: Optional[List[ItemData]] = None, + mapping: Optional[Dict[str, ItemData]] = None, + ): + self.single = single # pragma: no cover + + +@pytest.fixture +def data_subconfig(tmp_cwd): + pathlib.Path("data.yaml").write_text(json_or_yaml_dump({"x": 2, "y": "a"})) + return tmp_cwd + + +def assert_data_item(item): + assert item == Namespace(x=2, y="a", __path__=item["__path__"]) + assert str(item["__path__"]) == "data.yaml" + + +def test_sub_configs_optional_dataclass(parser, data_subconfig): + parser.add_argument("--data", type=Optional[ItemData], sub_configs=True) + + cfg = parser.parse_args(["--data=data.yaml"]) + assert_data_item(cfg.data) + + +def test_sub_configs_optional_dataclass_from_signature(parser, data_subconfig): + parser.add_class_arguments(ItemDataMain, "main", sub_configs=True) + + cfg = parser.parse_args(["--main.single=data.yaml"]) + assert_data_item(cfg.main.single) + init = parser.instantiate(cfg) + assert init.main.single == ItemData(x=2, y="a") + + +def test_sub_configs_list_dataclass_from_signature(parser, data_subconfig): + parser.add_class_arguments(ItemDataMain, "main", sub_configs=True) + + cfg = parser.parse_args(['--main.objects=["data.yaml"]']) + assert_data_item(cfg.main.objects[0]) + + +def test_sub_configs_dict_dataclass_from_signature(parser, data_subconfig): + parser.add_class_arguments(ItemDataMain, "main", sub_configs=True) + + cfg = parser.parse_args(['--main.mapping={"a": "data.yaml"}']) + assert_data_item(cfg.main.mapping["a"]) + + +def test_sub_configs_optional_dataclass_save_multifile(parser, data_subconfig): + main = {"main": {"single": "data.yaml"}} + pathlib.Path("config.yaml").write_text(json_or_yaml_dump(main)) + out_dir = data_subconfig / "out" + out_dir.mkdir() + + parser.add_argument("--cfg", action="config") + parser.add_class_arguments(ItemDataMain, "main", sub_configs=True) + + cfg = parser.parse_args(["--cfg=config.yaml"]) + parser.save(cfg, out_dir / "config.yaml", multifile=True) + + assert json_or_yaml_load((out_dir / "config.yaml").read_text())["main"] == main["main"] + assert json_or_yaml_load((out_dir / "data.yaml").read_text()) == {"x": 2, "y": "a"} + + +def test_sub_configs_optional_dataclass_path_not_exist(parser, data_subconfig): + parser.add_class_arguments(ItemDataMain, "main", sub_configs=True) + + with pytest.raises(ArgumentError) as ctx: + parser.parse_args(["--main.single=does-not-exist.yaml"]) + ctx.match("Unexpected import path format: does-not-exist.yaml") + + def test_sub_configs_list_subclass_path_not_exist(parser, item_subconfigs): parser.add_argument("--objects", type=List[ItemBase], sub_configs=True) diff --git a/jsonargparse_tests/test_pydantic.py b/jsonargparse_tests/test_pydantic.py index e8e7a04e..aac5c483 100644 --- a/jsonargparse_tests/test_pydantic.py +++ b/jsonargparse_tests/test_pydantic.py @@ -552,6 +552,19 @@ def test_model_argument_subclasses_enabled(parser, subtests, enable_subclasses): assert dump == expected +@pytest.mark.parametrize("optional", [False, True]) +def test_model_argument_symmetry_subclasses_disabled(parser, optional): + parser.add_argument("--cat", type=Optional[Cat] if optional else Cat) + + value = {"class_path": f"{__name__}.Cat", "init_args": {"name": "cc", "meows": 2}} + cfg = parser.parse_args([f"--cat={json.dumps(value)}"]) + assert cfg.cat == Namespace(name="cc", meows=2) + + value["class_path"] = f"{__name__}.SpecialCat" + with pytest.raises(ArgumentError, match="Subclasses are disabled for Cat"): + parser.parse_args([f"--cat={json.dumps(value)}"]) + + def test_convert_to_dict_closed_to_subclasses(): converted = convert_to_dict(person) assert converted == person_expected_dict diff --git a/jsonargparse_tests/test_shtab.py b/jsonargparse_tests/test_shtab.py index 5d4db154..15f555c1 100644 --- a/jsonargparse_tests/test_shtab.py +++ b/jsonargparse_tests/test_shtab.py @@ -1,3 +1,4 @@ +import dataclasses import re import shlex import subprocess @@ -466,6 +467,33 @@ def test_bash_nested_subclasses(parser, subtests): ) +@dataclasses.dataclass +class Area: + latitude: float + longitude: float + radius: float = 500.0 + + +@pytest.mark.parametrize("area_type", [Area, Optional[Area]]) +def test_bash_dataclass_fields(parser, area_type): + parser.add_argument("--area", type=area_type) + shtab_script = get_shtab_script(parser, "bash") + options = get_bash_array(shtab_script, "_shtab_tool_option_strings") + assert {"--area", "--area.latitude", "--area.longitude", "--area.radius"}.issubset(options) + + +def test_bash_optional_dataclass_field_types(parser, subtests): + parser.add_argument("--area", type=Optional[Area]) + assert_bash_typehint_completions( + subtests, + parser, + [ + ("area.latitude", float, "", [], None), + ("area.radius", float, "5", [], None), + ], + ) + + def test_bash_callable_return_class(parser, subtests): parser.add_argument("--cls", type=Callable[[int], Base]) shtab_script = get_shtab_script(parser, "bash") diff --git a/jsonargparse_tests/test_subcommands.py b/jsonargparse_tests/test_subcommands.py index 5c7c783a..72c23dd3 100644 --- a/jsonargparse_tests/test_subcommands.py +++ b/jsonargparse_tests/test_subcommands.py @@ -1,8 +1,10 @@ from __future__ import annotations +import dataclasses import json import os from pathlib import Path +from typing import Optional from unittest.mock import patch import pytest @@ -236,6 +238,47 @@ def test_subcommand_env_overrides_default_config(parser, subparser, tmp_cwd): assert cfg.create.stats is False +@dataclasses.dataclass +class EnvArea: + latitude: float + longitude: float + radius: float = 500.0 + + +def test_subcommand_env_dataclass_value(parser, subparser): + parser.env_prefix = "APP" + parser.default_env = True + subparser.add_argument("--area", type=Optional[EnvArea]) + subparser.add_argument("--limit", type=int, default=20) + subcommands = parser.add_subcommands() + subcommands.add_subcommand("search", subparser) + + expected = Namespace(latitude=35.7, longitude=139.7, radius=500.0) + with patch.dict(os.environ, {"APP_SEARCH__AREA": '{"latitude": 35.7, "longitude": 139.7}'}): + cfg = parser.parse_args(["search"]) + assert cfg.search.area == expected + assert parser.instantiate(cfg).search.area == EnvArea(latitude=35.7, longitude=139.7) + cfg = parser.parse_args(["search", "--limit=2"]) + assert cfg.search.area == expected + + env = {"APP_SUBCOMMAND": "search", "APP_SEARCH__AREA": '{"latitude": 1.0, "longitude": 2.0}'} + with patch.dict(os.environ, env): + cfg = parser.parse_env() + assert cfg.search.area == Namespace(latitude=1.0, longitude=2.0, radius=500.0) + + +def test_subcommand_env_dataclass_value_overridden_by_command_line(parser, subparser): + parser.env_prefix = "APP" + parser.default_env = True + subparser.add_argument("--area", type=Optional[EnvArea]) + subcommands = parser.add_subcommands() + subcommands.add_subcommand("search", subparser) + + with patch.dict(os.environ, {"APP_SEARCH__AREA": '{"latitude": 35.7, "longitude": 139.7}'}): + cfg = parser.parse_args(["search", '--area={"latitude": 1.0, "longitude": 2.0, "radius": 10.0}']) + assert cfg.search.area == Namespace(latitude=1.0, longitude=2.0, radius=10.0) + + def test_subcommand_required_false(parser, subparser): subcommands = parser.add_subcommands(required=False) subcommands.add_subcommand("foo", subparser)