From 0737346f9f1d86d745b41ad9b327a9cdcfdc26a8 Mon Sep 17 00:00:00 2001 From: Mauricio Villegas <5780272+mauvilsa@users.noreply.github.com> Date: Mon, 10 Aug 2026 07:22:42 +0200 Subject: [PATCH 1/2] Use docstrings of base classes to document inherited parameters and attributes --- CHANGELOG.rst | 73 ++++++++++++-------------- DOCUMENTATION.rst | 8 +++ jsonargparse/_optionals.py | 51 +++++++++++++----- jsonargparse_tests/test_attrs.py | 24 ++++++++- jsonargparse_tests/test_dataclasses.py | 31 +++++++++++ jsonargparse_tests/test_pydantic.py | 49 +++++++++++++++++ jsonargparse_tests/test_signatures.py | 23 ++++++++ jsonargparse_tests/test_typehints.py | 4 +- 8 files changed, 208 insertions(+), 55 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 43c83d33..8c4f0b9b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -39,10 +39,8 @@ Added `__). - Support ``types.UnionType`` and ``types.GenericAlias`` as types, often seen in third party libraries in unions such as ``type | UnionType | dict``. The value - is a string with a type expression, e.g. ``"int | str"`` and ``"list[int]"``. - Previously adding an argument with these types failed with ``TypeError: - 'member_descriptor' object is not iterable`` (`#945 - `__). + is a string with a type expression, e.g. ``"int | str"`` and ``"list[int]"`` + (`#945 `__). - Support ``Collection``, ``Container`` and ``Reversible``, validated as a list, and ``AbstractSet``, validated as a set (`#950 `__). @@ -69,21 +67,17 @@ Fixed scripts themselves were not affected (`#947 `__). - ``fail_untyped=True`` failing for mandatory parameters that do have a type, - with an error that says the parameter "does not specify a type". This happened - for any type that jsonargparse can't validate, since the parameter was skipped, - making it indistinguishable from an untyped one. Now ``fail_untyped`` only fails - for parameters that have no type at all (`#948 + with an error that says the parameter "does not specify a type". Now it only + fails for parameters that have no type at all (`#948 `__). - Signature parameters with a pydantic type nested in a container, e.g. - ``list[HttpUrl]``, being skipped. Only pydantic types given as the entire type - of a parameter were registered for validation (`#948 + ``list[HttpUrl]``, being skipped (`#948 `__). - ``dump``, and thus ``--print_config``, failing when the value of an ``Any`` - typed argument is a class instance that the config format can't represent, e.g. - a default that is an arbitrary object. Now these values are serialized the same - as the instances given for a subclass type, i.e. as an import path when the - value can be imported back, otherwise as a message that says that it was not - serializable (`#948 `__). + typed argument is a class instance that the config format can't represent. Now + these values are serialized as an import path, or as a message that says that + it was not serializable, see :ref:`unvalidated-types` (`#948 + `__). - ``AssertionError`` without a message when adding an argument typed as a subscripted user defined generic class, e.g. ``Optional[Strategy[T]]`` (`#950 `__). @@ -97,20 +91,27 @@ Fixed `__). - Parameters of a subscripted generic class being dropped when their type is a PEP 604 union, e.g. ``p: int | None`` in a ``Generic[T]`` class added as - ``MyClass[int]`` (`#950 `__). + ``MyClass[int]`` (`#950 + `__). +- Docstrings of base classes not being used to document inherited parameters and + attributes, e.g. the attribute docstrings of a pydantic model declared in a + base model not being shown in the help. Now the entire method resolution order + is searched (`#951 `__). +- The description of a group being taken from an inherited ``__init__`` + docstring of a base class from another package, most notably pydantic models + without a docstring getting ``Create a new model by parsing and validating + input data from keyword arguments``. Now the nearest class docstring in the + method resolution order is used, skipping base classes that only provide + machinery (`#951 `__). Changed ^^^^^^^ - Signature parameters with a type that jsonargparse can't validate are now - accepted instead of skipped. A type can't be validated when it fails to - resolve, e.g. a missing import or a typo in a postponed annotation, or when it - is not a supported type. Only the parts of the type that can't be validated - accept any value, e.g. a ``list[SomeType]`` still requires a list, and the - subtypes of a ``Union`` that can't be validated are no longer silently - discarded. These parts are shown in the help as ``Unvalidated<...>``, making - evident which type is not validated, and a debug log states the reason. See - the new documentation section :ref:`unvalidated-types` (`#936 - `__, `#944 + accepted instead of skipped. Only the parts of the type that can't be + validated accept any value, e.g. a ``list[SomeType]`` still requires a list. + These parts are shown in the help as ``Unvalidated<...>`` and a debug log + states the reason. See the new documentation section :ref:`unvalidated-types` + (`#936 `__, `#944 `__, `#948 `__). - ``Required`` and ``NotRequired`` given as the type of an argument are no @@ -120,11 +121,9 @@ Changed - Whether a class implements a ``Protocol`` is now decided by checking that its methods can be called in all the ways that the protocol methods can be called, similar to what static type checkers do, instead of requiring the parameter - lists to be identical. Among others, this means that names of positional-only - parameters are ignored, ``*args``/``**kwargs`` in the implementation can stand - in for protocol parameters, and extra optional parameters in the - implementation are accepted. Parameter and return types must still match - exactly, except when the protocol has no annotation or ``Any`` (`#941 + lists to be identical. This accepts more implementations than before. + Parameter and return types must still match exactly, except when the protocol + has no annotation or ``Any`` (`#941 `__). - The default print config argument name will remain as ``--print_config`` in v5.0.0, no longer changing as described in the deprecated section of v4.35.0. @@ -133,14 +132,12 @@ Changed silently skipped. ``Namespace`` is only intended for parsing results (`#948 `__). - The subtypes of a ``Union`` are now sorted when the argument is added, instead - of only while parsing. This means that the type shown in the help tells the - order in which the subtypes are attempted. The subtypes that accept any value, - i.e. ``Any`` and the ones that can't be validated, are now moved to the end, - so that they no longer prevent the remaining subtypes from being attempted. - The same is done for ``object``, which accepts the import path of any class. - The only sorting that still happens while parsing is for list append, since it - depends on the value. See the new documentation section :ref:`union-types` - (`#949 `__). + of only while parsing, so the type shown in the help tells the order in which + the subtypes are attempted. The subtypes that accept any value, i.e. ``Any``, + ``object`` and the ones that can't be validated, are now moved to the end, so + that they no longer prevent the remaining subtypes from being attempted. See + the new documentation section :ref:`union-types` (`#949 + `__). v4.50.0 (2026-07-22) diff --git a/DOCUMENTATION.rst b/DOCUMENTATION.rst index 17e59231..85ad86ac 100644 --- a/DOCUMENTATION.rst +++ b/DOCUMENTATION.rst @@ -1747,6 +1747,14 @@ that don't have attribute docstrings. To enable this, do as follows: prize: int = 100 """Amount won.""" +Docstrings are searched for in the entire class inheritance chain. Thus, +parameters and attributes that a class inherits are documented in the help by +the base class that declares them, and the description of a group is taken from +the nearest class in the method resolution order that has a docstring. Base +classes that only provide machinery, i.e. ``object``, ``abc.ABC``, +``typing.Generic``, ``enum.Enum``, ``pydantic.BaseModel`` and the like, are +skipped, since their docstrings describe themselves instead of the class being +added to the parser. .. testcleanup:: docstrings diff --git a/jsonargparse/_optionals.py b/jsonargparse/_optionals.py index c57f88dc..c9c1d44e 100644 --- a/jsonargparse/_optionals.py +++ b/jsonargparse/_optionals.py @@ -6,7 +6,6 @@ import sys from contextlib import contextmanager from copy import deepcopy -from dataclasses import is_dataclass from importlib.metadata import version from importlib.util import find_spec from typing import Any, Union @@ -229,33 +228,59 @@ def parse_docstring(component, params=False, logger=None): return None +# modules of base classes that only provide machinery, e.g. object, abc.ABC, +# typing.Generic, enum.Enum, pydantic.BaseModel, whose docstrings describe +# themselves instead of the parameters of the derived classes +docstring_skip_modules = {"abc", "attr", "attrs", "builtins", "enum", "pydantic", "typing", "typing_extensions"} + + +def is_docstring_base_source(cls) -> bool: + """Whether a base class can have docstrings that document the parameters of its subclasses.""" + return cls.__module__.split(".", 1)[0] not in docstring_skip_modules + + +def get_mro_doc_sources(cls) -> list: + """Docstring sources for a class, from the most base class to the class itself.""" + bases = [b for b in inspect.getmro(cls)[1:] if is_docstring_base_source(b)] + return bases[::-1] + [cls] + + def parse_docs(component, parent, logger): docs = {} if docstring_parser_support: - if is_dataclass(parent) and component.__name__ == "__init__": - next_mro = inspect.getmro(parent)[1] - if is_dataclass(next_mro): - docs.update(parse_docs(next_mro, next_mro.__init__, logger)) - doc_sources = [component] if inspect.isclass(parent) and component.__name__ == "__init__": - doc_sources += [parent] + # base classes first, so that descriptions in derived classes take precedence + doc_sources = get_mro_doc_sources(parent)[:-1] + [component, parent] + elif inspect.isclass(component): + doc_sources = get_mro_doc_sources(component) + else: + doc_sources = [component] for src in doc_sources: doc = parse_docstring(src, params=True, logger=logger) if doc: for param in doc.params: - docs[param.arg_name] = param.description + if param.description: + docs[param.arg_name] = param.description return docs def get_doc_short_description(function_or_class, method_name=None, logger=None): if docstring_parser_support: - component = function_or_class - if inspect.isclass(function_or_class): - if not method_name: - docstring = parse_docstring(function_or_class, params=False, logger=logger) + if inspect.isclass(function_or_class) and not method_name: + # nearest short description in the mro, since a derived class often inherits the constructor + for cls in get_mro_doc_sources(function_or_class)[::-1]: + docstring = parse_docstring(cls, params=False, logger=logger) if docstring and docstring.short_description: return docstring.short_description - component = getattr(function_or_class, method_name or "__init__") + init = cls.__dict__.get("__init__") + if init is not None: + # the class defines its own constructor, so base classes don't describe it + docstring = parse_docstring(init, params=False, logger=logger) + return docstring.short_description if docstring else None + return None + component = function_or_class + if inspect.isclass(function_or_class): + component = getattr(function_or_class, method_name) docstring = parse_docstring(component, params=False, logger=logger) if docstring: return docstring.short_description diff --git a/jsonargparse_tests/test_attrs.py b/jsonargparse_tests/test_attrs.py index f83196f6..9ce02d2a 100644 --- a/jsonargparse_tests/test_attrs.py +++ b/jsonargparse_tests/test_attrs.py @@ -1,12 +1,13 @@ from __future__ import annotations from typing import List +from unittest.mock import patch import pytest -from jsonargparse import Namespace +from jsonargparse import Namespace, set_parsing_settings from jsonargparse._optionals import attrs_support -from jsonargparse_tests.conftest import get_parser_help +from jsonargparse_tests.conftest import get_parser_help, skip_if_docstring_parser_unavailable if attrs_support: import attrs @@ -46,6 +47,16 @@ class AttrsWithNestedDataclassNoDefault: p1: float subfield: AttrsSubField + @attrs.define + class AttrsAttrDocsBase: + p1: str = "-" + """p1 description""" + + @attrs.define + class AttrsAttrDocsSub(AttrsAttrDocsBase): + p2: int = 2 + """p2 description""" + @pytest.mark.skipif(not attrs_support, reason="attrs package is required") class TestAttrs: @@ -87,3 +98,12 @@ def test_nested_without_default(self, parser): parser.add_argument("--data", type=AttrsWithNestedDataclassNoDefault) cfg = parser.parse_args(["--data.p1=1.23"]) assert cfg.data == Namespace(p1=1.23, subfield=Namespace(p1="-", p2=0)) + + @skip_if_docstring_parser_unavailable + @patch.dict("jsonargparse._optionals._docstring_parse_options") + def test_attribute_docstrings_inherited(self, parser): + set_parsing_settings(docstring_parse_attribute_docstrings=True) + parser.add_class_arguments(AttrsAttrDocsSub, "d") + help_str = get_parser_help(parser) + assert "p1 description (type: str, default: -)" in help_str + assert "p2 description (type: int, default: 2)" in help_str diff --git a/jsonargparse_tests/test_dataclasses.py b/jsonargparse_tests/test_dataclasses.py index d050dad8..ced35778 100644 --- a/jsonargparse_tests/test_dataclasses.py +++ b/jsonargparse_tests/test_dataclasses.py @@ -414,6 +414,37 @@ def test_attribute_docstrings(parser): assert "attr_int description (type: int, default: 1)" in help_str +@dataclasses.dataclass +class WithAttrDocsBase: + """Base description.""" + + attr_base: str = "b" + "attr_base description" + + +@dataclasses.dataclass +class WithAttrDocsMid(WithAttrDocsBase): + attr_mid: int = 1 + "attr_mid description" + + +@dataclasses.dataclass +class WithAttrDocsSub(WithAttrDocsMid): + attr_sub: float = 0.1 + "attr_sub description" + + +@skip_if_docstring_parser_unavailable +@patch.dict("jsonargparse._optionals._docstring_parse_options") +def test_attribute_docstrings_inherited(parser): + set_parsing_settings(docstring_parse_attribute_docstrings=True) + parser.add_class_arguments(WithAttrDocsSub) + help_str = get_parser_help(parser) + assert "attr_base description (type: str, default: b)" in help_str + assert "attr_mid description (type: int, default: 1)" in help_str + assert "attr_sub description (type: float, default: 0.1)" in help_str + + @dataclasses.dataclass class Data: p1: str diff --git a/jsonargparse_tests/test_pydantic.py b/jsonargparse_tests/test_pydantic.py index a856bd89..e8e7a04e 100644 --- a/jsonargparse_tests/test_pydantic.py +++ b/jsonargparse_tests/test_pydantic.py @@ -5,6 +5,7 @@ import pathlib from copy import deepcopy from typing import Dict, List, Literal, Optional, Union +from unittest.mock import patch import pytest @@ -21,6 +22,7 @@ get_parse_args_stdout, get_parser_help, json_or_yaml_load, + skip_if_docstring_parser_unavailable, ) if pydantic_support: @@ -416,6 +418,53 @@ def test_pydantic_model_path_fields(parser, file_r): parser.parse_args([f"--model.file={file_r}", "--model.dir=not_exist"]) +if pydantic_support: + + class ModelAttrDocsBase(pydantic.BaseModel): + """Base model description.""" + + p1: str = "-" + """p1 description""" + + class ModelAttrDocsMid(ModelAttrDocsBase): + p2: int = 2 + """p2 description""" + + class ModelAttrDocsSub(ModelAttrDocsMid): + p3: float = 0.3 + """p3 description""" + + class ModelWithoutDocs(pydantic.BaseModel): + p1: str = "-" + + +@skip_if_docstring_parser_unavailable +@patch.dict("jsonargparse._optionals._docstring_parse_options") +def test_pydantic_attribute_docstrings_inherited(parser): + set_parsing_settings(docstring_parse_attribute_docstrings=True) + parser.add_class_arguments(ModelAttrDocsSub, "s") + help_str = get_parser_help(parser) + assert "p1 description (type: str, default: -)" in help_str + assert "p2 description (type: int, default: 2)" in help_str + assert "p3 description (type: float, default: 0.3)" in help_str + + +def test_pydantic_group_description_from_base(parser): + parser.add_class_arguments(ModelAttrDocsSub, "s") + help_str = get_parser_help(parser) + assert "Create a new model by parsing" not in help_str + if docstring_parser_support: + assert "Base model description:" in help_str + + +def test_pydantic_group_description_without_docstrings(parser): + parser.add_class_arguments(ModelWithoutDocs, "n") + help_str = get_parser_help(parser) + assert "Create a new model by parsing" not in help_str + assert "A base class for creating Pydantic models" not in help_str + assert f":" in help_str + + if pydantic_support: class Pet(pydantic.BaseModel): diff --git a/jsonargparse_tests/test_signatures.py b/jsonargparse_tests/test_signatures.py index 7a469cc3..cacff913 100644 --- a/jsonargparse_tests/test_signatures.py +++ b/jsonargparse_tests/test_signatures.py @@ -414,6 +414,29 @@ def test_add_class_docstring_parse_fail(parser, logger): assert "a1 description" not in help_str +class WithDocstringBase: + """WithDocstringBase short description. + + Args: + b1: b1 description + """ + + def __init__(self, b1: int = 1): + pass # pragma: no cover + + +class WithoutOwnDocstring(WithDocstringBase): + pass + + +@skip_if_docstring_parser_unavailable +def test_add_class_group_description_from_base(parser): + parser.add_class_arguments(WithoutOwnDocstring, "w") + help_str = get_parser_help(parser) + assert "WithDocstringBase short description:" in help_str + assert "b1 description" in help_str + + def test_add_class_custom_instantiator(parser, clear_instantiators): def instantiate(cls, **kwargs): instance = cls(**kwargs) diff --git a/jsonargparse_tests/test_typehints.py b/jsonargparse_tests/test_typehints.py index 672ce7d5..1852120a 100644 --- a/jsonargparse_tests/test_typehints.py +++ b/jsonargparse_tests/test_typehints.py @@ -519,7 +519,7 @@ def test_list_variants(parser, list_type): class WithCollection: def __init__(self, allowed: Optional[Collection[str]] = None): - self.allowed = allowed + self.allowed = allowed # pragma: no cover def test_collection_signature_parameter(parser): @@ -1062,7 +1062,7 @@ def test_module_type_union_with_callable_dump(parser): class WithCallableDefault: def __init__(self, cb: Callable = uuid.uuid4): - self.cb = cb + self.cb = cb # pragma: no cover def test_module_type_union_with_class_dump(parser): From 3fc73f23e48d96005c1a61f1f80e652c62fa1008 Mon Sep 17 00:00:00 2001 From: Mauricio Villegas <5780272+mauvilsa@users.noreply.github.com> Date: Mon, 10 Aug 2026 07:28:44 +0200 Subject: [PATCH 2/2] Tox coverage in parallel --- .pre-commit-config.yaml | 4 ++-- pyproject.toml | 13 +++++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5244fc59..a597e797 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -96,8 +96,8 @@ repos: verbose: true - id: tox - name: tox --parallel - entry: tox --parallel + name: tox --parallel --parallel-no-spinner + entry: tox --parallel --parallel-no-spinner stages: [pre-push] language: system pass_filenames: false diff --git a/pyproject.toml b/pyproject.toml index 4048d1e9..1a6415a7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -189,6 +189,8 @@ Villegas = "Villegas" legacy_tox_ini = """ [tox] envlist = py{310,311,312,313,314}-{all-extras,no-extras,argparse},omegaconf,pydantic-v1,without-pyyaml,without-future-annotations +labels = + coverage = py{310,311,312,313,314}-{all-extras,no-extras},pydantic-v1,without-pyyaml,without-future-annotations skip_missing_interpreters = true [testenv] @@ -202,6 +204,7 @@ passenv = UV_EXCLUDE_NEWER # Ensure uv installs into the tox env even if UV_SYSTEM_PYTHON=1 set outside setenv = UV_SYSTEM_PYTHON = 0 + COVERAGE_FILE = {tox_root}/jsonargparse_tests/.coverage.{env_name} commands = all-extras: python -m pytest {posargs} no-extras: python -m pytest {posargs} @@ -271,4 +274,14 @@ commands = python -m pytest /tmp/_without_future_annotations {posargs} commands_post = sh -c "rm -rf /tmp/_without_future_annotations" + +[testenv:coverage-report] +skip_install = true +deps = coverage[toml] +changedir = jsonargparse_tests +setenv = + COVERAGE_FILE = {tox_root}/jsonargparse_tests/.coverage +commands = + coverage combine + coverage html """